From d309109f2b6b90d511118eaa54181d62d6ce6acc Mon Sep 17 00:00:00 2001 From: bench Date: Sat, 19 Sep 2026 17:52:03 +0800 Subject: [PATCH] pre-grind snapshot 2026-09-19: batches 1-7 adapters/probes/recipes + eval additive changes --- CONTINUE.md | 91 + CONTRACT.md | 38 +- HANDBOOK.md | 29 +- README.md | 53 +- agent/SYSTEM.md | 11 +- agent/VERSION | 2 +- agent/prompts/e2e.md | 15 +- bin/stageclient.py | 2 +- config.yaml | 8 +- docs/BPU量化轨迹工厂-技术分享讲稿-20260916.md | 475 ++ docs/BPU量化轨迹工厂-方法论.html | 498 ++ docs/BPU量化轨迹工厂-设计方案.html | 790 +++ docs/BPU量化轨迹工厂-闭环方法论.html | 460 ++ docs/README.md | 18 + PLAN.md => docs/archive/PLAN.md | 0 .../archive/REPORT-codex.md | 2 +- .../archive/SPEC-two-phase-compile.md | 5 + {scripts => docs/archive}/evidence-codex.md | 0 docs/board-process-lifecycle.md | 26 + export/reward.py | 64 +- export/test_materialization.py | 106 +- export/to_agentrl.py | 71 +- harness/adapters/__init__.py | 45 +- harness/adapters/aliked.py | 12 + harness/adapters/atss_r50_fpn.py | 12 + harness/adapters/bisenetv1_r18.py | 7 + harness/adapters/bisenetv2.py | 7 + harness/adapters/conditional_detr_r50.py | 114 + harness/adapters/d2_net.py | 69 + harness/adapters/dab_detr_r50.py | 12 + harness/adapters/ddrnet_23.py | 7 + harness/adapters/ddrnet_23_slim.py | 7 + harness/adapters/dedode_v2.py | 74 + harness/adapters/deeplabv3_resnet101.py | 7 + harness/adapters/deeplabv3_resnet50.py | 7 + harness/adapters/deeplabv3plus_resnet101.py | 7 + harness/adapters/deeplabv3plus_resnet50.py | 7 + harness/adapters/deformable_detr_r50.py | 12 + harness/adapters/depth_anything_v1_base.py | 10 + harness/adapters/depth_anything_v1_common.py | 87 + harness/adapters/depth_anything_v1_large.py | 10 + harness/adapters/depth_anything_v1_small.py | 10 + harness/adapters/depth_anything_v2_base.py | 10 + harness/adapters/depth_anything_v2_common.py | 94 + harness/adapters/depth_anything_v2_large.py | 10 + .../depth_anything_v2_metric_common.py | 120 + .../depth_anything_v2_metric_hypersim_base.py | 12 + ...depth_anything_v2_metric_hypersim_large.py | 12 + ...depth_anything_v2_metric_hypersim_small.py | 12 + .../depth_anything_v2_metric_vkitti_base.py | 12 + .../depth_anything_v2_metric_vkitti_large.py | 12 + .../depth_anything_v2_metric_vkitti_small.py | 12 + harness/adapters/depth_anything_v2_small.py | 10 + harness/adapters/detr_r50.py | 12 + harness/adapters/disk.py | 12 + harness/adapters/dpt_hybrid.py | 19 + harness/adapters/dpt_large.py | 19 + harness/adapters/efficientvit_b0.py | 10 + harness/adapters/efficientvit_b1.py | 10 + harness/adapters/efficientvit_b2.py | 10 + harness/adapters/efficientvit_b3.py | 10 + harness/adapters/efficientvit_common.py | 163 + harness/adapters/efficientvit_seg_b0.py | 14 + harness/adapters/efficientvit_seg_b1.py | 14 + harness/adapters/efficientvit_seg_b2.py | 14 + harness/adapters/efficientvit_seg_b3.py | 14 + harness/adapters/efficientvit_seg_common.py | 94 + harness/adapters/fast_scnn.py | 13 + harness/adapters/fcn_resnet101.py | 7 + harness/adapters/fcn_resnet50.py | 7 + harness/adapters/fcos_r50_fpn.py | 12 + harness/adapters/feature_common.py | 215 + harness/adapters/feature_sources.py | 79 + harness/adapters/feature_sources3.py | 39 + harness/adapters/foveabox_r50_fpn.py | 12 + harness/adapters/fsaf_r50_fpn.py | 12 + harness/adapters/gfl_r50_fpn.py | 12 + harness/adapters/hrnet_w18.py | 7 + harness/adapters/hrnet_w32.py | 9 + harness/adapters/hrnet_w48.py | 7 + harness/adapters/hrnet_w48_2.py | 9 + harness/adapters/keynetplushardnet.py | 129 + .../adapters/mask2former_swin_b_instance.py | 22 + .../adapters/mask2former_swin_s_instance.py | 17 + .../adapters/mask2former_swin_t_instance.py | 17 + harness/adapters/midas_common.py | 242 + harness/adapters/midas_small.py | 19 + harness/adapters/mmdet_dense_common.py | 222 + harness/adapters/mmdet_detr_common.py | 141 + harness/adapters/mmdet_iseg_common.py | 290 + harness/adapters/mmdet_solo_common.py | 209 + harness/adapters/mmseg_common.py | 83 + harness/adapters/ocrnet_hrnet_w18.py | 17 + harness/adapters/ocrnet_hrnet_w48.py | 17 + harness/adapters/openmmlab.py | 184 + harness/adapters/openmmlab_recipes.yaml | 5097 +++++++++++++++++ harness/adapters/pidnet_l.py | 17 + harness/adapters/pidnet_m.py | 17 + harness/adapters/pidnet_s.py | 17 + harness/adapters/pspnet_resnet101.py | 7 + harness/adapters/pspnet_resnet50.py | 7 + harness/adapters/r2d2.py | 76 + harness/adapters/raft_common.py | 88 + harness/adapters/raft_large.py | 11 + harness/adapters/raft_small.py | 11 + harness/adapters/reppoints_r50_fpn.py | 12 + harness/adapters/retinanet_r101_fpn.py | 12 + harness/adapters/retinanet_r50_fpn.py | 12 + harness/adapters/rtmdet_common.py | 146 + harness/adapters/rtmdet_l.py | 7 + harness/adapters/rtmdet_m.py | 7 + harness/adapters/rtmdet_s.py | 7 + harness/adapters/rtmdet_tiny.py | 7 + harness/adapters/rtmdet_x.py | 7 + harness/adapters/rtmpose_common.py | 228 + harness/adapters/rtmpose_l.py | 7 + harness/adapters/rtmpose_m.py | 7 + harness/adapters/rtmpose_s.py | 7 + harness/adapters/rtmpose_tiny.py | 7 + harness/adapters/rtmpose_x.py | 8 + harness/adapters/segformer_b0.py | 8 + harness/adapters/segformer_b1.py | 8 + harness/adapters/segformer_b2.py | 8 + harness/adapters/segformer_b3.py | 8 + harness/adapters/segformer_b4.py | 8 + harness/adapters/segformer_b5.py | 8 + harness/adapters/segnext_base.py | 13 + harness/adapters/segnext_large.py | 13 + harness/adapters/segnext_small.py | 13 + harness/adapters/segnext_tiny.py | 13 + harness/adapters/simcc_resnet50.py | 9 + harness/adapters/solo_r50.py | 16 + harness/adapters/solov2_r50.py | 16 + harness/adapters/ssd300_vgg16.py | 108 + .../adapters/ssdlite320_mobilenetv3_large.py | 19 + harness/adapters/ssdlite_common.py | 123 + harness/adapters/stdc1_seg.py | 17 + harness/adapters/stdc2_seg.py | 17 + harness/adapters/superpoint.py | 12 + harness/adapters/topdown_pose_common.py | 149 + harness/adapters/upernet_resnet50.py | 7 + harness/adapters/upernet_swin_b.py | 7 + harness/adapters/upernet_swin_s.py | 7 + harness/adapters/upernet_swin_t.py | 7 + harness/adapters/vfnet_deformconv.py | 64 + harness/adapters/vfnet_r50_fpn.py | 12 + harness/adapters/xfeat.py | 70 + harness/adapters/yolox_common.py | 153 + harness/adapters/yolox_l.py | 29 + harness/adapters/yolox_m.py | 29 + harness/adapters/yolox_s.py | 29 + harness/adapters/yolox_tiny.py | 29 + harness/adapters/yolox_x.py | 29 + harness/backends/ptq.py | 2 +- harness/cli.py | 81 +- harness/continuation.py | 305 + harness/coverage.py | 245 +- harness/db.py | 25 +- harness/errors.py | 3 +- harness/eval/__init__.py | 9 +- harness/eval/cls.py | 9 +- harness/eval/coco_extra.py | 292 + harness/eval/common.py | 7 +- harness/eval/det.py | 24 +- harness/eval/feat.py | 127 + harness/eval/flow.py | 121 + harness/eval/mde.py | 167 + harness/eval/run.py | 28 +- harness/eval/seg.py | 152 + harness/gates.py | 114 +- harness/paths.py | 6 + harness/pipeline.py | 293 +- harness/registry.py | 19 +- harness/rollout.py | 91 +- harness/scheduler.py | 7 +- harness/stagerun.py | 5 +- harness/stages/__init__.py | 2 +- harness/stages/base.py | 22 +- harness/stages/s01_fetch.py | 24 +- harness/stages/s02_baseline.py | 15 +- harness/stages/s03_export.py | 8 +- harness/stages/s04_calibrate.py | 18 +- harness/stages/s05_compile.py | 7 +- harness/stages/s06_hbm.py | 18 +- harness/stages/s08_eval.py | 2 +- harness/tools/board_bench.py | 29 +- harness/tools/board_guard.py | 155 + harness/tools/board_perf.py | 73 +- harness/tools/export_onnx.py | 35 +- harness/verify.py | 46 +- harness/watchdog.py | 71 +- scripts/_probe.py | 24 - scripts/audit_rollouts.py | 9 +- scripts/gate_summary.py | 49 - scripts/jail_setup.py | 5 +- scripts/make_calib.py | 47 +- scripts/make_t1.py | 100 +- scripts/probe_classification_adapters.py | 126 + scripts/probe_coco_adapters.py | 119 + scripts/probe_da_metric_adapters.py | 417 ++ scripts/probe_depth_adapters.py | 432 ++ scripts/probe_feature_adapters.py | 161 + scripts/probe_midas_adapters.py | 420 ++ scripts/probe_mmdet_dense_adapters.py | 375 ++ scripts/probe_mmdet_detr2_adapters.py | 410 ++ scripts/probe_mmdet_detr_adapters.py | 403 ++ scripts/probe_mmdet_iseg_adapters.py | 479 ++ scripts/probe_raft_adapter.py | 400 ++ scripts/probe_rtmdet_adapters.py | 312 + scripts/probe_rtmpose_adapters.py | 299 + scripts/probe_seg_adapters.py | 147 + scripts/probe_ssd_yolox_adapters.py | 281 + scripts/probe_ssdlite_adapter.py | 252 + scripts/probe_topdown_adapters.py | 237 + scripts/start_run.sh | 8 +- scripts/sync.sh | 12 +- scripts/sync_patch.py | 81 + scripts/sync_status.py | 149 + scripts/test_accuracy_policy.py | 43 +- scripts/test_adapter_recipes.py | 130 + scripts/test_agent_first.py | 163 + scripts/test_board_guard.py | 144 + scripts/test_board_pipeline.py | 3 +- scripts/test_board_runtime.py | 29 +- scripts/test_coco_extra.py | 58 + scripts/test_codex.py | 14 +- scripts/test_collection.py | 91 +- scripts/test_completion.py | 14 + scripts/test_continuation.py | 611 ++ scripts/test_coverage.py | 20 + scripts/test_depth_gate.py | 59 + scripts/test_eval_provenance.py | 155 + scripts/test_eval_subset.py | 225 + scripts/test_explicit_batch.py | 41 + scripts/test_generation.py | 26 +- scripts/test_mmseg_static_pool.py | 31 + scripts/test_pose_gate.py | 296 + scripts/test_reward_board_offline.py | 12 +- scripts/test_rollout_backfill.py | 3 +- scripts/test_sync_patch.py | 36 + 240 files changed, 23005 insertions(+), 564 deletions(-) create mode 100644 CONTINUE.md create mode 100644 docs/BPU量化轨迹工厂-技术分享讲稿-20260916.md create mode 100644 docs/BPU量化轨迹工厂-方法论.html create mode 100644 docs/BPU量化轨迹工厂-设计方案.html create mode 100644 docs/BPU量化轨迹工厂-闭环方法论.html create mode 100644 docs/README.md rename PLAN.md => docs/archive/PLAN.md (100%) rename REPORT-codex.md => docs/archive/REPORT-codex.md (99%) rename SPEC-two-phase-compile.md => docs/archive/SPEC-two-phase-compile.md (97%) rename {scripts => docs/archive}/evidence-codex.md (100%) create mode 100644 docs/board-process-lifecycle.md create mode 100644 harness/adapters/aliked.py create mode 100644 harness/adapters/atss_r50_fpn.py create mode 100644 harness/adapters/bisenetv1_r18.py create mode 100644 harness/adapters/bisenetv2.py create mode 100644 harness/adapters/conditional_detr_r50.py create mode 100644 harness/adapters/d2_net.py create mode 100644 harness/adapters/dab_detr_r50.py create mode 100644 harness/adapters/ddrnet_23.py create mode 100644 harness/adapters/ddrnet_23_slim.py create mode 100644 harness/adapters/dedode_v2.py create mode 100644 harness/adapters/deeplabv3_resnet101.py create mode 100644 harness/adapters/deeplabv3_resnet50.py create mode 100644 harness/adapters/deeplabv3plus_resnet101.py create mode 100644 harness/adapters/deeplabv3plus_resnet50.py create mode 100644 harness/adapters/deformable_detr_r50.py create mode 100644 harness/adapters/depth_anything_v1_base.py create mode 100644 harness/adapters/depth_anything_v1_common.py create mode 100644 harness/adapters/depth_anything_v1_large.py create mode 100644 harness/adapters/depth_anything_v1_small.py create mode 100644 harness/adapters/depth_anything_v2_base.py create mode 100644 harness/adapters/depth_anything_v2_common.py create mode 100644 harness/adapters/depth_anything_v2_large.py create mode 100644 harness/adapters/depth_anything_v2_metric_common.py create mode 100644 harness/adapters/depth_anything_v2_metric_hypersim_base.py create mode 100644 harness/adapters/depth_anything_v2_metric_hypersim_large.py create mode 100644 harness/adapters/depth_anything_v2_metric_hypersim_small.py create mode 100644 harness/adapters/depth_anything_v2_metric_vkitti_base.py create mode 100644 harness/adapters/depth_anything_v2_metric_vkitti_large.py create mode 100644 harness/adapters/depth_anything_v2_metric_vkitti_small.py create mode 100644 harness/adapters/depth_anything_v2_small.py create mode 100644 harness/adapters/detr_r50.py create mode 100644 harness/adapters/disk.py create mode 100644 harness/adapters/dpt_hybrid.py create mode 100644 harness/adapters/dpt_large.py create mode 100644 harness/adapters/efficientvit_b0.py create mode 100644 harness/adapters/efficientvit_b1.py create mode 100644 harness/adapters/efficientvit_b2.py create mode 100644 harness/adapters/efficientvit_b3.py create mode 100644 harness/adapters/efficientvit_common.py create mode 100644 harness/adapters/efficientvit_seg_b0.py create mode 100644 harness/adapters/efficientvit_seg_b1.py create mode 100644 harness/adapters/efficientvit_seg_b2.py create mode 100644 harness/adapters/efficientvit_seg_b3.py create mode 100644 harness/adapters/efficientvit_seg_common.py create mode 100644 harness/adapters/fast_scnn.py create mode 100644 harness/adapters/fcn_resnet101.py create mode 100644 harness/adapters/fcn_resnet50.py create mode 100644 harness/adapters/fcos_r50_fpn.py create mode 100644 harness/adapters/feature_common.py create mode 100644 harness/adapters/feature_sources.py create mode 100644 harness/adapters/feature_sources3.py create mode 100644 harness/adapters/foveabox_r50_fpn.py create mode 100644 harness/adapters/fsaf_r50_fpn.py create mode 100644 harness/adapters/gfl_r50_fpn.py create mode 100644 harness/adapters/hrnet_w18.py create mode 100644 harness/adapters/hrnet_w32.py create mode 100644 harness/adapters/hrnet_w48.py create mode 100644 harness/adapters/hrnet_w48_2.py create mode 100644 harness/adapters/keynetplushardnet.py create mode 100644 harness/adapters/mask2former_swin_b_instance.py create mode 100644 harness/adapters/mask2former_swin_s_instance.py create mode 100644 harness/adapters/mask2former_swin_t_instance.py create mode 100644 harness/adapters/midas_common.py create mode 100644 harness/adapters/midas_small.py create mode 100644 harness/adapters/mmdet_dense_common.py create mode 100644 harness/adapters/mmdet_detr_common.py create mode 100644 harness/adapters/mmdet_iseg_common.py create mode 100644 harness/adapters/mmdet_solo_common.py create mode 100644 harness/adapters/mmseg_common.py create mode 100644 harness/adapters/ocrnet_hrnet_w18.py create mode 100644 harness/adapters/ocrnet_hrnet_w48.py create mode 100644 harness/adapters/openmmlab.py create mode 100644 harness/adapters/openmmlab_recipes.yaml create mode 100644 harness/adapters/pidnet_l.py create mode 100644 harness/adapters/pidnet_m.py create mode 100644 harness/adapters/pidnet_s.py create mode 100644 harness/adapters/pspnet_resnet101.py create mode 100644 harness/adapters/pspnet_resnet50.py create mode 100644 harness/adapters/r2d2.py create mode 100644 harness/adapters/raft_common.py create mode 100644 harness/adapters/raft_large.py create mode 100644 harness/adapters/raft_small.py create mode 100644 harness/adapters/reppoints_r50_fpn.py create mode 100644 harness/adapters/retinanet_r101_fpn.py create mode 100644 harness/adapters/retinanet_r50_fpn.py create mode 100644 harness/adapters/rtmdet_common.py create mode 100644 harness/adapters/rtmdet_l.py create mode 100644 harness/adapters/rtmdet_m.py create mode 100644 harness/adapters/rtmdet_s.py create mode 100644 harness/adapters/rtmdet_tiny.py create mode 100644 harness/adapters/rtmdet_x.py create mode 100644 harness/adapters/rtmpose_common.py create mode 100644 harness/adapters/rtmpose_l.py create mode 100644 harness/adapters/rtmpose_m.py create mode 100644 harness/adapters/rtmpose_s.py create mode 100644 harness/adapters/rtmpose_tiny.py create mode 100644 harness/adapters/rtmpose_x.py create mode 100644 harness/adapters/segformer_b0.py create mode 100644 harness/adapters/segformer_b1.py create mode 100644 harness/adapters/segformer_b2.py create mode 100644 harness/adapters/segformer_b3.py create mode 100644 harness/adapters/segformer_b4.py create mode 100644 harness/adapters/segformer_b5.py create mode 100644 harness/adapters/segnext_base.py create mode 100644 harness/adapters/segnext_large.py create mode 100644 harness/adapters/segnext_small.py create mode 100644 harness/adapters/segnext_tiny.py create mode 100644 harness/adapters/simcc_resnet50.py create mode 100644 harness/adapters/solo_r50.py create mode 100644 harness/adapters/solov2_r50.py create mode 100644 harness/adapters/ssd300_vgg16.py create mode 100644 harness/adapters/ssdlite320_mobilenetv3_large.py create mode 100644 harness/adapters/ssdlite_common.py create mode 100644 harness/adapters/stdc1_seg.py create mode 100644 harness/adapters/stdc2_seg.py create mode 100644 harness/adapters/superpoint.py create mode 100644 harness/adapters/topdown_pose_common.py create mode 100644 harness/adapters/upernet_resnet50.py create mode 100644 harness/adapters/upernet_swin_b.py create mode 100644 harness/adapters/upernet_swin_s.py create mode 100644 harness/adapters/upernet_swin_t.py create mode 100644 harness/adapters/vfnet_deformconv.py create mode 100644 harness/adapters/vfnet_r50_fpn.py create mode 100644 harness/adapters/xfeat.py create mode 100644 harness/adapters/yolox_common.py create mode 100644 harness/adapters/yolox_l.py create mode 100644 harness/adapters/yolox_m.py create mode 100644 harness/adapters/yolox_s.py create mode 100644 harness/adapters/yolox_tiny.py create mode 100644 harness/adapters/yolox_x.py create mode 100644 harness/continuation.py create mode 100644 harness/eval/coco_extra.py create mode 100644 harness/eval/feat.py create mode 100644 harness/eval/flow.py create mode 100644 harness/eval/mde.py create mode 100644 harness/eval/seg.py create mode 100644 harness/tools/board_guard.py delete mode 100644 scripts/_probe.py delete mode 100644 scripts/gate_summary.py create mode 100644 scripts/probe_classification_adapters.py create mode 100644 scripts/probe_coco_adapters.py create mode 100644 scripts/probe_da_metric_adapters.py create mode 100644 scripts/probe_depth_adapters.py create mode 100644 scripts/probe_feature_adapters.py create mode 100644 scripts/probe_midas_adapters.py create mode 100644 scripts/probe_mmdet_dense_adapters.py create mode 100644 scripts/probe_mmdet_detr2_adapters.py create mode 100644 scripts/probe_mmdet_detr_adapters.py create mode 100644 scripts/probe_mmdet_iseg_adapters.py create mode 100644 scripts/probe_raft_adapter.py create mode 100644 scripts/probe_rtmdet_adapters.py create mode 100644 scripts/probe_rtmpose_adapters.py create mode 100644 scripts/probe_seg_adapters.py create mode 100644 scripts/probe_ssd_yolox_adapters.py create mode 100644 scripts/probe_ssdlite_adapter.py create mode 100644 scripts/probe_topdown_adapters.py create mode 100644 scripts/sync_patch.py create mode 100644 scripts/sync_status.py create mode 100644 scripts/test_adapter_recipes.py create mode 100644 scripts/test_agent_first.py create mode 100644 scripts/test_board_guard.py create mode 100644 scripts/test_coco_extra.py create mode 100644 scripts/test_continuation.py create mode 100644 scripts/test_depth_gate.py create mode 100644 scripts/test_eval_provenance.py create mode 100644 scripts/test_eval_subset.py create mode 100644 scripts/test_explicit_batch.py create mode 100644 scripts/test_mmseg_static_pool.py create mode 100644 scripts/test_pose_gate.py create mode 100644 scripts/test_sync_patch.py diff --git a/CONTINUE.md b/CONTINUE.md new file mode 100644 index 0000000..cfc38b2 --- /dev/null +++ b/CONTINUE.md @@ -0,0 +1,91 @@ +# 剩余任务切到 100 张 + +旧批次保存已经发生的执行,新批次只接剩下的任务。流程是:**生成独立评测包 → 预览清单 → 停止领取并收尾 → 更新清单并应用 → 单条试跑 → 恢复并发**。 + +普通流水线和 Agent 仍使用原来的 `bench run`、`bench collect`。不全量 `expand`,不使用 `collect --coverage all`。注册表默认值和校准集不改;500 张复核按需另做,这里不加入自动升级机制。 + +## 1. 先准备,旧批次继续运行 + +在有原始数据池的机器上生成固定子集。ImageNet 随机选类,每类取一张;COCO 固定随机抽图。 + +```bash +python scripts/make_t1.py --dataset imagenet1k --dataset coco2017 --n 100 --seed 20260914 +``` + +新包在 `data/t1//n100-s20260914/`,原包保留。同一份清单用于 FP32 和板端 INT8;新缓存按内容哈希和评测器版本分开存放。 + +下例的 `OLD_RUN`、`NEW_RUN` 和 `JOB_ID` 都替换成清单里的实际值。下面的 `plan` 只读数据库和文件,不领取任务、不创建 run、不停止进程: + +```bash +bench continue plan --run OLD_RUN \ + --t1 data/t1/imagenet1k/n100-s20260914/manifest.json \ + --t1 data/t1/coco2017/n100-s20260914/manifest.json > preview.json +``` + +只提供剩余任务涉及的数据集;缺少某个数据集的替换包会报错。默认生成新的 run ID,也可用 `--to NEW_RUN` 指定。清单含新旧任务、原采样序号、模式、策略、评测包身份及阻塞原因。服务器仍在执行时出现 `blockers` 是正常的,最终清单在收尾后重新生成。 + +> `bench reset-stale` 必须跑在 `plan` **之前**,不能夹在 `plan` 和 `apply` 中间:plan 把源 run 的 stages 全行打进 `source_sha256`,reset-stale 改任何一行 stage 状态都会让已生成的清单在 apply 时被拒(fail-closed,报 `source changed`),只能重新 plan。 + +`counts` 分四项: + +| 项目 | 含义 | +|---|---| +| `oracle` | 从未开始的普通任务 | +| `agent_queued` | 已经排队、从未开始的 Agent 采样 | +| `agent_due_repairs` | 新清单固定为 0,不再按 Oracle 失败补任务 | +| `conditional_repairs` | 新清单固定为 0;旧清单保留,但不再激活 | + +成功和失败都消耗原采样次数。计划 4 条、完成 1 条就只接余下 3 条,并保留它们的原 `kk`,不从零重新编号。首轮优先顺序及 `collect --first-pass` 也计入旧批次的已启动、已完成记录。 + +## 2. 工具验证完成后,停止领取并完成交接 + +```bash +bench drain --run OLD_RUN --all +bench status --run OLD_RUN +``` + +`--all` 同时设置普通流水线和 Agent 的停止领取标记。等已经领取的整个任务及核验结束、两个调度进程退出;不等全部待办跑完。运行中的旧进程必须已支持 drain,不能靠同步新代码改变它的内存逻辑。 + +收尾后,用第 1 步的 `plan` 命令重新输出 `handoff.json`,核对清单,然后: + +```bash +bench continue apply handoff.json +bench continue check --run NEW_RUN +``` + +应用时再次校验评测包、源任务状态及源注册表指纹。过期清单、还在运行、曾执行过又重置成待办的记录都会阻止交接。成功后源待办记为 `TRANSFERRED`,源 run 不再允许启动或重新铺队列;完成的结果、日志和轨迹保留原处。重复应用同一清单只核对结果,不重复创建任务。 + +工具只处理一次从原始批次到接续批次的交接;不提供连续多级迁移框架。数据库事务提交前的文件准备失败,可用同一清单重试。目标目录保存 `continuation.json` 作为交接回执。 + +## 3. 先试一条,再恢复并发 + +如果选中的任务还没跑普通流水线,先执行它;如果只剩 Agent,跳过这一条命令: + +```bash +bench run --run NEW_RUN --job JOB_ID --parallel 1 +bench collect --run NEW_RUN --job JOB_ID --parallel 1 --limit 1 +bench export --run NEW_RUN +bench continue check --run NEW_RUN +``` + +确认 FP32 和 INT8 的包哈希一致、实际评测 `n=100`、结果及 Agent 训练记录包含 `eval_set`,再按原资源配置恢复: + +```bash +bench run --run NEW_RUN +bench collect --run NEW_RUN --watch +``` + +两条恢复命令分别在现有终端/进程管理方式下启动;不新增守护服务。`check` 比较具体任务及采样序号,不只比较总数。试跑后待办数会减少,但“待办 + 执行中 + 已结束”仍与交接数量对应。历史 QUEUED repair 保留编号但不再领取,须与可执行的 e2e 待办分开统计。 + +已有任务快照中的 `oracle_ref` 保留历史身份;新执行与判定不读取旧 Oracle。Agent 按需调用已有输入准备阶段,缓存按评测集区分,不重跑整条普通流水线。新交接只保留实际存在的采样行,不补 Oracle 失败驱动的 repair。 + +100 张保留现有 `pass-screen` 筛查口径和原精度门槛。奖励 v0.6 只使用本次实测,不再与 Oracle 比较。训练记录保留原 run、job、采样序号和评测集身份,训练端按 `eval_set` 区分可比较的组。 + +## 本地验证范围 + +单元与集成测试使用临时 SQLite、合成评测包和模拟执行器,验证抽样、任务守恒、交接恢复、调度、缓存命令及导出。它们不能替代真实 100 张图片上的 FP32/INT8 试跑,也不能替代 Linux 板端和 Agent 沙箱验证。 + +## 已知未修问题(从 PLAN.md §7 迁入) + +- **events.jsonl 在爆**:convnext.small #0 才 7 轮就 6.0 MB,其中 5.53 MB 是 1548 条 `message_update`(流式 token 增量)。落盘时过滤掉,或跑完压成 `.raw.zst` 只留结构事件。124 条 rollout 就是几个 GB,而 audit / 导出每次都要全文扫。 +- **heartbeat 健康时零输出**,无法区分"都健康"和"它什么都没看到"。健康时也打一行。 diff --git a/CONTRACT.md b/CONTRACT.md index a66b404..89fa0b0 100644 --- a/CONTRACT.md +++ b/CONTRACT.md @@ -4,9 +4,19 @@ 2026-09-16 按当前目标收敛:默认运行 CSV 中 ready 模型的 S100P(`nash-m`)量化、编译、上板与精度评测,并为这些模型采集真实 Agent 轨迹。`collect --coverage all` 为本 run 的 oracle 模型补齐 e2e 计划:只去掉 matrix 的 priority 限制,保留任务类型、march、backend、policy 和 k;默认 `matrix` 仍使用原选择范围。已结束的流程和 Agent 轨迹都必须导出为数据,成功、精度未达标、预算耗尽分别标注,导出不能阻塞上板。 +2026-09-16 全 338 项覆盖补充:保持原 CSV 标签和历史 run。新批次可用 `expand --model` 显式选择模型, +前置检查依据实际权重入口、构建/导出代码、评测器、校准、数据与附加门;新增 Ultralytics 任务还必须有真实 +checkpoint/ONNX/指标对照证据,缺项即拒绝入队。具名批次只放宽 priority/task/status 选择,保留原 march、 +backend、policy、k 和精度门。任务快照记录 admission 来源,不能把 admission 或 probe 等同于编译/上板通过。 +`collect --first-pass` 每个 job 只领取首条,重复采样仍保持 QUEUED,不改 kk、不补造轨迹。 +`coverage --all-runs` 总表包含全部 CSV 行,独立列出前置阻塞、实际阶段、原始响应中的 teacher 身份和训练导出; +传统算法仍保留条目并说明 PTQ 不适用。FastSAM 的类别语义未补齐前不能入队。 +Pose 须有真实导出的关键点对应证据;板端同时判定 OKS AP 与平均原图坐标偏移 ≤2px。 +对应由同图人框 IoU≥0.5 一对一匹配确定,统计全部17点,保留未匹配记录;无匹配或缺证据为 UNVERIFIED。 + ## 0. 三条原则 -1. **harness 是 oracle。** 每个 job 先由确定性流水线跑出 ground truth(`oracle/result.json`)。agent 的表现永远相对 oracle 衡量。 +1. **Agent 独立完成任务。** GLM-5.3 可直接领取 e2e,自行调用输入准备、量化、编译和上板工具;`bench run` 可选地预热 01–03 缓存。harness 分配资源、核验本次 HBM 的实测精度并保存轨迹。完整 Oracle 仅用显式 `bench run --oracle` 作历史对照或排查,不参与 Agent 执行、成功判定、奖励或 SFT 筛选。 2. **agent 只有一个可写区。** `rollouts//workspace/`。`job.json`、`harness/eval/`、`data/`、`cache/` 全部只读。agent 说什么不算数,verdict 才算数。 3. **Append-only。** `run_id` 冻结;换工具链、改配置、修 bug 都开新 run。`manifest.json` 记两套 venv 的 `pip freeze` sha256,job 启动前比对,不一致拒跑。状态只在 `state/jobs.sqlite`,文件系统只存内容。 @@ -58,7 +68,8 @@ bench/ ## 3. 状态机 -**job(由 oracle 决定)**:`PENDING → ORACLE_RUNNING → PASS | FAIL | UNVERIFIED` +**job 输入准备**:`PENDING → ORACLE_RUNNING → PREPARED | FAIL | UNVERIFIED`(沿用现有运行态名称)。 +`PREPARED` 只表示 01–03 成功,写 `oracle/preparation.json`,不产生 Oracle verdict,也不代表适配或精度通过。显式 `--oracle` 可从 PENDING/PREPARED 跑完整 01–08,写 `oracle/result.json`,终态仍为 PASS/FAIL/UNVERIFIED。默认准备命令不覆盖已结束任务。 **rollout(由实际核验结果决定)**:`QUEUED → RUNNING → VERIFYING → PASS | FAIL | BLOCKED | UNVERIFIED` Agent 执行是否结束/超预算独立记录在 `verdict.execution_terminal`、`timeout_reason`,不再用 `TIMEOUT` 覆盖已经核验的结果。`completion` 分别记录编译、HBM、真实上板、精度评测是否完成及精度是否达标。预算用尽但必选门已通过仍为 PASS;没做到的阶段为未完成,不凭预算耗尽推断模型失败。旧 TIMEOUT 可用 `bench verify` 按原 job 门槛重核,无需重编。 @@ -66,7 +77,7 @@ Agent 执行是否结束/超预算独立记录在 `verdict.execution_terminal` - PASS = 必选门全部 pass。 - 门无法判定(数据缺、板未接)→ 该门 `unverified`,job `UNVERIFIED`。 - `na`(本轮设计上不评)≠ `unverified`(本该评没评成)。na 不阻塞 PASS。 -- BLOCKED 判定:oracle 同阶段同类 error_class 失败 → 正确;oracle 通过 → 误判。 +- BLOCKED 记录 Agent 的阻塞报告和本次实测。失败日志不能证明任务不可解,`blocked_correct` 不作推断;不进入成功 SFT 或奖励学习候选。 ## 4. 四道门 @@ -97,9 +108,10 @@ Agent 执行是否结束/超预算独立记录在 `verdict.execution_terminal` - 板卡登记使用固定且唯一的 `slot`,对应 `board-.lock`;现有板保留 slot 0 以兼容在途旧进程。新增板使用新槽位,删除/禁用板卡后不复用槽位;重复接入同一地址不能增加容量。只把实际独立、运行库兼容的物理板加入池。新增/禁用板卡是调度资源变更,不重写冻结的 job 或已生成 HBM。新调度进程等待时重读板卡登记;旧进程仍需自然收尾后更换。 - 新 run 的任务指纹不包含板卡设备列表,但仍包含 march 映射与其他 registry 内容。旧 run 扩板前把原始 `boards.yaml` 保存为 `board_registry.initial.yaml`;续跑时用该快照验证旧整体指纹,证明除板卡设备列表外的输入未变,不能跳过模型、数据、门槛、工具链校验。 - **没有固定超时。** STALLED = 日志 `stall_s`(900 s)不增长 且 进程树 CPU < 5%。硬上限 24 h 只是保险。 +- 板端 RPC 的原有单次 3600 秒上限由独立 `board_guard` 在板上执行;SSH stdin 租约每 5 秒刷新,断连或 30 秒失联回收本次进程组。守护进程不加载模型,避免原生运行库阻塞清理。板端互斥锁由守护进程和推理子进程共同持有,未清空不能交给新推理。租约存活日志与样本/迭代进度分开记录,不能把心跳当作推理进展。主机阶段锁只代表调度占用,不能代替板端进程核验。 - 重阶段(compile / board / eval)由 harness 执行。agent 走 `bench stage run` / `bench stage wait`,不在 bash 里跑 hb_compile。 - rollout 预算 = turns + tokens;墙钟 12 h 仅防挂死。 -- Agent 动态领取任务:先覆盖尚无完整轨迹的不同模型,同等覆盖下优先 oracle PASS 和预计耗时较短的模型,再做重复采样;同一 job 同时最多一条 Agent 轨迹。每次有空位都重新读取队列,`collect --watch` 跟随尚未结束的 oracle,避免新完成的模型一直排在旧批次后面。 +- Agent 动态领取任务:先覆盖尚无完整轨迹的不同模型,同等覆盖下优先预计耗时较短的模型,再做重复采样;同一 job 同时最多一条 Agent 轨迹。每次有空位都重新读取队列,`collect --watch` 等待输入准备和在途任务,避免新完成的模型一直排在旧批次后面。 - 同一 run 只运行一个新 collector;任务领取原子化。SIGTERM 或 `state/drain-collect-` 停止领取,已开始的任务自然收尾。导出在独立线程刷新,不占 Agent 名额。 ## 7. 重试与轨迹边界 @@ -109,7 +121,7 @@ Agent 执行是否结束/超预算独立记录在 `verdict.execution_terminal` | oracle 阶段 | harness | 瞬态 error_class | 不是轨迹 | | rollout 内 | agent | 自己改配置重提 | 同一条轨迹 | | 新 rollout | harness | matrix k | 全新轨迹,永不串联 | -| repair | harness | oracle FAIL | 全新轨迹,输入是 oracle 现场 | +| 历史 repair | 不再生成或领取 | 保留既有编号和结果 | 有 Oracle 现场的修复案例,仅保留分析用途 | 显式扩大已有 run 的 Agent 覆盖时,仅追加缺少的计划和范围说明 annotation;编号接在该 job 的最大 kk 之后,重复执行不重复添加。冻结的 job、已执行轨迹、oracle 结果及评测门槛不变。确定性 oracle 的 PASS 不代表该模型已有 Agent 训练轨迹。 @@ -126,19 +138,17 @@ bash 里 `cd/pushd/popd` 一律拒(唯一例外 `cd <自己的 workspace>`, ## 8. error_class `unsupported_op shape_infer export_fail calib_nan verifier_mismatch`(确定性,不重试) -`oom board_offline stalled`(瞬态,重试 ≤ 2 / ≤ 2 / 1) +`oom board_offline board_busy stalled`(瞬态,前三者重试 ≤ 2,stalled 重试 ≤ 1) `unknown`(进人工分类队列) -## 9. reward(v0.4,离线算,配方版本化) +## 9. reward(v0.6,离线算,配方版本化) ``` r = 1.0[G1] + 1.0[G2] + 2.0[G3] -r += 1.0·clip(accuracy_improvement/|tol|, −1, 1) -r += 0.5·clip((lat_oracle − lat_agent)/lat_oracle, −1, 1) r −= 0.5·turns/max_turns + 0.5·tokens/max_tokens r −= 2.0[claim ≠ verdict] -BLOCKED: +1.0 若 oracle 同类失败 else −2.0 ;TIMEOUT: −1.0 -G3 = na 时其 2.0 权重按比例分给 G1/G2,精度相对项取 0 +TIMEOUT: −1.0;BLOCKED 无特殊奖惩,仍保存本次真实阶段得分 +G3 = na 时其 2.0 权重按比例分给 G1/G2 板卡不可用(G3 unverified 且最后一次 07-board/08-eval 为 board_offline):G2/G3 权重取 0,G1 保持 1.0 不加分, 免 TIMEOUT 罚分,reward 输出带 unverifiable="board_offline"(v0.4,2026-09-16) ``` @@ -146,7 +156,7 @@ G3 = na 时其 2.0 权重按比例分给 G1/G2,精度相对项取 0 板卡不可用的轨迹在导出器里仍按 UNVERIFIED 排除在 RL/SFT 候选之外;回收它们的办法是 `bench board-backfill --run R --rollout JOB:KK` 用其现成 HBM 补跑 07/08 再 verify,而不是改分。 -`abs_pp` 的 accuracy_improvement = Δ_agent − Δ_oracle;`rel_pct` 的 Δ 表示劣化,故取 Δ_oracle − Δ_agent。不同 kind 不互比。 +不再计算相对 Oracle 的精度或延时奖励。G3 仍比较同一评测集上的 FP32 与本次 INT8,沿用任务冻结的门槛。 TIMEOUT 扣分根据执行终止原因独立相加,已完成的 G1/G2/G3 成果照常得分,不再整条清零。 ## 10. 两套 venv @@ -164,13 +174,15 @@ harness 按 backend 选解释器。**禁止把 backend 判断交给 agent。** 数据默认按 `purpose=all` 保留已结束的成功、失败及部分轨迹;SFT 是其中的用途筛选,不能代表全部数据量。Agent 的 `teacher.jsonl` / `student.jsonl` 只含真实对话、工具调用和观测,缺失内容不得编造。`pipeline.jsonl` 保留确定性流程的实际输入、阶段动作与测量,明确标记其来源,不能计成模型生成的 Agent 轨迹。每个 run 使用同一导出目录,显式刷新,按稳定 ID 去重;导出失败报错并可补导,不回滚已完成任务。 +`bench.verdict/3` 不读取 Oracle;旧判定在导出时只读重核,源 verdict、数据库、消息和产物不改。SFT 候选要求本次实测 PASS、声明一致、完整对话和工具定义,并保留污染注释与版本过滤。Oracle FAIL 不再触发待复核;历史 repair 保留原模式,仅供分析。导出是候选数据,仍需训练端分组、去重和独立评测。 + ## 12. 协作规则(2026-09-15 起,两个会话同时改一棵树踩过一次) - **只有一个写者能改一层。** 分层:`harness/`(oracle 流水线、阶段、门、调度)· `agent/`(pi 配置、扩展、提示词)· `export/`(清洗、脱敏、reward)· `registry/`(清单与阈值,改了就是新 run)· `scripts/`。谁在改哪层写进下面的表;别的会话对那层只读。 - **改动只发生在本地副本,机器上不直接改代码。** 流程:改 → `git commit`(`scripts/sync.sh` 会自动提交)→ `scripts/sync.sh` 推送。sync 带守卫:远端自上次推送后被动过就拒绝,先合并再推。 - `registry/models.csv` 是在机器上由 `excel_to_models.py` 生成的,方向反过来:`scripts/sync.sh --pull`。 - 机器上的 `runs/ state/ data/ cache/ datasets/` 是数据,不同步、不手改;改判走 `bench rejudge`,补测走 `bench board-backfill`。 -- 正在跑的调度进程(`bench run` / `bench collect`)加载的是启动时的代码;改了 harness 只对新进程生效,子进程工具(`harness.tools.*`、`harness.eval.run`)立刻生效——改评测口径必须同时升 `EVALUATOR_VERSION`。 +- 正在跑的调度进程(`bench run` / `bench collect`)加载的是启动时的代码;改了 harness 只对新进程生效,子进程工具(`harness.tools.*`、`harness.eval.run`)立刻生效——改评测口径必须同时升 `EVALUATOR_VERSION`。**换代必须 `bench drain` 等它跑完再起新进程,只 sync 文件不算换代**(踩过:一个调度进程跑了 13 小时的旧代码)。 | 层 | 当前写者 | |---|---| diff --git a/HANDBOOK.md b/HANDBOOK.md index 7ea6079..4f2568a 100644 --- a/HANDBOOK.md +++ b/HANDBOOK.md @@ -5,20 +5,13 @@ ## 一、它在做什么(一段话) -把 338 个视觉模型逐个量化编译到 BPU(nash-e/m/p),每个"模型×芯片×配置"叫一个 **job**。每个 job 先由 **harness** 自己跑一遍标准流程(叫 **oracle**,产出"标准答案"和四道门的判定),然后让 **agent**(一个调大模型的 pi 进程)在看不到答案的情况下**自己做一遍**,harness 独立核验它做得对不对。agent 做一遍的过程记录下来就是一条 **rollout(轨迹)**,带着 harness 给的可核验分数,这就是训练数据。 +把 338 个视觉模型逐个量化编译到 BPU,每个“模型×芯片×配置”叫一个 **job**。GLM-5.3 Agent 自行准备输入、量化、编译和上板;harness 管理资源并用本次真实产物核验。一次执行的对话、工具调用和观测是一条 **rollout(轨迹)**。 -``` -registry/models.csv ──▶ bench new-run/expand ──▶ jobs(PENDING) - │ - bench run(后台调度) ──▶ oracle:01 fetch → 02 baseline → 03 export → 04 calibrate → 05 compile → 06 hbm → 07 board → 08 eval - │ 四道门 G1 编译 / G2 余弦 / G3 任务指标 / G4 上板延时 - ▼ - bench collect(后台采集) ──▶ 每个 job k 条 rollout:pi 进程在 /rollouts//workspace 里作答 - │ → verify_rollout 独立核验 → verdict.json(reward 的唯一来源) - ▼ - scripts/audit_rollouts.py ──▶ 哪些轨迹干净、能进 RL/SFT - export/to_agentrl.py ──▶ 训练格式 -``` +流程:**领取任务 → Agent 执行 → 本次实测判定 → 导出并筛选训练候选**。 + +`bench run` 可以预热权重、FP32 基线和 ONNX 缓存。完整 Oracle 仅供显式对照,不是任务前置,也不参与成功判定、奖励或 SFT 筛选。FP32 仍是 INT8 精度的比较基线,两者使用同一评测集。 + +旧 Oracle、旧 repair 和已有结果保留。历史 repair 已获得失败现场,适合分析修复过程,不能当成独立完成任务的示范;尚未开始的旧 repair 保留编号但不再领取。 ## 二、你能控制的东西(改这些文件,其他都是派生的) @@ -48,7 +41,7 @@ python3 scripts/trace.py 一条轨迹逐轮回放:a bench report --run 静态 HTML 报告(reports/.html) ``` -启动/续跑:`scripts/start_run.sh --note "..."`(oracle)、`bench collect --run [--job …] [--limit N]`(采轨迹)。 +启动/续跑:`scripts/start_run.sh --note "..."`(准备与采集)、`bench collect --run [--job …] [--limit N]`(采轨迹)。 换代码/改配置要重启调度器时:`bench drain --run `(跑完手里的不再认领新 job)→ 等它退出 → `scripts/start_run.sh --resume`。不要 kill 正在编译的进程。 注意常驻进程里的代码是启动那一刻的:改了 harness/ 或 config.yaml,正在跑的 `bench run` / `bench collect` 不会变,新起的才会。 两条自检:`node agent/extensions/guard-core.test.js`(边界判定单测)、`scripts/guard_smoke.sh`(起真 pi 进程试图越界,应全部被拒)。 @@ -67,10 +60,10 @@ bench report --run 静态 HTML 报告(reports/.html | `old_version(vX` 正常无输出退出0,异常退出1;`bench watch` 非TTY仅输出一次。 -完整实测、审计前后对比和未覆盖项见 [REPORT-codex.md](REPORT-codex.md)。本节取代上面 v4 的“当前”状态描述。 +完整实测、审计前后对比和未覆盖项见 [REPORT-codex.md](docs/archive/REPORT-codex.md)。本节取代上面 v4 的“当前”状态描述。 diff --git a/README.md b/README.md index 3387de3..4456cff 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,41 @@ # bench/ — 视觉模型量化与上板评测 -设计:`../BPU量化轨迹工厂-设计方案.html`(v0.4)。契约:`CONTRACT.md`。本 README 只讲怎么把它放到机器上跑起来。 +## 338 模型分批覆盖 -主流程按 CSV 中已就绪的模型运行:FP32 基线 → 导出 → 校准、编译 → S100P(`nash-m`)上板 → T1 精度评测。新任务默认要求任务指标相对 FP32 劣化不超过 5%,改善也通过;G2 余弦用于诊断,性能只记录。Agent / RLVR 采集按需单独启动。 +`bench coverage --all-runs --csv reports/model-coverage.csv` 维护原 CSV 全部 338 项; +相邻 JSON 保存所有 run/job/kk、逐阶段记录、真实轨迹路径、teacher 身份和训练导出关联。 +静态前置齐备、FP32/ONNX 对照通过、量化编译、真实上板、任务精度通过分别记录,不能互相替代。 +SIFT/ORB/AKAZE 仍占总表三行,按传统算法记不适用 PTQ,不虚构 HBM 或 Agent 轨迹。 + +新接入模型先核查具体权重、构建/导出、输入、校准、标注和评测门;不修改原 CSV 标签或旧 run。 +Ultralytics COCO 分割可用 `scripts/probe_coco_adapters.py --model yolov8n.seg` 做真实权重、 +ONNX 数值与 COCO 指标对照。Pose 另存原图坐标关键点对应及偏移,必须通过真实导出对照才能入队; +板端 G3 同时要求 OKS AP 和平均偏移 ≤2px,缺几何证据则 UNVERIFIED。probe 不是板端验收。 +FastSAM 的类别语义仍属阻塞。EfficientViT 构建器文件存在不代表依赖、预处理和导出已验证。 + +通过前置核查的具名批次沿用现有命令: + +```bash +bench new-run --note '338 coverage: validated model batch' +bench expand --run RUN --modes oracle,e2e --marches nash-m --model yolov8n.seg +bench run --run RUN --parallel 2 +bench collect --run RUN --modes e2e --first-pass --watch --parallel 2 +``` + +显式 `--model` 放宽该批的 priority/task/status 选择,但保留 march/backend/qcfg、teacher、e2e k=4、 +冻结精度门槛及原采样编号;不再生成 Oracle repair,缺真实 probe 或其他前置时整批拒绝。 +`--first-pass` 只消费每个 job 的首条,原重复采样仍保持 QUEUED;等可执行模型完成首轮再用普通 collect 接续。 +当前旧 collector 若需切换,只设置其 `state/drain-collect-RUN`,等在途任务与导出自然完成再启新进程。 +不能直接 kill 或用新代码同步冒充旧进程已加载首轮策略。 + +创建 `state/coverage-master.enabled` 后,新进程每次训练导出会刷新总表;旧进程可沿用 +`bench export --run RUN --follow` 接入相同刷新。总表失败会显式报错,不回滚已完成任务和训练数据。 + +设计:[BPU 量化轨迹工厂设计方案](docs/BPU量化轨迹工厂-设计方案.html)(v0.4)。契约:`CONTRACT.md`。方法论、闭环说明和技术分享讲稿见 [项目文档](docs/README.md)。本 README 只讲怎么把它放到机器上跑起来。 + +主流程:GLM-5.3 领取任务,按需准备输入、校准、编译、上板和评测 → harness 按本次实测独立判定 → 导出原始轨迹并筛选 SFT 候选。`bench run` 可选地预热权重、FP32 和 ONNX 缓存;`--oracle` 只作显式对照。Oracle 不参与 Agent 判定、奖励或 SFT 筛选,也不再生成 repair。历史文件和排队编号保留,旧 repair 不再领取。新任务仍要求相对 FP32 劣化不超过 5%,改善也通过;历史任务沿用冻结门槛。 + +已有批次只把剩余任务切到 100 张测试集,见 [剩余任务交接](CONTINUE.md)。使用交接清单保留原采样序号,不全量展开或重新补采。 ## 0. 上机 @@ -33,20 +66,20 @@ for d in state data cache; do mkdir -p /home/$USER/bench-$d; [ -L $d ] || { rm - | 4 | `~/toolchains/oe-3.7.0/venv/bin/python scripts/ingest_pool.py --dataset imagenet1k --dataset coco2017` | `data/pool/…/manifest.json`,只读 | | 5 | `… scripts/make_t1.py --dataset imagenet1k --dataset coco2017` | `data/t1//20260914.tar` | | 6 | `… scripts/make_calib.py --task cls --input-spec 1x3x224x224` / `--task det --input-spec 1x3x640x640` | `data/calib/…` | -| 7 | `bench preflight --modes oracle --oracle-only` | 0 errors 才往下;不要求 agent 环境 | -| 8 | `bench new-run --note "S100P"` → `bench expand --modes oracle --marches nash-m` | 每个就绪模型的 `runs/r…//job.json`,`state/jobs.sqlite` | +| 7 | `bench preflight --modes oracle,e2e` | 0 errors 才往下;包含 Agent 环境检查 | +| 8 | `bench new-run --note "S100P"` → `bench expand --modes oracle,e2e --marches nash-m` | 每个就绪模型的 `runs/r…//job.json`,`state/jobs.sqlite` | | 9 | `bench run --dry-run` | 状态机空转一遍,不碰工具链 | 步骤 4–6 用 PTQ venv 的 python(要 pyarrow / pillow / numpy)。ImageNet 首选 HF parquet 14 片;tar 是备用源,入池时会 `tar -tzf | wc -l` 校验。 ## 2. P2 切片(真跑) -环境和数据准备好后,`scripts/start_run.sh --note "S100P" --parallel 2` 会检查环境、展开 nash-m oracle 并后台运行;`--resume` 续跑已有 run。其他板型需先在 `registry/matrix.yaml` 的 `marches` 中加入对应值,再用 `bench expand --marches ...` 选择。 +环境和数据准备好后,`scripts/start_run.sh --note "S100P" --parallel 2` 会检查环境、展开 nash-m 输入准备和 Agent 计划,后台准备 01–03 并按首轮优先采集;`--resume` 续跑已有 run。采样次数、GLM-5.3 身份和精度门槛不变。其他板型需先在 `registry/matrix.yaml` 的 `marches` 中加入对应值,再用 `bench expand --marches ...` 选择。 ```bash -bench run --job cls--resnet18--nash-m--ptq--int8 --parallel 1 +bench run --oracle --job cls--resnet18--nash-m--ptq--int8 --parallel 1 # 仅需要完整对照时 bench show cls--resnet18--nash-m--ptq--int8 # 阶段时间线、stderr 尾巴 -bench run --job det--yolov8n--nash-m--ptq--int8 --parallel 1 +bench run --oracle --job det--yolov8n--nash-m--ptq--int8 --parallel 1 bench status; bench status --fail; bench report # reports/.html ``` @@ -112,15 +145,15 @@ march / 输出前缀 / 工作目录被 harness 钉死;`--config` 必须在 `wo - **机器是共享的**(2026-09-14 实测):244 核 / 1.4 TB / **8 张 32 GB 卡**(不是 4 张),4/5/7 被别人跑满、6 占 29 GB;12 个登录用户。`config.yaml pools.gpu.devices: [0,1,2,3]` 钉死只用空闲卡,槽位 → `CUDA_VISIBLE_DEVICES`。 - 资源池是跨进程的(`state/locks/-.lock` flock):oracle 调度进程、agent 的 `bench stage run`、多个 `bench run` 共用同一组槽位;board 池 1 槽 = 严格串行。model-level 阶段按模型加锁,三个 march 并行时只有一个真跑 01–03。job 认领是原子的(UPDATE … WHERE state='PENDING'),两个调度进程不会重跑同一个 job。 - 新 run 的 manifest 记 `harness_code_sha256`;harness 代码变了只告警,venv / registry / EVALUATOR_VERSION 变了拒跑(实测拦下过一次:改 config.yaml 的 venv 列表后 `bench run` 拒绝,开了新 run)。 -- 沙箱板是 S100P(`nash-m`);历史 job 未记录 board 时,阶段按 march 查询当前板卡配置。没有匹配板卡时无法完成板端 G3,结果保持 UNVERIFIED。`board_perf.py` 开测前探测板上有无别人的推理进程 / BPU ratio,最多等 10 分钟,仍忙则照测但标 `contended: true`,reward 的延时相对项忽略这类样本。 +- 沙箱板是 S100P(`nash-m`);历史 job 未记录 board 时,阶段按 march 查询当前板卡配置。没有匹配板卡时无法完成板端 G3,结果保持 UNVERIFIED。`board_perf.py` 开测前探测板上推理进程 / BPU ratio,最多等 10 分钟,仍忙则以 `board_busy` 拒绝测量。07/08 使用独立板端守护进程,持有板端互斥锁,并在 SSH 断连、30 秒未收到租约或原有单次 RPC 3600 秒上限到达时回收推理进程组。详见 [板端进程生命周期](docs/board-process-lifecycle.md)。 - `job.json` 与 mode 无关:一个 job_id 一份,`rollout_plans` 列出 e2e/repair 计划;oracle 与所有 rollout 共用。 - sqlite 默认 TRUNCATE 日志(WAL 在 FUSE 上会 disk I/O error);`BENCH_STATE_DIR` 可把 state 放本地盘。 - 校准集来源:本机 ImageNet / COCO 只有 val,`make_calib.py` 退到 val-holdout(排除 T1)并在 manifest 记 `source`;出 T2 交付数前必须换 train。 - 07-board 失败重试后仍失败 → G4 unverified、继续 08-eval;08-eval 失败 → job UNVERIFIED 带 error_class;01–06 失败 → FAIL。 - claim 缺席(agent 没调 `claim` 工具)→ claim.json 记 `ABSENT`,reward −0.25;claim=PASS 但没有可核验产物 → verdict FAIL 并计 claim≠verdict。 -- BLOCKED 正确性:claim.error_class 与 oracle 同类(`unknown` 不算同类)且阶段一致(claim 未填阶段则不比阶段)。 -- reward 的精度相对项按 G3 的 `kind` 解释方向:`rel_pct` 是劣化百分比,越小越好;`abs_pp` 是改善百分点,越大越好。 +- BLOCKED 仅记录阻塞报告,不根据 Oracle 推断正确性,不进入成功 SFT 或奖励学习候选。 +- reward v0.6 只看本次实测门、成本和声明,不与 Oracle 比较。导出时只读重核旧判定,保留源结果;历史 repair 单列为分析案例。 ## 6. 目录里什么进 git diff --git a/agent/SYSTEM.md b/agent/SYSTEM.md index e147e40..39ddf94 100644 --- a/agent/SYSTEM.md +++ b/agent/SYSTEM.md @@ -1,6 +1,6 @@ # bench agent · 系统提示(追加在 pi 默认系统提示之后) -你在一个受控的量化上板环境里工作。规则不是建议,违反会被工具层拒绝或被判负。 +你负责从权重到导出、FP32 基线、量化、编译、上板和精度评测的完整任务。harness 提供执行工具、资源池与独立核验,不会预先替你跑通模型。有效缓存可以复用。规则不是建议,违反会被工具层拒绝或被判负。 ## 你能写的地方 只有当前目录(`workspace/`)。其他一切只读。越界写入会被 path-guard 拒绝并记录。 @@ -49,9 +49,12 @@ compiler_parameters: 诊断信号在哪:`bench stage wait` 结束后给你 stdout/stderr 路径;hb_compile 日志末尾有逐节点 `Calibrated Cosine / Quantized Cosine` 表和模型输出余弦。余弦偏低时可检查敏感节点、尝试 `node_config` 提到 int16;最终任务精度以 G3 实测与 job 中的阈值为准。 ## 重阶段不在 bash 里跑 -编译、上板、评测由 harness 执行。你提交、你等待、你看日志: +导出、基线、编译、上板和评测均由你提交工具,harness 执行并保存日志: ``` +bench stage run fetch # 获取或复用正式权重 +bench stage run export # 导出 ONNX;有效缓存直接复用 +bench stage run baseline # 当前 T1 的 FP32 参考;同子集缓存直接复用 bench stage run calibrate # 生成编译配置(也可先看 job.json 自己写一份) bench stage run compile --config compile.yaml # cwd 就是 workspace/,相对路径按 workspace 解析(写 workspace/compile.yaml 也认) bench stage wait --poll 60 @@ -73,10 +76,10 @@ claim(terminal="PASS"|"FAIL"|"BLOCKED", stage=..., error_class=..., summary="… ``` **预算规则**:剩 5 轮时你会收到一条预算提醒——收到后无论进展如何,先调用 `claim` 汇报当前状态(不确定就报 FAIL 并写清到了哪一步),再决定要不要继续。达到轮数上限会立即结束进程,没有额外 grace 轮;请提前 claim。token 预算耗尽后只允许 claim,仍受轮数硬上限约束。没有 claim 就结束的轨迹被记为 claim 缺席,另扣分。 -你的 claim 会与 harness 独立核验的 verdict 比对。claim=PASS 而 verdict=FAIL 是最重的扣分项,比老实报 FAIL 差得多。不确定就报 FAIL 并说明。BLOCKED 只在你确认是工具链/算子层面无法完成时使用,并给出 error_class(unsupported_op / shape_infer / calib_nan / verifier_mismatch 之一)。 +你的 claim 会与 harness 独立核验的 verdict 比对。claim=PASS 而 verdict=FAIL 是最重的扣分项,比老实报 FAIL 差得多。不确定就报 FAIL 并说明。BLOCKED 用于有实际错误证据的工具、适配器或材料缺口;给出具体阶段、错误和所需材料。无法归入工具已有错误类别时用 unknown,不能把缺权重或缺适配器说成 unsupported_op。 ## 不要做的事 -不下载数据集或权重(都在只读区);不改 `job.json`;不猜指标——指标只来自 `bench stage run eval` 的输出;不在 bash 里 `sleep` 等编译,用 `bench stage wait`。 +不直接写共享数据或权重缓存,权重获取用 `bench stage run fetch`;不改 `job.json`;不猜指标——指标只来自阶段工具的实测输出;不在 bash 里 `sleep` 等编译,用 `bench stage wait`。权重、适配器或数据缺失时如实报告具体缺口;未尝试编译就不能声称工具链不支持该模型。 ## 工具链参考 `.horizon/skills/` 下的 skill 是这条工具链的文档。`oe-doc ` 可以查 OE 文档。两套 venv 的选择由 harness 按 job.json 的 backend 决定,你不需要也不能选。 diff --git a/agent/VERSION b/agent/VERSION index 1e8b314..7f8f011 100644 --- a/agent/VERSION +++ b/agent/VERSION @@ -1 +1 @@ -6 +7 diff --git a/agent/prompts/e2e.md b/agent/prompts/e2e.md index 3fcb195..a5081b9 100644 --- a/agent/prompts/e2e.md +++ b/agent/prompts/e2e.md @@ -1,4 +1,4 @@ -# 任务:把 {model}({task})量化并编译到 {march}({backend} / {qcfg}),通过全部必选门 +# 任务:从模型权重出发,将 {model}({task})部署到 {march}({backend} / {qcfg}),通过全部必选门 job:`{job_id}` run:`{run_id}` job.json:`{job_json}`(只读) 数据:`{data_dir}`(只读) 你的可写区:当前目录 @@ -11,13 +11,16 @@ job.json:`{job_json}`(只读) 数据:`{data_dir}`(只读) 你的 必选项以 job.json 的 `gates.required` 为准。 -## 你手上有什么 -- FP32 ONNX:`{onnx}`(03-export 已产出) -- FP32 基线:`{baseline_dir}/metrics.json` -- 校准集:`{calib_dir}`({calib_n} 个 npy,与 val 零交集) +## 输入和可复用缓存 +- 权重来源、模型身份和输入约定:见 job.json 的 model_spec +- ONNX 缓存位置:`{onnx}`(可能尚未生成) +- FP32 基线缓存位置:`{baseline_dir}/metrics.json`(可能尚未生成或属于旧子集) +- 校准集位置:`{calib_dir}`(目标 {calib_n} 个 npy,与 val 零交集) - T1 评测子集:`{t1_tar}`({t1_n} 张) ## 流程 -先读 job.json;系统提示里有 compile.yaml 全部可改的旋钮,不必再读 harness 或工具链源码。默认配置(calibration_type: default,全 int8)先跑一次拿到基线,看日志里的逐节点余弦再决定调什么。提交 `bench stage run compile --config <你的>.yaml`(cwd 就是 workspace/,不要再加 workspace/ 前缀),`bench stage wait`。失败就读日志改配置;同一方法最多重试 1 次。编译过了依次跑 `bench stage run hbm`、`bench stage run board`、`bench stage run eval`,每步等待完成;性能记录失败仍继续评测。最后调用 `claim`,缺少板端精度结果时不能报 PASS。 +先读 job.json,然后依次提交 `bench stage run fetch`、`bench stage run export`、`bench stage run baseline`,每步用 `bench stage wait` 等待并检查结果。这些阶段由你发起并记入本条轨迹;有效缓存会直接返回 SKIPPED,不重复计算。没有人预先替你判断能否量化成功。 + +输入准备完成后,由你选择量化方案并提交 `bench stage run compile --config <你的>.yaml`(cwd 就是 workspace/),再 `bench stage wait`。系统提示里列有 compile.yaml 可改参数。失败就读日志改方案;同一方法最多重试 1 次。编译通过后依次提交 `hbm`、`board`、`eval`,每步等待完成;性能记录失败仍继续评测。遇到缺权重、缺适配器或缺评测工具,明确记录实际错误和缺失材料,不把它写成量化失败。最后调用 `claim`,缺少板端精度结果时不能报 PASS。 预算:{max_turns} 轮、{max_tokens} tokens。等待编译不消耗预算。剩 5 轮会收到提醒——那时先 claim,别等到耗尽。 diff --git a/bin/stageclient.py b/bin/stageclient.py index ce58404..3420dee 100644 --- a/bin/stageclient.py +++ b/bin/stageclient.py @@ -9,7 +9,7 @@ import time import uuid from pathlib import Path -NAMES = {n.split('-', 1)[1]: n for n in ('04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval')} +NAMES = {n.split('-', 1)[1]: n for n in ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval')} def rollout_dir(): return Path(os.environ['BENCH_ROLLOUT_DIR']) diff --git a/config.yaml b/config.yaml index a4a30ae..b84ec87 100644 --- a/config.yaml +++ b/config.yaml @@ -39,6 +39,12 @@ watchdog: cpu_idle_pct: 5 poll_s: 30 hard_ceiling_s: 86400 # 24h 纯保险 + io_stall_s: 21600 # 树内持续有 D 线程(卡在 I/O)这么久 → 也判定异常。 + # 别往下调:线程级 D 检测上线前这条是死代码(d_count 恒 0), + # 上线后它第一次生效,收紧会把"链接期正在写盘"换个理由再杀一次。 + no_progress_ceiling_s: 28800 # 日志与 CPU 都不动这么久 → 兜底,免疫 D 抖动。 + # 必须 > io_stall_s,否则它抢先命中、盖掉更准确的 STALLED 原因。 + # 四条阈值单调:stall_s(long 3600) < io_stall_s < 本条 < hard_ceiling_s soft_alert: p95x3 # 超过历史 p95 × 3 只告警,不杀 perf: @@ -77,7 +83,7 @@ data: endpoints: teacher: provider: openai-compatible - base_url: https://ai-api.d-robotics.cc/v1 + base_url: https://open.bigmodel.cn/api/coding/paas/v4 model: glm-5.3 api_key_env: TEACHER_API_KEY student: diff --git a/docs/BPU量化轨迹工厂-技术分享讲稿-20260916.md b/docs/BPU量化轨迹工厂-技术分享讲稿-20260916.md new file mode 100644 index 0000000..e1ab835 --- /dev/null +++ b/docs/BPU量化轨迹工厂-技术分享讲稿-20260916.md @@ -0,0 +1,475 @@ +# 嵌入式大模型后训练 · 第一阶段汇报讲稿 + +面向:技术团队 / 同事(领导在场) | 完整版约 30 分钟,速用版 3 分钟 | 数字截面:2026-09-16 21:03 运行态快照 + 09-17 设计变更 + +> 用法:每节先是"讲什么"(可以直接念),后面是"提示"(只给你自己看:数字来源、措辞纪律、被追问时怎么接)。 +> +> 全篇措辞原则:讲进展用具体数字,讲未完成用"在建 / 已排期",讲经验用"这一类工程的共性规律"。不评价任何既往决策,不做"早期做错了"的归因——第一阶段的目标本来就是打通,现在是按计划进入第二阶段。 + +--- + +# 速用 · 三分钟口头版 + +> 临时汇报、例会同步、走廊被问到,用这一版。可以直接念,念完约 3 分钟。 + +**背景。** 前期我们调研了嵌入式方向上能做的题目,可选的其实不少。我们的判断是,第一个锚点要选一条能自己闭环的——判据能被程序判定、验证比求解便宜、失败原因能分类,这三条同时成立,才撑得起大量采数据。BPU 模型量化上板这条线三条都满足,和楠总对齐之后,方向就定在这里了。 + +**起点。** 方向定下来,这个阶段的重点就是把上板这件事的数据积累起来。起点是三样现成的东西:HBDK 4 工具链、S100P 板子,以及我们整理的 338 个公开视觉模型清单——其中已完成接入的 63 个(分类 48、检测 15),其余的在逐个做导出适配和评测器补齐。 + +**做了什么。** 搭了两套东西。 + +一套是**数据收集系统**:agent 在隔离环境里拿到一个模型,自己决定怎么量化、怎么编译,harness 负责执行工具、分配板卡、独立核验它做出来的产物,全过程留痕。agent 这一层用的是 pi,teacher 模型是 GLM-5.3。 + +另一套是**板端评测环境**,这块的投入产出比超出预期:板子上只做一件事——跑张量。读图、预处理、后处理、算指标全部留在主机侧,标注和评测代码永远不上板。好处是加一类新任务板端零改动,而且 FP32 基线和量化模型吃同一份输入、走同一个评测器,比较才有意义。顺带一个意外收获:**板端推理比服务器模拟快 45 倍**——所以"必须上真板"不是成本,是提速。 + +**这里有一个重要的设计调整,上周做的。** 原本每个模型要先用确定性流水线跑一遍产出标准答案,再让 agent 做。实测下来,这一遍吃掉了三分之二的板时,而它一条轨迹也不产。所以我们把它改成了 **agent 主导**:默认只准备输入——权重、FP32 基线、ONNX 导出,这三步结果可缓存复用;输入齐备 agent 就直接上手。完整的确定性流水线保留成一个显式开关,只在做对照和排查问题的时候跑。**精度判据一点没放松**,因为参照物从来就是 FP32 基线,不是默认配置的量化结果——判的还是板端实测相对 FP32 劣化不超过 5%。 + +**现在的状态,三个数。** + +第一,**板端终判打通了**。精度数字来自 S100P 真实推理,不是服务器模拟——这是这个阶段最关键的一步,在此之前所有精度都是模拟出来的,不能作为交付依据。目前已有 18 个模型完成板端实测并通过,这是第一批可以对外讲的上板基准。 + +第二,**agent 轨迹开始产出**。已结束 34 条,其中判定通过 22 条,覆盖 14 个不同模型。作为对比,上一个截面这个数是 3 条,而且全部集中在同一个最简单的模型上——那时候只能说明流程能走通,现在能说明 agent 确实在解决问题。最好的一条,板端 top-1 相对 FP32 只掉了 0.06%。 + +第三,**出现了第一条修复模式的成功轨迹**:确定性流水线做不出来的模型,agent 调出来了。明细还在核验,核完它就是我们第一条黄金样例。 + +**资源。** 现在是 4 块板加一台 8 卡机器。板卡是这条线的咽喉——严格串行,一块板同时只能跑一个任务,利用率已经到 93%,基本是满的。刚才那个设计调整把三分之二的板时让给了采集,等于产能翻了两倍多,但这是一次性红利,规模再往上走还是要靠扩容。一轮完整实验需要多少板时是能算出来的〔填入下一节算好的数〕。所以后续主要的诉求是板卡扩容,我这边也会去协调。 + +**下一步三件事。** 把 agent 单次实验从一个多小时压到 5 分钟以内,方案已经验证过;接学生模型,跑通在线训练闭环;把覆盖从 63 个模型往更多任务类扩,分割和姿态估计的适配已经在做了。 + +--- + +# 速用 · 数字卡片 + +> 汇报前存手机或打印一页。**打✱的项汇报当天早上要重新取一次**,它们会随采集推进变化。 + +| 会被问 | 直接答 | +|---|---| +| 模型清单多大 | 338 个公开视觉模型,10 个任务类;已完成接入 63 个(分类 48、检测 15),其余在补导出适配和评测器 | +| 板端跑通了吗 ✱ | 通了。已有 18 个模型完成板端实测并通过 | +| 采了多少轨迹 ✱ | 已结束 34 条(通过 22 / 失败 8 / 未验证 4),另有在跑和排队的 | +| 覆盖多少模型 ✱ | 通过的 22 条覆盖 14 个不同模型,最简单的 resnet18 只占 4 条 | +| 最好的一条结果 | regnet.y.800mf 板端 top-1 77.85 对 FP32 77.90,相对劣化 0.06% | +| 精度判据是什么 | 板端实测的任务指标,相对 FP32 基线劣化 ≤ 5%,改善也算通过 | +| 一次上板多久 | 性能测试 31 秒;完整精度评测(2000 张)约 8 分钟 | +| 板子忙不忙 | 12.6 小时窗口内利用率 93% | +| 板端比模拟快多少 | 45 倍(0.889 毫秒 vs 40 毫秒一张) | +| 编译要多久 | 单次 35 到 118 分钟;其中量化只占 107 秒,BPU 编译占 78 分钟,比例 1:44 | +| 测试集为什么 2000 张 | 配对比较下差值标准误约 0.5 个百分点;500 张时约 1 个点,和容差同量级会误判 | +| 隔离安不安全 | 内核级隔离,红队 11 种路径逃逸全部挡住 | +| 现在还跑标准流水线吗 | 默认不跑了。只准备输入(权重 / FP32 基线 / ONNX),agent 直接上手;完整流水线保留成显式对照开关 | + +--- + +# 速用 · 资源诉求怎么提 + +> "能扩到更多会更好一些"这种说法领导没法决策。给算法、给数、给不扩的后果,三样都要有。 + +**算法**(当场能写在白板上): + +``` +一轮实验的板时 = 题目数 × 每题采样条数 × 每条里的板端测量次数 × 单次测量耗时 +``` + +**现有实测单价**:单次完整精度评测约 8 分钟,性能测试 31 秒,板卡严格串行。 + +**代入现在的规模**(63 个模型 × 4 条采样,每条按 2 次板端评测算):约 500 次测量 × 8 分钟 ≈ 67 小时,**1 块板要 2.8 天,4 块板约 17 小时**。 + +**怎么讲**:现在 4 块板对应的节奏是一天一轮。上周那次设计调整已经把三分之二的板时从"跑标准答案"让给了"采轨迹",这是一次性红利,已经吃掉了。接下来采集规模按计划往上走——覆盖从 63 个模型扩到更多任务类、每题采样数从 4 提到 8——板时是线性涨的,届时一轮就变成三四天。这是训练迭代速度的直接上限,所以板卡扩容不是效率优化,是决定我们多久能验证一次假设。 + +**两个配套动作一起提**(显得诉求是算过的,不是单纯要东西): +- 两阶段编译落地后,agent 单次实验从一个多小时压到 5 分钟以内,同样的板子能多跑几倍任务——先把现有资源的效率榨出来,再谈扩容。 +- 板卡要按物理身份登记、一块板一把锁。同一地址的多个容器不能算成额外容量,否则排队系统以为有多块板,两个任务同时推上去互相干扰,测出来的延时全废。 + +--- + +# 速用 · 资料清单 + +**A. 手边必备**:上面的数字卡片,一页纸。 + +**B. 投屏随时能打开**(按最可能被问的顺序排好标签页): + +1. `BPU量化轨迹工厂-方法论.html` —— 被问"整条线怎么运转",打开 §0 流程图 +2. `BPU量化轨迹工厂-闭环方法论.html` —— 被问"怎么算走通了",打开 §5 分辨力四例表 +3. `CPU算子优化Agent-立项说明.html` —— 被问"下一个方向",打开 §5 缺口总表 +4. 本讲稿 —— 被问细节时按节号跳 + +> ⚠️ 前两份 HTML 是按"确定性流水线先出标准答案"那一版设计写的,和现在的 agent 主导不一致。投屏前要么先说一句"这是上一版设计,第 X 节讲的调整就是针对这里",要么抽空把两页的对应段落更新掉。**别让听众自己发现不一致。** + +**C. 被追问才调的证据**: + +| 问题 | 文件 | +|---|---| +| 板端利用率、故障统计、补测结果 | `outputs/plan-verify-20260916/REVIEW.md` | +| 板卡连通性、已有产物清单 | `outputs/s100p-gap-20260916/` | +| 当前运行态计数 | `outputs/glm53-endpoint-switch/runtime-after.json` | +| 四道门口径、agent 主导的新原则 | `bench/CONTRACT.md` §0 和 §4 | +| 338 项覆盖总表 | `bench coverage --all-runs --csv reports/model-coverage.csv` | + +**D. 汇报前必须自查的四件事**: + +1. **板卡登记状态。** 容量等于"注册表里启用的板数",不是物理板数。最近一次记录里启用的只有 1 块,另一块因 BPU 打不开被停用。物理上现在有 4 块,要确认是否都已登记并启用——否则说"4 块板"但调度容量是 1,会被追问。 +2. **修复模式那条轨迹的明细。** verdict、claim 一致性、审计洁净度三项,核完才能讲"黄金样例"。 +3. **重新取一次计数。** 数字卡片里打✱的会变,用当天早上的数。 +4. **repair 的触发条件。** 代码里 repair 仍然挂在"oracle 失败"上。默认不跑完整流水线之后,repair 默认也就不触发了——确认一下现在是靠显式 `--oracle` 补现场,还是已经改成吃 agent 自己的失败。这个点很可能被问到(见问答第 3 条)。 + +--- + +## 0. 开场(1 分钟) + +**讲什么** + +今天汇报三段:我们的主线是什么、为什么第一个锚点选在 BPU 部署这条链路上、这条链路现在跑出了什么。 + +先给结论:**主线是训练一个能干嵌入式工程活的大模型**。这件事成不成,不取决于我们用多大的底座模型,取决于一件更基础的事——我们能不能把"做得好不好"变成一个机器判得准的数字。判不出来就没有训练信号,再大的模型也只是在猜;判得不准,训练就是在拟合噪声。 + +所以第一阶段我们没有直接去训模型,而是先把这个"判得准的数字"造出来。现在它造出来了,并且已经在产出真实轨迹。 + +**提示** + +- 开场不报通过率,第 6 节会专门讲为什么通过率不是指标。 +- "不取决于用多大的模型"是整场的主心骨,后面每节都回扣它。 + +--- + +## 1. 主线:我们要训的是什么模型(4 分钟) + +**讲什么** + +我们的定位是做行业的底座——供应商逻辑。放到模型上,就是训一个**嵌入式领域的工程 agent**:客户把模型交给我们要往板上部署,中间那一串工程活——量化配置怎么调、算子不支持怎么办、落到 CPU 上的算子怎么优化、端到端延时怎么压——现在靠工程师一个个手工做,我们要让模型接手。 + +这类模型和通用对话模型的训练方式不一样,差别就在监督信号: + +| | 通用对话 / 创意类 | 我们这类嵌入式工程任务 | +|---|---|---| +| 对错谁判 | 没有客观判据,要训一个奖励模型当裁判 | 程序判定:返回码、精度掉点、板端延时 | +| 主要风险 | 长期和"裁判被钻空子"缠斗 | 判据本身要造准,造准了就不会被钻 | +| 数据来源 | 人工标注 / 偏好对 | 真实执行留下的轨迹,自带可核验结果 | + +这是我们的结构性优势,不用白不用。但要兑现它,前提是先有一个**可机器判定的验证器**,而且要有分辨力——不只是能盖章通过,还要能把该拒的拒掉。 + +所以训练路径是这样一条,顺序不能换: + +1. **造验证器**:把工程任务变成有判据的题,判定完全由程序做。 +2. **采轨迹**:让 agent 在隔离环境里做题,执行过程和核验结果一起落盘。 +3. **评测与打分**:同一套代码独立核验,核验结果就是训练信号。 +4. **训练**:先用做对且诚实的轨迹做 SFT 打底,再接 RLVR 做在线强化。 +5. **回到 1**:模型变强之后,题目难度和判据跟着升级。 + +第一阶段做的是第 1 到第 3 步,第 4 步的数据和接口已经备好。 + +**提示** + +- 这段是领导最关心的"我们到底在干什么",讲慢一点。 +- 表格不要念完,念第一列和最后一列的对比,重点落在"我们不在那个约束里"。 +- 被问"什么时候开训":答案在第 9 节,判据是"分辨力四例各有一例被正确落位",不是"攒够多少条"。 + +--- + +## 2. 锚点:为什么第一个选 BPU 部署这条链路(3 分钟) + +**讲什么** + +嵌入式的工程任务很多,我们前期接触到的也不少。第一个锚点选在 BPU 部署链路上,理由很实在:**它是目前唯一一条能自己闭环的**。 + +判断一个任务适不适合做成训练场,有三条硬标准,三条同时成立才行: + +| 标准 | 含义 | BPU 部署链路满足吗 | +|---|---|---| +| 判据可机器化 | 对错由程序判,不靠人评、不靠模型当裁判 | 满足:返回码、逐输出余弦、Top-1 / mAP 差值,全是数值比较 | +| 验证远比求解便宜 | 跑一遍很贵、判一次很便宜,才撑得起大量采轨迹 | 满足:跑一个模型要编译加上板评测,判一次只是读一份结果文件 | +| 失败模式是有限集 | 失败能分类,才能区分"判断错"与"工具链真做不到" | 满足:确定性五类、瞬态三类,其余进人工队列 | + +第三条最容易被忽略,但它决定了"正确地放弃"这种能力能不能被学到。没有失败分类,agent 说"这个做不了"我们无法判真假,只能一律当失败处理。 + +除了这三条,还有现实理由:模型清单是现成的、数据集是现成的、板卡是现成的——环境搭起来就能跑,不需要先等别的条件成熟。 + +这条链路上有两个落点,一个在跑、一个在立项: + +- **落点一:量化编译(在跑)。** 模型往板上走,量化配置怎么调才能既过精度又跑得快。这是本次汇报的主体。 +- **落点二:CPU fallback 算子优化(立项中)。** BPU 支持不了的算子会落到 CPU 上,往往吃掉端到端延时的大头。骨架可以直接复用,判据要按新任务重做——量化任务判的是精度,算子优化判的是正确性和速度,判据怎么定要重新设计。立项说明已经写好,等第一个落点的数据链跑顺再启动。 + +**提示** + +- 说"第一个锚点",不说"只能做这个",留出横向扩展余地。 +- 落点二一句话带过,有兴趣的会单独找你。 +- 被问"为什么不先做更有业务价值的":答"这条链路本身就是业务交付——跑出来的就是模型在 S100P 上的上板基准数据,这是要交付的东西;训练数据是同一遍执行的副产物,不额外花钱"。这句能解释"为什么不是纯投入"。 + +--- + +## 3. 怎么做:三步走(5 分钟) + +**讲什么** + +方法上就三步:**搭环境 → 跑任务采数据 → 评测**。每一步里真正花时间的地方我说一下。 + +### 第一步:搭环境 + +不是装几个包的意思,是把"可复现"做成机械规则: + +- 数据集复制入池、设只读、写清单(记哈希 / 数量 / 来源 / 日期),agent 永远不下载。三个理由:可复现(线上数据集版本会飘)、防作弊(agent 会挑好算的子集)、轨迹干净(网络波动不该进训练数据)。 +- 资源分池:GPU、CPU、板卡、agent 各自设并发,跨进程文件锁。板卡严格串行,一块板一个槽位——用一个并发数管所有会把 GPU 堵死。 +- 两套工具链两套虚拟环境,启动前比对依赖哈希,不一致直接拒跑而不是告警——否则"只追加不覆盖"就是句空话。 +- agent 的作答区隔离在内核层,隔离起不来就拒绝启动,不静默降级。 + +### 第二步:跑任务采数据 + +这里上周做了一次**设计调整,是这个阶段最重要的一次决策**。 + +原来的做法是每个模型先由确定性流水线完整跑一遍产出标准答案,再让 agent 做。实测下来这一遍代价很大:在一个 12.6 小时的观测窗口里,标准答案占了 7.9 小时板时,agent 采集只占 3.8 小时——**三分之二的板时花在一条轨迹也不产出的事情上**,而板卡是整条线的咽喉。 + +所以改成 **agent 主导**:默认只做输入准备——取权重、跑 FP32 基线、导出 ONNX,这三步的结果可缓存、可复用;输入齐备 agent 就直接领任务上手。完整的确定性流水线保留成一个显式开关,需要做对照或者排查问题的时候单独跑。 + +这个调整有一条边界要说清楚,因为它是最容易被误解的地方:**精度判据没有因此变松**。G3 判的是"板端实测的任务指标相对 **FP32 基线** 劣化不超过 5%"——参照物从来就是 FP32,不是默认配置的量化结果。FP32 基线属于输入准备的一部分,照常产出。所以去掉的只是"默认配置能做到多少"这个参考点,判定本身一点没动。 + +顺带一个收益:交付口径其实变好了。原来的基准数字回答的是"默认配置能不能上板",现在回答的是"这个模型在 S100P 上到底能做到什么水平"——后者才是客户真正想知道的。 + +### 第三步:评测 + +**这一层的投入产出比超出预期,值得多说两句。** + +核心设计是一条:**板上只跑张量,别的都不上板。** + +``` +主机:读图 → 预处理 → 输入张量 + ↓ 传张量 +板子:跑推理 ← 板上只做这一件事 + ↑ 回传输出张量 +主机:后处理 → 算指标 → 判门 +``` + +标注、评测器、后处理、指标计算永远留在主机侧。这带来四个好处: + +1. **加新任务类板端零改动。** 不管是分割还是姿态估计,板子看到的都只是张量。我们现在正在补分割和姿态的适配,板端一行代码没动。 +2. **评测口径只有一份。** FP32 基线和量化模型吃同一份输入、走同一个评测器、比同一批图,比较才有意义——不会出现"板上一套、主机一套"。 +3. **板上不用装依赖。** 不需要 pycocotools、不需要各种数据集工具包,接一块新板的成本很低。 +4. **意外收获:板端比服务器模拟快 45 倍。** 单张 0.889 毫秒对 40 毫秒。所以"必须上真板"不是成本,是提速。 + +评测这层还有两件事是刻意做的: + +- **数据分三档,规模是算出来的不是拍的。** 校准集、日常筛查集、交付全量集。日常筛查集分类任务用 2000 张——因为 500 张评 Top-1 单次标准误约 1.9 个百分点,分辨不出 1 个点的掉点;而 FP32 和量化是配对比较,差值标准误由结论不一致的图决定,2000 张时约 0.5 个点。所以 2000 张上只记"筛过了",交付数字要用完整验证集。 +- **入队门槛严格,探测不等于验收。** 新接入的模型要先核查权重入口、导出代码、评测器、校准数据、标注和附加门,缺一项就拒绝入队。姿态估计还额外要求板端同时判 OKS AP 和平均关键点偏移不超过 2 像素。有一个模型的类别语义没对齐,现在还卡在门外没让进——这是故意的。 + +最后,评测这层的验收标准要特别说一句:**看分辨力,不看通过数**。一个只会盖章通过的验证器,和一个真能分辨好坏的验证器,从成功样本上看是一模一样的。所以每一步的签收都要包含反向对照——故意造一个错的、造一个说谎的,确认系统把它们挑出来。 + +**提示** + +- 三步的标题要报清楚,细节可按时间裁。 +- **第二步那个设计调整要主动讲,不要等人问。** 理由有三:一是已经写进契约的第一条原则,是有记录的决策不是漂移;二是文件夹里那几份 HTML 还是旧设计,不说清楚会被发现不一致;三是"我们发现三分之二板时花在不产数据的事情上,砍掉了"本身是个漂亮的决策故事,藏起来可惜。 +- 讲的时候重音放在"判据没变松"这句上,这是唯一可能被质疑的点,主动堵住。 +- 第三步是这次新增的重点,领导对"投入产出比高"这种话敏感,把 45 倍和"板端零改动"这两个具体的点说出来。 + +--- + +## 4. 这条线长什么样:五个构件(4 分钟) + +**讲什么** + +具体到实现,是一条单向链,五个构件,职责是硬边界:**出题的不作答,作答的不判分,判分的不看自述。** + +1. **冻结的任务单元。** 一道题 = 任务类型 × 模型 × 芯片 × 后端 × 量化配置,拼成一个编号,比如 `cls--resnet18--nash-m--ptq--int8`。它既是磁盘上的目录名,也是所有表的关联键。换工具链、改阈值、修 bug 一律开新批次,老批次留着对照。这样任何一条轨迹永远能追溯到它当时面对的确切环境。 + +2. **可复用的输入准备。** 取权重、跑 FP32 基线、导出 ONNX。这三步的产物按内容哈希缓存,同一个模型在不同批次之间复用,不重复付钱。**要强调一点:准备产物是"可复用的输入证据",不是"标准答案"**——它不含任何量化结果,不参与判定。完整的确定性流水线是一个显式开关,做对照和排查时才跑。 + +3. **隔离的作答区。** agent 只有一个可写目录。隔离靠内核的命名空间机制,把兄弟轨迹、凭证、评测代码从文件系统里摘掉,再加进程隔离和权限清空。 + +4. **独立核验。** 四道门:编译、数值、任务指标、板端延时。其中编译和任务指标是必选门,数值余弦降为诊断,延时只记录。判定完全不读 agent 的自述,自述只用来和核验结果比对,不一致重罚——训练集里出现一次谎报成功,就是在教它谎报。 + +5. **离线打分和分级导出。** 打分是纯函数、带版本号,改配方可以对历史数据全量重打分,不用重跑任何实验。导出分两级:强化学习收正负例(要的是区分度),监督微调只收"做对了且诚实"的。 + +还有一条贯穿的原则:**每条轨迹彼此独立,永不串联。** 第 k+1 条看得见第 k 条,组内就不独立了,组内比较失去意义——而组内比较是整个训练方法的地基。所以"读了同一道题别人的作答"在审计里直接判不可用,和"读了评测代码"同等对待。 + +**提示** + +- 构件 2 是这次改动最大的一块,措辞已经按新设计写好,照念即可。"可复用的输入证据,不是标准答案"这句是契约里的原话,很准确。 +- 四道门的现行必选/可选:编译和任务指标必选,余弦诊断,延时记录。直接讲现行的,不铺垫历史。 +- "出题的不作答,作答的不判分,判分的不看自述"这三句念出来效果好。 + +--- + +## 5. 路线与目标收敛(2 分钟) + +**讲什么** + +实施顺序是严格有序的:地基 → 一条切片打穿全流程并人肉核对每个数字 → 闭环(正常和修复各跑通一次,并能构造出"应该被拒"的例子)→ 横向铺一批压测资源池 → 全量。 + +第二步为什么要人肉核对:因为后面所有打分都建立在核验结果上,核验可信是整个计分体系的前提。这一步省掉,后面采的所有数据都建在沙子上。 + +采集分两条线,成功指标不同:一条是覆盖交付,求覆盖率;一条是训练数据,精选题目、每题多条采样,求分辨力。两条线按阶段分开调度,互不挤占。 + +目标板已经收敛到一块:**S100P**。配套口径也定了:精度必须板端实测,不认服务器模拟;任务指标相对 FP32 劣化 ≤ 5%;延时记录但不设硬门;不允许靠把算子挪下 BPU 来换精度;其他板型暂时无板,只判编译、精度标未验证。 + +这次收敛省下的算力是实打实的:任务数从两百多降到 63 个完整加编译检查,算力省掉三分之二。加上第 3 节讲的 agent 主导那次调整,同样的板子现在能产出的轨迹是原来的三倍多。 + +**提示** + +- 措辞纪律:说"收敛",不说"之前太散"。收敛是阶段性动作,是好事。 +- 最后那句"三倍多"是两次调整的叠加效果(省 2/3 任务数 + 省 2/3 板时),是这次汇报里效率提升最有力的一句,别漏。 + +--- + +## 6. 跑到哪了:第一阶段战果(8 分钟,重点) + +**讲什么** + +**6.1 板端终判接通了。** 这是本阶段最关键的一步。精度评测现在走真实板端推理,主机预处理、板上跑张量、主机后处理算指标。在此之前所有精度都是服务器模拟出来的,不能作为交付依据。性能测试 31 秒,完整精度评测(2000 张)约 8 分钟。 + +**6.2 第一批可交付的上板基准。** 目前已有 18 个模型完成板端实测并通过,精度按相对 FP32 劣化 5% 判。这是第一批可以对外讲的数字。 + +**6.3 agent 轨迹开始产出。** 已结束 34 条:通过 22、失败 8、未验证 4,另有在跑和排队的。 + +**6.4 核心验收信号达成。** 上一阶段定的验收是"非最简单模型的通过轨迹 ≥ 5 条"——当时是 0,三条通过全集中在 resnet18 上,说明系统只证明了"流程能走通",还没证明"agent 能解决问题"。现在 22 条通过覆盖 14 个不同模型,resnet18 只占 4 条,其余覆盖 resnet 系列、resnext、wide-resnet、mobilenet、efficientnet 四个规格、regnet 三个规格、shufflenet 四个规格。 + +**6.5 修复模式跑出第一条成功轨迹。** 确定性流水线做不出来的模型,agent 调出来了——这正是我们最想采到的样本形态。明细核验还在做,核完可以定为第一条黄金样例。 + +**6.6 板卡抖动的数据回收。** 中途板子的网络网关有一段拒连,几十次板端评测失败,两条轨迹跑满预算也没测成。处理方式不是改分数,而是加了一个补测入口:用 agent 自己编出来的产物补跑板端两步再重新核验。两条都是真实通过——最好的一条板端 top-1 相对 FP32 只掉 0.06%,是全批次最小的;另一条掉 3.69%,也在门内。同时打分规则补了一条:板卡不可用时只免罚、不加分,并标明不可验证的原因。 + +**6.7 效率上的两次调整。** 目标板收敛省掉三分之二任务数;agent 主导省掉三分之二板时。两项叠加,同样的硬件现在能产出的轨迹是原来的三倍多。 + +**6.8 覆盖在往外扩。** 338 项的覆盖总表已经建起来,逐阶段记录:静态前置齐备、FP32 对照通过、量化编译、真实上板、任务精度通过分别记录,不能互相替代。分割和姿态估计的适配在做,姿态的板端判据已经定死——同时看 OKS AP 和平均关键点偏移 ≤ 2 像素。传统算法那几项如实标"不适用 PTQ",不虚构产物。 + +**6.9 隔离与数据洁净度经得起查。** 内核级隔离做过红队:手写请求打了 11 种路径逃逸,11 种全部挡住。审计脚本对每条轨迹给机械理由,不合格的不进数据集;历史数据集全部加作废标记留档、不删除——轨迹是复盘证据,删了同样的问题要重踩一次才知道。 + +**提示(措辞纪律)** + +- 6.5 **只讲到"状态是通过、明细在核"**,不要说"核心假设已验证"。措辞用"这正是我们想要的形态,核完就是黄金样例"。 +- 6.5 还有一个坑:修复模式的输入是"确定性流水线的失败现场",而默认已经不跑它了。如果被追问"那以后还怎么产生修复任务",照第 10 节第 3 条答,不要临场编。 +- 不要引用早期那组 143 / 83 / 60 / 38 的统计。如果领导记得并问起,答"那是上一批不同板型的统计口径,这次全部切到板端终判后重新起的截面,两组不可比"。 +- 6.2 的 18 个,如果被问"这 18 个是 agent 跑出来的还是流水线跑出来的"——如实说是 09-16 那批完整流水线跑的,现在这批是 agent 在产出;两种都是板端实测,判据一样。**这个问题大概率会问,提前想好一句话。** +- 板端利用率 93% 这个数字为后面的板卡诉求做铺垫,一定要说到。 + +--- + +## 7. 这一类工程的共性规律(5 分钟) + +**讲什么** + +这些不是我们独有的问题,是所有"长时间运行 + 多进程 + 数值判定"系统的共性规律。我们把它们变成了规矩写进契约,后面接手的人不用再花时间。 + +1. **正在跑的代码不等于磁盘上的代码。** 常驻调度进程加载的是它启动那一刻的代码和配置,缓存还会把旧配置留在内存里。所以"改好了"指的是文件,不是正在跑的东西。规矩:改了流水线必须让旧进程收尾退出再用新代码起新进程,不能只同步文件;进程启动时自证代码指纹,不一致拒绝领新任务。 + +2. **隔离要做在内核层,不做在字符串层。** 解析 agent 写的命令做白名单,是一类永远追不平的防线——每一版都会出现新写法。改成"让它看不见"之后这类问题结构性地消失了。命令解析那层降级成教学层,只负责给一句人话的拒绝理由、省 agent 的预算。 + +3. **数值相似度是过程量,任务指标才是结论。** 多分支输出不能展平算一个整体余弦——检测模型的输出里框坐标量级大、类别分数在 0 到 1 之间,整体余弦会被前者主导。实测过一个:整体余弦最低 0.9976,而类别分支只有 0.36,对应 mAP 从 46 掉到 26。规矩:多语义输出逐分支判;余弦降为诊断,不当必选门。 + +4. **容差必须显式声明方向和单位。** "相对劣化百分比"越小越好,"绝对百分点改善"越大越好,符号相反。独立复现过:同样是掉 2.5%,容差写 -5 判失败、写 5 判通过。规矩:每条容差显式声明类型,评测器按类型算方向;百分点和百分比不相加。 + +5. **预算耗尽不等于失败。** 已经产出并核验过的结果照常记账,不能因为超预算把整条成果清零——否则一条做得很好但没来得及收尾的轨迹会被记成彻底失败,而那往往是最有训练价值的一条。规矩:执行终止原因和产物判定分开记,互不覆盖。 + +6. **工具链层面两个具体的坑,写出来给后面的人省时间。** 运行时返回的数组引用的是内部缓冲区,下次调用会原地覆盖,必须显式拷贝——有一版评测因此把 Top-1 算成了 2.6%。校准数据目录下只能放指定格式文件,工具会把目录里所有文件当样本读,清单文件放进去会炸。 + +**提示** + +- 整体是"经验沉淀"的口吻,不是"我们犯了什么错"。每条落点都是"变成了规矩",不是"造成了损失"。 +- 不要提具体进程号、不要说某批任务作废了多少个——那是内部复盘细节,汇报讲规矩就够。 +- 时间紧就讲 1、2、3,这三条最能说明"为什么这套系统要这么设计"。 + +--- + +## 8. 在建项(2 分钟) + +**讲什么** + +排在第二阶段的工作,都已定方案: + +1. **两阶段编译**(最大的提速项)。一次编译里量化占 107 秒、BPU 编译占 78 分钟,比例 1 比 44;而 agent 迭代要看的精度信号在量化结束那一刻就有了。拆开之后单次实验从 35 到 118 分钟压到 5 分钟以内。工具链原生支持这个拆分,已在一个模型上实测过等价性。落地前要先固定两件事:编译参数完整转发,候选产物用摘要关联成一份清单。 + +2. **修复模式的触发方式**。目前修复任务的输入是确定性流水线的失败现场,而默认已经不跑它了。要改成吃 agent 自己的失败现场,或者对失败的模型显式补跑一次对照来生成现场。这条直接关系到能不能持续产出黄金样本,优先级高。 + +3. **自述时序的处理**(约 1 小时)。板子断开时 agent 按预算提醒先写了失败、补测后结论变成通过,这种时序差目前会被记成不一致。方案已定:标为待复核而不扣分,复核前不进训练候选。 + +4. **判定的进一步拆分**。编译成功、输入输出契约、产物一致性目前共用一个字段,要拆开各自存;输入输出校验目前只比形状,要补数据类型和排布。 + +5. **打分锚点的补充**。原来的相对打分项以默认配置为参照,现在这个参照点按需才有。好消息是强化学习本身是组内比较——同一道题多条采样,模型难度天然抵消,这部分影响不大;但跨题比较和监督微调的绝对门槛还需要一个锚,方案是用 FP32 基线加门槛,必要时对少量代表性模型保留对照。 + +6. **学生策略接入**。目前全部是教师模型的轨迹,在线强化学习的闭环还没跑过。接入后第一次跑的目标不是涨分,是确认同一道题的多条采样之间确实存在可区分的好坏。 + +7. **覆盖扩展**。分割和姿态估计的适配在做,按"模型数除以边际成本"排序——这两类复用已有的图像和缓存,边际成本最低。 + +8. **板卡健康检查与扩容**。目前只看配置里的启用标记,没有连通性探测。加上探测和退避之后,板卡数量直接决定实验周期。 + +**提示** + +- 口吻是"已排期的工作",不是"欠账"。每条带方案和量级,显得可控。 +- 第 2 条和第 5 条是 agent 主导那次调整带出来的配套工作,主动列出来,说明这次改动是想清楚了才做的、连副作用都盘过——这比不提要好得多。 +- 第 8 条是唯一隐含资源诉求的点。领导接话再给数,不要主动伸手要。 + +--- + +## 9. 下一步与阶段验收(2 分钟) + +**讲什么** + +第二阶段按这个顺序推进,不并行: + +1. 补齐修复模式的触发方式和打分锚点——这两条是上次调整的配套,先收尾。 +2. 两阶段编译落地。先在一个分类切片上验证参数、产物关联、板端数字一致,再扩到复杂模型和检测模型。 +3. 判定拆分和显式重判接口:历史批次按冻结规则判,新批次按新规则判,两套口径不混。 +4. 接学生策略,跑通一次在线采样,看同一道题多条采样之间打分有没有区分度。 +5. 规模化。 + +阶段验收我建议用两个信号,都不是通过数: + +- **分辨力四例各有一例被正确落位**:做对了、做错了、正确放弃、超预算——尤其是后两类"应当被拒"的例子。这是方法论走通的签收点。 +- **可用轨迹比例**:版本合格、没越界、没被误判的轨迹占比。这才是这条流水线的产能指标。 + +通过数受模型难度、门槛松紧、工具链成熟度共同影响,想让它涨把阈值放松就行——所以它不能当项目指标。 + +**提示** + +- 最后一句要说,但语气放平,说成"这是我们给自己定的纪律",不是"别拿通过率考核我们"。 +- 需要对外时间点时可以说:两阶段编译落地后采集效率提升一个数量级,届时给出第一版训练数据集的规模和分辨力指标。 + +--- + +## 10. 问答预案 + +**1. "不跑确定性流水线了,判定会不会变松?"** +不会。精度门判的是"板端实测的任务指标相对 **FP32 基线** 劣化不超过 5%",参照物从来就是 FP32,不是默认配置的量化结果。FP32 基线属于输入准备,照常产出。去掉的只是"默认配置能做到多少"这个参考点,四道门一个没动,板端实测一步没省。 + +**2. "那怎么知道一个模型是 agent 做不好,还是我们的流水线本身有问题?"** +三条:一是完整流水线保留成显式开关,怀疑的时候单独跑一次对照,几十分钟出结果;二是同一道题有多条独立采样,如果多条以完全相同的方式失败,那更可能是环境问题不是 agent 问题;三是失败有分类,环境类和判断类的错误码是分开的。另外这也正是我们保留这个开关而不是删掉的原因。 + +**3. "修复模式的输入是流水线的失败现场,不跑了以后怎么产生修复任务?"** +这是这次调整带出来的配套工作,已经排进第二阶段第一项。两个方案:改成吃 agent 自己的失败现场,或者对失败的模型显式补跑一次对照生成现场。倾向前者——agent 自己的失败现场对训练更有价值,因为那才是它真实会遇到的情况。 + +**4. "什么时候能真正训出模型?"** +数据侧的接口和格式已经备好,轨迹在持续产出。开训的判据不是攒够多少条,而是分辨力四例各有一例被正确落位——因为只产出正样本证明不了验证器有分辨力,用没有分辨力的数据去训,花了算力学到的是测量误差。这四例里还差两类,在第二阶段第一项里。 + +**5. "通过率是不是偏低?是不是说明模型大多上不了板?"** +不能这么推。失败里混着配置层级的问题、进程终止、评测口径三类,都和模型本身能不能上 BPU 无关,要给这个结论得先把这几类剔干净。现在能确定说的是:通过的那些,精度数字是板端实测、按 5% 相对劣化判的,这部分可以直接作为交付基准。 + +**6. "agent 到底比默认配置强在哪?"** +有明确的局部证据:一个分类模型默认量化掉 8.9 个百分点,agent 通过调位宽和校准参数做到掉 0.1 个百分点,诊断链条完整——定位误差累积位置、做过对照实验。同时我们也主动给它加了约束:如果延时完全不设限,"整网提到 int16"就是一行代码的通用解,任务会失去难度。所以位宽构成要进打分,对照要做三组:默认、全 int16、agent 方案。 + +**7. "评测这块投入大吗?"** +不大,而且设计上是一次投入长期复用。板子上只跑张量,读图、预处理、后处理、算指标全在主机侧,所以加一类新任务板端零改动——我们现在补分割和姿态,板端一行没动。板上也不用装各种数据集工具包,接新板成本很低。顺带板端推理比服务器模拟快 45 倍,上真板反而是提速。 + +**8. "隔离真的安全吗?数据会不会泄漏?"** +内核级隔离加进程隔离加权限清空,红队 11 种路径逃逸全部挡住。唯一藏不住的是大模型的访问凭证——agent 进程自己要读它,同进程的命令行就一定读得到,这是结构性限制,靠定期轮换兜底。另外审计层还有一道:即使读到了,内容指纹会命中,那条轨迹直接判不可用。 + +**9. "这套东西只能用在量化上吗?"** +骨架和任务无关,可以迁出去,但有三条前提:判据可机器化、验证远比求解便宜、失败模式是有限集。CPU 算子优化那条满足,立项说明已经写了;判据要按新任务重做,这是迁移时最容易犯也最贵的错。 + +**10. "这件事对外能讲的价值是什么?"** +四样产出:一是后训练出来的模型;二是带打分和失败分类的数据集;三是这套基准和流水线——换一块板、换一类任务,别的组能直接接着用;四是业务数字,端到端延时降低多少、覆盖多少客户模型。按我们做底座的定位,第三样的战略价值不低于第一样:模型会过时,基准不会,而且基准定义了整个行业用什么标准衡量这件事。 + +--- + +## 附:数字来源 + +| 讲稿里的数字 | 来源 | 截面 | +|---|---|---| +| agent 主导、输入准备只做 01–03、完整流水线改显式开关 | `bench/CONTRACT.md` §0 第一条原则 | 09-17 | +| oracle 占板 7.92 h vs rollout 3.83 h、利用率 93%、板端故障统计 | `outputs/plan-verify-20260916/REVIEW.md` | 09-16 15:26 | +| 轨迹计数、板端通过数、端点切换 | `outputs/glm53-endpoint-switch/runtime-after.json` | 09-16 21:03 | +| 0.06% / 3.69% 补测结果 | `outputs/plan-verify-20260916/evidence/backfill_result.jsonl` | 09-16 16:35 | +| 板端 31 s / 8 min、45 倍、2000 张统计依据 | `bench/PLAN.md` §4、`bench/README.md` | 09-15 | +| 107 s : 78 min、两阶段等价性 | `bench/SPEC-two-phase-compile.md` | 09-15 | +| 覆盖总表、姿态判据、入队门槛 | `bench/README.md` 头部、`bench/CONTRACT.md` 头部 | 09-17 | +| 红队 11/11、审计规则、历史数据集作废 | `bench/REPORT-codex.md` | 09-15 | +| 余弦 0.9976 对 0.36、分辨力四例 | `BPU量化轨迹工厂-闭环方法论.html` | 09-15 | +| 三条适用前提、CPU 算子优化落点 | `CPU算子优化Agent-立项说明.html` | 09-15 | + +**引用纪律**:旧截面的统计不能当现在的结论,引用前先确认截面时间;通过率不能当"模型能不能上 BPU"的结论。 + +**文档一致性提醒**:`方法论.html` 和 `闭环方法论.html` 仍是"确定性流水线先出标准答案"那一版设计,与现行的 agent 主导不一致。投屏前先说明,或抽空更新对应段落。 diff --git a/docs/BPU量化轨迹工厂-方法论.html b/docs/BPU量化轨迹工厂-方法论.html new file mode 100644 index 0000000..a0bc5a4 --- /dev/null +++ b/docs/BPU量化轨迹工厂-方法论.html @@ -0,0 +1,498 @@ + + + + + +把工程任务做成训练数据 + + + + +
+ +
+
方法论 · BPU 量化轨迹工厂
+

把工程任务做成训练数据

+

这套东西从头到尾怎么做、每一步要注意什么。四个部分:任务怎么定、资源怎么摆、实验怎么跑、最后交付什么。每一条都是已经做过的,后面附着当初为什么这么定。

+
+
规模
338 模型 / 10 类
+
已就绪
63(cls 48 · det 15)
+
主 run 已结束
168 oracle
+
目标板
S100P · nash-m
+
数据去向
RLVR · SFT
+
+
+ +
+

§0整条线长什么样

+

一句话:把 338 个模型的量化上板,逐个变成一道有标准答案的题;让 agent 在看不见答案的情况下做一遍;用同一套代码独立核验它做得对不对;核验结果就是训练信号。

+
+
registry/models.csv ──▶ new-run / expand ──▶ jobs(PENDING)
+                                                │
+              bench run(后台调度)──▶ oracle 八阶段:01 fetch → 02 baseline → 03 export
+                                                │              → 04 calibrate → 05 compile
+                                                │              → 06 hbm → 07 board → 08 eval
+                                                │              四道门:G1 编译 · G2 余弦 · G3 任务指标 · G4 延时
+                                                ▼
+              bench collect(后台采集)──▶ 每个 job k 条 rollout:agent 在自己的 workspace 里作答
+                                                │              → 独立核验 → verdict.json(reward 的唯一来源)
+                                                ▼
+              audit_rollouts.py ──▶ 哪些轨迹干净、能进 RL / SFT
+              export/to_agentrl.py ──▶ datasets/<dataset_id>/
+
+

这条线上每一环的职责是硬边界,不能交叉:出题的不作答,作答的不判分,判分的不看自述。下面四节就是沿着这条线,把「怎么做、为什么这么做」写下来。

+
+ +
+

§1任务:怎么把一个工程活变成一道能训练的题

+

这一节回答的是:什么算一道题、什么算做对了、做不出来怎么办。

+
+ +
+
1
+
+

一道题 = 一个冻结的组合,题号既是目录名也是关联键

+

一道题叫一个 job:任务类型 × 模型 × 芯片 × 后端 × 量化配置,五段拼成 job_id,比如 det--yolov8n--nash-e--ptq--int8。它同时是磁盘上的目录名,和所有表里的关联键。

+
为什么后面要把轨迹、判定、奖励、数据集四类东西对起来。只要题号是唯一的、不变的,这四样永远能 join 回去;题号一变,历史数据就全断了。同理 run 也有号(r20260914-01),工具链指纹不进题号——两套 hbdk4 塞进一个号里,号本身就会说谎。
+
+
+ +
+
2
+
+

先由确定性流水线跑出标准答案,再让 agent 做

+

每道题先由 harness 自己走一遍八个阶段,产出 oracle/result.json,这就是标准答案。agent 之后做的每一次,都相对它衡量。

+
为什么同一份标准答案有三个用途,少一个都不行:① 对错的依据;② 相对基准——奖励里的精度和延时都算相对 oracle 的差,因为绝对延时是模型自身的属性,不是 agent 的功劳;③ 「做不到」的裁判——agent 说这道题做不了的时候,看 oracle 是不是在同一阶段以同类错误失败。不需要人判,也不需要另一个模型来判。
+
+
+ +
+
3
+
+

题的状态和答的状态分开记,永不互相覆盖

+

job 的状态由 oracle 决定(PASS / FAIL / UNVERIFIED),rollout 的状态由核验结果决定(PASS / FAIL / BLOCKED / UNVERIFIED)。一道题只有一份标准答案,每次作答各有一份判定。

+
为什么塞进一个字段就会出数据事故:要么标准答案被某次作答覆盖,要么两种 PASS 混成一个。而且「oracle 做不出来、agent 做出来了」是最值钱的样本——它要么是黄金数据,要么说明我们的流水线有 bug。这类自动打 review 标记进人工抽查,查完反过来修 harness,形成回路。
+
+
+ +
+
4
+
+

判据分成四道门,必选和可选分清楚

+

G1 编译(产物在不在、芯片型号和输入格式对不对、日志有没有 ERROR)· G2 数值(量化前后逐输出余弦)· G3 任务指标(同一子集、同一预处理、同一评测器下相对 FP32 的掉点)· G4 板端延时。必选门全过才算 PASS。

+
为什么门是绝对判断——这个模型能不能交付;奖励是相对判断——agent 比标准答案做得好还是差。两件事都要,但不能混成一个数。
+
最容易出事的一条:G1 里的芯片型号不一致直接判死。这是 agent 最容易钻的口子——换个芯片编译当然更容易过。
+
+
+ +
+
5
+
+

「这轮不评」和「本该评没评成」必须分开记

+

门有两种非结果:na 是本轮设计上就不评(比如 3D 占据第一轮只编译不评精度),不阻塞 PASS;unverified 是本该评但没评成(数据缺、板没接),阻塞 PASS。

+
为什么这是交付文档里「未判定 ≠ 不适用」的直接映射。两者混成一个「没数」,对外就会说成「这个模型不行」,而实际上只是我们没测。
+
+
+ +
+
6
+
+

失败必须有名字,因为它要同时喂三个地方

+

每个失败由正则分类器从 stderr 打一个标签,存进数据库和结果文件。确定性的五类:unsupported_op · shape_infer · export_fail · calib_nan · verifier_mismatch;瞬态的三类:oom · board_offline · stalled;没匹配上的进 unknown 人工队列,规则库随之增长。

+
为什么要分类不是为了好看,是因为这个标签同时被三处用到:① 决定要不要重试——确定性的不重试(重试一百次也一样),瞬态的重试;② 喂给 repair 模式的提示词——修复任务要告诉 agent 现场是什么错;③ 判定「做不到」是真是假——agent 报 BLOCKED 时,比对 oracle 是不是同阶段同类错误。没有这三处用途,确实不用分类;有了这三处,不分类就没法做。
+
+
+ +
+
7
+
+

难度不是事先定的,是 oracle 跑完自己浮出来的

+

顺序是:先把所有就绪模型的 oracle 铺一遍 → 得到失败率分布和错误分类分布 → 难度自然分成三层:默认配置就过的、改配置能救的、工具链真做不到的。采轨迹的重点是中间那层,但这一层只能等 oracle 跑完才知道是谁。

+
为什么第一轮预期失败率就是高的,尤其局部特征、光流、3D 占据这三类——当初就写明了「这是数据,不是事故」。事先按模型大小或者主观印象去猜哪道题难,猜不准;而 oracle 本来就要全铺一遍(它是 benchmark 交付本身),难度分层是这一遍的副产品,不额外花钱。
+
+
+ +
+
+ +
+

§2资源:机器、数据、板子怎么摆

+

这一节回答的是:算力怎么分、数据从哪来、慢的东西怎么办。

+
+ +
+
1
+
+

资源分池,一个并发数不能管所有

+

四个池各自设并发:GPU(基线、评测)· CPU(导出、编译)· board(板卡)· agent(受 LLM 端点限速)。池是跨进程的文件锁,调度进程、agent 提交的阶段、多个 bench run 共用同一组槽位。

+
为什么用一个数字管所有,板端会把 GPU 堵死——板卡是严格串行的,一块板就是一个槽位,而 GPU 阶段本可以并行好几路。
+
+
+ +
+
2
+
+

板卡按物理身份登记,一块板一把锁,槽位不复用

+

每块独立物理板登记唯一槽位、芯片型号、运行库路径和各自的连接信息;容量等于启用的板数。停用的板标记禁用,槽位不回收再用。同一地址、同一物理板的多个容器,不能登记成额外容量。

+
为什么容量是用来排队的。把同一块板登记两次,排队系统会以为有两块,两个任务同时推上去互相干扰,测出来的延时全废。
+
+
+ +
+
3
+
+

数据复制入池、设只读、写清单;agent 永远不下载

+

数据集从机器上别人的目录复制进自己的池子(不是软链),复制完 chmod -R a-w,写 manifest.json 记样本清单的 sha256、数量、来源路径、日期。

+
为什么复制不软链别人的目录随时可能被清掉。几十 GB 的冗余换来的是零耦合。为什么 agent 不下载三条:可复现(线上数据集版本会飘)、防作弊(agent 会挑好算的子集)、轨迹干净(网络波动不该进训练数据)。
另外实测过:官方 COCO 源 26 KB/s,815 MB 要下 9 小时,而同样的东西 HF 是 8 MB/s、阿里云 OSS 是 43.8 MB/s。照着任务清单里给的官方入口下载是下不动的,所以每个数据集声明有序多源,逐个试、成功后校验 sha256。
+
+
+ +
+
4
+
+

数据分三档,用途不同,规模是算出来的

+
+ + + + + + + +
档用途规模
T0 calib量化校准128–512 张,从 train 抽,与验证集零交集
T1 smoke日常筛cls 2000 · det/iseg/pose 1000 · seg 500
T2 full交付判完整验证集,开新 run
+
+
为什么 cls 是 2000 不是 500这是算出来的,不是拍的:500 张评 Top-1,单次测量的标准误约 1.9 个百分点,根本分辨不出 1 个点的掉点。但 FP32 和量化是配对比较——同一批图,误差高度相关,差值的标准误由「结论不一致的那些图」决定,2000 张时约 0.5 个点,够用;500 张时约 1 个点,和容差同量级,会误判。
所以 T1 上只能记 pass-screen(筛过了),T2 上才是 pass(判过了)。对训练信号来说 T1 够用,对交付数字来说必须 T2。
+
+
+ +
+
5
+
+

没有固定超时,只有看门狗

+

判定卡死的条件是两个同时成立:日志 15 分钟不增长,且整个进程树 CPU 空闲。24 小时硬上限只是保险,正常永远碰不到。编译走单独的长车道(1 小时)。

+
为什么不同模型的编译时长差两个数量级——实测 convnext.small 118 分钟、efficientnet.b4 74 分钟、resnet50 61 分钟。固定超时一定会误杀慢模型,而误杀产生的失败样本不是 agent 的错,却会被当成 agent 的失败学进去。
判据本身也有坑:CPU 采样必须递归覆盖所有后代进程(编译的实际计算在孙子进程里),D 状态(等 I/O)不能算作空闲。
+
+
+ +
+
6
+
+

重活由 harness 执行,agent 只提交和等待

+

agent 不在自己的 bash 里跑编译,而是 bench stage run compile --config workspace/x.yaml 提交,拿一个句柄,再 bench stage wait 等结果。它的 bash 仍然可用,用来看日志、改配置、跑小脚本。

+
四个好处① 编译时长和 agent 工具的超时脱钩;② 后端和虚拟环境的选择不可能被 agent 选错(两套工具链版本完全不同,禁止把这个判断交给 agent);③ 预算和看门狗对 oracle 与 agent 一视同仁;④ 日志位置统一,回放的时候找得到。
+
+
+ +
+
7
+
+

便宜的信号用来迭代,贵的信号用来终判

+

同一个编译命令里,量化 107 秒,BPU 编译 78 分钟,比例 1:44(resnet50 实测日志时间戳)。而 agent 迭代要看的精度信号,在量化结束那一刻就有了。所以把这两步拆开:agent 用量化后的中间产物快速试,确认可行了才付那一次昂贵的 BPU 编译,产出板端权威数字。

+
实测依据resnet18 两阶段:量化 23 秒 + 编译 73 秒,产出的 .hbm 字节数与一次性编译完全一致,所以拆分没有正确性风险。
另一头:x86 上跑中间产物 40.5 毫秒一张,板端 BPU 0.889 毫秒一张,板端快 45 倍——但要用板端就得先付编译的钱。所以分工是:便宜的当快速筛,板端当终判,权威数字只认板端。
+
+
+ +
+
+ +
+

§3实验方案:怎么跑,怎么保证跑出来的东西可信

+

这一节是踩坑最多的一节。可信度不是靠小心,是靠几条机械规则。

+
+ +
+
1
+
+

只追加,不覆盖:口径变了就开新 run

+

换工具链、改配置、修 bug 之后重跑,一律开新 run,老 run 留着对照。run 的清单文件记录两套虚拟环境 pip freeze 的 sha256,每个任务启动前比对,不一致直接拒跑。状态只存在数据库里,文件系统只存内容。

+
为什么要拒跑而不是告警否则「只追加」是句空话——同一个 run 里混着两套工具链跑出来的结果,事后没有任何办法区分。
+
+
+ +
+
2
+
+

正在跑的代码 ≠ 磁盘上的代码

+

常驻调度进程加载的是它启动那一刻的代码和配置。改完 harness 必须让旧进程收尾退出(bench drain,不要 kill 正在编译的进程),再用新代码起新进程。

+
踩过的坑一个调度进程连续跑了 23 小时多的旧代码,期间生成的编译配置一直是修好之前的写法;磁盘上的文件早就改对了,看文件完全看不出问题。「改好了」指的是文件,不是正在跑的东西。缓存也会把旧配置留在内存里。
+
+
+ +
+
3
+
+

agent 只有一个可写区,隔离必须在内核层

+

agent 的工作目录只有 rollouts/<kk>/workspace/,标准答案、评测代码、别的作答、凭证全在外面。隔离靠 mount namespace 把这些东西从文件系统里摘掉,加上 PID 命名空间和权限清空;jail 起不来就拒绝启动,不静默降级。

+
踩过的坑之前那版隔离是解析 agent 写的 shell 命令做白名单,连续四版被绕过:换行后的 cd、$IFS、路径里拼 $VAR/、在自己目录里做个软链接指回去。这类防线永远追不平所有写法。改成「让它看不见」之后,这一类 bug 就没有了——看不见的东西,再多绕法也没用。
命令解析那层降级成教学层:只负责给一句人话的拒绝理由,省 agent 的预算。
藏不住的一样东西LLM 凭证。agent 进程自己要读它,同进程的 bash 就一定读得到。这是结构性限制,靠定期轮换 key 兜底,别指望隔离。
+
+
+ +
+
4
+
+

agent 说的不算数,独立核验才算数

+

判定完全不读 agent 的自述。它交的自述只用来和核验结果比对,比对结果单独记一个字段。自述与核验不一致要重罚,而且这类样本绝不进监督训练集。

+
为什么训练集里出现一次「谎报成功」,就是在教它谎报。没交自述只小额扣分——不报比报错轻,但不能没有代价。
还有一条同类的:编译返回 0 不等于验收通过。编译日志里的百分比、校准余弦、静态性能估算都不能代替四道门。
+
+
+ +
+
5
+
+

每条轨迹彼此独立,永不串联

+

三层重试,边界是硬的:① harness 对瞬态错误的重试不算轨迹(没有 agent 参与);② agent 在一条轨迹里看到报错自己改配置重提,是同一条轨迹——这正是我们要采的诊断过程;③ 新的 rollout 是全新轨迹,全新上下文,不知道其他作答存在;④ 修复模式只吃 oracle 的失败现场,不吃别人作答的失败。

+
为什么组内比较是整个训练方法的地基。第 k+1 条看得见第 k 条,组内就不独立了,比较失去意义。所以「读了同一道题别的作答」在审计里直接判不可用,和「读了标准答案」同等对待。
+
+
+ +
+
6
+
+

失败的轨迹不删,过滤发生在导出那一刻

+

磁盘上留什么,和数据集里收什么,是两件事。轨迹全留,导出时按版本号、污染标注和资格函数筛。改了提示词、边界或扩展,就把 agent 版本号 +1,旧版本的轨迹自然被过滤掉,不需要删。

+
为什么三条:① 轨迹是复盘证据——好几个 bug 都是靠原始轨迹定位的,删了同样的坑要重踩一次才知道;② 数据洁净度由导出时的规则保证,和磁盘上留不留无关;③ 版本号是用来追溯的,它记着这条轨迹是在哪一版规则下跑的。
同理,job 和 run 也不删,历史数据集作废也是加作废标记(连同原清单的哈希),不是删除。
+
+
+ +
+
7
+
+

每条轨迹能不能用,由审计逐条给理由

+

审计脚本每条轨迹输出一行,理由是机械判据:读了标准答案、读了别人的作答、被看门狗误杀、旧版本、没交自述、自述与核验不符、去读评判代码或工具链、内容指纹命中、碰了状态库、读了别的题。干净且版本合格的标 RL ✓,再加上核验通过且自述一致才标 SFT ✓。

+
注意一个反例做内容指纹匹配时,agent 自己合法拿到的阶段结果可能和标准答案数值完全相同(因为本来就该相同),这类不能算泄漏证据,否则正常轨迹会被误报。判泄漏要看来源,不能只看数值撞上了。
+
+
+ +
+
8
+
+

先打穿一条切片,人肉核对每一个数,再扩量

+

实施顺序是:地基(骨架、契约、数据入池、预检全绿)→ 一条 oracle 切片打穿八个阶段,人工核对每一个数字 → 闭环(端到端和修复各跑通一次)→ 横向铺一批压测资源池 → 全量。

+
为什么人肉核对因为后面所有奖励都是相对 oracle 算的——oracle 可信是整个计分体系的前提。这一步省掉,后面采的所有数据都建在沙子上。
+
+
+ +
+
9
+
+

假定 oracle 自己也可能是错的,留三道防线

+

① 四道门是绝对判断,不依赖 oracle;② 相对项全部 clip 在有界区间,单条错值不会把奖励打飞;③ 「oracle 失败而 agent 成功」的样本强制人工抽查,查完反过来修 harness。

+
为什么harness 的 bug 会系统性地把奖励打歪,而且不容易被发现——它不像崩溃,它安静地产出看起来合理的数字。
+
+
+ +
+
10
+
+

改判定口径必须同时升版本号并重判

+

评测器改了口径就升评测器版本;门的阈值改了就重跑判定(历史任务按它冻结时的阈值重判,新任务用新阈值)。两者都不做,数据库里就会同时存在两套口径的数字,而且分不出来。

+
踩过的坑某个阈值改了之后没有重判,一批任务就一直按旧阈值挂着失败。
还有一类更隐蔽的:容差的方向和单位写反。「相对劣化百分比」越小越好,「绝对百分点改善」越大越好,两者符号相反;独立复现过一次——同样是掉 2.5%,容差写 -5 判失败、写 5 判通过。每条容差必须显式声明 kind,评测器按 kind 计算,不允许省略。另外百分点和百分比不能相加,统计误差的放宽单位也要跟着换算。
+
+
+ +
+
11
+
+

指标要和被测对象的结构匹配

+

多分支输出不能展平成一个整体余弦。检测模型的输出里同时有框坐标(量级大)和类别分数(0–1),整体余弦会被前者主导。实测过一个:整体余弦最低 0.9976,而类别分支只有 0.36。

+
怎么办凡是多语义输出,逐分支判、逐分支记。这条同时提醒一件事:不要拿一个高余弦就推断精度没问题——数值相似度是过程量,任务指标才是结论。
+
+
+ +
+
12
+
+

一层只能有一个写者

+

代码按层划分归属:流水线与门、agent 配置、导出、清单与阈值、脚本。谁在改哪层要写明,别人对那层只读。改动只在本地副本做,通过同步脚本推上机器;同步脚本带守卫,机器上被人动过就拒绝推送。

+
踩过的坑两个会话同时改一棵树,互相覆盖过一次;清单文件还丢过两次。机器上的运行数据(runs、state、data、cache)一律不手改——改判走重判命令,补测走补测命令。
+
+
+ +
+
+ +
+

§4产出:交付什么,怎么算做成了

+

这条线上其实同时长出四样东西,四样都是产出,验收方式各不相同。

+
+ + + + + + + + +
产出是什么怎么算做成了
① 量化交付数据每个模型能不能上板、掉多少点、板上多快——oracle 这一遍本身就是 benchmark四道门有明确判定;T2 上的数字才叫交付数
② 训练数据带可核验分数的 agent 轨迹(教师、学生分开落)看干净可用的比例,不看通过数
③ 这套流水线本身目录契约、四道门、核验与导出——换个任务也能用换一类任务不用改环境契约
④ 复盘证据每次失败的分类、每条轨迹的回放任何结论都能追到原始记录
+
+
+ +
+
1
+
+

奖励离线算,纯函数,配方带版本号

+

核验结果文件里只存原始测量值,奖励在导出阶段离线算。配方是纯函数、带版本号,改了配方可以对历史数据全量重打分,不用重跑任何实验。

+
为什么配方一定会改好几版。如果奖励在跑的时候就写死进结果里,每改一次配方就要重跑一次实验——那是几个星期的机器时间。
+
+
+ +
+
2
+
+

导出资格分两级:强化学习收正负例,监督训练只收做对且诚实的

+

默认保留全部已结束的终态(成功、精度不达标、预算耗尽分别标注)。能不能进监督集是一层筛选,不代表数据总量。自述与核验不一致、需要人工复核这两条,是优先级最高的硬排除。

+
注意预算耗尽不等于失败。已经产出并核验过的门结果照常记账,不能因为超预算把整条成果清零——否则一条做得很好但没来得及收尾的轨迹,会被记成彻底失败。
+
+
+ +
+
3
+
+

每个数字都要带着证据边界一起流转

+

必须显式区分的几组:pass-screen(筛子上过了)≠ pass(完整验证过);工具的静态性能估算 ≠ 板端实测;服务器上的模拟运行 ≠ 物理板端运行;用随机输入测的延时 ≠ 用真实数据集测的精度。

+
为什么弱证据会在传递过程中逐级升格成强结论——第一手的人知道那是筛出来的,传到第三个人就变成「这个模型通过了」。这条不产出代码,但它决定这批数据将来能不能被别人信。
+
+
+ +
+
4
+
+

验收看分辨力,不看通过数

+

通过数受模型难度、门槛松紧、工具链成熟度共同影响——想让它涨,把阈值放松就行。真正的验收是:给这套系统四类终态(做对了、做错了、正确放弃、超预算),它能不能把四类都正确落位,包括把该拒的拒掉。

+
为什么一个只会盖章通过的验证器,和一个真能分辨好坏的验证器,从成功样本上看是一模一样的。所以只产出正样本,证明不了任何事。
+
+
+ +
+
5
+
+

教师和学生是两种数据,分开采、分开落

+

教师模型的轨迹加核验筛选,用于监督训练和拒绝采样;学生模型(自己的模型)的轨迹用于在线强化学习。提示词模板相同,在矩阵里按策略维度分开采,导出时落到不同数据集。

+
还有一条真正做在线强化学习时,需要采样时的对数概率,而现在这套 agent 进程只有对话记录。到那一步,驱动层会换成训练框架自己的循环。这套方案的持久资产是环境契约——目录结构、四道门、核验文件格式——不是当前这个 agent 进程。
+
+
+ +
+
+ +
+

§5这套方法论现在走到哪一步

+

截至 2026-09-16 的核验快照,用来说明这套方法论走到了哪一步,也用来提醒哪些数字是旧口径。

+
+ + + + + + + + + + + + + +
环节状态说明
目录契约与命名已落地job / run / rollout / 阶段 / 数据集五套编号都在用
oracle 八阶段已落地主 run 已结束 168 个(3 通过 / 165 失败),另有在跑和排队的
四道门与核验在修口径容差方向与单位、多分支指标、一致性检查的拆分正在改
内核级隔离已落地外部执行、只读产物、票据校验均已实现并有回归
轨迹审计与导出已落地按版本、污染标注、资格函数三重过滤;旧数据集已作废留档
板端终判接线中板卡可登录;37 份 nash-m 产物中 9 个有板端性能;板端精度路径尚未接上
两阶段编译计划内 · 待改方案已验证等价(字节一致),代码仍走一次性完整编译
修复模式 / 学生策略计划内 · 待启用数据库里全部是教师的端到端轨迹
任务覆盖63 / 338其余 257 个需要导出适配;评测器目前只有分类和检测两种
+
+
+

引用数字时的两条纪律

+

一、旧截面的统计不能当现在的结论。之前复盘里的 143 / 83 / 60 / 38 都是更早的截面和口径,现在的数是 168 个已结束 oracle。引用前先确认截面时间。

+

二、通过率不能当作「模型本身能不能上 BPU」的结论。这批失败里混着配置层级的错误、看门狗终止、评测口径问题——都和模型本身能不能上板无关。要给这个结论,得先把这几类剔干净。

+
+
+ +
+

来源:本文的每一条都出自 bench/ 已有的文档与实现——CONTRACT.md(契约 v0.4 落地版)、HANDBOOK.md(掌控手册)、README.md、SPEC-two-phase-compile.md(两阶段编译实测)、REPORT-codex.md(v5 隔离与导出修复报告)、《BPU 量化轨迹工厂 设计方案 v0.4》,以及 2026-09-16 的两份核验(方案审查、S100P 现状核验)。

+

证据边界:§5 的状态是 2026-09-16 01:17–01:48 的远端快照,不是实时;本文没有重跑任何编译、推理或评测。文中实测数字(编译耗时比、两阶段字节一致、推理速度、外网带宽、统计标准误)均引自上述文档中已记录的实测,未重新验证。

+
+ +
+ + diff --git a/docs/BPU量化轨迹工厂-设计方案.html b/docs/BPU量化轨迹工厂-设计方案.html new file mode 100644 index 0000000..07bc0ae --- /dev/null +++ b/docs/BPU量化轨迹工厂-设计方案.html @@ -0,0 +1,790 @@ + + + + + + +BPU 量化轨迹工厂 + + + + +
+ +
+
model-compile-hbdk / bench · 设计方案 v0.4
+

BPU 量化轨迹工厂

+

确定性 harness 作为 oracle 跑完 338 个视觉模型的量化上板,agent 在受控子集上端到端执行,独立核验给出可验证奖励。一个 job 一个目录,agent 只有一个可写区,产物零散落。

+
+
模型
338 / 10 类
+
优先级
A 166 · B 128 · C 44
+
目标板
s100 · s100p · s600
+
march
nash-e · m · p
+
后端
PTQ + QAT
+
训练目标
RLVR
+
+
+ +
+

§0v0.4 改了什么

+

v0.3 → v0.4 的四处实质变化,全部来自机器实测或对 v0.3 的追问:

+
+
A

验证集层重写:大部分数据集已在 /mnt/data 上,外网带宽分化极大

ImageNet val、COCO 2017、Cityscapes、KITTI、ETH3D、SceneFlow、nuScenes 都在别的用户目录里,world-readable。官方 COCO 源实测 26 KB/s 等于不通,HF 8 MB/s、OSS 44 MB/s。数据分 T0 calib / T1 smoke / T2 full 三档,job 默认只跑 T1。§7

+
B

固定超时废除,改看门狗;重编译一律由 harness 执行

不同模型编译时长差两个数量级,固定超时会误杀。现在只在「日志不增长且 CPU 空闲 15 分钟」时判 STALLED。agent 通过 bench stage run 提交编译、bench stage wait 等结果,不在 bash 里直接跑 hb_compile。§8

+
C

明确「失败之后会怎样」与「重试在哪条轨迹里」

错误分类表、bench status 可见性、三层重试的轨迹边界(oracle 瞬态重试不是轨迹;rollout 内自愈是同一条;新 rollout 是全新轨迹,永不串联)。§9

+
D

3D 占据 23 个模型第一轮只编译不评精度的具体协议

G3 显式记 na 而非 unverified,reward 重归一化,校准输入来源分级。§6 末尾

+
+

v0.3 的变化(harness 改 oracle、claim/verdict 分离、四道门、stage 分层)保留,不再重复。

+
+ +
+

§1三条不可违反的原则

+
+
+
01
+
+

harness 是 oracle,不是 agent 的替身

+

每个 job 先由确定性流水线跑出 ground truth:能不能编译、精度掉多少、板端多快。这份结果既是 benchmark 数据,也是给 agent 打分的参照系。agent 的表现永远相对 oracle 衡量——绝对延时是模型的属性,不是 agent 的功劳。

+
+
+
+
02
+
+

agent 只有一个可写区,一切可信输入只读

+

job.json、评测代码、校准集、验证集、FP32 基线全部在 agent 可写区之外。agent 写 workspace/,越界写入由 extension 拒绝。它说什么不算数,harness 用干净代码重新验它产出的 .hbm 才算数。

+
+
+
+
03
+
+

Append-only:重跑即新 run,永不原地覆盖

+

run_id 冻结。换工具链、改配置、修 bug 后重跑,都开新 run。manifest 记录两套 venv 的 pip freeze 哈希,job 启动前比对,不一致拒跑——否则"append-only"是空话。状态只在 jobs.sqlite,文件系统只存内容。

+
+
+
+
+ +
+

§2五层架构

+
+
+
L0
+
+

清单、矩阵与门限

+

models.csv(338 行,不可变)、matrix.yaml(march × backend × qcfg × mode × policy 的组合与裁剪)、gates.yaml(逐任务通过容差)、budgets.yaml(逐任务类超时)。展开后写入 jobs.sqlite。

+
mode: oracle | e2e | repair · policy: teacher | student
+
+
+
+
L1
+
+

oracle harness(无 agent)

+

model-level 阶段跑一次进缓存;target-level 阶段每 job 一遍。每阶段独立子目录,落 stdout / stderr / exit code / meta.json / 耗时。产出 oracle/result.json。

+
model-level: fetch → baseline → export | target-level: calibrate → compile → hbm → board → eval
+
+
+
+
L2
+
+

policy driver(pi)

+

e2e 模式:给自然语言任务、job.json、只读数据路径,agent 自己走完全流程。repair 模式:给 oracle 的失败现场。每次尝试一个 rollout 目录,K 次独立 rollout 构成 GRPO 的组。

+
pi --mode json -a --session-dir <rollout>/agent "<task>" > events.jsonl
+
+
+
+
L3
+
+

四道门与 verdict

+

对 agent 产出的 .hbm,用与 oracle 完全相同的只读评测代码重跑 G1–G4,写 verdict.json。与 claim.json 比对,记录是否一致。

+
G1 compile · G2 numeric · G3 task · G4 perf(optional)
+
+
+
+
L4
+
+

数据物化

+

扫描 rollouts,脱敏、清洗、线性化轨迹,与 verdict 和 oracle 配对,按 reward 配方打分,输出到 datasets/<dataset_id>/。reward 配方版本化,可全量重打分。

+
→ (prompt, trajectory, verdict, oracle_ref, reward) · 兼容 DemyAgent SFT
+
+
+
+
+ +
+

§3目录契约

+

所有新增物只进 bench/,.horizon/ 只读。与 v0.2 相比新增 cache/、data/,job 内拆 oracle/ 与 rollouts/。

+
+
/mnt/data/ming01.wang/model-compile-hbdk
+
├── AGENTS.md                # 已有 · pi 向上遍历时加载
+├── .horizon/                # 已有 · 只读
+├── .agents/skills -> .horizon/skills   # 新增软链 · pi 的 skill 发现路径
+└── bench/
+    ├── CONTRACT.md          # 本文档落地版
+    ├── config.yaml          # 资源池并发、端点、板卡地址
+    ├── registry/            # 输入 · 不可变 · git 管理
+    │   ├── models.csv
+    │   ├── matrix.yaml      #   march × backend × qcfg × mode × policy
+    │   ├── gates.yaml       #   逐任务通过容差(§6)
+    │   ├── budgets.yaml     #   逐任务类估算与看门狗参数(§8)
+    │   ├── datasets.yaml    #   每个数据集的有序来源、sha、T1 子集 seed(§7)
+    │   └── boards.yaml      #   s100→nash-e · s100p→nash-m · s600→nash-p
+    ├── data/                # 只读 · 从本机其他目录复制入池 · agent 永不下载(§7)
+    │   ├── pool/<dataset>/<version>/   # 全量 + manifest.json(sha, 数量, 来源路径)
+    │   ├── t1/<dataset>/<seed>.tar       # smoke 子集打包,避开 JuiceFS 小文件
+    │   └── calib/<task>/<input_spec>/    # 固定 seed 从 train 抽的 npy,与 val 零交集
+    ├── cache/               # model-level 产物 · content-addressed · 只读
+    │   ├── weights/<sha256>/
+    │   ├── onnx/<sha256>/
+    │   └── baseline/<model>--<dataset>/   # FP32 指标 + 逐输出张量样本
+    ├── harness/             # L1 + L3 代码
+    │   ├── pipeline.py  scheduler.py
+    │   ├── stages/  backends/{ptq,qat}/  adapters/
+    │   └── eval/            #   逐任务评测器 · agent 不可写 · verdict 唯一依据
+    ├── agent/               # L2 pi 配置
+    │   ├── SYSTEM.md  settings.json      # compaction 关闭
+    │   ├── prompts/e2e.md  repair.md      # 任务描述模板
+    │   └── extensions/  # path-guard · claim-card · oe-doc(CLI) · budget-kill
+    ├── export/              # L4 · 规则移植自 data_collect,不 import
+    │   ├── to_agentrl.py  redact.py  reward.py
+    ├── state/jobs.sqlite    # 状态、error_class、耗时历史的唯一真相
+    ├── reports/             # bench report 生成的静态看板,按 run_id
+    ├── datasets/ds20260914-01/   # 一次导出一个版本 · 记录覆盖的 run 与 reward 配方版本
+    └── runs/r20260914-01/
+        ├── manifest.json    # 两套 venv 的 pip freeze 哈希 · 板卡固件 · 数据 manifest sha
+        └── det--yolov8n--nash-e--ptq--int8/
+            ├── job.json     # 只读 · 输入契约 · 含数据路径与 gate 阈值快照
+            ├── oracle/      # harness 确定性一遍 → ground truth
+            │   ├── stages/04-calibrate/ 05-compile/ 06-hbm/ 07-board/ 08-eval/
+            │   ├── artifacts/                  # .bc .hbm · 命名由 job_id 决定
+            │   └── result.json                 # 四道门结果 + 原始测量值
+            └── rollouts/00/ 01/ …   # 每次 agent 尝试一个目录
+                ├── workspace/              # agent 唯一可写区
+                ├── agent/session.jsonl events.jsonl
+                ├── claim.json              # agent 自报 · 不可信
+                └── verdict.json            # L3 独立核验 · reward 唯一来源
+
+
+
两个 venv 必须隔离
+

PTQ 用 ~/toolchains/oe-3.7.0/venv(hbdk4 4.7.5 + horizon_tc_ui 3.5.3 + hmct 2.6.5),QAT 用 ~/toolchains/oellm2/venv(hbdk4 4.12.3a2.dev + horizon_plugin_pytorch 3.3.5)。harness 按 backend 选解释器,禁止把 backend 判断交给 agent。板端 UCP 3.15.2 可加载两套产物。

+
+
+ +
+

§4命名规范

+
+ + + + + + + + + + + + +
标识格式示例说明
run_idr<YYYYMMDD>-<seq>r20260914-01工具链指纹在 manifest.json,不进 id——两套 hbdk4 塞一个会说谎
job_id<task>--<model>--<march>--<backend>--<qcfg>det--yolov8n--nash-e--ptq--int8五段全小写,段内 a-z0-9.,段间双横线。既是目录名也是 join key
task 前缀10 类固定缩写cls det seg iseg pose mde stereo flow feat occ对应交付文档 T01–T10,不允许新增
march板型映射s100→nash-e · s100p→nash-m · s600→nash-p写死在 boards.yaml;默认 qcfg:e/m 走 int8,p 走 fp16+int8
rollout<job_id>/rollouts/<kk>…/rollouts/03两位序号,K 次独立 rollout 构成一组
artifact<job_id>.<ext>det--yolov8n--nash-e--ptq--int8.hbm单独拿出来也不撞名、不丢身份
stage 目录<NN>-<name>05-compile01–03 是 model-level(缓存),04–08 是 target-level
dataset_idds<YYYYMMDD>-<seq>ds20260914-01记录覆盖的 run 列表与 reward 配方版本
+
+
+ +
+

§5状态机:job 与 rollout 分开

+

job 的状态由 oracle 决定;rollout 的状态由 verdict 决定。两者互不覆盖,一个 job 可以 oracle FAIL 而某次 rollout PASS——那正是最有价值的样本。

+
+ + + + + JOB · 由 oracle 决定 + PENDING + ORACLE_RUNNING + PASS + FAIL + UNVERIFIED + + + + + 四道必选门全过 + 任一必选门 fail + 数据缺失、板未接 → 门 = 未判定,不是 PASS + + + ROLLOUT · 由 verdict 决定 · 每个 job 可有 K 个 + QUEUED + RUNNING + VERIFYING + PASS + FAIL + BLOCKED + TIMEOUT + + + + + + + claim 与 verdict 一致性另记 + 含 claim=PASS 而 verdict=FAIL 的说谎样本 + 预算耗尽,外部 kill + +
+
+
BLOCKED 怎么判,现在有机械规则了
+

agent 报 BLOCKED 时,看同一 job 的 oracle:oracle 也在同一阶段以同类错误失败 → agent 判断正确,正分;oracle 通过了 → agent 是误判或放弃,负分。不需要人判,也不需要 LLM 判。

+
+ +

job 与 rollout 为什么要分开——用一个例子说

+

job 是「一个模型 × 一块板 × 一种后端 × 一种量化配置」这个问题本身,比如 det--yolov8n--nash-e--ptq--int8。rollout 是 agent 对这个问题的一次作答。一个问题可以被答很多次:K 次 e2e、若干次 repair、teacher 答几次、student 答几次。所以:

+
+
→

job 只有一个 oracle 结果,rollout 每次一个 verdict

oracle 是确定性流水线跑出来的,跑一百次都一样,所以只存一份;它回答「这个问题的标准答案是什么」。verdict 每个 rollout 一份,回答「这次作答对不对、比标准答案好还是差」。把两者塞进一个状态机,要么 oracle 被某次 rollout 覆盖,要么 rollout 的 PASS/FAIL 和 job 的 PASS/FAIL 混成一个字段,都是数据事故。

+
→

oracle FAIL 而 rollout PASS:最值钱,也最需要复核

这种样本意味着 agent 做到了确定性流水线没做到的事——通常是它换了导出方式、改了不支持的算子、调了校准策略。它要么是黄金样本,要么是 harness 有 bug(oracle 本该能过)。所以这类 rollout 自动打上 review 标记,进人工抽查队列,抽查通过后 oracle 的实现也要相应修正,形成 harness 自身的改进回路。

+
→

oracle PASS 而 rollout FAIL:正常,但要看它错在哪

这是 agent 能力不足的样本,reward 为负。verdict 记录它在哪道门失败、error_class 是什么,这决定了它在 SFT 里是负样本还是被丢弃(例如超预算的 TIMEOUT 通常丢弃,说谎的 claim≠verdict 通常保留作对比对)。

+
→

同一 job 的 oracle 和所有 rollout 共用同一份数据、同一段评测代码

否则 delta 不可比。job.json 里钉死 eval 子集 manifest sha 和评测器版本,任何一方跑的时候都校验。

+
+
+
一个隐患:oracle 本身可能是错的
+

相对 oracle 计分的前提是 oracle 可信。harness 的 bug 会系统性地把 reward 打歪。三道防线:四道门是绝对判断不依赖 oracle;相对项经过 clip 有界;oracle FAIL / rollout PASS 的样本强制人工抽查。P2 垂直切片阶段必须人肉核对 oracle 的每一个数。

+
+
+ +
+

§6什么算通过:四道门

+

通过是绝对判断(这个量化模型能不能交付),reward 是相对判断(agent 比 oracle 做得好还是差)。两者都需要,不能混。门的阈值住在 registry/gates.yaml,随 run 快照进 job.json。

+
+ + + + + + + + +
门检查什么必选判定依据
G1 compile.hbm 存在;hb_model_info 读出的 march、输入 dtype/layout 与 job.json 一致;hb_verifier 通过;编译日志无 ERROR是全部满足 → pass。march 不一致直接 fail——这是 agent 最容易"作弊"的口子
G2 numeric量化模型 vs FP32 在固定 N 张校准外样本上,逐输出余弦相似度是所有输出 ≥ 阈值(默认 0.99,T09 按输出单列)。只报 min,不报均值
G3 task任务指标:量化实测 − FP32 基线(同一评测器、同一子集、同一预处理)是退化在逐任务容差内。基线来自 cache/baseline/ 自跑,不采信官方数
G4 perf板端单线程纯推理时延 mean 与 P95;BPU 利用率;DDR 带宽否仅当 job.json 声明 latency_budget_ms 才判;否则只记录。mean 与 P95 分开存,永不互推
+
+

PASS = 必选门全部 pass。任一必选门因数据缺失、板未接而无法判定 → 该门记 unverified,job 状态为 UNVERIFIED 而不是 PASS。unverified 与 na(该任务明确不用此门)分开记——这是交付文档"未判定 ≠ 不适用"的直接映射。

+ +

G3 逐任务容差(建议默认值,可在 gates.yaml 覆盖)

+

容差来源标在最后一列。凡标"借用"的,交付文档里没有找到该任务的阈值记录,属于本文提议,须在第一个 run 后校准。

+
+ + + + + + + + + + + + + + +
task主指标(评测集)量化容差必须固定的口径来源
cls T01Top-1(ImageNet-1K val);Top-5 只记录Δ ≥ −1.0 ppresize / crop / 归一化、类别映射;多标签不适用S9 MobileNetv2 −0.73 pp
det T02box mAP@[.5:.95](COCO val2017);AP50 附Δ ≥ −1.5 mAP 点IoU 匹配、NMS 阈值、pycocotools 版本;开放词汇另固定词表与 embeddingS2 DOSODv2 目标差 1.3 / 1.0
seg T03mIoU(ADE20K val / Cityscapes val);附各类 IoUΔ ≥ −1.0 ppignore 标签、背景是否参与、输出分辨率与 resizeS9 Unet −0.44 pp
iseg T04mask AP(COCO/LVIS);提示分割用固定协议 mIoUΔ ≥ −1.5掩码 IoU 阈值、每图最大实例数;提示:点/框生成规则、多 mask 选择借用 T02
pose T05OKS AP(COCO keypoints val)Δ ≥ −1.5;关键点均偏移 ≤ 2 pxsigma、可见性、ROI 对齐到原图还是输入图S11 1–2 px 可接受;AP 借用
mde T06AbsRel / RMSE / δ1(KITTI Eigen / NYUv2)AbsRel 相对劣化 ≤ 10%;δ1 Δ ≥ −1.0 pp必须声明尺度对齐方式;相对深度先对齐再算;非正值处理S12 仅有 0.90 相关性;阈值为提议
stereo T07EPE / D1-all(KITTI 2015)每项相对劣化 ≤ 15%逐帧均值口径;有效像素范围;A99 / Infinity 只在客户集上算S4 SOW:量化允许 ≤ 15%
flow T08EPE(2D) / Fl-all(Sintel / KITTI 2015)相对劣化 ≤ 15%输出上采样与位移缩放;遮挡/无效像素。光流 EPE ≠ 双目 EPE,分开存借用 T07
feat T09逐输出余弦相似度;HPatches 单应 AUCdesc ≥ 0.99;heatmap ≥ 0.92;AUC Δ ≥ −2 pp提取与匹配分开评;选点规则与点数S14 DFeature 0.9909 / 0.9282
occ T10占据 IoU / 语义 mIoU(Occ3D-nuScenes)Δ ≥ −1.5 pp体素大小、空间范围、类别集合、可见性 maskS15 无阈值;借用
+
+
+
pp 与相对 % 不是一回事
+

mAP 32.5 → 31.2 是"下降 1.3 个 mAP 点",不是"相对下降 1.3%"。gates.yaml 里每条容差都显式写 kind: abs_pp | rel_pct,评测器按 kind 计算,不允许省略。

+
+ +

任务特例:3D 占据 23 个模型,第一轮只编译不评精度

+

理由:Occ3D 评测依赖 nuScenes 多相机输入 + 标定 + 占据标签,预处理逐模型不同,是所有任务里最重的;而这 23 个模型的价值主要在算子兼容性(BEV pooling、体素化、稀疏卷积),编译与板端两维就能采到。nuScenes Full 已在机器上,第二轮再补 Occ3D 标签。具体做法:

+
+ + + + + + + + + + + +
环节第一轮做法写进哪里
matrixtask: occ 的 job 声明 gates: [G1, G2, G4],G3 显式 na,reason round1-compile-onlymatrix.yaml · job.json
校准输入三级来源,逐级回退并记录用了哪级:① 模型仓库自带的 demo 样本或 nuScenes 多相机加载器;② 从 nuScenes Full 抽 128 帧、按模型输入规格 resize;③ 输入范围内的合成随机张量。第 ③ 级在 job.json 记 calib.source: synthetic,G2 结果只作参考04-calibrate/meta.json
G2 数值门用与校准同级来源的另一批样本算余弦,阈值放宽到 0.97;synthetic 来源时 G2 记 pass-weak,不阻塞 PASS 但进 verdictgates.yaml occ 段
G4 板端照常测 latency / BPU util / DDR,这是第一轮 occ 最有价值的数字07-board
PASS 定义G1 ∧ G2(含 pass-weak);job 状态 PASS,但 result.json 里 round: 1,第二轮跑 G3 时开新 run,不覆盖jobs.sqlite
rewardreward.py 遇 G3=na 时把 G3 的 2.0 权重按比例分给 G1/G2,相对 oracle 的精度项取 0;不因缺门而整体缩水export/reward.py
agent 提示e2e prompt 模板明说「本轮目标:编译通过 + 板端可运行;不评精度」。.horizon 没有 occ 专用 skill,预期 BLOCKED 比例高,这是正常的、有价值的agent/prompts/e2e-occ.md
+
+

na 与 unverified 在这里的区别很关键:na 是「这一轮设计上就不评」,job 可以 PASS;unverified 是「本该评但没评成」,job 不能 PASS。第二轮补 Occ3D 标签后,occ 的 G3 从 na 变回必选,那是新 run。

+
+ +
+

§7数据集层:机器上已有什么,缺什么,怎么分档

+

agent 永远不下载数据。可复现(版本会飘)、防作弊(agent 会挑好算的子集)、轨迹干净(网络噪声不进训练数据)。这一节的两条实测事实改写了 v0.3 的假设。

+ +

事实一:外网带宽分化极大,官方源等于不通

+
+ + + + + + + + +
源实测含义
aliyun OSS43.8 MB/sOE 包来源,最快
huggingface.co8.0 MB/s直连 0.32 s,hf CLI 已装,不用换镜像
github releases5.1 MB/s可用
images.cocodataset.org26 KB/s815 MB 要 9 小时。Excel 的"数据集入口"列给的全是这类官方站点,不能照着下
+
+

所以 datasets.yaml 里每个数据集声明有序的多个源:本地 → HF → OSS 镜像 → GitHub → 官方。逐个尝试,成功后校验 sha256。

+ +

事实二:大部分数据集已经在 /mnt/data 上

+

JuiceFS 951 TB 可用,写入 482 MB/s。以下都在别的用户目录里,world-readable,实测 ls 通。

+
+ + + + + + + + + + + + +
数据集位置(/mnt/data/…)内容入 pool 时要做的
ImageNet-1K valchao01.wu/Data/Datasets/ImageNet/val_images.tar.gz(6.67 GB);HuggingFace/ImageNet_HF/imagenet/data/validation-*.parquet(14 片全)50,000 张 + classes.py校验 tar 完整性(旁边有个 .tmp 残片,说明下过两次);解压到 pool;parquet 作第二来源
COCO 2017chao01.wu/Data/Datasets/COCO2017/val2017/(已解压)+ annotations_trainval2017.zip;guannan.he/coco2017/val2017.zip5,000 张 + instances / person_keypoints 标注复制 val2017 与两份 json;det · iseg · pose 三类共用
Cityscapesguannan.he/CityScapes/{leftImg8bit,gtFine}_trainvaltest.zipval 500 张 + gtFine labelIds只解压 */val/,其余不动
KITTI 2012 / 2015qi.xiong/Dataset/StereoDataset/KITTI/data_scene_flow/stereo 视差 GT + flow GT,各 200 帧 trainstereo 与 flow 两个任务共用一份,manifest 分开
ETH3D two-viewqi.xiong/Dataset/StereoDataset/eth3d/双目训练/测试对stereo 补充集
SceneFlow · InStereo2K · FallingThingsguannan.he/SceneFlow/,qi.xiong/…/StereoDataset/合成双目备用,第一轮不用
nuScenes Fullqi.xiong/Dataset/AutoDriveDataset/nuScenes/nuScenes_Full多相机原始数据occ 第一轮只取 128 帧做校准;第二轮补 Occ3D 标签
bdd100kguannan.he/bdd100k/驾驶场景 det / seg备用
+
+
+
复制,不软链
+

别人的目录随时可能被清。全部复制进 bench/data/pool/<dataset>/<version>/,复制完 chmod -R a-w,写 manifest.json(样本清单 sha256、数量、来源路径、复制日期)。951 TB 面前,几十 GB 的冗余换来的是与他人目录的零耦合。

+
+ +

仍需下载的(合计约 15 GB,HF 速率半小时)

+
+ + + + + + + + + + +
数据集任务规模首选源
ADE20K valseg2,000 张,~1 GBHF
LVIS v1 val 标注iseg · 开放词汇json ~1 GB,图片复用 COCOHF / GitHub
NYUv2 testmde654 张,~4 GBHF
MPI-Sintel trainflow1,041 ×2,~5 GBHF 镜像 → 官方
HPatchesfeat116 seq,~4 GBGitHub / HF
Occ3D-nuScenes 标签occ 第二轮数 GBGitHub release
+
+ +

三档数据,job 默认只跑 T1

+
+ + + + + + + +
档用途规模来源谁用
T0 calibPTQ 校准;QAT calibration-only128–512 张,按任务train 划分固定 seed 抽,与 val 零交集每个 job 必用
T1 smokeG2 数值门 + G3 任务门的筛cls 2,000 · det/iseg/pose 1,000 · seg 500 · 其余全量(本就 ≤ 1k)val 固定 seed 分层抽,清单入 git每个 job 默认
T2 full正式数字,G3 的判完整 valpool 全量只给要出交付数的模型;开新 run
+
+
+
T1 的统计边界
+

500 张评 Top-1,单次测量标准误约 1.9 pp——分辨不出 1 pp 的掉点。但 FP32 与 INT8 是配对比较,同一批图,误差高度相关,delta 的标准误由不一致对决定,通常 ~5%:2,000 张时约 0.5 pp,够用;500 张时约 1 pp,与 −1.0 pp 容差同量级,会误判。所以 cls 的 T1 取 2,000 而不是 500,且 T1 上的 G3 容差自动放宽 tol − 2·SE,记 pass-screen;T2 才是 pass。对 reward 来说 T1 的信号足够;对交付来说 T2 才算数。

+
+ +
+
→

校准集细则

存 npy,shape 含 batch 维,layout 与 job 输入规格一致,按 <task>/<input_spec> 组织,同规格模型共用。G2 的数值对比样本从 T1 里另抽,不能用校准图自己验自己。hb_compile 的校准张数影响编译时长,默认 128。

+
→

注入方式

job.json.data 写明 calib / T1 / T2 的 pool 路径、version、manifest sha。目录对 agent 只读(权限 + path-guard 双保险)。job 目录里只放一条 data -> 软链方便 agent 引用——这不违反"一个 job 一个目录",那条原则约束产出,不约束输入。

+
→

小文件与 JuiceFS

JuiceFS 是 FUSE,max_read=128K,几万张小 JPEG 随机读会慢。T1 子集(≤ 2k 文件)打包成单个 tar 或 npz 放在 pool 里,评测器直接读包;T2 全量走原目录。/home 是本地盘还剩 1.2 TB,热子集可选镜像一份,不是必须。

+
→

preflight

pipeline.py --preflight 在跑任何 job 前把 matrix 展开后需要的全部数据集查一遍:pool 里有没有、manifest sha 对不对、T0/T1 子集生成了没有。缺什么一次性报出来。跑到 08-eval 才发现没数据,浪费的是前面七个阶段。

+
+
+ +
+

§8调度、预算与看门狗

+

并行数按资源池分开设,一个数字管所有会让板端把 GPU 堵死。耗时不均不靠固定超时解决——固定超时会误杀慢模型,v0.3 这一条是错的。

+
+
bench/config.yaml(节选)
+
pools:
+  gpu:    { workers: 4, gpu_per_worker: 1 }     # baseline / eval / qat-calibrate
+  cpu:    { workers: 8 }                        # export / hb_compile(CPU 密集)
+  board:  { workers: 1, target: s100 }          # 一块板 = 1,严格串行,带锁
+  agent:  { workers: 3, rpm: 60 }               # 受 LLM 端点限速
+
+watchdog:
+  stall_s: 900            # 日志不增长 且 进程树 CPU < 5% 持续 15 min → STALLED
+  hard_ceiling_s: 86400   # 24h 纯保险,正常永远碰不到
+  soft_alert: p95x3       # 超过历史 p95 × 3 只告警,不杀
+
+perf:   { warmup: 20, runs: 200, stats: [mean, p95] }
+
+rollouts: { k: 4, max_turns: 40, max_tokens: 200000, hard_ceiling_s: 43200 }
+                                                # 真正的预算是 turns 和 tokens;墙钟只是 12h 保险
+retry:  { transient: 2, deterministic: 0 }
+
+
+
→

看门狗而不是超时

一个阶段只在「日志 15 分钟不增长 且 进程树 CPU 空闲」时才被判 STALLED——真正在算的编译,哪怕跑 6 小时也不会被杀。STALLED 归 transient,重试一次进 long 车道(只有看门狗、没有软预算),再 STALLED 才记 FAIL:stalled。历史 p95 × 3 只用来告警和调度估算,不用来杀。

+
→

所有重阶段由 harness 执行,agent 只提交和等待

agent 不在 bash 里直接跑 hb_compile。它调 bench stage run compile --job … --config workspace/x.yaml,harness 把任务丢进 cpu 池、按 backend 选对 venv、日志落 stage 目录,返回一个 handle;agent 用 bench stage wait <handle> 拿结果。四个好处:编译时长与 pi 的 bash 工具超时脱钩;backend/venv 选择不可能被 agent 选错;预算与看门狗对 oracle 和 rollout 一视同仁;日志位置统一。agent 的 bash 仍然可用,用来看日志、改配置、跑小脚本。

+
→

调度顺序

先按 A / B / C 优先级,同级内按估算耗时短作业优先。C 类 44 个走独立低优先级车道,永远不堵 A 类。首轮估算用参数量,之后用 jobs.sqlite 里同模型同阶段的实测耗时——每个 job 跑完都回填。

+
→

板卡串行

07-board 只有一个 worker,拿锁才能推 hbm。板掉线归 transient,重试后仍失败 → 该 job 的 G4 记 unverified,不阻塞其他阶段。

+
→

rollout 预算

turns 和 tokens 是真正的预算——它们是 agent 自己能控制的量。墙钟 12h 只防 agent 进程本身挂死。因为编译等待发生在 harness 侧,一个 rollout 等一次 3 小时的编译不会吃掉它的预算。触顶由 budget-kill extension 结束会话,状态 TIMEOUT,轨迹保留。

+
+
+ +
+

§9失败之后会怎样:可见性、重试、轨迹边界

+

没量化成功的模型不是丢了,是一个带分类的数据点。这一节回答三件事:在哪看、重试几次、重试算不算同一条轨迹。

+ +

在哪看

+
+ + + + + + + + +
命令给什么
bench status按 state × task × march 的计数表;--fail 只列失败,附 error_class 分布
bench show <job_id>阶段时间线、每阶段 exit code 与耗时、error_class、stderr 末 50 行、rollouts 的 verdict 一览
bench annotate <job_id> --note人工备注("已知 hbdk 4.7.5 不支持 DeformConv"),只记录不改 verdict
bench report从 jobs.sqlite 生成静态 HTML 到 bench/reports/<run_id>.html,给不上机器的人看
+
+ +

error_class:失败必须有名字

+

每个 FAIL 由正则分类器从 stderr 打一个标签,存进 jobs.sqlite 和 result.json。它同时喂三个地方:看板、repair 模式的 prompt、BLOCKED 判定里的"同类错误"。

+
+ + + + + + + + + + + + + +
error_class含义性质处置
unsupported_op算子不在 march 支持列表确定性不重试;进 repair 队列;最有价值的一类
shape_infer动态 shape / 维度推导失败确定性不重试;进 repair 队列
export_failONNX 导出或 fx trace 失败确定性不重试;model-level,一次影响该模型所有 job
calib_nan校准出 NaN / 溢出确定性不重试;进 repair 队列
verifier_mismatchhb_verifier 或 G2 数值不一致确定性不重试;进 repair 队列
oomGPU / 主机内存不足瞬态重试 ≤ 2,第二次降并发
board_offline板子失联瞬态重试 ≤ 2;仍失败 → G4 unverified
stalled看门狗判定挂死瞬态重试 1 次进 long 车道
unknown没匹配到规则—不重试;进人工分类队列,规则库随之增长
+
+ +

重试在哪一层,是不是同一条轨迹

+
+ + + + + + + + +
层谁重试触发轨迹归属次数
oracle 阶段harness瞬态 error_class不是轨迹,没有 agent 参与;meta.json 记 attempt 号≤ 2
rollout 内agent 自己它看到编译报错,改配置再提交同一条轨迹。这就是要采的诊断过程;HORIZON.md 的"同一方法最多重试 1 次"在这里作为 prompt 约束生效受 max_turns 约束
新 rolloutharness 起新进程matrix 声明 k 次全新轨迹:全新上下文,同一初始状态,不知道其他 rollout 存在k
repair 模式harness 起新进程oracle FAIL全新轨迹,输入是 oracle 的失败现场,不是某次 rollout 的按 matrix
+
+
+
三条硬边界
+

不串联:rollout k+1 永远看不到 rollout k 的历史,否则轨迹不独立,GRPO 的组内比较失去意义。不续跑:TIMEOUT 的 rollout 是一条完整轨迹,reward 记负;想要更长就在下一个 run 调大 max_turns。repair 不吃 rollout 的失败:只吃 oracle 的,保证 repair 轨迹之间也独立。

+
+ +

没量化成功的模型最后会怎样

+

oracle FAIL 的 job 状态就停在 FAIL,带 error_class,不影响其他 job。它会:出现在 bench status --fail;按 matrix 进入 repair 队列生成轨迹;被 BLOCKED 规则引用。人可以 annotate 它。它不会被自动重跑——重跑意味着新 run,那是人的决定。第一轮预期 FAIL 率不低,尤其局部特征、光流、占据三类;这是数据,不是事故。

+
+ +
+

§10pi 作为 policy driver

+

pi 是可替换的一层。job.json、claim.json、verdict.json 与 driver 无关,将来换成 verl 的 agent loop 或 RPC 模式下的工具执行器,环境契约不变。

+
+ + + + + + + + + + + + + +
要点做法
启动命令pi --mode json -a --session-dir <rollout>/agent "<task>" > events.jsonl。不是 -p --mode json,两者是互斥模式;-a 让非交互模式加载项目级 extension
轨迹源用 events.jsonl 最后一条 agent_end.messages[],它已是线性化的完整对话;session.jsonl 是树,只做备份
compaction.pi/settings.json 里 "compaction": {"enabled": false}(已核对是正式设置项)。训练数据里出现 CompactionSummaryMessage 是脏的;上下文靠 max_turns 控制
skills 发现不用软链。.pi/settings.json 的 skills 数组支持 glob:"skills": ["../.horizon/skills/*/*", "../.horizon/skills/horizon-router"],路径相对 .pi/ 解析。嵌套两层的问题由 glob 直接解决
重阶段agent 不直接跑 hb_compile / 板端推理,通过 bench stage run / wait 提交给 harness(§8)。pi bash 工具的默认超时未查到文档,P1 实测;但因为编译已不在 bash 里跑,这个值只影响小命令
工具白名单--tools read,edit,write,bash,grep;write/edit 受 path-guard 限制在 workspace/
extensionspath-guard(越界拒绝)· claim-card(强制结构化汇报,写 claim.json;未调用则 claim 缺席,小额扣分)· oe-doc(把 oe-mcp 包成 CLI,pi 不支持 MCP)· budget-kill
模型端点policy: teacher 走 Claude/GPT 直连;policy: student 走 ~/.pi/agent/models.json 指向你的 vLLM。matrix 里显式选,不混采
环境变量agent 的 bash 只暴露 job 目录路径与 PI_SESSION_ID。板卡凭证不进 env,板端推送由 harness 在 agent 进程外完成
+
+
+ +
+

§11RLVR:verdict 定 reward,相对 oracle 计分

+

result.json / verdict.json 只存原始测量值,reward 在 L4 离线算,配方版本化在 export/reward.py。

+
+
rollouts/03/verdict.json
+
{
+  "job_id": "det--yolov8n--nash-e--ptq--int8", "rollout": 3, "mode": "e2e", "policy": "teacher",
+  "gates": {
+    "G1": { "verdict": "pass", "march_ok": true, "verifier_ok": true },
+    "G2": { "verdict": "pass", "cos_min": 0.9961, "threshold": 0.99 },
+    "G3": { "verdict": "pass", "metric": "mAP@[.5:.95]", "fp32": 0.372, "quant": 0.361,
+            "delta": -0.011, "kind": "abs_pp", "tolerance": -0.015 },
+    "G4": { "verdict": "na",   "latency_ms_mean": 4.21, "latency_ms_p95": 4.60, "bpu_util": 0.78 }
+  },
+  "terminal": "PASS",
+  "claim_consistent": true,
+  "oracle_ref": { "terminal": "PASS", "G3_delta": -0.013, "latency_ms_mean": 4.35 },
+  "cost": { "turns": 17, "tokens": 91233, "wall_s": 612 }
+}
+
+
+
export/reward.py · v0 配方(示意,权重待第一批数据后校准)
+
r  = 1.0·[G1 pass] + 1.0·[G2 pass] + 2.0·[G3 pass]              # 绝对门
+r += 1.0·clip((Δ_agent − Δ_oracle) / |tolerance|, −1, +1)         # 相对 oracle 的精度
+r += 0.5·clip((lat_oracle − lat_agent) / lat_oracle, −1, +1)      # 相对 oracle 的延时
+r −= 0.5·(turns / max_turns) + 0.5·(tokens / max_tokens)          # 代价
+r −= 2.0·[claim ≠ verdict]                                        # 说谎
+BLOCKED: +1.0 若 oracle 同阶段同类失败;否则 −2.0
+TIMEOUT: −1.0
+
+
+
→

teacher 与 student 是两种数据

teacher(Claude/GPT)轨迹 + verdict 筛选 → SFT / 拒绝采样微调。student(你的模型)轨迹 → on-policy。两者 prompt 模板相同,matrix 里 policy 维度分开采,导出时分开落 dataset。

+
→

真 on-policy GRPO 时,rollout 驱动者会变

GRPO 需要 rollout 时的 logprobs,pi session 只有 messages。到那一步,verl/AgentRL 自己的循环做 rollout,pi 退化为 RPC 模式的工具执行器或被替换。本方案的持久资产是环境契约(§3 目录 + §6 门 + verdict.json),不是 pi。

+
→

采样去重

同一模型跑 nash-e/m/p 的 e2e 轨迹高度重复。e2e 模式按 (model, backend) 只采主 march(默认 nash-e),全 march 只在 oracle 模式铺。

+
+
+ +
+

§12下游边界:搬规则,不搬代码

+

数据全部留在本项目。~/data_collect 与 data-litellm-1.0 零耦合。

+
+ + + + + + + + +
data_collect 的东西处置原因
rebuild / probe / backfill / consumer不要网关 raw 日志重建会话边界;pi 的 agent_end 已是完整线性对话
judge_sessions.py不要LLM 主观打分。我们有 verdict,不需要也不应该让模型判
clean_for_agentrl.py只搬规则 R1–R8双方言归一、thinking 剥离、system 首条、悬空尾轮截断——踩过的坑
redact_archive.py搬(v0.2 漏了)agent 的 bash 输出会进轨迹,路径、主机名、任何凭证都要脱敏
+
+

落地为 bench/export/{to_agentrl,redact,reward}.py:独立脚本,不 import 那边任何东西。那套代码 ROOT 硬编码指向 litellm 目录,一 import 两个项目的路径与数据版本就缠死。

+
+ +
+

§13待决与阻塞项

+
+
+
已解除

验证集落盘(v0.3 的头号阻塞)

+

ImageNet val、COCO、Cityscapes、KITTI、ETH3D、nuScenes 都在 /mnt/data 上(§7 表)。剩 ADE20K / LVIS 标注 / NYUv2 / Sintel / HPatches 约 15 GB 走 HF,半小时。HF token 不再需要。

+
+
+
待验证

ImageNet val tar 的完整性

+

val_images.tar.gz 旁边躺着一个 1.5 GB 的 .tmp 残片,说明下过两次。入 pool 前 tar -tzf | wc -l 必须等于 50000,或者直接用 HF parquet 那 14 片作首选源,二者取其一校验通过即可。

+
+
+
要修

.env.oe-package 里的死路径

+

PYTHON_BIN 指向 <项目根>/oellm2/venv/bin/python,项目根下没有 oellm2/——QAT venv 实际在 ~/toolchains/oellm2/venv。agent 按规则会读这个文件。P1 第一件事。

+
+
+
待验证

pi bash 工具的默认超时

+

settings.md 里没有这一项。因为重编译已改由 harness 执行,它只影响 agent 跑的小命令;但 bench stage wait 本身是长阻塞,需要确认它不会被 bash 工具掐断——不行就改成轮询式 wait --poll 60。

+
+
+
待确认

rdk-sandbox 是哪块板

+

映射已定:s100 → nash-e,s100p → nash-m,s600 → nash-p。沙箱那块具体是哪个,板上 hrut_somstatus 或 /proc/device-tree/model 看一眼。改密钥登录,口令用完轮换。

+
+
+
待定

teacher 模型选谁、student 端点在哪

+

决定第一批数据的性质。建议 P3 先用 teacher 跑通闭环,student 端点等 vLLM 起来再接。

+
+
+
已解除

pi 对嵌套 skills 目录的发现

+

settings.json 的 skills 数组支持 glob,"../.horizon/skills/*/*" 直接覆盖两层嵌套,不需要软链,也不需要逐个 --skill。

+
+
+
已解除

PTQ 工具链就位

+

OE 3.7.0 s100/s600 包已装,TinyNet → hb_compile → .hbm 实测跑通,HBRT4 加载校验通过。338 个公开权重走标准 PTQ。

+
+
+
留意

Board-Agent 三个中间产物已被删

+

model-codegen-info.json、qconfig_setting/、tmp_1_original_float_perf.json,可再生。记在这里防误判。

+
+
+
+ +
+

§14实施顺序

+
+
+
P1 · 地基
+

骨架、契约、数据入池

+

建 bench/、写 CONTRACT.md、修死路径、Excel → models.csv、写 gates/budgets/boards/datasets.yaml、装 pi + settings glob 验证 skills、COCO / ImageNet / Cityscapes / KITTI 从本机复制入池并校验,其余 15 GB 走 HF,--preflight 全绿。产出:空转的 pipeline.py。

+
+
+
P2 · oracle 切片
+

yolov8n + resnet18 各打穿八阶段

+

det 用 COCO T1,cls 用 ImageNet T1(现在都在本机)。PTQ、nash-e。人肉核对 oracle 的每一个数——这是相对 oracle 计分的信任基础。产出:两份 oracle/result.json,看门狗与 bench stage CLI 跑通。

+
+
+
P3 · 闭环
+

e2e + repair 各跑通一次

+

teacher 模型。e2e 跑 yolov8n;repair 拿一个 oracle 必失败的模型。验证 path-guard、claim/verdict 分离、budget-kill、导出脱敏。产出:第一批带 reward 的轨迹。

+
+
+
P4 · 横向
+

TorchVision 51 + COCO 系 ~60

+

oracle 全 march;e2e 每类 6–10 个。压测资源池、板卡锁、耗时估算回填。产出:第一个完整 run、失败率分布、gates 容差校准。

+
+
+
P5 · 全量
+

A 类 166 → B → C

+

局部特征 28、3D 占据 23、光流 14 算子特殊,是失败重灾区也是轨迹金矿,放最后。C 类走低优先级车道。

+
+
+
+ +
+ 环境指纹 · instance-mzr1k6rw · 4 × RTX 5090 · .horizon skill pack v0.2.0
+ PTQ 后端 ~/toolchains/oe-3.7.0/venv · horizon_tc_ui 3.5.3 · hmct 2.6.5 · hbdk4 4.7.5
+ QAT 后端 ~/toolchains/oellm2/venv · horizon_plugin_pytorch 3.3.5+cu128.torch280 · hbdk4 4.12.3a2.dev+cd3ae44 · hbm_infer 3.15.3
+ 板端 rdk-sandbox · UCP 3.15.2 · s100→nash-e · s100p→nash-m · s600→nash-p
+ 存储 JuiceFS:robotwin-archive 951 TB 可用 · /home 本地 1.2 TB · 带宽 OSS 43.8 / HF 8.0 / GitHub 5.1 MB/s · COCO 官方 26 KB/s
+ G3 容差来源 · 视觉模型交付指标说明 2026-09-14 · S2 S4 S9 S11 S12 S14 S15 +
+ +
+ + diff --git a/docs/BPU量化轨迹工厂-闭环方法论.html b/docs/BPU量化轨迹工厂-闭环方法论.html new file mode 100644 index 0000000..26b714e --- /dev/null +++ b/docs/BPU量化轨迹工厂-闭环方法论.html @@ -0,0 +1,460 @@ + + + + + + +闭环走通的判据 + + + + + +
+ +
+
方法论 · BPU 量化轨迹工厂配套
+

闭环走通的判据

+

把「垂直工程任务 → 可训练轨迹」沉淀成可复用范式。走通的标准不是产出一条 PASS 样本,而是四类终态都能被正确落位。

+
+
主批次终态
108
+
oracle PASS
3
+
可用轨迹
0 / 11
+
repair 入队
0
+
导出样本
1(tools 0)
+
+
+ +
+

§1先说结论

+

这套东西方法论上是成立的,而且已经把最难的那一半做完了——可机器判定的验证器(四道门 + oracle 相对基准)和内核级隔离都已落地。没走通的是另一半:闭环的最后三个接头是断的——FAIL 不进 repair、采集缺失不报错、运行进程与磁盘代码不一致。这三处都不是难题,是接线问题。

+
+

「走通」不等于「跑出一条正样本」

+

今天已经有 1 条 PASS 并导出成 teacher 样本了。但它不能证明闭环成立:它的 tools 数是 0,sft_eligible 仍被标成 1——说明验证器在该拦的地方没有拦。一个只会盖 PASS 的验证器和一个坏掉的验证器,从正样本上看不出区别。

+

闭环成立的判据只有一个:给它四类终态(PASS / FAIL / BLOCKED / TIMEOUT),它能把四类都正确落位,包括把该拒的样本拒掉。这是本页第 5 节 MVL 清单的全部依据。

+
+
+ +
+

§2范式:什么任务能这样工业化

+

这套做法可以从量化编译迁出去,但有前提。三条同时成立才适用:

+
+ + + + + + + +
前提含义量化编译任务为何满足
判据可机器化对错由程序判定,不靠人评、不靠模型当裁判返回码、逐输出余弦、Top1 / mAP 差值,全部是数值比较
验证远比求解便宜跑一遍很贵、判一次很便宜,才撑得起大量采轨迹一个 job 的 oracle 要编译加评测;判四道门只是读 result.json
失败模式是有限集失败能分类,才能区分「agent 判断错」与「工具链真做不到」error_class 确定性五类、瞬态三类,其余归 unknown 进人工队列
+
+

第三条最容易被忽略,但它决定了 BLOCKED 这一类终态能不能存在。没有失败分类,agent 说「这个做不了」就无法判真假,只能当失败处理——正确放弃的能力就永远学不到。

+
+ +
+

§3五个构件,缺一不闭合

+

范式本身是一条单向链:冻结输入 → oracle 出标准答案 → 隔离下作答 → 独立核验 → 分级导出。每一环的职责是硬边界,不能交叉。

+
+
+
L0
+
+

冻结的任务单元

+

job = 模型 × march × backend × qcfg,job_id 既是目录名也是 join key。run 口径(venv、registry、门阈值)任一变化就开新 run,不原地改。这让任何一条轨迹永远能追溯到它当时面对的确切世界。

+
+
+
+
L1
+
+

确定性 oracle —— 一份产出,三种用途

+

这是整个范式的杠杆点。同一份 oracle 同时是:①ground truth(agent 做得对不对);②相对基准(reward 里精度与延时都是相对 oracle 的差,不是绝对值);③BLOCKED 的裁判(agent 说做不到时,看 oracle 是不是在同阶段以同类 error_class 失败)。省掉 oracle 就同时失去这三样。

+
+
+
+
L2
+
+

隔离的作答区

+

agent 只有 rollouts/<kk>/workspace/ 一个可写区。隔离靠 unshare -rm 的 mount namespace 把答案、兄弟轨迹、凭证从文件系统里摘掉;解析 shell 字符串的 guard 退化为教学层,只负责给一句人话的拒绝理由、省 agent 的预算。

+
+
+
+
L3
+
+

独立核验的 verdict

+

四道门:G1 编译形式、G2 数值余弦、G3 任务指标、G4 板端延时(非必选)。判定完全不读 agent 的自述,claim 只用来和 verdict 比对。na(本轮设计上不评)与 unverified(本该评没评成)严格区分——前者不阻塞 PASS,后者阻塞。

+
+
+
+
L4
+
+

离线 reward + 分级导出资格

+

reward 只读 verdict.json,纯函数、配方版本化,可对历史全量重打分。导出资格分两级:RL 收正负例(要的是区分度),SFT 只收「做对了且诚实」——claim ≠ verdict 和 review=true 是两条优先级最高的硬排除。

+
+
+
+
+ +
+

§4八条不变量,每条都有今天的反例

+

这些不是设计偏好,是今天这批任务用 105 个 FAIL 换来的。每条后面附的是它被违反时实际发生了什么。

+
+ +
+
I1
+
+

运行中的版本必须可证明等于磁盘上的版本

+

常驻调度进程加载的是它启动那一刻的代码与配置。「改好了」指的是文件,不是正在跑的东西。进程启动时必须自证代码指纹,与 run manifest 不一致就拒绝领新任务——而不是打印一行 WARN。

+
反例 · 37 例编译失败oracle 调度 PID 3776582 于 01:41:58 启动,harness/backends/ptq.py 于 01:46:29 修好;进程没重启,下午新生成的 YAML 仍把 quant_config 放在 model_parameters 下。paths.config() 的 lru_cache 让旧配置也常驻内存。check_manifest() 对代码漂移只告警,没有拒跑。
+
+
+ +
+
I2
+
+

隔离必须在内核层,不能在字符串层

+

解析 agent 写的 shell 命令做白名单,是一类永远追不平的防线——每一版都会出现新写法。真正的隔离是让进程看不见。看不见的东西,再多绕法也没用。

+
反例 · guard 连续四版被穿透换行后的 cd、$IFS、$VAR/ 路径拼接、workspace 内软链回 ../../../oracle——四种手法各穿透一版。v5 起改为 mount namespace 摘除加 capability drop,guard 降级为教学层。
+
+
+ +
+
I3
+
+

采集链任何一环缺失,必须硬失败

+

数据管道里的静默降级比崩溃危险得多:崩溃会被发现,降级会安静地产出「看起来完整」的训练数据。凡是训练必需的字段,缺失即拒绝启动或拒绝导出。

+
反例 · 唯一一条导出样本工具定义为空rollout.py 起 dump_tools.js 时没补 Node bin 的 PATH → npm: not found → 非零退出被忽略 → 不生成 tools.json → 导出回退也失败、返回 ([], 'missing') → sft_eligibility() 不检查这项 → 样本仍被标 eligible。A/B 对照:同一脚本补 PATH 后返回 0、导出 7 个工具。
+
+
+ +
+
I4
+
+

verdict 覆盖 claim,说谎是最贵的

+

agent 自述不参与任何判定,只参与一致性比对。claim ≠ verdict 扣 2.0 并绝不进 SFT——训练集里出现一次谎报成功,就是在教它谎报。claim 缺席扣 0.25:不报比报错轻,但不能没有代价。

+
同类风险 · 编译返回 0 不是验收通过编译日志里的百分比、校准余弦、静态 FPS 估算都不能代替 G1/G2/G3。三条 ConvNeXt 轨迹 exit_code=0 且已生成 HBM,仍没有最终 verdict。
+
+
+ +
+
I5
+
+

状态机必须闭合:每个终态都要有去处

+

FAIL 不是终点,是 repair 轨迹的输入——而且是更有价值的那种输入,因为修复现场比一次性成功更能教出能力。配置里开了开关,不等于链路接上了。

+
反例 · repair 0 / 0matrix.yaml 已配 when: oracle_fail;registry.expand_matrix() 见到该值直接跳过,注释声称由 pipeline 动态追加;而 pipeline.run_oracle_job() 存完 FAIL 就返回,collect_rollouts() 只消费数据库里已有的 QUEUED 行。全库 repair job 与 rollout 均为 0。
+
+
+ +
+
I6
+
+

指标口径必须与被测对象的结构匹配

+

把多分支输出展平算一个整体余弦,结果会被量级最大的那一支主导,小量级分支的失真被完全掩盖。凡是多语义输出,门必须逐分支判。

+
反例 · YOLO11s 整体余弦 0.9976,类别分支 0.36输出 [1,84,8400] 同时含框坐标(量级大)与 0–1 类别分数。eval/common.py::cosine() 整体展平,G2 判为 pass;而实际 mAP 46.01 → 25.92。整段余弦高不等于精度达标。
+
+
+ +
+
I7
+
+

证据边界必须写在结论旁边

+

每个数字都要带着「它证明了什么、没证明什么」一起流转。否则弱证据会在传递中逐级升格成强结论。这条不产出代码,但它决定了这套数据将来能不能被别人信。

+
今天需要显式区分的三组pass-screen(T1 放宽门限初筛)≠ pass(T2 完整验证);hb_model_info 静态性能估算 ≠ 板端实测;服务器上 run_sim ≠ 物理板端运行。9 份板端记录用的是随机输入,没有板端数据集精度。
+
+
+ +
+
I8
+
+

失败的轨迹不删,过滤发生在导出时

+

磁盘上留什么和数据集里收什么是两件事。轨迹是复盘证据——今天两个 bug 都是靠原始轨迹定位的;数据洁净度由 --min-agent-version、annotations 和资格函数在导出那一刻保证。删了,同样的坑得重新踩一遍才知道。

+
当前状态r20260915-03 的 11 条已完成 rollout 全部不可用(7 条读了 oracle、3 条被误杀、其余旧版本),已 --annotate 标注保留;9 份旧数据集加 VOID 标记而非删除。
+
+
+ +
+
+ +
+

§5最小可复现闭环(MVL)

+

定义:在一个新冻结 run 上锁定一条切片(建议 cls--resnet18--nash-e--ptq--int8,它是唯一已知能走完全程的 job),走到一条可进 SFT 的样本,并且同一 run 内四类终态各有一例被正确落位。做到这一步,方法论就算走通——规模是之后的事。

+ +

5.1 四个前置闸门:全为真才允许启动

+
+
+
P1
+

版本自证

调度进程启动时打印自身加载的 harness 代码指纹与两套 venv 的 pip freeze sha256,与 run manifest 比对;不一致拒跑而非告警。

+
需改造
+
+
+
P2
+

真实产物校验

用已安装的 horizon_tc_ui 3.5.3 Schema 校验实际写到磁盘的第一份 compile YAML,而不是校验生成函数的返回值。这是今天 37 例失败唯一有效的拦截点。

+
需改造
+
+
+
P3
+

采集链完整性

tools.json 存在且工具定义数大于 0,否则拒绝启动 rollout;导出侧同样把它列为硬性资格条件。

+
缺失
+
+
+
P4
+

隔离自检

guard_smoke.sh 的越界尝试全部被拒、guard-core.test.js 通过;jail 不可用时拒绝启动,不静默退回 guard。

+
已具备
+
+
+ +

5.2 七步产物链:每步都要留下可核验的东西

+
+ + + + + + + + + + + +
步动作可核验产物判据
1oracle 跑完切片oracle/result.jsonG1/G2/G3 均有明确判定,error_class 非空
2agent 在 jail 内作答session.jsonl · events.jsonl · workspace 产物guard.log 无越界成功记录
3独立核验verdict.json不读 claim 即可算出四道门
4一致性比对claim.json ↔ verdictclaim_consistent 有明确真假,不是缺省
5污染审计audit_rollouts.py 输出行无 oracle_read / sibling_read / stalled_killed / old_version
6离线打分reward 与配方版本号同一 verdict 重算两次结果逐位一致
7导出teacher.jsonltools 数大于 0,sft_eligible 的理由可打印
+
+ +

5.3 分辨力四例:这才是走通的核心判据

+

只产出正样本证明不了验证器有分辨力。必须在同一个 run 里构造出四类终态,并确认每一类都被正确落位——尤其是第二、第四行这两个「应当被拒」的例子。

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
终态怎么构造期望落位现状
PASSResNet18 / nash-e 正常跑四道门齐全 · RL ✓ · SFT ✓ · tools 完整有 PASS,tools 缺
污染故意让一条 rollout 去读 oracle/jail 层面根本读不到;即便读到,audit 标 oracle_read 并判不可用v5 已隔离,未做对照
BLOCKED挑 oracle 确定性失败的 job(MaxViT-t 的 aten::einsum / opset 11)agent 正确判断放弃 → blocked_correct=true → reward +1 → SFT 收全库无 BLOCKED 样本
谎报构造 claim=PASS 而 verdict=FAIL 的轨迹reward −2.0 · SFT 拒收 · 理由可打印未验证
+
+

第三行同时解释了为什么 105 个 FAIL 是资产而不是负债:MaxViT-t 的导出不支持、YOLOv8n 的 f64 Cast、ResNet34 的权重哈希不匹配,这三类都是现成的 BLOCKED 教材——前提是 repair 链路接上,让它们进得了轨迹。

+
+ +
+

§6离 MVL 还差什么

+
+ + + + + + + + + +
缺口证据阻断了哪一步
运行进程仍是旧代码PID 3776582 / 2015257 未重启;7 个源文件两次采样哈希相同,修复未落到运行态P1;以及步 1 的 error_class 非空(28 条 FAIL 分类为空,用当前纯函数可全部分类)
tools 快照链路断PATH 缺 Node bin,A/B 对照已复现P3、步 7
FAIL → repair 未接线expand_matrix() 跳过、run_oracle_job() 不追加5.3 第三行(BLOCKED 无样本可造)
G2 口径对多分支输出失真YOLO11s 类别分支 0.36 而整体 0.9976步 3 的判定可信度
17 条 STALLED 真因未定日志终止标记为 log unchanged and cpu idle;任务仍用 JuiceFS 工作目录,存在 I/O 等待误判风险,但缺历史进程树与 D 状态采样步 5 的 stalled_killed 判断依据
+
+
+ +
+

§7执行顺序

+

五步是严格有序的:前一步不做完,后一步的观测结果不可信。

+
+
+
1
+
+

统一运行版本,开新冻结 run

+

bench drain 让旧调度跑完手里的活并退出(不要 kill 正在编译的进程),保存当前证据,从修正后的代码起新进程。仅重启补不齐尚未实现的 repair 与导出校验。

+
验收:新 run 第一份落盘 compile YAML 过 Schema;进程指纹与 manifest 一致
+
+
+
+
2
+
+

接上三个断头

+

Node PATH 与 tools 硬校验;FAIL → repair 的幂等入队;G1 的 dtype / layout 完整校验。三处都是接线,不涉及算法。

+
验收:造一个 FAIL,repair job 自动出现且不重复;tools 缺失时 rollout 拒绝启动
+
+
+
+
3
+
+

跑通 MVL 的七步产物链

+

只跑 ResNet18 / nash-e 一条切片,不扩量。目标是一条 tools 完整、四门齐全、audit 干净的样本。

+
验收:audit 显示 RL ✓ 且 SFT ✓;导出样本 tools 数大于 0
+
+
+
+
4
+
+

补齐分辨力四例

+

污染、BLOCKED、谎报三个负向对照各造一例。这一步才是「方法论走通」的签收点。

+
验收:四类终态各一例,落位与 5.3 表格逐行相符
+
+
+
+
5
+
+

再谈规模

+

分别验收数值与任务指标:保留 FP32 ONNX 对照,按模型处理量化敏感层,检测模型另判框坐标与类别分数。不要为了提高 PASS 数放宽阈值——阈值一放,前面四步建立的分辨力就作废了。

+
验收:收敛指标看 RL ✓ 比例,不看 PASS 绝对数
+
+
+
+
+ +
+

§8规模化阶段的唯一指标

+
+

看 RL ✓ 比例,不看 PASS 数

+

PASS 数受模型难度、门阈值、工具链成熟度共同影响,涨跌说明不了 agent 学没学到东西。RL ✓(版本合格、没越界、没被误杀)衡量的是轨迹本身可用吗——它才是这条流水线的产能指标。当前该比例为 0/11。

+

同理,当前 3/108 约 2.8% 的通过率不能当作模型固有可编译率:其中 37 例是配置层级错误、17 例是看门狗终止,都与模型能不能上 BPU 无关。

+
+
+ +
+

数据来源:2026-09-15 的四份检查报告(远端状态复查 16:19–16:20、失败根因定位、板端执行证据复核、精度对比),以及 bench/ 仓库的 CONTRACT.md、HANDBOOK.md 与 harness/、export/ 源码。

+

证据边界:文中数字为已保存评测记录与当日快照,本页未重跑任何推理或编译。「37 例配置层级错误」与「tools 链路」两项有受控复现;「17 条 STALLED 全为误杀」未经证实,不应据此下结论。3/108 是当前判定口径下的比例。

+
+ +
+ + diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..3330f51 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,18 @@ +# 项目文档 + +本目录收录 BPU 量化轨迹工厂的设计与方法说明,原始文档内容保持不变: + +- [设计方案](BPU量化轨迹工厂-设计方案.html) +- [方法论](BPU量化轨迹工厂-方法论.html) +- [闭环方法论](BPU量化轨迹工厂-闭环方法论.html) +- [技术分享讲稿(2026-09-16)](BPU量化轨迹工厂-技术分享讲稿-20260916.md) + +运行入口与配置方式见 [项目 README](../README.md),实现约定见 [CONTRACT.md](../CONTRACT.md)。文档中的实验状态、服务器路径和实测结果对应其编写时间;当前运行状态需要重新核查。 + +## Git 收录范围 + +仓库收录 harness、模型适配器、Agent 配置模板、数据导出代码、脚本、测试、registry、配置和项目文档,保留现有提交历史。 + +模型权重、ONNX/HBM、数据集、缓存、运行状态、训练轨迹、评测输出、日志、第三方源码下载、压缩包以及本地凭据留在各运行环境,不随源码仓库分发。原工作目录下的 `outputs/` 和 `job_verdicts.csv` 属于运行产物;这些文件继续保留在本地。启动前按项目 README 准备依赖、数据、模型与本地环境变量。 + +`.env.local.example` 和 `agent/models.json.tmpl` 为可提交模板;真实 `.env.local` 和 `agent/models.json` 已被忽略。 diff --git a/PLAN.md b/docs/archive/PLAN.md similarity index 100% rename from PLAN.md rename to docs/archive/PLAN.md diff --git a/REPORT-codex.md b/docs/archive/REPORT-codex.md similarity index 99% rename from REPORT-codex.md rename to docs/archive/REPORT-codex.md index a1c78e7..30d74f1 100644 --- a/REPORT-codex.md +++ b/docs/archive/REPORT-codex.md @@ -4,7 +4,7 @@ ## 结论与边界 -B/A、D、E、G 均已实现;agent/VERSION=5。已通过真实 jail + pi smoke、外部 worker 的真实 calibrate 闭环、guard-core 单测、预算边界测试和 Python 回归。最终探针复测的原始输出见 [验证记录](scripts/evidence-codex.md)。 +B/A、D、E、G 均已实现;agent/VERSION=5。已通过真实 jail + pi smoke、外部 worker 的真实 calibrate 闭环、guard-core 单测、预算边界测试和 Python 回归。最终探针复测的原始输出见 [验证记录](evidence-codex.md)。 没有启动 `bench collect` / `bench rollout`,没有 kill 生产编译、pi、stagerun 或调度进程,没有改 config.yaml / registry 数值,没有手工 UPDATE/DELETE 生产 DB。canary #02 仅供只读审计,未重新核验或改写其文件。旧数据集只新增作废文件。 diff --git a/SPEC-two-phase-compile.md b/docs/archive/SPEC-two-phase-compile.md similarity index 97% rename from SPEC-two-phase-compile.md rename to docs/archive/SPEC-two-phase-compile.md index db44e26..ffd9b65 100644 --- a/SPEC-two-phase-compile.md +++ b/docs/archive/SPEC-two-phase-compile.md @@ -1,3 +1,8 @@ +> **已归档(2026-09-17):未实现,暂不做。** +> 注意不是"被 long 车道替代"——long 只是 STALLED 后把看门狗阈值抬到 3600s 的重试车道 +> (`stagerun.py` 对 05-compile 无条件设 `ctx.lane="long"`),与两阶段编译无关。 +> 本文的动机测量(量化 107s / BPU 编译 4680s)至今仍然成立,问题也仍未解决。 + # SPEC · 05-compile 拆成两阶段(交接文档) 给下一个负责改这块的 agent。所有数字都是 2026-09-15 在 `120.48.151.164` 实测的, diff --git a/scripts/evidence-codex.md b/docs/archive/evidence-codex.md similarity index 100% rename from scripts/evidence-codex.md rename to docs/archive/evidence-codex.md diff --git a/docs/board-process-lifecycle.md b/docs/board-process-lifecycle.md new file mode 100644 index 0000000..14ce96d --- /dev/null +++ b/docs/board-process-lifecycle.md @@ -0,0 +1,26 @@ +# 板端进程生命周期 + +主机的 `board-.lock` 只覆盖主机阶段,SSH 退出不保证板上 Python 退出。 +2026-09-19 的实测故障是:板端 CPU 算子执行较长,主机 `capture_output=True` 没有实时日志, +看门狗按主机空闲 900 秒终止本地进程组并释放锁;板端留成 PPID=1,重试又在同一板上启动推理。 +旧退出函数仅删除临时目录,也无法回收运行中的模型。不能用 CPU 忙证明模型正常推进。 + +`RemoteSession.bench` 现在上传并调用独立的 `board_guard.py`,用于 07 性能和 08 精度两条入口: + +- 守护进程持有 runtime_dir 下 `.board-inference.lock`;子进程继承锁,在未退出时不能再次开测。 +- 开始前检查已有 `board_bench.py` / `hrt_model_exec`,发现未经守护的推理返回 `board_busy`。 +- 主机每 5 秒续租;EOF、30 秒无租约、单次 RPC 3600 秒上限或终止信号,都回收本次进程组。 +- 先 TERM,5 秒后 KILL;不因组长先退出而跳过组内其他进程。Linux parent-death signal 在守护进程被 KILL 时终止直接模型子进程。 +- 原生运行库在子进程内,不能阻塞守护进程。无法立即清除的内核不可中断进程仍持有板端锁,后续测量失败退出。 +- 板端 stderr 实时回传。`board_progress` 是加载、样本、warmup、计时迭代的实际边界;`alive` 只是传输存活,单次 RPC 上限独立计时。 +- 成功后清理本次临时目录;失败目录保留 `guard-result.json` 和 HBM 供定位。日志记录 session 路径,结果记录 `board_guard_version`。 +- 性能阶段等 10 分钟仍繁忙,拒绝生成新的性能数值。历史 `contended` 数据保留原始证据,不能静默改成正常测量。 + +验证:`PYTHONPATH=. python3 -m unittest scripts.test_board_runtime scripts.test_board_guard`。 +Linux 上的 guard 测试会真实注入断连、停止续租、超时、TERM/KILL 和锁争用;不访问 SSH/BPU。 +本地 macOS 会跳过 Linux 进程测试,应额外在 Linux 隔离临时目录执行。 + +部署走 `scripts/sync.sh`。若远端已有其他任务的未合并改动,可以提交本次文件后使用 +`scripts/sync.sh --files `,仅比较所选文件与上次同步的前像,保存远端备份并校验落盘哈希。 +新文件先于引用方安装;不会推进其他文件的同步基线。旧的已启动评测不会重新 import, +新启动的工具子进程会加载修复,无需重启 collector。现役任务的结果须分别检查,不得宣称所有旧进程已经换代。 diff --git a/export/reward.py b/export/reward.py index f93bf52..163cfff 100644 --- a/export/reward.py +++ b/export/reward.py @@ -1,18 +1,7 @@ -"""reward v0.4 配方。离线算,只读 verdict.json;配方版本化,可全量重打分。 +"""reward v0.6:只按本次真实门证据、成本、claim 一致性和超时计算。 -r = 1.0·[G1] + 1.0·[G2] + 2.0·[G3] 绝对门 -r += 1.0·clip(精度改善/|tol|, −1, +1) 相对 oracle 的精度 -r += 0.5·clip((lat_oracle − lat_agent)/lat_oracle, −1, +1) 相对 oracle 的延时 -r −= 0.5·turns/max_turns + 0.5·tokens/max_tokens 代价 -r −= 2.0·[claim ≠ verdict] 说谎 -r −= 0.25·[claim 缺席] -BLOCKED: +1.0 若 oracle 同阶段同类失败;否则 −2.0 -执行 TIMEOUT: 另扣 −1.0,已经完成的真实阶段仍保留得分。 -G3 = na:其 2.0 权重按比例分给 G1/G2(各 2.0),精度相对项取 0 -板卡不可用(v0.4):G3 unverified 且最后一次 07-board / 08-eval 以 board_offline 失败 → G2/G3 权重取 0(板上测的两门 - 都没法测,不是 agent 的错),G1 保持 1.0 不加分,免 TIMEOUT 罚分,返回值带 unverifiable="board_offline"。 - 实测(2026-09-16 r20260916-01):regnet.y.800mf 14 次 eval 全 board_offline,v0.3 给 −0.97,等于把基础设施故障记在 - agent 头上。注意这类轨迹在导出器里仍按 UNVERIFIED 排除在 RL/SFT 之外;真正回收要用 bench board-backfill --rollout 补测。 +不读取 Oracle 对照或 BLOCKED 正确性标签。BLOCKED 的已测阶段仍保留分数, +但导出器不将未证实的 BLOCKED 纳入奖励训练。板卡不可用仍按既有规则免超时罚分。 """ from __future__ import annotations @@ -20,15 +9,11 @@ import json import sys from pathlib import Path -RECIPE_VERSION = "v0.4" +RECIPE_VERSION = "v0.6" BOARD_STAGES = ("07-board", "08-eval") PASSING = {"pass", "pass-screen", "pass-weak"} -W = {"G1": 1.0, "G2": 1.0, "G3": 2.0, "acc_rel": 1.0, "lat_rel": 0.5, "cost_turns": 0.5, "cost_tokens": 0.5, - "lie": 2.0, "claim_absent": 0.25, "blocked_ok": 1.0, "blocked_bad": 2.0, "timeout": 1.0} - - -def clip(x: float, lo: float = -1.0, hi: float = 1.0) -> float: - return max(lo, min(hi, x)) +W = {"G1": 1.0, "G2": 1.0, "G3": 2.0, "cost_turns": 0.5, "cost_tokens": 0.5, + "lie": 2.0, "claim_absent": 0.25, "timeout": 1.0} def board_unavailable(verdict: dict) -> bool: @@ -46,37 +31,18 @@ def compute(verdict: dict, max_turns: int = 40, max_tokens: int = 200000) -> dic g = verdict.get("gates") or {} terminal = verdict.get("terminal") claim = verdict.get("claim") or {} - oref = verdict.get("oracle_ref") or {} cost = verdict.get("cost") or {} unverifiable = None - if terminal == "BLOCKED": - terms["blocked"] = W["blocked_ok"] if verdict.get("blocked_correct") else -W["blocked_bad"] + g3_na = (g.get("G3") or {}).get("verdict") == "na" + if board_unavailable(verdict): + unverifiable = "board_offline" + w1, w2, w3 = W["G1"], 0.0, 0.0 else: - g3_na = (g.get("G3") or {}).get("verdict") == "na" - if board_unavailable(verdict): - unverifiable = "board_offline" - w1, w2, w3 = W["G1"], 0.0, 0.0 # 不加分只免罚:板上测的两门不计,G1 原样 - else: - w1, w2, w3 = (2.0, 2.0, 0.0) if g3_na else (W["G1"], W["G2"], W["G3"]) - terms["G1"] = w1 * float((g.get("G1") or {}).get("verdict") in PASSING) - terms["G2"] = w2 * float((g.get("G2") or {}).get("verdict") in PASSING) - terms["G3"] = w3 * float((g.get("G3") or {}).get("verdict") in PASSING) - # abs_pp 的 delta 越大越好;rel_pct 的 delta 是劣化百分比,越小越好。 - g3 = g.get("G3") or {} - same_kind = oref.get("G3_kind", g3.get("kind")) == g3.get("kind") - if not g3_na and same_kind and g3.get("delta") is not None and oref.get("G3_delta") is not None and g3.get("tolerance"): - improvement = g3["delta"] - oref["G3_delta"] - if g3.get("kind") == "rel_pct": - improvement = -improvement - terms["acc_rel"] = W["acc_rel"] * clip(improvement / abs(g3["tolerance"])) - else: - terms["acc_rel"] = 0.0 - g4 = g.get("G4") or {} - if g4.get("latency_ms_mean") and oref.get("latency_ms_mean") and not g4.get("contended"): - terms["lat_rel"] = W["lat_rel"] * clip((oref["latency_ms_mean"] - g4["latency_ms_mean"]) / oref["latency_ms_mean"]) - else: - terms["lat_rel"] = 0.0 + w1, w2, w3 = (2.0, 2.0, 0.0) if g3_na else (W["G1"], W["G2"], W["G3"]) + terms["G1"] = w1 * float((g.get("G1") or {}).get("verdict") in PASSING) + terms["G2"] = w2 * float((g.get("G2") or {}).get("verdict") in PASSING) + terms["G3"] = w3 * float((g.get("G3") or {}).get("verdict") in PASSING) if (verdict.get("execution_terminal") == "TIMEOUT" or terminal == "TIMEOUT") and not unverifiable: terms["timeout"] = -W["timeout"] # 板子不通时把预算耗在重试上,不算 agent 超时 @@ -95,7 +61,7 @@ def compute(verdict: dict, max_turns: int = 40, max_tokens: int = 200000) -> dic terms["lie"] = -W["lie"] r = sum(terms.values()) return {"reward": round(r, 4), "terms": {k: round(v, 4) for k, v in terms.items()}, "recipe": RECIPE_VERSION, - "terminal": terminal, "review": bool(verdict.get("review")), "note": terms_note, + "terminal": terminal, "note": terms_note, "unverifiable": unverifiable} diff --git a/export/test_materialization.py b/export/test_materialization.py index f0d6adc..372ea48 100644 --- a/export/test_materialization.py +++ b/export/test_materialization.py @@ -11,6 +11,7 @@ from unittest.mock import patch from export import to_agentrl as ex from export.reward import compute from harness.db import SCHEMA +from harness.verify import VERDICT_SCHEMA, VERIFICATION_RULE def dialogue(): @@ -36,15 +37,17 @@ class MaterializationTests(unittest.TestCase): def tearDown(self): self.env.stop(); self.patch.stop(); self.con.close(); self.tmp.cleanup() - def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6): + def add(self, kk=0, terminal="PASS", execution="DONE", review=False, ended=True, version=6, mode="e2e"): jd = self.root / "runs/r/j" rd = jd / "rollouts" / f"{kk:02d}" (rd / "agent").mkdir(parents=True) - job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m", "backend": "ptq"} + job = {"run_id": "r", "job_id": "j", "task": "cls", "model": "resnet18", "march": "nash-m", "backend": "ptq", + "gates": {"required": ["G1", "G3"], "G1": {}, "G2": {}, + "G3": {"kind": "rel_pct", "tol": 5, "require_source": "board"}, "G4": {}}} (jd / "job.json").write_text(json.dumps(job)) - (rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher", + (rd / "rollout.json").write_text(json.dumps({"state": execution, "model": "glm-5.3", "policy": "teacher", "mode": mode, "agent_config_version": version, "max_turns": 60, "max_tokens": 100})) - verdict = {"run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal, + verdict = {"schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE, "mode": mode, "run_id": "r", "job_id": "j", "rollout": kk, "terminal": terminal, "execution_terminal": execution, "timeout_reason": "budget" if execution == "TIMEOUT" else None, "claim_consistent": True, "claim": {"terminal": terminal}, "review": review, "gates": {"G1": {"verdict": "pass"}, "G2": {"verdict": "pass"}, "G3": {"verdict": "pass-screen"}}, @@ -75,7 +78,7 @@ class MaterializationTests(unittest.TestCase): def test_timeout_keeps_verified_stage_rewards(self): rd = self.add(execution="TIMEOUT") reward = compute(json.loads((rd / "verdict.json").read_text()), 60, 100) - self.assertEqual(reward["recipe"], "v0.4") + self.assertEqual(reward["recipe"], "v0.6") self.assertEqual(reward["terms"]["G3"], 2.0) self.assertEqual(reward["terms"]["timeout"], -1.0) manifest = ex.materialize(["r"], "data") @@ -95,13 +98,87 @@ class MaterializationTests(unittest.TestCase): self.assertEqual(recs[0]["trajectory"]["source"], "message_end") self.assertEqual(sum(m["role"] == "assistant" for m in recs[0]["messages"]), 1) - def test_review_is_retained_but_not_positive_training_candidate(self): - self.add(review=True) - manifest = ex.materialize(["r"], "data") - self.assertEqual(manifest["stats"]["review"], 1) + def legacy_evidence(self, rd, *, mode="e2e", claim="PASS"): + original = json.loads((rd / "verdict.json").read_text()) + original.update(schema="bench.verdict/2", review=True, + oracle_ref={"terminal": "FAIL", "G3_delta": -99}) + (rd / "verdict.json").write_text(json.dumps(original)) + (rd / "claim.json").write_text(json.dumps({"terminal": claim})) + for stage, measurements in { + "06-hbm": {"G1": {"hbm_exists": True, "march_ok": True, "io_ok": True, + "verifier_ok": True, "log_has_error": False}}, + "08-eval": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}}, + }.items(): + sd = rd / "stages" / stage + sd.mkdir(parents=True, exist_ok=True) + (sd / "meta.json").write_text(json.dumps({"state": "OK", "measurements": measurements})) + return original + + def test_legacy_oracle_review_is_reverified_without_modifying_sources(self): + rd = self.add(review=True) + self.legacy_evidence(rd) + before = {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in rd.rglob("*") if p.is_file()} + before_db = list(self.con.execute("SELECT * FROM rollouts")) + manifest = ex.materialize(["r"], "data", purpose="sft") + self.assertEqual(manifest["stats"]["reverified"], 1) rec = json.loads((self.root / "datasets/data/teacher.jsonl").read_text()) + self.assertTrue(rec["sft_eligible"]) + self.assertTrue(rec["training_uses"]["reward_learning_candidate"]) + self.assertEqual(rec["verification"]["method"], "readonly-reverification") + self.assertEqual(rec["verdict"]["schema"], VERDICT_SCHEMA) + self.assertFalse(rec["verdict"]["review"]) + self.assertNotIn("oracle_ref", rec) + self.assertNotIn("history", rec) + self.assertTrue(rec["provenance"]["original_verdict"]["review"]) + self.assertEqual(rec["provenance"]["original_verdict"]["sha256"], before[rd / "verdict.json"]) + self.assertEqual(before, {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in before}) + self.assertEqual(before_db, list(self.con.execute("SELECT * FROM rollouts"))) + + def test_failed_reverification_is_excluded_without_legacy_fallback(self): + rd = self.add(review=True) + self.legacy_evidence(rd) + (rd / "stages/08-eval/meta.json").write_text("broken evidence") + stats = ex.materialize(["r"], "data")["stats"] + self.assertEqual(stats["written"], 0) + self.assertEqual(stats["dropped"], {"reverification_failed:JSONDecodeError": 1}) + + def test_historical_repair_remains_analysis_only(self): + rd = self.add(mode="repair") + self.legacy_evidence(rd) + ex.materialize(["r"], "all") + rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text()) + self.assertEqual(rec["mode"], "repair") + self.assertFalse(rec["sft_eligible"]) + self.assertEqual(rec["sft_reason"], "historical-repair-context") + self.assertFalse(rec["training_uses"]["reward_learning_candidate"]) + self.assertEqual(ex.materialize(["r"], "sft", purpose="sft")["stats"]["written"], 0) + self.assertEqual(ex.materialize(["r"], "rl", purpose="rl")["stats"]["written"], 0) + + def test_blocked_is_not_a_success_or_reward_training_target(self): + rd = self.add(terminal="BLOCKED") + v = json.loads((rd / "verdict.json").read_text()) + v.update(blocked_correct=True, oracle_ref={"terminal": "FAIL"}) + (rd / "verdict.json").write_text(json.dumps(v)) + ex.materialize(["r"], "all") + rec = json.loads((self.root / "datasets/all/teacher.jsonl").read_text()) self.assertFalse(rec["sft_eligible"]) self.assertFalse(rec["training_uses"]["reward_learning_candidate"]) + self.assertNotIn("blocked", rec["reward_terms"]) + + def test_oracle_independence_does_not_bypass_transcript_or_annotation_checks(self): + first = self.add(ended=False) + self.legacy_evidence(first) + second = self.add(kk=1) + self.legacy_evidence(second) + (second / "claim.json").unlink() + third = self.add(kk=2) + self.legacy_evidence(third) + self.con.execute("INSERT INTO annotations(run_id,job_id,note) VALUES('r','j','rollout#2 contaminated: copied solution')") + self.con.commit() + stats = ex.materialize(["r"], "data", purpose="sft")["stats"] + self.assertEqual(stats["written"], 0) + self.assertEqual(stats["dropped"], {"sft:partial-transcript": 1, "sft:claim-missing-or-inconsistent": 1, + "contaminated": 1}) def test_old_version_and_missing_verdict_have_explicit_exclusions(self): self.add(version=5) @@ -113,6 +190,17 @@ class MaterializationTests(unittest.TestCase): self.assertIn("old_version", exclusions) self.assertIn("missing_verdict", exclusions) + def test_sft_and_rl_exports_do_not_read_historical_pipeline(self): + self.add() + with patch.object(ex, "pipeline_records", side_effect=OSError("unreadable old pipeline")) as pipeline: + for purpose in ("sft", "rl"): + with self.subTest(purpose=purpose): + result = ex.materialize(["r"], purpose, purpose=purpose) + self.assertEqual(result["stats"]["agent_written"], 1) + self.assertEqual(result["stats"]["pipeline_written"], 0) + self.assertEqual((self.root / "datasets" / purpose / "pipeline.jsonl").read_text(), "") + pipeline.assert_not_called() + def test_pipeline_is_real_separate_and_failed_jobs_are_retained(self): self.add() jd = self.root / "runs/r/j" diff --git a/export/to_agentrl.py b/export/to_agentrl.py index b8f6c8a..6e42e1d 100644 --- a/export/to_agentrl.py +++ b/export/to_agentrl.py @@ -1,4 +1,4 @@ -"""L4 数据物化:扫描 rollouts → 脱敏、清洗(R1–R8)、线性化 → 与 verdict / oracle 配对 → reward → datasets//。 +"""L4 数据物化:扫描 rollouts → 脱敏、清洗(R1–R8)、线性化 → 独立核验 → reward → datasets//。 python export/to_agentrl.py --runs r20260914-01 [r…] [--policy teacher] [--include-timeout] @@ -267,15 +267,15 @@ def sft_eligibility(verdict: dict, reward: float) -> tuple[bool, str]: honestly claimed success is not rejected solely for exhausting its budget. Legacy TIMEOUT verdicts stay evidence until reverified, never assumed PASS. """ - if verdict.get("review"): - return False, "review-pending(oracle FAIL / rollout PASS,待人工抽查)" + if verdict.get("mode") == "repair": + return False, "historical-repair-context" if verdict.get("claim_consistent") is not True: return False, "claim-missing-or-inconsistent" t = verdict.get("terminal") if t == "PASS": return True, "PASS" if t == "BLOCKED": - return (True, "BLOCKED-correct") if verdict.get("blocked_correct") else (False, "BLOCKED-misjudged") + return False, "BLOCKED-unproven" if t == "TIMEOUT": return False, "legacy-TIMEOUT-needs-reverification" return False, f"{t}-not-success-demonstration" @@ -292,7 +292,7 @@ def scan(run_ids: list[str]) -> list[Path]: # DB index only: never enumerate the JuiceFS run tree. with readonly_db() as con: rows = [row for rid in run_ids for row in con.execute( - "SELECT run_id, job_id, kk FROM rollouts WHERE run_id=? AND state NOT IN ('RUNNING','QUEUED','VERIFYING')", (rid,))] + "SELECT run_id, job_id, kk FROM rollouts WHERE run_id=? AND state NOT IN ('RUNNING','QUEUED','VERIFYING','TRANSFERRED')", (rid,))] return sorted(BENCH / 'runs' / rid / jid / 'rollouts' / f'{kk:02d}' for rid, jid, kk in rows) @@ -363,7 +363,7 @@ def pipeline_records(run_ids: list[str]): DB-indexed only. No recursive traversal of model/data trees and no simulated LLM dialogue. Missing evidence is recorded rather than hiding a finished job. """ - from harness.verify import completion_summary + from harness.verify import completion_summary, eval_set with readonly_db() as con: jobs = [dict(row) for rid in run_ids for row in con.execute( "SELECT * FROM jobs WHERE run_id=? AND mode='oracle' AND state IN ('PASS','FAIL','UNVERIFIED')", (rid,))] @@ -402,6 +402,7 @@ def pipeline_records(run_ids: list[str]): "schema": "bench.pipeline-training/1", "source": "deterministic_pipeline", "id": hashlib.sha1(f"{rid}/{jid}/oracle".encode()).hexdigest()[:16], "run_id": rid, "job_id": jid, "model": job.get("model", row.get("model")), + "eval_set": eval_set(job), "continuation": job.get("continuation"), "task": job.get("task", row.get("task")), "march": job.get("march", row.get("march")), "backend": job.get("backend", row.get("backend")), "terminal": result.get("terminal", row["state"]), "completion": completion_summary(result.get("gates") or {}, result.get("stages") or {}), @@ -429,7 +430,8 @@ def _write_dataset(a, work: Path, ds_id: str) -> dict: } for rid in a.runs} stats = {"scanned": 0, "written": 0, "agent_written": 0, "teacher_written": 0, "student_written": 0, "pipeline_written": 0, "excluded": 0, "dropped": {}, "review": 0, "partial": 0, - "by_terminal": {}, "by_execution_terminal": {}, "pipeline_by_terminal": {}, "sft_eligible": 0} + "by_terminal": {}, "by_execution_terminal": {}, "pipeline_by_terminal": {}, "sft_eligible": 0, + "reverified": 0} with contextlib.ExitStack() as stack: files = {name: stack.enter_context((work / f"{name}.jsonl").open("w")) for name in ("teacher", "student", "pipeline", "exclusions")} @@ -442,10 +444,11 @@ def _write_dataset(a, work: Path, ds_id: str) -> dict: write("exclusions", {"run_id": rd.parents[2].name, "job_id": rd.parents[1].name, "rollout": int(rd.name), "source": str(rd.relative_to(BENCH)), "reasons": sorted(reasons)}) - for record in pipeline_records(a.runs): - write("pipeline", record) - stats["pipeline_written"] += 1 - _inc(stats["pipeline_by_terminal"], record["terminal"]) + if a.purpose == "all": + for record in pipeline_records(a.runs): + write("pipeline", record) + stats["pipeline_written"] += 1 + _inc(stats["pipeline_by_terminal"], record["terminal"]) for rd in scan(a.runs): _write_agent(rd, a, stats, annotations, write, exclude) file_info = {} @@ -463,7 +466,8 @@ def _write_dataset(a, work: Path, ds_id: str) -> dict: "annotations_read": True, "dropped_counts_overlap": True, "policy": a.policy, "include_timeout": True, "purpose": a.purpose, "min_reward": a.min_reward}, "stats": stats, "files": file_info, - "notes": ["pipeline.jsonl is deterministic execution evidence, not Agent dialogue.", + "notes": ["pipeline.jsonl contains historical deterministic evidence only for purpose=all; SFT/RL export reads Agent evidence only.", + "Verdict v3 and reward v0.6 use only this rollout evidence; Oracle references are history only.", "SFT/reward candidates are labels; no train split, framework conversion, or online RL readiness is implied.", "Writers hold .export.lock; manifest is published last. Readers can lock or verify file hashes for a coherent refresh."]} (work / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False)) @@ -471,8 +475,10 @@ def _write_dataset(a, work: Path, ds_id: str) -> dict: def _write_agent(rd, a, stats, annotations, write, exclude): + from harness.verify import eval_set, verify_rollout, VERDICT_SCHEMA, VERIFICATION_RULE stats["scanned"] += 1 - verdict = _read_json(rd / "verdict.json") + original_verdict = _read_json(rd / "verdict.json") + verdict = original_verdict rj = _read_json(rd / "rollout.json") if not verdict: exclude(rd, {"missing_verdict"}); return @@ -487,6 +493,17 @@ def _write_agent(rd, a, stats, annotations, write, exclude): reasons.add('old_version') if reasons: exclude(rd, reasons); return + original_sha = hashlib.sha256((rd / "verdict.json").read_bytes()).hexdigest() + reverified = verdict.get("schema") != VERDICT_SCHEMA + if reverified: + try: + verdict = verify_rollout(rd, write=False) + stats["reverified"] += 1 + except Exception as error: + exclude(rd, {f"reverification_failed:{type(error).__name__}"}); return + verification = {"schema": verdict.get("schema"), + "rule": verdict.get("verification_rule", VERIFICATION_RULE), + "method": "readonly-reverification" if reverified else "recorded-verdict"} raw, trajectory = load_trajectory(rd) if not raw: exclude(rd, {"no_trajectory"}); return @@ -502,6 +519,10 @@ def _write_agent(rd, a, stats, annotations, write, exclude): if cst["drop"]: exclude(rd, {"no_usable_messages"}); return msgs, rst = redact_messages(msgs) + jd = rd.parent.parent + job = json.loads((jd / "job.json").read_text()) + mode = verdict.get("mode") or rj.get("mode") + verdict = {**verdict, "mode": mode} rw = compute_reward(verdict, rj.get("max_turns", a.max_turns), rj.get("max_tokens", a.max_tokens)) sft_ok, sft_reason = sft_eligibility(verdict, rw["reward"]) trajectory["partial"] = trajectory["partial"] or not cst["tool_pairing"]["valid"] or cst.get("trailing_user", False) @@ -517,36 +538,39 @@ def _write_agent(rd, a, stats, annotations, write, exclude): sft_ok, sft_reason = False, "missing-called-tool-definitions" if a.purpose == "sft" and not sft_ok: exclude(rd, {f"sft:{sft_reason}"}); return - reward_candidate = verdict["terminal"] in ("PASS", "FAIL", "BLOCKED") and not verdict.get("review") + reward_candidate = verdict["terminal"] in ("PASS", "FAIL") and mode != "repair" and not rw.get("unverifiable") if a.purpose == "rl" and not reward_candidate: - exclude(rd, {"rl:review-or-unverified-outcome"}); return + exclude(rd, {"rl:historical-repair-or-unverified-outcome"}); return if a.min_reward is not None and rw["reward"] < a.min_reward: exclude(rd, {"min_reward"}); return - jd = rd.parent.parent - job = json.loads((jd / "job.json").read_text()) rec = { "schema": "bench.agent-training/2", "source": "agent_events", "id": hashlib.sha1(f"{verdict['run_id']}/{verdict['job_id']}/{verdict['rollout']}".encode()).hexdigest()[:16], "run_id": verdict["run_id"], "job_id": verdict["job_id"], "rollout": verdict["rollout"], - "mode": verdict.get("mode") or rj.get("mode"), "policy": policy, "model": rj.get("model"), + "eval_set": verdict.get("eval_set") or eval_set(job), + "continuation": {**job["continuation"], "source_rollout": verdict["rollout"]} if job.get("continuation") else None, + "mode": mode, "policy": policy, "model": rj.get("model"), "agent_config_version": str(rj.get("agent_config_version", "1")), "task": job["task"], "march": job["march"], "backend": job["backend"], "messages": msgs, - "verdict": {k: verdict[k] for k in ("terminal", "execution_terminal", "timeout_reason", "completion", "failed_gate", "claim_consistent", "blocked_correct", "review") if k in verdict}, + "verdict": {k: verdict[k] for k in ("schema", "verification_rule", "terminal", "measured_terminal", "execution_terminal", "timeout_reason", "completion", "failed_gate", "claim_consistent", "blocked_correct", "review") if k in verdict}, "gates": {g: (verdict["gates"].get(g) or {}).get("verdict") for g in ("G1", "G2", "G3", "G4")}, - "oracle_ref": verdict.get("oracle_ref"), + "verification": verification, "reward": rw["reward"], "reward_terms": rw["terms"], "reward_recipe": rw["recipe"], "reward_unverifiable": rw.get("unverifiable"), # v0.4:board_offline = 板卡故障导致没法测,不是 agent 的结果 "sft_eligible": sft_ok, "sft_reason": sft_reason, "training_uses": {"analysis": True, "sft_candidate": sft_ok, "reward_learning_candidate": reward_candidate, - "review_required": bool(verdict.get("review"))}, + "review_required": False}, "trajectory": trajectory, "execution": {"terminal": verdict.get("execution_terminal") or rj.get("state"), "timeout_reason": verdict.get("timeout_reason") or rj.get("timeout_reason"), "started_at": rj.get("started_at"), "finished_at": rj.get("finished_at"), "max_turns": rj.get("max_turns"), "max_tokens": rj.get("max_tokens")}, "provenance": {"events": str((rd / "agent" / "events.jsonl").relative_to(BENCH)), - "verdict": str((rd / "verdict.json").relative_to(BENCH))}, + "verdict": str((rd / "verdict.json").relative_to(BENCH)), + "original_verdict": {"sha256": original_sha, "schema": original_verdict.get("schema"), + "terminal": original_verdict.get("terminal"), + "review": original_verdict.get("review")}}, "cost": verdict.get("cost"), "tools": tools, "tools_source": tools_src, @@ -560,7 +584,6 @@ def _write_agent(rd, a, stats, annotations, write, exclude): stats["agent_written"] += 1 stats[f"{policy}_written"] += 1 stats["partial"] += int(trajectory["partial"]) - stats["review"] += int(bool(verdict.get("review"))) _inc(stats["by_terminal"], verdict["terminal"]) _inc(stats["by_execution_terminal"], rec["execution"]["terminal"]) stats["sft_eligible"] += int(sft_ok) @@ -615,7 +638,7 @@ def main() -> int: ap.add_argument("--policy", choices=["teacher", "student"]) ap.add_argument("--include-timeout", action="store_true", help="compatibility flag: timeout evidence is now included by default") ap.add_argument("--purpose", choices=["all", "sft", "rl"], default="all", - help="all=保留终态轨迹并标注用途;sft=完整诚实成功候选;rl=有核验结果且不待复核的候选") + help="all=保留终态轨迹并标注用途;sft=完整诚实成功候选;rl=本次可核验 PASS/FAIL 候选") ap.add_argument("--min-reward", type=float) ap.add_argument("--max-turns", type=int, default=40) ap.add_argument("--max-tokens", type=int, default=200000) diff --git a/harness/adapters/__init__.py b/harness/adapters/__init__.py index 58b59f8..5116202 100644 --- a/harness/adapters/__init__.py +++ b/harness/adapters/__init__.py @@ -1,2 +1,43 @@ -"""custom 模型的构建适配器:每个模型一个 .py,提供 build(weights: Path, input_spec: str) -> nn.Module。 -非 torchvision / ultralytics / timm 的模型走这里。""" +"""Optional model recipes, shared by input preparation and evaluation.""" +from __future__ import annotations + +import hashlib +import importlib +import json +from functools import lru_cache +from pathlib import Path + + +@lru_cache(maxsize=None) +def get(model: str): + name = model.replace(".", "_").replace("-", "_") + if not (Path(__file__).parent / f"{name}.py").is_file(): + return None + return importlib.import_module(f"{__name__}.{name}") + + +def model_spec(row: dict) -> dict: + """Resolve concrete adapter inputs without changing the source CSV.""" + adapter = get(row["model_id"]) + if adapter is None: + return row + result = dict(row) + result.update(getattr(adapter, "MODEL_SPEC", {})) + if checkpoint := getattr(adapter, "CHECKPOINT", None): + result.update(weights=checkpoint["url"], checkpoint=dict(checkpoint)) + if recipe := getattr(adapter, "PREPROCESS", None): + result["preprocess"] = dict(recipe) + if runtime := getattr(adapter, "RUNTIME_PYTHON", None): + result["runtime_python"] = str(runtime) + return result + + +def calibration_dir(row: dict) -> Path: + from .. import paths + directory = paths.DATA / "calib" / row["task"] / row["input_spec"] + recipe = model_spec(row).get("preprocess") + if recipe: + # Equal image recipes share calibration; existing jobs retain their path. + digest = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest() + directory /= digest[:16] + return directory diff --git a/harness/adapters/aliked.py b/harness/adapters/aliked.py new file mode 100644 index 0000000..7429342 --- /dev/null +++ b/harness/adapters/aliked.py @@ -0,0 +1,12 @@ +"""Original ALIKED-N16; static 1024 keypoints, all learned descriptor layers in graph.""" +from . import feature_common as common +EXPORTER_VERSION = common.EXPORTER_VERSION +INPUT_SPEC = common.input_spec('aliked') +MODEL_SPEC = {'input_spec': INPUT_SPEC, 'dataset': 'hpatches', 'input_override_reason': 'Official ALIKED expects RGB; CSV grayscale is incorrect.'} +CHECKPOINT = common.checkpoint_info('aliked') +PREPROCESS = common.recipe('aliked') +OPSET = 17 +preprocess = common.preprocess + +def build(weights, input_spec): + return common.build('aliked', weights, input_spec) diff --git a/harness/adapters/atss_r50_fpn.py b/harness/adapters/atss_r50_fpn.py new file mode 100644 index 0000000..c219496 --- /dev/null +++ b/harness/adapters/atss_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection atss.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "atss.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-atss.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/bisenetv1_r18.py b/harness/adapters/bisenetv1_r18.py new file mode 100644 index 0000000..2f65d2a --- /dev/null +++ b/harness/adapters/bisenetv1_r18.py @@ -0,0 +1,7 @@ +"""Official Cityscapes bisenetv1.r18; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("bisenetv1.r18") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "bisenetv1.r18") diff --git a/harness/adapters/bisenetv2.py b/harness/adapters/bisenetv2.py new file mode 100644 index 0000000..41f3a64 --- /dev/null +++ b/harness/adapters/bisenetv2.py @@ -0,0 +1,7 @@ +"""Official Cityscapes bisenetv2; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("bisenetv2") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "bisenetv2") diff --git a/harness/adapters/conditional_detr_r50.py b/harness/adapters/conditional_detr_r50.py new file mode 100644 index 0000000..531d395 --- /dev/null +++ b/harness/adapters/conditional_detr_r50.py @@ -0,0 +1,114 @@ +"""Exact MMDetection conditional.detr.r50; see mmdet_detr_common for the +static deployment geometry this batch reuses without editing it. + +ConditionalDETR keeps the DETR single-scale encoder path (real per-image +padding mask retained in-graph) and adds the official conditional decoder +with learned query scaling and reference points. Outputs stay the final +decoder layer's raw class logits and normalized cxcywh; only the official +parameter-free DETRHead selection runs in host decode. +""" +from functools import partial +from types import SimpleNamespace + +from .openmmlab import build as build_official, get_recipe +from .mmdet_detr_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, OPSET, + transformer_inputs) +from .mmdet_dense_common import (recipe as image_recipe, + preprocess_variant as image_preprocess, + geometry) + +MODEL_ID = "conditional.detr.r50" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +QUERIES, CHANNELS, TOPK = 300, 80, 100 + + +def recipe(): + dp = get_recipe(MODEL_ID)["preprocess"]["official_data_preprocessor"] + meta = image_recipe("retinanet.r50.fpn") + if (dp["mean"] != meta["mean"] or dp["std"] != meta["std"] + or not dp["bgr_to_rgb"] or dp["pad_size_divisor"] != 1): + raise ValueError("Official conditional-detr image normalization changed") + # Same image transformation permits existing calibration cache reuse. + return meta + + +PREPROCESS = recipe() +EXPORTER_VERSION = "mmdet-conditional.detr.r50-640-raw-v1" + + +def preprocess(image, input_spec=INPUT_SPEC): + if input_spec != INPUT_SPEC: + raise ValueError(f"Expected {INPUT_SPEC}, got {input_spec}") + recipe() # verify exact equality before reusing the image-only helper + return image_preprocess("retinanet.r50.fpn", image, input_spec) + + +def output_names(): + return ["class_logits", "boxes_cxcywh"] + + +def _decoder(): + from mmdet.models.dense_heads.detr_head import DETRHead + context = SimpleNamespace(test_cfg={"max_per_img": TOPK}, num_classes=80, + loss_cls=SimpleNamespace(use_sigmoid=True)) + return partial(DETRHead._predict_by_feat_single, context) + + +def decode(outputs, original_size): + import numpy as np + import torch + if len(outputs) != 2: + raise ValueError("Expected final query logits and cxcywh tensors") + tensors = [] + for value, shape in zip(outputs, ((1, QUERIES, CHANNELS), (1, QUERIES, 4))): + value = np.asarray(value) + if value.shape != shape or not np.isfinite(value).all(): + raise ValueError(f"Invalid {MODEL_ID} raw output: {value.shape}, expected {shape}") + tensors.append(torch.tensor(value, dtype=torch.float32)[0]) + _, scales, img_shape = geometry(original_size) + with torch.inference_mode(): + result = _decoder()(*tensors, dict(img_shape=img_shape, scale_factor=scales), rescale=True) + return result.bboxes.numpy(), result.scores.numpy(), result.labels.numpy() + + +def build(weights, input_spec): + import torch + raw = build_official(MODEL_ID, weights, input_spec) + network, cfg = raw.upstream, raw.cfg + meta = recipe() + dp = cfg.model.data_preprocessor + if (type(network).__name__ != "ConditionalDETR" + or type(network.bbox_head).__name__ != "ConditionalDETRHead" + or network.num_queries != QUERIES + or network.bbox_head.cls_out_channels != CHANNELS + or network.bbox_head.num_classes != 80 + or dict(cfg.model.test_cfg) != {"max_per_img": TOPK} + or list(dp.mean) != meta["mean"] or list(dp.std) != meta["std"] + or not dp.bgr_to_rgb or dp.pad_size_divisor != 1): + raise ValueError("Pinned official conditional-detr contract changed") + state = torch.load(str(weights), map_location="cpu")["state_dict"] + for name, parameter in network.named_parameters(): + torch.testing.assert_close(parameter, state[name], rtol=0, atol=0) + del state + + class RawQueries(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = network + self.upstream.cfg = cfg + self.cfg = cfg + self.checkpoint_sha256 = raw.checkpoint_sha256 + + def forward(self, x): + features = self.upstream.extract_feat(x) + # Single-scale DETR path; the official conditional decoder consumes + # the same memory/mask/pos tensors via its own reference points. + encoder, decoder = transformer_inputs(self.upstream, features, x, False) + memory = self.upstream.forward_encoder(**encoder) + query, head_inputs = self.upstream.pre_decoder(**memory) + decoder.update(query) + head_inputs.update(self.upstream.forward_decoder(**decoder)) + scores, boxes = self.upstream.bbox_head(**head_inputs) + return scores[-1], boxes[-1] + + return RawQueries().eval() diff --git a/harness/adapters/d2_net.py b/harness/adapters/d2_net.py new file mode 100644 index 0000000..2b863ed --- /dev/null +++ b/harness/adapters/d2_net.py @@ -0,0 +1,69 @@ +"""Official D2-Net d2_tf single-scale CNN with exact official host postprocessing.""" +from functools import lru_cache +from . import feature_sources as sources + +INPUT_SPEC = '1x3x480x640' +MODEL_SPEC = {'input_spec': INPUT_SPEC, 'dataset': 'hpatches', 'input_override_reason': 'Official D2-Net Caffe preprocessing uses three BGR channels.'} +EXPORTER_VERSION = 'd2net-static-1' +OPSET = 17 +CHECKPOINT = {'url': 'https://dusmanu.com/files/d2-net/d2_tf.pth', + 'sha256': 'c2d32c3bc53b6588d40bc5325d536a159e529b7492198a2cfa4b11913c615c80', + 'filename': 'd2_tf.pth', 'official_relative_path': str(sources.ROOT/'d2net/models/d2_tf.pth')} +PREPROCESS = {'input_spec': INPUT_SPEC, 'color': 'BGR', 'scale': 1., 'mean': [103.939,116.779,123.68], + 'resize': 'opencv_linear_640x480', 'output_layout': 'NCHW', 'protocol': 'd2net-official-caffe-single-scale-v1'} + + +def official_module(): + return sources.module('d2net/lib/model_test.py') + + +@lru_cache(None) +def _functions(): + exception = sources.module('d2net/lib/exceptions.py').EmptyTensorError + ns = sources.definitions('d2net/lib/utils.py', ['preprocess_image','interpolate_dense_features','upscale_positions'], + {'EmptyTensorError': exception}) + return sources.definitions('d2net/lib/pyramid.py', ['process_multiscale'], ns) + + +def preprocess(image, input_spec=INPUT_SPEC): + import cv2 + import numpy as np + if input_spec != INPUT_SPEC: raise ValueError(input_spec) + rgb = cv2.resize(np.asarray(image.convert('RGB')), (640,480), interpolation=cv2.INTER_LINEAR) + return np.ascontiguousarray(_functions()['preprocess_image'](rgb, preprocessing='caffe')[None], dtype=np.float32) + + +def official_model(weights): + import torch + sources.verify_weights(weights, CHECKPOINT['sha256']) + m = official_module().D2Net(use_cuda=False, use_relu=True) + m.load_state_dict(torch.load(weights, map_location='cpu', weights_only=False)['model'], strict=True) + return m.eval() + + +def official_features(m, x): + import torch + k,s,d = _functions()['process_multiscale'](x, m, scales=[1]) + # The official extractor swaps (row,col) to (x,y) before saving. + out = sources.top_features(k[:,[1,0]], s, d) + return tuple(torch.from_numpy(out[key]) for key in ('keypoints','scores','descriptors')) + + +def build(weights, input_spec): + if input_spec != INPUT_SPEC: raise ValueError(input_spec) + return official_model(weights).dense_feature_extraction + + +def decode(outputs): + import torch + from types import SimpleNamespace + dense, = sources.tensors(outputs) + assert tuple(dense.shape) == (1,512,119,159) + class DenseMap(torch.nn.Module): + num_channels = 512 + def forward(self, x): return dense + module = official_module() + decoder = SimpleNamespace(dense_feature_extraction=DenseMap(), detection=module.HardDetectionModule(), + localization=module.HandcraftedLocalizationModule()) + with torch.no_grad(): k,s,d = official_features(decoder, torch.zeros(1,3,480,640)) + return {'keypoints': k.numpy(), 'scores': s.numpy(), 'descriptors': d.numpy()} diff --git a/harness/adapters/dab_detr_r50.py b/harness/adapters/dab_detr_r50.py new file mode 100644 index 0000000..8d8a103 --- /dev/null +++ b/harness/adapters/dab_detr_r50.py @@ -0,0 +1,12 @@ +"""Exact MMDetection dab.detr.r50; see mmdet_detr_common for static deployment geometry.""" +from functools import partial +from .mmdet_detr_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, OPSET, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "dab.detr.r50" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-dab.detr.r50-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/ddrnet_23.py b/harness/adapters/ddrnet_23.py new file mode 100644 index 0000000..3196d35 --- /dev/null +++ b/harness/adapters/ddrnet_23.py @@ -0,0 +1,7 @@ +"""Official Cityscapes ddrnet.23; exact static benchmark recipe.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("ddrnet.23") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "ddrnet.23") diff --git a/harness/adapters/ddrnet_23_slim.py b/harness/adapters/ddrnet_23_slim.py new file mode 100644 index 0000000..f61b9f9 --- /dev/null +++ b/harness/adapters/ddrnet_23_slim.py @@ -0,0 +1,7 @@ +"""Official Cityscapes ddrnet.23.slim; exact static benchmark recipe.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("ddrnet.23.slim") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "ddrnet.23.slim") diff --git a/harness/adapters/dedode_v2.py b/harness/adapters/dedode_v2.py new file mode 100644 index 0000000..fe0c10c --- /dev/null +++ b/harness/adapters/dedode_v2.py @@ -0,0 +1,74 @@ +"""Official DeDoDe v2 detector-L + descriptor-B; both learned nets in one graph.""" +from . import feature_sources3 as sources +from .feature_common import MAX_KEYPOINTS + +INPUT_SPEC='1x3x480x640' +MODEL_SPEC={'input_spec':INPUT_SPEC,'dataset':'hpatches', + 'input_override_reason':'Official DeDoDe RGB ImageNet-normalized input; detector-L v2 plus descriptor-B.'} +EXPORTER_VERSION='dedode-v2-l-b-static-1' +OPSET=17 +CHECKPOINT={'url':'https://github.com/Parskatt/DeDoDe/releases/download/v2/dedode_detector_L_v2.pth', + 'sha256':'4113809dd9e0367af013a45fc2255a6b243ff241cd06520d17a65d9e231bdc17', + 'filename':'dedode_detector_L_v2.pth', + 'official_relative_path':str(sources.ROOT/'dedode/weights/dedode_detector_L_v2.pth')} +DESCRIPTOR_CHECKPOINT={'url':'https://github.com/Parskatt/DeDoDe/releases/download/dedode_pretrained_models/dedode_descriptor_B.pth', + 'sha256':'8eccfc270ec990ced60cd54a434411a47c4c504de13586f596d042e005b3022b', + 'path':str(sources.ROOT/'dedode/weights/dedode_descriptor_B.pth')} +PREPROCESS={'input_spec':INPUT_SPEC,'color':'RGB','scale':1/255,'mean':[.485,.456,.406], + 'std':[.229,.224,.225],'resize':'opencv_linear_640x480','output_layout':'NCHW', + 'protocol':'dedode-v2-official-rgb-l-b-static-v1'} + +def preprocess(image,input_spec=INPUT_SPEC): + import cv2 + import numpy as np + if input_spec!=INPUT_SPEC:raise ValueError(input_spec) + x=cv2.resize(np.asarray(image.convert('RGB')),(640,480),interpolation=cv2.INTER_LINEAR).astype(np.float64)/255 + x=(x-np.asarray(PREPROCESS['mean']))/np.asarray(PREPROCESS['std']) + return np.ascontiguousarray(x.transpose(2,0,1)[None],dtype=np.float32) + +def official_model(weights): + import torch + sources.check(weights,CHECKPOINT['sha256']) + sources.check(DESCRIPTOR_CHECKPOINT['path'],DESCRIPTOR_CHECKPOINT['sha256']) + module=sources.package('dedode','DeDoDe') + detector=module.dedode_detector_L(device='cpu',weights=torch.load(weights,map_location='cpu',weights_only=True),remove_borders=False) + descriptor=module.dedode_descriptor_B(device='cpu',weights=torch.load(DESCRIPTOR_CHECKPOINT['path'],map_location='cpu',weights_only=True)) + nets=torch.nn.ModuleDict({'detector':detector,'descriptor':descriptor}).eval() + # The benchmark is FP32; CPU upstream autocast would be disabled anyway. + for m in nets.modules(): + if hasattr(m,'amp'):m.amp=False + return nets + +def official_features(m,x): + import torch.nn.functional as F + module=sources.package('dedode','DeDoDe.utils') + result=m['detector'].detect({'image':x},num_keypoints=MAX_KEYPOINTS) + desc=m['descriptor'].describe_keypoints({'image':x},result['keypoints'])['descriptions'] + return module.to_pixel_coords(result['keypoints'],480,640)[0].cpu(),result['confidence'][0].cpu(),F.normalize(desc[0],dim=-1).cpu() + +def build(weights,input_spec): + import torch + if input_spec!=INPUT_SPEC:raise ValueError(input_spec) + class Graph(torch.nn.Module): + def __init__(self):super().__init__();self.nets=official_model(weights) + def forward(self,x): + return self.nets['detector']({'image':x})['keypoint_logits'],self.nets['descriptor']({'image':x})['description_grid'] + return Graph().eval() + +def decode(outputs): + import numpy as np + import torch + import torch.nn.functional as F + logits,descriptors=[torch.as_tensor(np.asarray(o,np.float32)) for o in outputs] + assert logits.shape==(1,1,480,640) and descriptors.shape==(1,256,480,640) + assert torch.isfinite(logits).all() and torch.isfinite(descriptors).all() + module=sources.package('dedode','DeDoDe.utils') + # Exact detector.detect / descriptor.describe_keypoints operations; explicit + # CPU map device allows host decoding even when a server GPU is visible. + probability=logits.reshape(1,-1).softmax(-1).reshape(1,1,480*640).sum(1) + k,s=module.sample_keypoints(probability.reshape(1,480,640),device=logits.device, + use_nms=False,sample_topk=True,num_samples=MAX_KEYPOINTS,return_scoremap=True, + sharpen=False,upsample=False,increase_coverage=True,remove_borders=False) + d=F.grid_sample(descriptors,k[:,None],mode='bilinear',align_corners=False)[:,:,0].mT + return {'keypoints':module.to_pixel_coords(k,480,640)[0].numpy(), + 'scores':s[0].numpy(),'descriptors':F.normalize(d[0],dim=-1).numpy()} diff --git a/harness/adapters/deeplabv3_resnet101.py b/harness/adapters/deeplabv3_resnet101.py new file mode 100644 index 0000000..c33237e --- /dev/null +++ b/harness/adapters/deeplabv3_resnet101.py @@ -0,0 +1,7 @@ +"""Exact official ADE20K DeepLabV3 ResNet101; admission requires actual validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("deeplabv3.resnet101") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "deeplabv3.resnet101") diff --git a/harness/adapters/deeplabv3_resnet50.py b/harness/adapters/deeplabv3_resnet50.py new file mode 100644 index 0000000..636e2d2 --- /dev/null +++ b/harness/adapters/deeplabv3_resnet50.py @@ -0,0 +1,7 @@ +"""Exact official ADE20K DeepLabV3 ResNet50; admission requires actual validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("deeplabv3.resnet50") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "deeplabv3.resnet50") diff --git a/harness/adapters/deeplabv3plus_resnet101.py b/harness/adapters/deeplabv3plus_resnet101.py new file mode 100644 index 0000000..6ca142f --- /dev/null +++ b/harness/adapters/deeplabv3plus_resnet101.py @@ -0,0 +1,7 @@ +"""Exact official ADE20K DeepLabV3+ ResNet101; admission requires actual validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("deeplabv3plus.resnet101") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "deeplabv3plus.resnet101") diff --git a/harness/adapters/deeplabv3plus_resnet50.py b/harness/adapters/deeplabv3plus_resnet50.py new file mode 100644 index 0000000..41ec512 --- /dev/null +++ b/harness/adapters/deeplabv3plus_resnet50.py @@ -0,0 +1,7 @@ +"""Exact official ADE20K DeepLabV3+ ResNet50; admission requires actual validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("deeplabv3plus.resnet50") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "deeplabv3plus.resnet50") diff --git a/harness/adapters/deformable_detr_r50.py b/harness/adapters/deformable_detr_r50.py new file mode 100644 index 0000000..b414457 --- /dev/null +++ b/harness/adapters/deformable_detr_r50.py @@ -0,0 +1,12 @@ +"""Exact MMDetection deformable.detr.r50; see mmdet_detr_common for static deployment geometry.""" +from functools import partial +from .mmdet_detr_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, OPSET, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "deformable.detr.r50" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-deformable.detr.r50-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/depth_anything_v1_base.py b/harness/adapters/depth_anything_v1_base.py new file mode 100644 index 0000000..e71d850 --- /dev/null +++ b/harness/adapters/depth_anything_v1_base.py @@ -0,0 +1,10 @@ +"""Official Depth Anything V1 base: relative inverse depth.""" +from pathlib import Path +from .depth_anything_v1_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + RESTORE_ALIGN_CORNERS, build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("base") + + +def build(weights: Path, input_spec: str): + return build_variant("base", weights, input_spec) diff --git a/harness/adapters/depth_anything_v1_common.py b/harness/adapters/depth_anything_v1_common.py new file mode 100644 index 0000000..298362f --- /dev/null +++ b/harness/adapters/depth_anything_v1_common.py @@ -0,0 +1,87 @@ +"""Official relative Depth Anything V1; pinned source and strict checkpoints.""" +from __future__ import annotations + +import hashlib +import importlib +import os +from pathlib import Path +import sys + +from .depth_anything_v2_common import INPUT_SPEC, preprocess + +UPSTREAM_COMMIT = "1d03336771fe09c5398ffdd211441e33941a97dc" +SOURCE_TREE_SHA256 = "0196533fdd23b8031f85b7ea542bd0400a4c362916b16b2e5ea91678144f65b6" +RUNTIME_ROOT = Path.home() / "bench-cache/sources/depth-anything-v1" / UPSTREAM_COMMIT +MODEL_SPEC = {"source": "custom", "dataset": "nyuv2", "input_spec": INPUT_SPEC, "resize": 0} +OPSET = 17 +EXPORTER_VERSION = "depth-anything-v1-relative-v1-opset17-static518" +RESTORE_ALIGN_CORNERS = False +PREPROCESS = { + "id": "depth-anything-v1-rgb-static518-v1", + "layout": "NCHW", "dtype": "float32", "color": "RGB", + "resize": "OpenCV INTER_CUBIC to 518x518; keep_aspect_ratio=False for static deployment", + "normalize": "RGB/255; mean=[0.485,0.456,0.406]; std=[0.229,0.224,0.225]", + "output": "N,H,W nonnegative affine-invariant inverse depth; NOT metric depth", + "restore": "bilinear to original H,W, align_corners=False; no min/max image normalization", + "source_commit": UPSTREAM_COMMIT, +} +_MODELS = { + "small": {"encoder": "vits", "features": 64, "out_channels": [48, 96, 192, 384], + "revision": "1bee667edbfec7f9b345abaffec8c9c2bd5293c2", + "sha256": "ba0ab5365610467de4fea1ffe64b4ae3a756561a4df9e11bd7369205241f2082"}, + "base": {"encoder": "vitb", "features": 128, "out_channels": [96, 192, 384, 768], + "revision": "a7dff65359777209cb99e1361df8a76144f87e49", + "sha256": "d25d4cf2b50e63198c42e684abb10b9d242714cec0fe3e2eb8f7da41fada25b3"}, + "large": {"encoder": "vitl", "features": 256, "out_channels": [256, 512, 1024, 1024], + "revision": "973948530e4e4f4afd6d1913f670d9f96071dcaa", + "sha256": "6fda5199f0043e91e1c6117df00313db57c630d8ed0306070b43da1ecfffbd9d"}, +} + + +def checkpoint_info(variant: str) -> dict: + info = _MODELS[variant] + repo = f"LiheYoung/depth_anything_{info['encoder']}14" + return {"url": f"https://huggingface.co/{repo}/resolve/{info['revision']}/pytorch_model.bin", + "sha256": info["sha256"], "revision": info["revision"], "output_type": "relative_inverse_depth"} + + +def source_root() -> Path: + root = Path(os.environ.get("BENCH_DEPTH_ANYTHING_V1_SOURCE", str(RUNTIME_ROOT))).resolve() + digest = hashlib.sha256() + files = sorted(list((root / "depth_anything").rglob("*.py")) + list((root / "torchhub").rglob("*.py"))) + for path in files: + digest.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes()) + if digest.hexdigest() != SOURCE_TREE_SHA256: + raise RuntimeError(f"missing or changed official Depth Anything V1 source {UPSTREAM_COMMIT}: {root}") + return root + + +def build_variant(variant: str, weights: Path, input_spec: str): + import torch + + if input_spec != INPUT_SPEC: + raise ValueError(f"expected fixed {INPUT_SPEC}, got {input_spec}") + root = source_root() + weights = Path(weights).resolve() + config = _MODELS[variant] + digest = hashlib.sha256() + with weights.open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + digest.update(block) + if digest.hexdigest() != config["sha256"]: + raise ValueError(f"wrong official relative Depth Anything V1 {variant} checkpoint") + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + module = importlib.import_module("depth_anything.dpt") + if Path(module.__file__).resolve() != root / "depth_anything/dpt.py": + raise RuntimeError("a different Depth Anything V1 source is already imported") + # Upstream localhub resolves its bundled DINOv2 relative to the checkout. + # The harness builds each model in a separate worker process. + previous = Path.cwd() + try: + os.chdir(root) + model = module.DepthAnything({key: config[key] for key in ("encoder", "features", "out_channels")}) + finally: + os.chdir(previous) + model.load_state_dict(torch.load(weights, map_location="cpu", weights_only=True), strict=True) + return model.eval() diff --git a/harness/adapters/depth_anything_v1_large.py b/harness/adapters/depth_anything_v1_large.py new file mode 100644 index 0000000..ec9bba1 --- /dev/null +++ b/harness/adapters/depth_anything_v1_large.py @@ -0,0 +1,10 @@ +"""Official Depth Anything V1 large: relative inverse depth.""" +from pathlib import Path +from .depth_anything_v1_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + RESTORE_ALIGN_CORNERS, build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("large") + + +def build(weights: Path, input_spec: str): + return build_variant("large", weights, input_spec) diff --git a/harness/adapters/depth_anything_v1_small.py b/harness/adapters/depth_anything_v1_small.py new file mode 100644 index 0000000..1fc34af --- /dev/null +++ b/harness/adapters/depth_anything_v1_small.py @@ -0,0 +1,10 @@ +"""Official Depth Anything V1 small: relative inverse depth.""" +from pathlib import Path +from .depth_anything_v1_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + RESTORE_ALIGN_CORNERS, build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("small") + + +def build(weights: Path, input_spec: str): + return build_variant("small", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_base.py b/harness/adapters/depth_anything_v2_base.py new file mode 100644 index 0000000..c2723a7 --- /dev/null +++ b/harness/adapters/depth_anything_v2_base.py @@ -0,0 +1,10 @@ +"""Official Depth Anything V2 base: affine-invariant inverse depth.""" +from pathlib import Path +from .depth_anything_v2_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("base") + + +def build(weights: Path, input_spec: str): + return build_variant("base", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_common.py b/harness/adapters/depth_anything_v2_common.py new file mode 100644 index 0000000..d1afabf --- /dev/null +++ b/harness/adapters/depth_anything_v2_common.py @@ -0,0 +1,94 @@ +"""Pinned official Depth Anything V2 relative inverse-depth models. + +The original source stays in a private checkout. No timm/transformers install, +replacement architecture, metric-depth checkpoint, or synthetic weights. +""" +from __future__ import annotations + +import hashlib +import importlib +import os +from pathlib import Path +import sys + +UPSTREAM_COMMIT = "a561b849ebae10a6f5ef49e26c83cbbcd36c71bf" +SOURCE_TREE_SHA256 = "92af3e54e49dac0216ac039d7234c81082ec902329691e7aa57d30b2219f612b" +RUNTIME_ROOT = Path.home() / "bench-cache/sources/depth-anything-v2" / UPSTREAM_COMMIT +INPUT_SPEC = "1x3x518x518" +MODEL_SPEC = {"source": "custom", "dataset": "nyuv2", "input_spec": INPUT_SPEC, "resize": 0} +OPSET = 17 +EXPORTER_VERSION = "depth-anything-v2-relative-v1-opset17-static518" +PREPROCESS = { + "id": "depth-anything-v2-rgb-static518-v1", + "layout": "NCHW", "dtype": "float32", "color": "RGB", + "resize": "OpenCV INTER_CUBIC to 518x518; keep_aspect_ratio=False for static deployment", + "normalize": "RGB/255; mean=[0.485,0.456,0.406]; std=[0.229,0.224,0.225]", + "output": "N,H,W nonnegative affine-invariant inverse depth; NOT metric depth", + "restore": "bilinear to original H,W, align_corners=True; no min/max image normalization", + "source_commit": UPSTREAM_COMMIT, +} + +_MODELS = { + "small": {"encoder": "vits", "features": 64, "out_channels": [48, 96, 192, 384], + "revision": "03876f8651c73a60fe4c2c48294e09fcb6838fcf", + "sha256": "715fade13be8f229f8a70cc02066f656f2423a59effd0579197bbf57860e1378"}, + "base": {"encoder": "vitb", "features": 128, "out_channels": [96, 192, 384, 768], + "revision": "a4e71a6c2ce52fe50df0f212066b0d4a87be9b5e", + "sha256": "0d2b7002e62d39d655571c371333340bd88f67ab95050c03591555aa05645328"}, + "large": {"encoder": "vitl", "features": 256, "out_channels": [256, 512, 1024, 1024], + "revision": "cbbb86a30ce19b5684b7a05155dc7e6cbc7685b9", + "sha256": "a7ea19fa0ed99244e67b624c72b8580b7e9553043245905be58796a608eb9345"}, +} + + +def checkpoint_info(variant: str) -> dict: + info = _MODELS[variant] + filename = f"depth_anything_v2_{info['encoder']}.pth" + repo = f"depth-anything/Depth-Anything-V2-{variant.capitalize()}" + return {"url": f"https://huggingface.co/{repo}/resolve/{info['revision']}/{filename}", + "sha256": info["sha256"], "revision": info["revision"], "output_type": "relative_inverse_depth"} + + +def source_root() -> Path: + root = Path(os.environ.get("BENCH_DEPTH_ANYTHING_V2_SOURCE", str(RUNTIME_ROOT))).resolve() + digest = hashlib.sha256() + for path in sorted((root / "depth_anything_v2").rglob("*.py")): + digest.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes()) + if digest.hexdigest() != SOURCE_TREE_SHA256: + raise RuntimeError(f"missing or changed official Depth Anything V2 source {UPSTREAM_COMMIT}: {root}") + return root + + +def preprocess(image, input_spec: str = INPUT_SPEC): + import cv2 + import numpy as np + + if input_spec != INPUT_SPEC: + raise ValueError(f"Depth Anything V2 recipe is fixed at {INPUT_SPEC}, got {input_spec}") + rgb = np.asarray(image.convert("RGB"), dtype=np.float64) / 255.0 + rgb = cv2.resize(rgb, (518, 518), interpolation=cv2.INTER_CUBIC) + rgb = (rgb - np.array([.485, .456, .406])) / np.array([.229, .224, .225]) + return np.ascontiguousarray(rgb.transpose(2, 0, 1)[None], dtype=np.float32) + + +def build_variant(variant: str, weights: Path, input_spec: str): + import torch + + if input_spec != INPUT_SPEC: + raise ValueError(f"expected fixed {INPUT_SPEC}, got {input_spec}") + root = source_root() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + module = importlib.import_module("depth_anything_v2.dpt") + if Path(module.__file__).resolve() != root / "depth_anything_v2/dpt.py": + raise RuntimeError("a different Depth Anything V2 source is already imported") + digest = hashlib.sha256() + with Path(weights).open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + digest.update(block) + config = _MODELS[variant] + if digest.hexdigest() != config["sha256"]: + raise ValueError(f"wrong official relative Depth Anything V2 {variant} checkpoint") + model = module.DepthAnythingV2(**{key: config[key] for key in ("encoder", "features", "out_channels")}) + model.load_state_dict(torch.load(weights, map_location="cpu", weights_only=True), strict=True) + return model.eval() diff --git a/harness/adapters/depth_anything_v2_large.py b/harness/adapters/depth_anything_v2_large.py new file mode 100644 index 0000000..06e2887 --- /dev/null +++ b/harness/adapters/depth_anything_v2_large.py @@ -0,0 +1,10 @@ +"""Official Depth Anything V2 large: affine-invariant inverse depth.""" +from pathlib import Path +from .depth_anything_v2_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("large") + + +def build(weights: Path, input_spec: str): + return build_variant("large", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_metric_common.py b/harness/adapters/depth_anything_v2_metric_common.py new file mode 100644 index 0000000..91320b4 --- /dev/null +++ b/harness/adapters/depth_anything_v2_metric_common.py @@ -0,0 +1,120 @@ +"""Pinned official Depth Anything V2 metric-depth models (Hypersim / VKITTI). + +Same pinned upstream checkout as the relative V2 adapters; the metric head +lives in metric_depth/ of that tree. Sigmoid head times max_depth gives +nonnegative metric depth in metres. Static 518x518 deployment resize with the +official normalization; restore is the author's bilinear align_corners=True. +""" +from __future__ import annotations + +import hashlib +import importlib +import os +from pathlib import Path +import sys + +UPSTREAM_COMMIT = "a561b849ebae10a6f5ef49e26c83cbbcd36c71bf" +# sha256 over "\0" of every *.py under metric_depth/ of the pinned V2 checkout. +SOURCE_TREE_SHA256 = "5c2a223435082664012aa38ae11f3810738a4b6cba1efbb3c09a0405b8f6f17d" +RUNTIME_ROOT = Path.home() / "bench-cache/sources/depth-anything-v2" / UPSTREAM_COMMIT +INPUT_SPEC = "1x3x518x518" +MODEL_SPEC = {"source": "custom", "dataset": "nyuv2", "input_spec": INPUT_SPEC, "resize": 0} +OPSET = 17 +EXPORTER_VERSION = "depth-anything-v2-metric-v1-opset17-static518" +RESTORE_ALIGN_CORNERS = True +PREPROCESS = { + "id": "depth-anything-v2-metric-rgb-static518-v1", + "layout": "NCHW", "dtype": "float32", "color": "RGB", + "resize": "OpenCV INTER_CUBIC to 518x518; keep_aspect_ratio=False for static deployment", + "normalize": "RGB/255; mean=[0.485,0.456,0.406]; std=[0.229,0.224,0.225]", + "output": "N,H,W nonnegative metric depth in metres (Sigmoid head times max_depth)", + "restore": "bilinear to original H,W, align_corners=True; no min/max image normalization", + "source_commit": UPSTREAM_COMMIT, +} + +_ENCODERS = { + "small": {"encoder": "vits", "features": 64, "out_channels": [48, 96, 192, 384]}, + "base": {"encoder": "vitb", "features": 128, "out_channels": [96, 192, 384, 768]}, + "large": {"encoder": "vitl", "features": 256, "out_channels": [256, 512, 1024, 1024]}, +} +_DOMAINS = { + "hypersim": {"max_depth": 20.0, "dataset": "Hypersim (indoor)", "repo_tag": "Hypersim"}, + "vkitti": {"max_depth": 80.0, "dataset": "VKITTI (outdoor)", "repo_tag": "VKITTI"}, +} + +_MODELS = { + ("hypersim", "small"): ("3bc65d4e14a6786a61acec16453c50e12bf5f338", + "b782898d8a3e8be1f639de33837ed85e9b4b73e40f8f5e5cd99067588d722545"), + ("hypersim", "base"): ("4ebcd7fd0858cbb953451619ca943ea39ec08e00", + "9dc9e274c5eff55c6daf27b660c0ced0eca4e8593a6da90cdcb04d2b4d3f3fa2"), + ("hypersim", "large"): ("79720800638389a78b2defc92caa885104f69974", + "6f82ff2bc543ac02ddff4aa31fa363676a8305dd3ccf04e80e2af115a044cb6d"), + ("vkitti", "small"): ("c725b8589bdf6ab04072cab74c0467830db80d6d", + "9203e538d35255c90dda4b7fedb47ff33fe725497bcca3b1e53b3a65ee63f0cb"), + ("vkitti", "base"): ("e96fd0c8cedf1825a55b9c61309f754982948df0", + "4dad67a7cc10b462bca48e6b8569c762b8eb3c1adada170a3851a6d3ba37bb3e"), + ("vkitti", "large"): ("070e97e4b80e317ec1d03c19927304f4091a180b", + "239b1054a369e66da2576e9a118d6d7c12d90dc8ebe609579a9a09cd8e05fe38"), +} + + +def checkpoint_info(domain: str, variant: str) -> dict: + revision, sha256 = _MODELS[(domain, variant)] + encoder = _ENCODERS[variant]["encoder"] + filename = f"depth_anything_v2_metric_{domain}_{encoder}.pth" + tag = _DOMAINS[domain]["repo_tag"] + repo = f"depth-anything/Depth-Anything-V2-Metric-{tag}-{variant.capitalize()}" + return {"url": f"https://huggingface.co/{repo}/resolve/{revision}/{filename}", + "sha256": sha256, "revision": revision, "max_depth": _DOMAINS[domain]["max_depth"], + "trained_on": _DOMAINS[domain]["dataset"], "output_type": "metric_depth_m"} + + +def source_root() -> Path: + root = Path(os.environ.get("BENCH_DEPTH_ANYTHING_V2_METRIC_SOURCE", str(RUNTIME_ROOT))).resolve() + digest = hashlib.sha256() + files = sorted(path for path in (root / "metric_depth").rglob("*.py") + if "__pycache__" not in path.parts) + if not files: + raise RuntimeError(f"missing official Depth Anything V2 metric source {UPSTREAM_COMMIT}: {root}") + for path in files: + digest.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes()) + if digest.hexdigest() != SOURCE_TREE_SHA256: + raise RuntimeError(f"missing or changed official Depth Anything V2 metric source {UPSTREAM_COMMIT}: {root}") + return root + + +def preprocess(image, input_spec: str = INPUT_SPEC): + import cv2 + import numpy as np + + if input_spec != INPUT_SPEC: + raise ValueError(f"Depth Anything V2 metric recipe is fixed at {INPUT_SPEC}, got {input_spec}") + rgb = np.asarray(image.convert("RGB"), dtype=np.float64) / 255.0 + rgb = cv2.resize(rgb, (518, 518), interpolation=cv2.INTER_CUBIC) + rgb = (rgb - np.array([.485, .456, .406])) / np.array([.229, .224, .225]) + return np.ascontiguousarray(rgb.transpose(2, 0, 1)[None], dtype=np.float32) + + +def build_variant(domain: str, variant: str, weights: Path, input_spec: str): + import torch + + if input_spec != INPUT_SPEC: + raise ValueError(f"expected fixed {INPUT_SPEC}, got {input_spec}") + root = source_root() + config = _MODELS[(domain, variant)] + digest = hashlib.sha256() + with Path(weights).open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + digest.update(block) + if digest.hexdigest() != config[1]: + raise ValueError(f"wrong official metric Depth Anything V2 {domain}-{variant} checkpoint") + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + module = importlib.import_module("metric_depth.depth_anything_v2.dpt") + if Path(module.__file__).resolve() != root / "metric_depth/depth_anything_v2/dpt.py": + raise RuntimeError("a different Depth Anything V2 metric source is already imported") + model = module.DepthAnythingV2(**{key: value for key, value in _ENCODERS[variant].items() + if key in ("encoder", "features", "out_channels")}, + max_depth=_DOMAINS[domain]["max_depth"]) + model.load_state_dict(torch.load(weights, map_location="cpu", weights_only=True), strict=True) + return model.eval() diff --git a/harness/adapters/depth_anything_v2_metric_hypersim_base.py b/harness/adapters/depth_anything_v2_metric_hypersim_base.py new file mode 100644 index 0000000..ada5004 --- /dev/null +++ b/harness/adapters/depth_anything_v2_metric_hypersim_base.py @@ -0,0 +1,12 @@ +"""Official metric Depth Anything V2 Hypersim-base: metric depth, static 518.""" +from pathlib import Path + +from .depth_anything_v2_metric_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, + PREPROCESS, RESTORE_ALIGN_CORNERS, build_variant, + checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("hypersim", "base") + + +def build(weights: Path, input_spec: str): + return build_variant("hypersim", "base", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_metric_hypersim_large.py b/harness/adapters/depth_anything_v2_metric_hypersim_large.py new file mode 100644 index 0000000..c18dbcb --- /dev/null +++ b/harness/adapters/depth_anything_v2_metric_hypersim_large.py @@ -0,0 +1,12 @@ +"""Official metric Depth Anything V2 Hypersim-large: metric depth, static 518.""" +from pathlib import Path + +from .depth_anything_v2_metric_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, + PREPROCESS, RESTORE_ALIGN_CORNERS, build_variant, + checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("hypersim", "large") + + +def build(weights: Path, input_spec: str): + return build_variant("hypersim", "large", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_metric_hypersim_small.py b/harness/adapters/depth_anything_v2_metric_hypersim_small.py new file mode 100644 index 0000000..7e51c90 --- /dev/null +++ b/harness/adapters/depth_anything_v2_metric_hypersim_small.py @@ -0,0 +1,12 @@ +"""Official metric Depth Anything V2 Hypersim-small: metric depth, static 518.""" +from pathlib import Path + +from .depth_anything_v2_metric_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, + PREPROCESS, RESTORE_ALIGN_CORNERS, build_variant, + checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("hypersim", "small") + + +def build(weights: Path, input_spec: str): + return build_variant("hypersim", "small", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_metric_vkitti_base.py b/harness/adapters/depth_anything_v2_metric_vkitti_base.py new file mode 100644 index 0000000..118fa4f --- /dev/null +++ b/harness/adapters/depth_anything_v2_metric_vkitti_base.py @@ -0,0 +1,12 @@ +"""Official metric Depth Anything V2 VKITTI-base: metric depth, static 518.""" +from pathlib import Path + +from .depth_anything_v2_metric_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, + PREPROCESS, RESTORE_ALIGN_CORNERS, build_variant, + checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("vkitti", "base") + + +def build(weights: Path, input_spec: str): + return build_variant("vkitti", "base", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_metric_vkitti_large.py b/harness/adapters/depth_anything_v2_metric_vkitti_large.py new file mode 100644 index 0000000..720cb6b --- /dev/null +++ b/harness/adapters/depth_anything_v2_metric_vkitti_large.py @@ -0,0 +1,12 @@ +"""Official metric Depth Anything V2 VKITTI-large: metric depth, static 518.""" +from pathlib import Path + +from .depth_anything_v2_metric_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, + PREPROCESS, RESTORE_ALIGN_CORNERS, build_variant, + checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("vkitti", "large") + + +def build(weights: Path, input_spec: str): + return build_variant("vkitti", "large", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_metric_vkitti_small.py b/harness/adapters/depth_anything_v2_metric_vkitti_small.py new file mode 100644 index 0000000..dca46b6 --- /dev/null +++ b/harness/adapters/depth_anything_v2_metric_vkitti_small.py @@ -0,0 +1,12 @@ +"""Official metric Depth Anything V2 VKITTI-small: metric depth, static 518.""" +from pathlib import Path + +from .depth_anything_v2_metric_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, + PREPROCESS, RESTORE_ALIGN_CORNERS, build_variant, + checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("vkitti", "small") + + +def build(weights: Path, input_spec: str): + return build_variant("vkitti", "small", weights, input_spec) diff --git a/harness/adapters/depth_anything_v2_small.py b/harness/adapters/depth_anything_v2_small.py new file mode 100644 index 0000000..c2daa24 --- /dev/null +++ b/harness/adapters/depth_anything_v2_small.py @@ -0,0 +1,10 @@ +"""Official Depth Anything V2 small: affine-invariant inverse depth.""" +from pathlib import Path +from .depth_anything_v2_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("small") + + +def build(weights: Path, input_spec: str): + return build_variant("small", weights, input_spec) diff --git a/harness/adapters/detr_r50.py b/harness/adapters/detr_r50.py new file mode 100644 index 0000000..a555b68 --- /dev/null +++ b/harness/adapters/detr_r50.py @@ -0,0 +1,12 @@ +"""Exact MMDetection detr.r50; see mmdet_detr_common for static deployment geometry.""" +from functools import partial +from .mmdet_detr_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, OPSET, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "detr.r50" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-detr.r50-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/disk.py b/harness/adapters/disk.py new file mode 100644 index 0000000..7dea85d --- /dev/null +++ b/harness/adapters/disk.py @@ -0,0 +1,12 @@ +"""Original EPFL DISK depth checkpoint/Unet; official host NMS and descriptor lookup.""" +from . import feature_common as common +EXPORTER_VERSION = common.EXPORTER_VERSION +INPUT_SPEC = common.input_spec('disk') +MODEL_SPEC = {'input_spec': INPUT_SPEC, 'dataset': 'hpatches', 'input_override_reason': 'Official DISK expects RGB; CSV grayscale is incorrect.'} +CHECKPOINT = common.checkpoint_info('disk') +PREPROCESS = common.recipe('disk') +OPSET = 17 +preprocess = common.preprocess + +def build(weights, input_spec): + return common.build('disk', weights, input_spec) diff --git a/harness/adapters/dpt_hybrid.py b/harness/adapters/dpt_hybrid.py new file mode 100644 index 0000000..b13c5b6 --- /dev/null +++ b/harness/adapters/dpt_hybrid.py @@ -0,0 +1,19 @@ +"""Official MiDaS v3 DPT-Hybrid: relative inverse depth at static 384x384.""" +from pathlib import Path + +from .midas_common import (EXPORTER_VERSION, INPUT_SPECS, MODEL_SPECS, OPSET, PREPROCESSES, + RESTORE_ALIGN_CORNERS, RESTORE_INTERPOLATION, build_variant, + checkpoint_info, preprocess_for) + +CHECKPOINT = checkpoint_info("hybrid") +INPUT_SPEC = INPUT_SPECS["hybrid"] +MODEL_SPEC = MODEL_SPECS["hybrid"] +PREPROCESS = PREPROCESSES["hybrid"] + + +def build(weights: Path, input_spec: str): + return build_variant("hybrid", weights, input_spec) + + +def preprocess(image, input_spec: str = INPUT_SPEC): + return preprocess_for(image, input_spec, "hybrid") diff --git a/harness/adapters/dpt_large.py b/harness/adapters/dpt_large.py new file mode 100644 index 0000000..8890f17 --- /dev/null +++ b/harness/adapters/dpt_large.py @@ -0,0 +1,19 @@ +"""Official MiDaS v3 DPT-Large: relative inverse depth at static 384x384.""" +from pathlib import Path + +from .midas_common import (EXPORTER_VERSION, INPUT_SPECS, MODEL_SPECS, OPSET, PREPROCESSES, + RESTORE_ALIGN_CORNERS, RESTORE_INTERPOLATION, build_variant, + checkpoint_info, preprocess_for) + +CHECKPOINT = checkpoint_info("large") +INPUT_SPEC = INPUT_SPECS["large"] +MODEL_SPEC = MODEL_SPECS["large"] +PREPROCESS = PREPROCESSES["large"] + + +def build(weights: Path, input_spec: str): + return build_variant("large", weights, input_spec) + + +def preprocess(image, input_spec: str = INPUT_SPEC): + return preprocess_for(image, input_spec, "large") diff --git a/harness/adapters/efficientvit_b0.py b/harness/adapters/efficientvit_b0.py new file mode 100644 index 0000000..eea8aea --- /dev/null +++ b/harness/adapters/efficientvit_b0.py @@ -0,0 +1,10 @@ +"""CSV efficientvit.b0: MIT Han Lab ImageNet-1K B0, official r224 checkpoint.""" +from pathlib import Path + +from .efficientvit_common import PREPROCESS, build_variant, checkpoint_info, preprocess + +CHECKPOINT = checkpoint_info("b0") + + +def build(weights: Path, input_spec: str): + return build_variant("b0", weights, input_spec) diff --git a/harness/adapters/efficientvit_b1.py b/harness/adapters/efficientvit_b1.py new file mode 100644 index 0000000..6d79a5e --- /dev/null +++ b/harness/adapters/efficientvit_b1.py @@ -0,0 +1,10 @@ +"""CSV efficientvit.b1: MIT Han Lab ImageNet-1K B1, official r224 checkpoint.""" +from pathlib import Path + +from .efficientvit_common import PREPROCESS, build_variant, checkpoint_info, preprocess + +CHECKPOINT = checkpoint_info("b1") + + +def build(weights: Path, input_spec: str): + return build_variant("b1", weights, input_spec) diff --git a/harness/adapters/efficientvit_b2.py b/harness/adapters/efficientvit_b2.py new file mode 100644 index 0000000..800aeb6 --- /dev/null +++ b/harness/adapters/efficientvit_b2.py @@ -0,0 +1,10 @@ +"""CSV efficientvit.b2: MIT Han Lab ImageNet-1K B2, official r224 checkpoint.""" +from pathlib import Path + +from .efficientvit_common import PREPROCESS, build_variant, checkpoint_info, preprocess + +CHECKPOINT = checkpoint_info("b2") + + +def build(weights: Path, input_spec: str): + return build_variant("b2", weights, input_spec) diff --git a/harness/adapters/efficientvit_b3.py b/harness/adapters/efficientvit_b3.py new file mode 100644 index 0000000..4597b03 --- /dev/null +++ b/harness/adapters/efficientvit_b3.py @@ -0,0 +1,10 @@ +"""CSV efficientvit.b3: MIT Han Lab ImageNet-1K B3, official r224 checkpoint.""" +from pathlib import Path + +from .efficientvit_common import PREPROCESS, build_variant, checkpoint_info, preprocess + +CHECKPOINT = checkpoint_info("b3") + + +def build(weights: Path, input_spec: str): + return build_variant("b3", weights, input_spec) diff --git a/harness/adapters/efficientvit_common.py b/harness/adapters/efficientvit_common.py new file mode 100644 index 0000000..d746cda --- /dev/null +++ b/harness/adapters/efficientvit_common.py @@ -0,0 +1,163 @@ +"""MIT Han Lab EfficientViT ImageNet classifiers, using the official factory. + +No downloads, package installation, key remapping, or architecture fallback occur +here. The pinned official package and its dependencies live in a private cache; +the shared venv is unchanged. Missing files remain errors, not readiness evidence. +""" +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +import sys +from pathlib import Path + +UPSTREAM_REPOSITORY = "https://github.com/mit-han-lab/efficientvit" +# The original classification release avoids later SAM/DC_AE eager imports. +# Its unmodified factory strictly accepts all four official r224 checkpoints. +UPSTREAM_COMMIT = "b3ed56d97ac8ff0cc6add7ad2533c42a7488d736" +RUNTIME_ROOT = Path.home() / "bench-cache/sources/efficientvit-cls" +CHECKPOINT_REPOSITORY = "han-cai/efficientvit-cls" +CHECKPOINT_REVISION = "df3d006c2567f9e322b03731f20fe4405a1ab090" +INPUT_SPEC = "1x3x224x224" + +# Hashes are the HF LFS object IDs, also verified against downloaded bytes. +_CHECKPOINTS = { + "b0": (13741687, "feebc4d3ebf610673c8ed1d92b21fa2c2e57661868173a37126b734ca5dc3771"), + "b1": (36575137, "bf8798aa03ed2bba21fc8d58f43ca9a92096f3f03bd700e1df1dcde11559f499"), + "b2": (97581671, "6ccfecc65460856cb6415f2356deb8b90e0216960f661f5055aef173f8ca421d"), + "b3": (194992573, "3c5be1a11a8a41b3ac4e30e5bff8bf99065c98b4b2d586e1803fd7571f76fb0d"), +} + +# Check the actual architecture, factory and loader, not just a package name. +# This pins core source identity; it is not a full environment attestation. +_SOURCE_SHA256 = { + "__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "cls_model_zoo.py": "c5e21fa447add578968ce6ed179c6e877aa3414b46f1d21af45e947610030599", + "models/efficientvit/__init__.py": "6ec6831645b992d01f479174f55df2c026a05adf099a9eebb45b9690c84b8051", + "models/efficientvit/cls.py": "41534fcd771aa42d9db2cb95d7580dfdf5ac5d54b57b2df8ac2b35dd4be7414e", + "models/efficientvit/backbone.py": "f30d4460b3c2000421fc3421d1cbb3d501233e13fa4f80f9df7bbc7ed5664863", + "models/nn/ops.py": "a5c177a1d3f9d1d8c85647c57d01295506f84ba53b763b0ba77095be34e174e0", + "models/nn/norm.py": "45d34b3b5002c8d76557ed3017c02ccfc31723f2d84efd2f0cd7074706132cf5", + "models/nn/act.py": "7b71a6626dd7ebd7bf0cba283f3f3f5ec3da6ebec0d7c04b2fbafa1969abd7f8", + "models/utils/network.py": "db63b048a48c1edd47e6f401d0cc7d96dfc388b7e37f568acdef5c7a83acc5c4", +} + +PREPROCESS = { + "input_spec": INPUT_SPEC, + "color": "RGB", + "crop_ratio": 0.95, + "resize_shorter": 236, # ceil(224 / 0.95) + "interpolation": "bicubic", + "center_crop": 224, + "scale": 1.0 / 255.0, + "mean": (0.485, 0.456, 0.406), + "std": (0.229, 0.224, 0.225), + "output_layout": "NCHW", +} + + +def checkpoint_info(variant: str) -> dict: + """Return an immutable-revision HTTPS URI suitable for existing fetch_http.""" + if variant not in _CHECKPOINTS: + raise ValueError(f"Unsupported MIT EfficientViT classification variant: {variant!r}") + size, sha256 = _CHECKPOINTS[variant] + filename = f"efficientvit_{variant}_r224.pt" + return { + "model_id": f"efficientvit.{variant}", + "factory_name": f"{variant}-r224", + "filename": filename, + "url": f"https://huggingface.co/{CHECKPOINT_REPOSITORY}/resolve/{CHECKPOINT_REVISION}/{filename}", + "size": size, + "sha256": sha256, + } + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_checkpoint(variant: str, weights: Path) -> dict: + """Reject HTML, wrong resolutions/variants, and converted or random weights.""" + info = checkpoint_info(variant) + weights = Path(weights) + if not weights.is_file(): + raise FileNotFoundError(f"Official checkpoint required: {weights}; source: {info['url']}") + if weights.stat().st_size != info["size"] or _sha256(weights) != info["sha256"]: + raise ValueError( + f"Checkpoint identity mismatch for {info['model_id']}: {weights}; " + f"expected {info['size']} bytes, SHA256 {info['sha256']} from {info['url']}" + ) + return info + + +def _check_input_spec(input_spec: str) -> None: + if input_spec != INPUT_SPEC: + raise ValueError(f"This adapter pins the official r224 recipe: expected {INPUT_SPEC}, got {input_spec!r}") + + +def _official_factory(): + source = RUNTIME_ROOT / f"efficientvit-{UPSTREAM_COMMIT}" + root = source / "efficientvit" + for relative, expected in _SOURCE_SHA256.items(): + path = root / relative + if not path.is_file() or _sha256(path) != expected: + raise RuntimeError( + f"Unverified EfficientViT source: {path}; expected core source from " + f"{UPSTREAM_REPOSITORY}/tree/{UPSTREAM_COMMIT} (SHA256 {expected})." + ) + # Only this process sees the private dependencies. Restore the import path + # after upstream's imports; never change PYTHONPATH or the shared venv. + original_path = sys.path[:] + try: + sys.path[:0] = [str(source), str(RUNTIME_ROOT / "deps")] + spec = importlib.util.find_spec("efficientvit") + if spec is None or spec.origin is None or Path(spec.origin).resolve() != (root / "__init__.py").resolve(): + raise RuntimeError("A different EfficientViT package is already loaded; use a fresh worker.") + module = importlib.import_module("efficientvit.cls_model_zoo") + return module.create_cls_model + except ImportError as exc: + raise RuntimeError( + f"Official EfficientViT import failed ({exc}). Prepare pinned private dependencies " + f"under {RUNTIME_ROOT / 'deps'}. No fallback is used." + ) from exc + finally: + sys.path[:] = original_path + + +def build_variant(variant: str, weights: Path, input_spec: str): + """Load a hash-verified checkpoint via upstream's strict state_dict loader. + + Upstream keeps norm epsilon at 1e-5 for the B series and reads the nested + state_dict with torch.load(map_location='cpu'), then load_state_dict(strict=True). + """ + _check_input_spec(input_spec) + info = verify_checkpoint(variant, weights) + factory = _official_factory() + model = factory( + name=info["factory_name"], pretrained=True, weight_url=str(Path(weights).resolve()) + ) + return model.eval() + + +def preprocess(image, input_spec: str = INPUT_SPEC): + """Official PIL/torchvision evaluation transform, returned as float32 NCHW. + + Shared by baseline, calibration, G2 and quantized evaluation through the + harness adapter hook; it is not an in-model normalization wrapper. + """ + _check_input_spec(input_spec) + from torchvision import transforms + from torchvision.transforms.functional import InterpolationMode + + transform = transforms.Compose([ + transforms.Resize(236, interpolation=InterpolationMode.BICUBIC), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean=PREPROCESS["mean"], std=PREPROCESS["std"]), + ]) + return transform(image.convert("RGB")).unsqueeze(0).numpy() diff --git a/harness/adapters/efficientvit_seg_b0.py b/harness/adapters/efficientvit_seg_b0.py new file mode 100644 index 0000000..5fa76c8 --- /dev/null +++ b/harness/adapters/efficientvit_seg_b0.py @@ -0,0 +1,14 @@ +"""Official MIT EfficientViT segmentation B0; readiness requires a real probe.""" +from pathlib import Path + +from .efficientvit_seg_common import ( + build_variant, checkpoint_info, model_spec, preprocess, preprocess_recipe, +) + +CHECKPOINT = checkpoint_info("b0") +MODEL_SPEC = model_spec("b0") +PREPROCESS = preprocess_recipe("b0") + + +def build(weights: Path, input_spec: str): + return build_variant("b0", weights, input_spec) diff --git a/harness/adapters/efficientvit_seg_b1.py b/harness/adapters/efficientvit_seg_b1.py new file mode 100644 index 0000000..14d2572 --- /dev/null +++ b/harness/adapters/efficientvit_seg_b1.py @@ -0,0 +1,14 @@ +"""Official MIT EfficientViT segmentation B1; readiness requires a real probe.""" +from pathlib import Path + +from .efficientvit_seg_common import ( + build_variant, checkpoint_info, model_spec, preprocess, preprocess_recipe, +) + +CHECKPOINT = checkpoint_info("b1") +MODEL_SPEC = model_spec("b1") +PREPROCESS = preprocess_recipe("b1") + + +def build(weights: Path, input_spec: str): + return build_variant("b1", weights, input_spec) diff --git a/harness/adapters/efficientvit_seg_b2.py b/harness/adapters/efficientvit_seg_b2.py new file mode 100644 index 0000000..799d12b --- /dev/null +++ b/harness/adapters/efficientvit_seg_b2.py @@ -0,0 +1,14 @@ +"""Official MIT EfficientViT segmentation B2; readiness requires a real probe.""" +from pathlib import Path + +from .efficientvit_seg_common import ( + build_variant, checkpoint_info, model_spec, preprocess, preprocess_recipe, +) + +CHECKPOINT = checkpoint_info("b2") +MODEL_SPEC = model_spec("b2") +PREPROCESS = preprocess_recipe("b2") + + +def build(weights: Path, input_spec: str): + return build_variant("b2", weights, input_spec) diff --git a/harness/adapters/efficientvit_seg_b3.py b/harness/adapters/efficientvit_seg_b3.py new file mode 100644 index 0000000..cab7547 --- /dev/null +++ b/harness/adapters/efficientvit_seg_b3.py @@ -0,0 +1,14 @@ +"""Official MIT EfficientViT segmentation B3; readiness requires a real probe.""" +from pathlib import Path + +from .efficientvit_seg_common import ( + build_variant, checkpoint_info, model_spec, preprocess, preprocess_recipe, +) + +CHECKPOINT = checkpoint_info("b3") +MODEL_SPEC = model_spec("b3") +PREPROCESS = preprocess_recipe("b3") + + +def build(weights: Path, input_spec: str): + return build_variant("b3", weights, input_spec) diff --git a/harness/adapters/efficientvit_seg_common.py b/harness/adapters/efficientvit_seg_common.py new file mode 100644 index 0000000..9c0fca7 --- /dev/null +++ b/harness/adapters/efficientvit_seg_common.py @@ -0,0 +1,94 @@ +"""Official MIT EfficientViT segmentation heads and fixed board input recipe. + +Reuse the unmodified 2023 package/private deps prepared for classification. +Weights are immutable-revision official releases, never classification weights. +The bench keeps its static 512x1024 input for both datasets; this is not the +published full-resolution Cityscapes / variable-aspect ADE20K evaluation. +""" +from __future__ import annotations + +import importlib +from pathlib import Path + +from .efficientvit_common import RUNTIME_ROOT, UPSTREAM_COMMIT, _official_factory, _sha256 + +SOURCE_COMMIT = "b3ed56d97ac8ff0cc6add7ad2533c42a7488d736" +CHECKPOINT_REPOSITORY = "han-cai/efficientvit-seg" +CHECKPOINT_REVISION = "cf3ccaf9cbaf670a2cb283612773546824ca0aa1" +INPUT_SPEC = "1x3x512x1024" +_SOURCE_SHA256 = { + "seg_model_zoo.py": "8cd5e99445dbf8c1270c37f93f6f4eca0ac3fefe66f263f325e0e23d50222922", + "models/efficientvit/seg.py": "a279c103c57f467de1eb93c7db3c170872a48faaf618aff58ada1af6bd5fd8bb", +} +_CHECKPOINTS = { + "b0": ("cityscapes", 19, 2924013, "923d6fdd5e93640cc0c2f3f213764f34e80b477cd98a6b294d870ea6df5acc50"), + "b1": ("ade20k", 150, 19339726, "59cca6a7ff4d1a6494b7f12072c6a68fee40ef401e971f26af01a2e0f3c8f5e7"), + "b2": ("ade20k", 150, 61436227, "fabfa7163e35d14888690969cd852069f399137be39697bfe5165f03ea76b510"), + "b3": ("ade20k", 150, 158594800, "fd2706e63ae9d425eb022bce26e10cebb56edd347644003f32f3ac0c9c3855a2"), +} + + +def checkpoint_info(variant: str) -> dict: + dataset, classes, size, sha256 = _CHECKPOINTS[variant] + filename = f"efficientvit_seg_{variant}_{dataset}.pt" + return {"model_id": f"efficientvit.seg.{variant}", "factory_name": variant, + "dataset": dataset, "classes": classes, "filename": filename, + "url": f"https://huggingface.co/{CHECKPOINT_REPOSITORY}/resolve/{CHECKPOINT_REVISION}/{filename}", + "size": size, "sha256": sha256, "source_commit": SOURCE_COMMIT} + + +def model_spec(variant: str) -> dict: + info = checkpoint_info(variant) + result = {"dataset": info["dataset"], "input_spec": INPUT_SPEC} + if variant == "b0": + result["dataset_override_reason"] = ( + "CSV says ADE20K; official B0 segmentation factory/weights only support Cityscapes (19 classes)." + ) + return result + + +def preprocess_recipe(variant: str) -> dict: + return {"input_spec": INPUT_SPEC, "dataset": checkpoint_info(variant)["dataset"], + "color": "RGB", "resize": "fixed_shape", "interpolation": "opencv_inter_cubic", + "scale": 1.0 / 255.0, "mean": [0.485, 0.456, 0.406], "std": [0.229, 0.224, 0.225], + "output_layout": "NCHW", "logits_resize": "bicubic", "align_corners": False, + "protocol": "bench-static-resize-v1; logits resized to original label size before argmax"} + + +def input_shape(input_spec: str) -> tuple[int, ...]: + if input_spec != INPUT_SPEC: + raise ValueError(f"EfficientViT segmentation recipe requires {INPUT_SPEC}, got {input_spec!r}") + return tuple(int(n) for n in input_spec.split("x")) + + +def preprocess(image, input_spec: str = INPUT_SPEC): + """Official cv2 cubic + float32 ImageNet normalization at the bench shape.""" + import cv2 + import numpy as np + + _, _, h, w = input_shape(input_spec) + rgb = np.asarray(image.convert("RGB")) + if rgb.shape[:2] != (h, w): + rgb = cv2.resize(rgb, (w, h), interpolation=cv2.INTER_CUBIC) + x = rgb.astype(np.float32) / np.float32(255.0) + x = (x - np.asarray((0.485, 0.456, 0.406), np.float32)) / np.asarray((0.229, 0.224, 0.225), np.float32) + return np.ascontiguousarray(x.transpose(2, 0, 1)[None]) + + +def build_variant(variant: str, weights: Path, input_spec: str): + input_shape(input_spec) + info, weights = checkpoint_info(variant), Path(weights) + if not weights.is_file() or weights.stat().st_size != info["size"] or _sha256(weights) != info["sha256"]: + raise ValueError(f"Official segmentation checkpoint identity mismatch: {weights}; expected {info}") + if UPSTREAM_COMMIT != SOURCE_COMMIT: + raise RuntimeError("EfficientViT segmentation requires the pinned 2023 shared source") + root = RUNTIME_ROOT / f"efficientvit-{SOURCE_COMMIT}" / "efficientvit" + # Check the segmentation code before the shared factory's eager imports. + for relative, expected in _SOURCE_SHA256.items(): + path = root / relative + if not path.is_file() or _sha256(path) != expected: + raise RuntimeError(f"Unverified official segmentation source: {path}") + _official_factory() # Verify shared core; load private dependencies; restore sys.path. + factory = importlib.import_module("efficientvit.seg_model_zoo").create_seg_model + return factory(name=variant, dataset=info["dataset"], pretrained=True, + weight_url=str(weights.resolve())).eval() diff --git a/harness/adapters/fast_scnn.py b/harness/adapters/fast_scnn.py new file mode 100644 index 0000000..291b67d --- /dev/null +++ b/harness/adapters/fast_scnn.py @@ -0,0 +1,13 @@ +"""Official Cityscapes Fast-SCNN with exact static pooling during export.""" +from functools import partial +from .mmseg_common import (metadata, preprocess_image, build_model, + lower_static_pools, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("fast.scnn") +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = build_model("fast.scnn", weights, input_spec) + lower_static_pools(network.upstream.backbone) + return network diff --git a/harness/adapters/fcn_resnet101.py b/harness/adapters/fcn_resnet101.py new file mode 100644 index 0000000..a5a481a --- /dev/null +++ b/harness/adapters/fcn_resnet101.py @@ -0,0 +1,7 @@ +"""Exact official ADE20K FCN ResNet101; admission requires actual validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("fcn.resnet101") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "fcn.resnet101") diff --git a/harness/adapters/fcn_resnet50.py b/harness/adapters/fcn_resnet50.py new file mode 100644 index 0000000..eceaf2a --- /dev/null +++ b/harness/adapters/fcn_resnet50.py @@ -0,0 +1,7 @@ +"""Exact official ADE20K FCN ResNet50; admission requires actual validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("fcn.resnet50") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "fcn.resnet50") diff --git a/harness/adapters/fcos_r50_fpn.py b/harness/adapters/fcos_r50_fpn.py new file mode 100644 index 0000000..8c3bde2 --- /dev/null +++ b/harness/adapters/fcos_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection fcos.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "fcos.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-fcos.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/feature_common.py b/harness/adapters/feature_common.py new file mode 100644 index 0000000..e196996 --- /dev/null +++ b/harness/adapters/feature_common.py @@ -0,0 +1,215 @@ +"""Pinned, unmodified feature sources. Static learned graphs; explicit host decoding.""" +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +import json +import os +from functools import lru_cache +from pathlib import Path +import sys + +ROOT = Path(os.environ.get('BENCH_FEATURE_SOURCES', '/home/ming01.wang/bench-cache/sources/feature')) +MAX_KEYPOINTS = 1024 +EXPORTER_VERSION = 'feature-static-1' +_CHECKPOINTS = { + 'superpoint': ('superglue/models/weights/superpoint_v1.pth', '52b6708629640ca883673b5d5c097c4ddad37d8048b33f09c8ca0d69db12c40e', + 'https://raw.githubusercontent.com/magicleap/SuperGluePretrainedNetwork/ddcf11f42e7e0732a0c4607648f9448ea8d73590/models/weights/superpoint_v1.pth'), + 'disk': ('disk/depth-save.pth', '9c2ee4ded238892dfa51569941372601e35e4a74aa6f84ea80053d2ab1c07abe', + 'https://raw.githubusercontent.com/cvlab-epfl/disk/8dc6d4d65b47210f629e21ad1fbfb56944b11696/depth-save.pth'), + 'aliked': ('aliked/models/aliked-n16.pth', '5be8704840ed662d9d8c561bf7279c222092674e7eb05fd0feab94899e9d82f2', + 'https://raw.githubusercontent.com/Shiaoming/ALIKED/683d7c65197395c0b3f01ebe76e1084a27e73a65/models/aliked-n16.pth'), +} + + +def checkpoint_info(model): + rel, digest, url = _CHECKPOINTS[model] + return {'url': url, 'sha256': digest, 'filename': Path(rel).name, 'official_relative_path': rel} + + +def input_spec(model): + if model in ('xfeat', 'd2.net', 'r2d2', 'dedode.v2', 'keynetplushardnet'): + from . import get + return get(model).INPUT_SPEC + return '1x1x480x640' if model == 'superpoint' else '1x3x480x640' + + +def recipe(model): + return {'input_spec': input_spec(model), 'color': 'GRAY' if model == 'superpoint' else 'RGB', + 'resize': 'opencv_linear_640x480', 'scale': 1 / 255, 'output_layout': 'NCHW', + 'protocol': 'feature-static-640x480-v1', 'gray': 'opencv_RGB2GRAY_uint8'} + + +def preprocess(image, spec, model=None): + if model in ('xfeat', 'd2.net', 'r2d2', 'dedode.v2', 'keynetplushardnet'): + from . import get + return get(model).preprocess(image, spec) + import cv2 + import numpy as np + if spec not in ('1x1x480x640', '1x3x480x640'): raise ValueError(spec) + x = np.asarray(image.convert('RGB')) + if spec.startswith('1x1x'): x = cv2.cvtColor(x, cv2.COLOR_RGB2GRAY) + x = cv2.resize(x, (640, 480), interpolation=cv2.INTER_LINEAR).astype(np.float32) / 255 + return np.ascontiguousarray(x[None, None] if x.ndim == 2 else x.transpose(2, 0, 1)[None]) + + +@lru_cache(None) +def official_module(model): + lock = json.loads((ROOT / 'source-lock.json').read_text()) + prefix = {'superpoint': 'superglue/', 'disk': 'disk/', 'aliked': 'aliked/'}[model] + for rel, expected in lock['files'].items(): + if rel.startswith(prefix) and hashlib.sha256((ROOT / rel).read_bytes()).hexdigest() != expected: + raise ValueError(f'upstream source modified: {rel}') + if model == 'superpoint': + spec = importlib.util.spec_from_file_location('_bench_official_superpoint', ROOT / 'superglue/models/superpoint.py') + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + roots = [ROOT / model] + if model == 'disk': roots += [ROOT / 'disk/submodules' / n for n in ('unets', 'torch-dimcheck', 'torch-localize')] + # Optional pure-Python dependencies live here, never in the shared venv. + roots.append(ROOT / 'deps') + old = sys.path[:] + try: + sys.path[:0] = [str(p) for p in roots] + name = 'disk' if model == 'disk' else 'nets.aliked' + mod = importlib.import_module(name) + if not Path(mod.__file__).resolve().is_relative_to((ROOT / model).resolve()): + raise RuntimeError(f'upstream module namespace collision: {mod.__file__}') + return mod + finally: + sys.path[:] = old + + +def official_model(model, weights): + import torch + weights = Path(weights) + if hashlib.sha256(weights.read_bytes()).hexdigest() != checkpoint_info(model)['sha256']: + raise ValueError('official checkpoint SHA mismatch') + module = official_module(model) + state = torch.load(weights, map_location='cpu', weights_only=False) + if model == 'superpoint': + m = module.SuperPoint({'max_keypoints': MAX_KEYPOINTS}) + elif model == 'disk': + m = module.DISK(); state = state['extractor'] + else: + m = module.ALIKED(model_name='aliked-n16', device='cpu', top_k=MAX_KEYPOINTS, load_pretrained=False) + m.load_state_dict(state, strict=True) + return m.eval() + + +def official_features(model, m, x): + """Reference forward without ALIKED's CUDA timing-only calls on CPU.""" + if model == 'superpoint': + d = m({'image': x}) + return d['keypoints'][0], d['scores'][0], d['descriptors'][0].T + if model == 'disk': + d = m.features(x, kind='nms', n=MAX_KEYPOINTS, window_size=5, cutoff=0.)[0] + return d.kp, d.kp_logp, d.desc + feature, score = m.extract_dense_map(x) + keypoints, scores, _ = m.dkd(score) + descriptors, _ = m.desc_head(feature, keypoints) + wh = x.new_tensor([x.shape[-1] - 1, x.shape[-2] - 1]) + return (keypoints[0] + 1) * wh / 2, scores[0], descriptors[0] + + +def replace_deformable(m): + """Exact torchvision DCNv1 sampling expressed as standard ONNX GridSample+Conv. + + Only ALIKED's ungrouped, unmodulated 3x3 stride-1 operator is supported. + This changes the wrapper instance, never the downloaded source or reference. + """ + import torch + from torch import nn + from torch.nn import functional as F + + class StaticDeform(nn.Module): + def __init__(self, source): + super().__init__() + assert not source.mask and source.regular_conv.kernel_size == (3, 3) + assert source.regular_conv.stride == (1, 1) and source.regular_conv.groups == 1 + self.offset_conv = source.offset_conv; self.regular_conv = source.regular_conv + + def forward(self, x): + # The adapter contract is static NCHW; freeze these dimensions so + # ONNX knows the 1x1 convolution kernel shape after sample stacking. + b, c, h, w = (int(v) for v in x.shape) + offset = self.offset_conv(x).clamp(-max(h, w) / 4., max(h, w) / 4.) + yy, xx = torch.meshgrid(torch.arange(h, device=x.device, dtype=x.dtype), + torch.arange(w, device=x.device, dtype=x.dtype), indexing='ij') + samples = [] + for k in range(9): + y = yy + (k // 3 - 1) + offset[:, 2 * k] + z = xx + (k % 3 - 1) + offset[:, 2 * k + 1] + grid = torch.stack((2 * z / (w - 1) - 1, 2 * y / (h - 1) - 1), -1) + samples.append(F.grid_sample(x, grid, align_corners=True, padding_mode='zeros')) + sampled = torch.stack(samples, 2).reshape(b, c * 9, h, w) + weights = self.regular_conv.weight.flatten(1).unsqueeze(-1).unsqueeze(-1) + return F.conv2d(sampled, weights, self.regular_conv.bias) + + for name, child in list(m.named_children()): + if child.__class__.__name__ == 'DeformableConv2d': setattr(m, name, StaticDeform(child)) + else: replace_deformable(child) + + +def build(model, weights, spec): + import torch + from torch import nn + from torch.nn import functional as F + if spec != input_spec(model): raise ValueError(f'{model} requires {input_spec(model)}, got {spec}') + original = official_model(model, weights) + + class Graph(nn.Module): + def __init__(self, m): + super().__init__(); self.m = m + + def forward(self, x): + m = self.m + if model == 'disk': return m.unet(x) + if model == 'aliked': + keypoints, scores, descriptors = official_features(model, m, x) + return (keypoints.reshape(1, MAX_KEYPOINTS, 2), + scores.reshape(1, MAX_KEYPOINTS), descriptors.reshape(1, MAX_KEYPOINTS, 128)) + z = m.relu(m.conv1a(x)); z = m.relu(m.conv1b(z)); z = m.pool(z) + z = m.relu(m.conv2a(z)); z = m.relu(m.conv2b(z)); z = m.pool(z) + z = m.relu(m.conv3a(z)); z = m.relu(m.conv3b(z)); z = m.pool(z) + z = m.relu(m.conv4a(z)); z = m.relu(m.conv4b(z)) + scores = F.softmax(m.convPb(m.relu(m.convPa(z))), 1)[:, :-1] + b, _, h, w = scores.shape + scores = scores.permute(0, 2, 3, 1).reshape(b, h, w, 8, 8) + scores = scores.permute(0, 1, 3, 2, 4).reshape(b, h * 8, w * 8) + desc = F.normalize(m.convDb(m.relu(m.convDa(z))), p=2, dim=1) + return scores, desc + + if model == 'aliked': replace_deformable(original) + return Graph(original).eval() + + +def decode(model, outputs): + """The same pinned official postprocessing for eager, ORT and board outputs.""" + if model in ('xfeat', 'd2.net', 'r2d2', 'dedode.v2', 'keynetplushardnet'): + from . import get + return get(model).decode(outputs) + import numpy as np + import torch + module = official_module(model) + tensors = [torch.as_tensor(np.asarray(o, dtype=np.float32)) for o in outputs] + if any(not torch.isfinite(t).all() for t in tensors): raise ValueError('nonfinite feature outputs') + if model == 'superpoint': + scores, descriptors = tensors + assert scores.shape == (1, 480, 640) and descriptors.shape == (1, 256, 60, 80) + scores = module.simple_nms(scores, 4)[0] + k = torch.nonzero(scores > .005); s = scores[tuple(k.T)] + k, s = module.remove_borders(k, s, 4, 480, 640) + k, s = module.top_k_keypoints(k, s, MAX_KEYPOINTS) + k = torch.flip(k, [1]).float() + d = module.sample_descriptors(k[None], descriptors, 8)[0].T + elif model == 'disk': + assert tensors[0].shape == (1, 129, 480, 640) + from disk.model.detector import Detector + kp = Detector().nms(tensors[0][:, 128:], n=MAX_KEYPOINTS, window_size=5, cutoff=0.)[0] + f = kp.merge_with_descriptors(tensors[0][0, :128]); k, s, d = f.kp, f.kp_logp, f.desc + else: + k, s, d = [t[0] for t in tensors] + assert k.shape == (MAX_KEYPOINTS, 2) and d.shape == (MAX_KEYPOINTS, 128) + return {'keypoints': k.numpy(), 'scores': s.numpy(), 'descriptors': d.numpy()} diff --git a/harness/adapters/feature_sources.py b/harness/adapters/feature_sources.py new file mode 100644 index 0000000..53d9a38 --- /dev/null +++ b/harness/adapters/feature_sources.py @@ -0,0 +1,79 @@ +"""Verified official sources for the second feature batch; no shared recipe changes.""" +import ast +import hashlib +import importlib.util +import json +import os +from functools import lru_cache +from pathlib import Path +import sys + +ROOT = Path(os.environ.get('BENCH_FEATURE_BATCH2_SOURCES', '/home/ming01.wang/bench-cache/sources/feature-batch2')) + + +@lru_cache(None) +def verify_sources(family): + manifest = json.loads((ROOT/'source-lock.json').read_text()) + for rel, digest in manifest['files'].items(): + if rel.startswith(family+'/') and hashlib.sha256((ROOT/rel).read_bytes()).hexdigest() != digest: + raise ValueError(f'official source mismatch: {rel}') + + +def verify_weights(path, expected): + if hashlib.sha256(Path(path).read_bytes()).hexdigest() != expected: + raise ValueError(f'checkpoint identity mismatch: {path}') + + +@lru_cache(None) +def module(relative): + verify_sources(relative.split('/')[0]) + name = '_bench_feature2_' + relative.replace('/', '_').replace('.', '_') + spec = importlib.util.spec_from_file_location(name, ROOT/relative) + value = importlib.util.module_from_spec(spec) + before = sys.path[:] + try: + sys.path.insert(0, str(ROOT/relative.split('/')[0])) + spec.loader.exec_module(value) + finally: + sys.path[:] = before + return value + + +def definitions(relative, names, namespace=None): + """Import exact upstream definitions without unrelated training/UI imports. + + Original function/class/assignment AST nodes are compiled unchanged. This + does not reimplement their numerical logic or edit the upstream files. + """ + import numpy as np + import torch + import torch.nn as nn + import torch.nn.functional as F + verify_sources(relative.split('/')[0]) + body = [] + for node in ast.parse((ROOT/relative).read_text()).body: + if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in names: + body.append(node) + elif isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id in names for t in node.targets): + body.append(node) + assert len(body) == len(names), (relative, names) + scope = {'np': np, 'torch': torch, 'nn': nn, 'F': F, **(namespace or {})} + exec(compile(ast.Module(body=body, type_ignores=[]), str(ROOT/relative), 'exec'), scope) + return scope + + +def tensors(outputs): + import numpy as np + import torch + result = [torch.as_tensor(np.asarray(v, np.float32)) for v in outputs] + if any(not torch.isfinite(v).all() for v in result): raise ValueError('nonfinite feature maps') + return result + + +def top_features(keypoints, scores, descriptors): + import numpy as np + from .feature_common import MAX_KEYPOINTS + indices = np.asarray(scores).argsort()[-MAX_KEYPOINTS:] + return {'keypoints': np.asarray(keypoints, np.float32)[indices, :2], + 'scores': np.asarray(scores, np.float32)[indices], + 'descriptors': np.asarray(descriptors, np.float32)[indices]} diff --git a/harness/adapters/feature_sources3.py b/harness/adapters/feature_sources3.py new file mode 100644 index 0000000..7e32b27 --- /dev/null +++ b/harness/adapters/feature_sources3.py @@ -0,0 +1,39 @@ +"""Private batch3 official source loader; never used from production during probes.""" +import hashlib +import importlib +import importlib.util +import json +from functools import lru_cache +from pathlib import Path +import sys + +ROOT=Path('/home/ming01.wang/bench-cache/sources/feature-batch3') + +@lru_cache(None) +def verify(family): + record=next(r for r in json.loads((ROOT/'source-lock.json').read_text()) if r['name']==family) + for relative,digest in record['files'].items(): + if hashlib.sha256((ROOT/family/relative).read_bytes()).hexdigest()!=digest: + raise ValueError(f'official source changed: {family}/{relative}') + +def check(path,digest): + if hashlib.sha256(Path(path).read_bytes()).hexdigest()!=digest:raise ValueError(f'weight mismatch: {path}') + +@lru_cache(None) +def package(family,name): + verify(family) + before=sys.path[:] + try: + sys.path[:0]=[str(ROOT/family),str(ROOT/'deps')] + module=importlib.import_module(name) + if not Path(module.__file__).resolve().is_relative_to((ROOT/family).resolve()): + raise RuntimeError(f'official namespace collision: {module.__file__}') + return module + finally:sys.path[:]=before + +@lru_cache(None) +def module(family,relative): + verify(family) + spec=importlib.util.spec_from_file_location('_official3_'+family+'_'+relative.replace('/','_'),ROOT/family/relative) + result=importlib.util.module_from_spec(spec);spec.loader.exec_module(result) + return result diff --git a/harness/adapters/foveabox_r50_fpn.py b/harness/adapters/foveabox_r50_fpn.py new file mode 100644 index 0000000..e157ca6 --- /dev/null +++ b/harness/adapters/foveabox_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection foveabox.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "foveabox.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-foveabox.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/fsaf_r50_fpn.py b/harness/adapters/fsaf_r50_fpn.py new file mode 100644 index 0000000..b8d708a --- /dev/null +++ b/harness/adapters/fsaf_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection fsaf.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "fsaf.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-fsaf.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/gfl_r50_fpn.py b/harness/adapters/gfl_r50_fpn.py new file mode 100644 index 0000000..bd93f5e --- /dev/null +++ b/harness/adapters/gfl_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection gfl.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "gfl.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-gfl.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/hrnet_w18.py b/harness/adapters/hrnet_w18.py new file mode 100644 index 0000000..bcf0105 --- /dev/null +++ b/harness/adapters/hrnet_w18.py @@ -0,0 +1,7 @@ +"""Official Cityscapes hrnet.w18; exact static benchmark recipe.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("hrnet.w18") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "hrnet.w18") diff --git a/harness/adapters/hrnet_w32.py b/harness/adapters/hrnet_w32.py new file mode 100644 index 0000000..af22eb3 --- /dev/null +++ b/harness/adapters/hrnet_w32.py @@ -0,0 +1,9 @@ +"""Official hrnet.w32 COCO17 at 256x192.""" +from functools import partial +from .topdown_pose_common import (metadata, build_model, predict as _predict, decode as _decode, + preprocess, g2_input, PREPROCESS, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata('hrnet.w32') +build = partial(build_model, 'hrnet.w32') +predict = partial(_predict, 'hrnet.w32') +decode = partial(_decode, 'hrnet.w32') diff --git a/harness/adapters/hrnet_w48.py b/harness/adapters/hrnet_w48.py new file mode 100644 index 0000000..9fe8eef --- /dev/null +++ b/harness/adapters/hrnet_w48.py @@ -0,0 +1,7 @@ +"""Official Cityscapes hrnet.w48; exact static benchmark recipe.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("hrnet.w48") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "hrnet.w48") diff --git a/harness/adapters/hrnet_w48_2.py b/harness/adapters/hrnet_w48_2.py new file mode 100644 index 0000000..a1bea43 --- /dev/null +++ b/harness/adapters/hrnet_w48_2.py @@ -0,0 +1,9 @@ +"""Official hrnet.w48.2 COCO17 at 256x192.""" +from functools import partial +from .topdown_pose_common import (metadata, build_model, predict as _predict, decode as _decode, + preprocess, g2_input, PREPROCESS, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata('hrnet.w48.2') +build = partial(build_model, 'hrnet.w48.2') +predict = partial(_predict, 'hrnet.w48.2') +decode = partial(_decode, 'hrnet.w48.2') diff --git a/harness/adapters/keynetplushardnet.py b/harness/adapters/keynetplushardnet.py new file mode 100644 index 0000000..4c3958b --- /dev/null +++ b/harness/adapters/keynetplushardnet.py @@ -0,0 +1,129 @@ +"""Author PyTorch Key.Net plus original HardNet++; no HyNet substitution.""" +from . import feature_sources3 as sources +from .feature_common import MAX_KEYPOINTS, preprocess as gray_preprocess + +INPUT_SPEC='1x1x480x640' +MODEL_SPEC={'input_spec':INPUT_SPEC,'dataset':'hpatches'} +EXPORTER_VERSION='keynet-hardnetpp-static-1' +OPSET=17 +CHECKPOINT={'url':'https://raw.githubusercontent.com/axelBarroso/Key.Net-Pytorch/74fe0a75998445bb47b74b642b764543b25dc90a/model/weights/keynet_pytorch.pth', + 'sha256':'59d3a46d57b439513acf531877fdfaa95eb550ccd10ec462e23d7c1f7a86d52a', + 'filename':'keynet_pytorch.pth','official_relative_path':str(sources.ROOT/'keynet-pytorch/model/weights/keynet_pytorch.pth')} +DESCRIPTOR_CHECKPOINT={'url':'https://raw.githubusercontent.com/axelBarroso/Key.Net/24808fb5bd7376e33f43e9f8f1029f4ef1a7ce3c/keyNet/pretrained_nets/HardNet%2B%2B.pth', + 'path':str(sources.ROOT/'keynet/keyNet/pretrained_nets/HardNet++.pth'), + 'sha256':'9fe80a335fe72d1225fdd1ae0943418b12489f95014bd51dddc9819d6ffa33c7'} +PREPROCESS={'input_spec':INPUT_SPEC,'color':'GRAY','gray':'opencv_RGB2GRAY_uint8','scale':1/255, + 'resize':'opencv_linear_640x480','output_layout':'NCHW', + 'protocol':'keynet-author-pytorch-hardnetpp-single-scale-s22-v1'} +OFFICIAL_REFERENCE_POLICY = { + 'selection': 'official positive NMS15, top1024; equal scores use ascending raster index', + 'tie_rule': 'ONNX TopK standard; official torch.sort is unstable for equal scores', + 'patches': 'official extract_patch, s_mult22, PS32, grayscale255, align_cornersFalse', + 'descriptor': 'official HardNet++ class and weights, no HyNet substitution', +} + +def preprocess(image,input_spec=INPUT_SPEC): + if input_spec!=INPUT_SPEC:raise ValueError(input_spec) + return gray_preprocess(image,input_spec) + +def official_model(weights): + import torch + sources.check(weights,CHECKPOINT['sha256']) + sources.check(DESCRIPTOR_CHECKPOINT['path'],DESCRIPTOR_CHECKPOINT['sha256']) + network=sources.package('keynet-pytorch','model.network') + detector=network.KeyNet({'num_filters':8,'num_levels':3,'kernel_size':5}) + detector.load_state_dict(torch.load(weights,map_location='cpu',weights_only=False)['state_dict'],strict=True) + descriptor=sources.module('keynet','keyNet/model/hardnet_pytorch.py').HardNet() + descriptor.load_state_dict(torch.load(DESCRIPTOR_CHECKPOINT['path'],map_location='cpu',weights_only=False)['state_dict'],strict=True) + return torch.nn.ModuleDict({'detector':detector,'descriptor':descriptor}).eval() + +def official_features(m,x): + import torch + module=sources.package('keynet-pytorch','model.extraction_tools') + score=module.remove_borders(m['detector'](x),15) + yx=module.NonMaxSuppression(nms_size=15)(score) + values=score[0,0,yx[0],yx[1]] + # Preserve official NMS/candidate ordering; specify only its undefined tie order. + order=torch.argsort(values,descending=True,stable=True)[:MAX_KEYPOINTS] + k=torch.stack([yx[1,order],yx[0,order]],-1).float();s=values[order] + n=len(k) + laf=module.to_laf(k[None],22*torch.ones(1,n,1,1),torch.zeros(1,n,1)) + patches=module.extract_patch(255*x,laf,PS=32,normalize_lafs_before_extraction=True)[0] + d=torch.cat([m['descriptor'](p) for p in patches.split(1000)],0) + return k,s,d + +def build(weights,input_spec): + import torch + import torch.nn.functional as F + if input_spec!=INPUT_SPEC:raise ValueError(input_spec) + utils=sources.package('keynet-pytorch','model.kornia_tools.utils') + class StaticHandcrafted(torch.nn.Module): + def __init__(self): + super().__init__() + # Original Kornia 0.1.4 Sobel kernel, fixed C=1 eliminates dynamic repeat. + self.register_buffer('kernel',utils.kornia.filters.SpatialGradient().kernel[None,None].flip(-3)) + + def gradient(self,x): + x=F.pad(x,[1,1,1,1],'replicate')[:,:,None] + return F.conv3d(F.pad(x,[0,0,0,0,1,1]),self.kernel) + + def forward(self,x): + sobel=self.gradient(x);dx,dy=sobel[:,:,0],sobel[:,:,1] + sobel_dx=self.gradient(dx);dxx,dxy=sobel_dx[:,:,0],sobel_dx[:,:,1] + dyy=self.gradient(dy)[:,:,1] + return torch.cat([dx,dy,dx**2.,dy**2.,dx*dy,dxy,dxy**2.,dxx,dyy,dxx*dyy],1) + + class StableTopK(torch.autograd.Function): + @staticmethod + def forward(ctx,values): + indices=torch.argsort(values,dim=1,descending=True,stable=True)[:,:MAX_KEYPOINTS] + return values.gather(1,indices),indices + + @staticmethod + def symbolic(g,values): + # ONNX TopK explicitly breaks ties by the lower original index. + k=g.op('Constant',value_t=torch.tensor([MAX_KEYPOINTS],dtype=torch.long)) + return g.op('TopK',values,k,axis_i=1,largest_i=1,sorted_i=1,outputs=2) + + class Graph(torch.nn.Module): + def __init__(self): + super().__init__();self.nets=official_model(weights) + self.nets['detector'].feature_extractor.hc_block=StaticHandcrafted() + self.register_buffer('pyramid_kernel',utils._get_pyramid_gaussian_kernel()[None]) + border=torch.zeros(1,1,480,640);border[:,:,15:-15,15:-15]=1 + self.register_buffer('border',border) + unit=F.affine_grid(torch.tensor([[[1.,0.,0.],[0.,1.,0.]]]),[1,1,32,32],align_corners=False) + self.register_buffer('patch_grid',torch.cat([unit,torch.ones(1,32,32,1)],-1).reshape(1,1024,3)) + + def forward(self,x): + # Same three internal pyramid levels as official KeyNet; fixed grayscale shapes. + detector=self.nets['detector'];level=x;features=[detector.feature_extractor(level)] + for h,w in ((400,533),(333,444)): + level=F.interpolate(F.conv2d(F.pad(level,[2,2,2,2],'reflect'),self.pyramid_kernel), + size=(h,w),mode='bilinear',align_corners=False) + features.append(F.interpolate(detector.feature_extractor(level),size=(480,640),mode='bilinear',align_corners=False)) + scores=detector.last_conv(torch.cat(features,1))*self.border + scores=scores*(scores==F.max_pool2d(scores,15,1,7)).to(scores.dtype) + s,ids=StableTopK.apply(scores.flatten(1)) + k=torch.stack((ids%640,torch.div(ids,640,rounding_mode='floor')),-1).float() + laf=utils.laf_from_center_scale_ori(k,22*torch.ones(1,MAX_KEYPOINTS,1,1,device=x.device),torch.zeros(1,MAX_KEYPOINTS,1,device=x.device)) + # At s=22, PS=32, every patch has official pyramid index 0. + # Precompute only the constant affine base grid; keep official LAF + # normalization and its floating-point operation order. + theta=utils.denormalize_laf(utils.normalize_laf(laf,x),x).reshape(MAX_KEYPOINTS,2,3) + grid=torch.bmm(self.patch_grid.expand(MAX_KEYPOINTS,-1,-1),theta.transpose(1,2)) + grid=torch.stack((2*grid[:,:,0]/640-1,2*grid[:,:,1]/480-1),-1) + # One tiled grid avoids materializing 1024 copies of the input. + patches=F.grid_sample(255*x,grid.reshape(1,MAX_KEYPOINTS*32,32,2), + padding_mode='border',align_corners=False).reshape(MAX_KEYPOINTS,1,32,32) + d=self.nets['descriptor'](patches) + return k.reshape(1,MAX_KEYPOINTS,2),s.reshape(1,MAX_KEYPOINTS),d[None] + return Graph().eval() + +def decode(outputs): + import numpy as np + k,s,d=[np.asarray(o,np.float32)[0] for o in outputs] + assert k.shape==(MAX_KEYPOINTS,2) and s.shape==(MAX_KEYPOINTS,) and d.shape==(MAX_KEYPOINTS,128) + assert all(np.isfinite(v).all() for v in (k,s,d)) + valid=s>0 + return {'keypoints':k[valid],'scores':s[valid],'descriptors':d[valid]} diff --git a/harness/adapters/mask2former_swin_b_instance.py b/harness/adapters/mask2former_swin_b_instance.py new file mode 100644 index 0000000..bf656bc --- /dev/null +++ b/harness/adapters/mask2former_swin_b_instance.py @@ -0,0 +1,22 @@ +"""Exact MMDetection mask2former.swin.b.instance; see mmdet_iseg_common. + +Backed by the official swin-b COCO-panoptic config and checkpoint (the model +zoo publishes no instance-only swin-b variant); instance outputs are the 80 +thing classes, which mmdet orders exactly like CocoDataset. +""" +from functools import partial + +from .mmdet_iseg_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, + build_variant, predict_variant, g2_input_variant) + +MODEL_ID = "mask2former.swin.b.instance" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-mask2former-640-raw-v1" +OPSET = 16 # ONNX GridSample for the multi-scale deformable attention layers. +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) +predict = partial(predict_variant, MODEL_ID) +g2_input = partial(g2_input_variant, MODEL_ID) diff --git a/harness/adapters/mask2former_swin_s_instance.py b/harness/adapters/mask2former_swin_s_instance.py new file mode 100644 index 0000000..c685fba --- /dev/null +++ b/harness/adapters/mask2former_swin_s_instance.py @@ -0,0 +1,17 @@ +"""Exact MMDetection mask2former.swin.s.instance; see mmdet_iseg_common.""" +from functools import partial + +from .mmdet_iseg_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, + build_variant, predict_variant, g2_input_variant) + +MODEL_ID = "mask2former.swin.s.instance" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-mask2former-640-raw-v1" +OPSET = 16 # ONNX GridSample for the multi-scale deformable attention layers. +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) +predict = partial(predict_variant, MODEL_ID) +g2_input = partial(g2_input_variant, MODEL_ID) diff --git a/harness/adapters/mask2former_swin_t_instance.py b/harness/adapters/mask2former_swin_t_instance.py new file mode 100644 index 0000000..a3e2134 --- /dev/null +++ b/harness/adapters/mask2former_swin_t_instance.py @@ -0,0 +1,17 @@ +"""Exact MMDetection mask2former.swin.t.instance; see mmdet_iseg_common.""" +from functools import partial + +from .mmdet_iseg_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, + build_variant, predict_variant, g2_input_variant) + +MODEL_ID = "mask2former.swin.t.instance" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-mask2former-640-raw-v1" +OPSET = 16 # ONNX GridSample for the multi-scale deformable attention layers. +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) +predict = partial(predict_variant, MODEL_ID) +g2_input = partial(g2_input_variant, MODEL_ID) diff --git a/harness/adapters/midas_common.py b/harness/adapters/midas_common.py new file mode 100644 index 0000000..236de73 --- /dev/null +++ b/harness/adapters/midas_common.py @@ -0,0 +1,242 @@ +"""Pinned official MiDaS relative-depth models (v2.1 small, v3 DPT hybrid/large). + +The pinned private runtime tree keeps the official MiDaS source, the timm +0.6.12 it requires and the gen-efficientnet backbone code, with only the +local-hub patch from the recorded receipt. No package install, replacement +architecture or synthetic weights. Outputs are nonnegative relative inverse +depth (disparity); restore follows the author's bicubic resize. +""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import sys + +UPSTREAM_COMMIT = "454597711a62eabcbf7d1e89f3fb9f569051ac9b" +# sha256 over "\0" of every *.py in the pinned runtime tree +# (midas + gen-efficientnet + timm), matching source_receipt.json. +SOURCE_TREE_SHA256 = "b27d6846712df86b72264c1df091d0693b4834155cfcc1d30777a359904d8b82" +RUNTIME_ROOT = Path.home() / "bench-cache/sources/midas" / UPSTREAM_COMMIT +OPSET = 17 +EXPORTER_VERSION = "midas-relative-v1-opset17-static" +RESTORE_INTERPOLATION = "bicubic" +RESTORE_ALIGN_CORNERS = False + +INPUT_SPECS = { + "small": "1x3x256x256", + "hybrid": "1x3x384x384", + "large": "1x3x384x384", +} +MODEL_SPECS = { + "small": {"source": "custom", "dataset": "nyuv2", "input_spec": INPUT_SPECS["small"], "resize": 0}, + "hybrid": {"source": "custom", "dataset": "nyuv2", "input_spec": INPUT_SPECS["hybrid"], "resize": 0}, + "large": {"source": "custom", "dataset": "nyuv2", "input_spec": INPUT_SPECS["large"], "resize": 0}, +} +PREPROCESSES = { + "small": { + "id": "midas-v21-small-rgb-static256-v1", + "layout": "NCHW", "dtype": "float32", "color": "RGB", + "resize": "OpenCV INTER_CUBIC to 256x256; keep_aspect_ratio=False for static deployment", + "normalize": "RGB/255; mean=[0.485,0.456,0.406]; std=[0.229,0.224,0.225]", + "output": "N,H,W nonnegative relative inverse depth (disparity); NOT metric depth", + "restore": "bicubic to original H,W, align_corners=False; no min/max image normalization", + "source_commit": UPSTREAM_COMMIT, + }, + "hybrid": { + "id": "midas-v3-hybrid-rgb-static384-v1", + "layout": "NCHW", "dtype": "float32", "color": "RGB", + "resize": "OpenCV INTER_CUBIC to 384x384; keep_aspect_ratio=False for static deployment", + "normalize": "RGB/255; mean=[0.5,0.5,0.5]; std=[0.5,0.5,0.5]", + "output": "N,H,W nonnegative relative inverse depth (disparity); NOT metric depth", + "restore": "bicubic to original H,W, align_corners=False; no min/max image normalization", + "source_commit": UPSTREAM_COMMIT, + }, + "large": { + "id": "midas-v3-large-rgb-static384-v1", + "layout": "NCHW", "dtype": "float32", "color": "RGB", + "resize": "OpenCV INTER_CUBIC to 384x384; keep_aspect_ratio=False for static deployment", + "normalize": "RGB/255; mean=[0.5,0.5,0.5]; std=[0.5,0.5,0.5]", + "output": "N,H,W nonnegative relative inverse depth (disparity); NOT metric depth", + "restore": "bicubic to original H,W, align_corners=False; no min/max image normalization", + "source_commit": UPSTREAM_COMMIT, + }, +} + +_CHECKPOINTS = { + "small": {"url": "https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_small_256.pt", + "sha256": "70d6b9c891758c67f974a6097fb0c608c7ee67fb81ac3e5588847d5596d56fca", + "release": "v2_1", "output_type": "relative_inverse_depth"}, + "hybrid": {"url": "https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt", + "sha256": "501f0c75b3bca7daec6b3682c5054c09b366765aef6fa3a09d03a5cb4b230853", + "release": "v3", "output_type": "relative_inverse_depth"}, + "large": {"url": "https://github.com/isl-org/MiDaS/releases/download/v3/dpt_large_384.pt", + "sha256": "2f21e586477d90cb9624c7eef5df7891edca49a1c4795ee2cb631fd4daa6ca69", + "release": "v3", "output_type": "relative_inverse_depth"}, +} + + +def checkpoint_info(variant: str) -> dict: + return dict(_CHECKPOINTS[variant]) + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def source_root() -> Path: + root = Path(os.environ.get("BENCH_MIDAS_SOURCE", str(RUNTIME_ROOT))).resolve() + digest = hashlib.sha256() + files = sorted(root.rglob("*.py")) + for path in files: + digest.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes()) + if digest.hexdigest() != SOURCE_TREE_SHA256: + raise RuntimeError(f"missing or changed pinned MiDaS runtime source {UPSTREAM_COMMIT}: {root}") + return root + + +def preprocess_for(image, input_spec: str, variant: str): + """Static deployment resize; per-variant normalization from midas.model_loader. + + The v3 DPT checkpoints train with mean=std=0.5, the v2.1 small checkpoint + with ImageNet statistics; both follow the official loader branches. + """ + import cv2 + import numpy as np + + size = int(INPUT_SPECS[variant].split("x")[-1]) + if input_spec != INPUT_SPECS[variant]: + raise ValueError(f"MiDaS {variant} recipe is fixed at {INPUT_SPECS[variant]}, got {input_spec}") + mean, std = (([.5, .5, .5], [.5, .5, .5]) if variant != "small" + else ([.485, .456, .406], [.229, .224, .225])) + rgb = np.asarray(image.convert("RGB"), dtype=np.float64) / 255.0 + rgb = cv2.resize(rgb, (size, size), interpolation=cv2.INTER_CUBIC) + rgb = (rgb - np.array(mean)) / np.array(std) + return np.ascontiguousarray(rgb.transpose(2, 0, 1)[None], dtype=np.float32) + + +def _static_forward_flex(self, x): + """Official midas backbones/vit.py forward_flex with python-int grid sizes. + + x.shape entries become Tensors under torch.onnx tracing (torch >= 2.x) and + then break _resize_pos_embed's F.interpolate and nn.Unflatten validation. + The pinned deployment resolution is static, so the ints are constants. + """ + import torch + + h, w = int(x.shape[2]), int(x.shape[3]) + pos_embed = self._resize_pos_embed(self.pos_embed, h // self.patch_size[1], w // self.patch_size[0]) + b = x.shape[0] + if hasattr(self.patch_embed, "backbone"): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[-1] + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + if getattr(self, "dist_token", None) is not None: + cls_tokens = self.cls_token.expand(b, -1, -1) + dist_token = self.dist_token.expand(b, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + if self.no_embed_class: + x = x + pos_embed + cls_tokens = self.cls_token.expand(b, -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + if not self.no_embed_class: + x = x + pos_embed + x = self.pos_drop(x) + for blk in self.blocks: + x = blk(x) + x = self.norm(x) + return x + + +def _static_forward_vit(pretrained, x): + """Official midas backbones/utils.py forward_adapted_unflatten, int sizes.""" + import torch + import torch.nn as nn + + h, w = int(x.shape[2]), int(x.shape[3]) + glob = pretrained.model.forward_flex(x) + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + unflatten = nn.Sequential( + nn.Unflatten(2, torch.Size([ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ]))) + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + layer_1 = pretrained.act_postprocess1[3: len(pretrained.act_postprocess1)](layer_1) + layer_2 = pretrained.act_postprocess2[3: len(pretrained.act_postprocess2)](layer_2) + layer_3 = pretrained.act_postprocess3[3: len(pretrained.act_postprocess3)](layer_3) + layer_4 = pretrained.act_postprocess4[3: len(pretrained.act_postprocess4)](layer_4) + return layer_1, layer_2, layer_3, layer_4 + + +def _install_static_vit_path(model, input_spec: str): + """Swap the ViT flex path for the int-size twins and prove bitwise equality. + + The official modules, weights and math are untouched: the twins only turn + the traced Tensor shape entries into python ints for the fixed deployment + resolution. A fixed deterministic input verifies the swapped forward stays + bitwise identical to the untouched official forward before returning. + """ + import types + + import torch + + size = int(input_spec.split("x")[-1]) + generator = torch.Generator().manual_seed(20260918) + probe = torch.rand((1, 3, size, size), generator=generator) + with torch.inference_mode(): + official = model(probe).clone() + model.pretrained.model.forward_flex = types.MethodType(_static_forward_flex, model.pretrained.model) + model.forward_transformer = _static_forward_vit + with torch.inference_mode(): + replaced = model(probe) + if not torch.equal(official, replaced): + raise RuntimeError("static-grid ViT path is not bitwise identical to the official forward") + return model + + +def build_variant(variant: str, weights: Path, input_spec: str): + import torch + + if input_spec != INPUT_SPECS[variant]: + raise ValueError(f"expected fixed {INPUT_SPECS[variant]}, got {input_spec}") + root = source_root() + if file_sha256(weights) != _CHECKPOINTS[variant]["sha256"]: + raise ValueError(f"wrong official MiDaS {variant} checkpoint") + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + # Constructing with the checkpoint path keeps use_pretrained=False: the + # official ImageNet backbone download never runs and the strict load is + # performed by MiDaS BaseModel.load itself. + if variant == "small": + from midas.midas_net_custom import MidasNet_small + + model = MidasNet_small(Path(weights), features=64, backbone="efficientnet_lite3", + exportable=True, non_negative=True, blocks={"expand": True}) + else: + from midas.dpt_depth import DPTDepthModel + + backbone = "vitl16_384" if variant == "large" else "vitb_rn50_384" + model = DPTDepthModel(path=str(weights), backbone=backbone, non_negative=True) + _install_static_vit_path(model, input_spec) + return model.eval() diff --git a/harness/adapters/midas_small.py b/harness/adapters/midas_small.py new file mode 100644 index 0000000..6b3478c --- /dev/null +++ b/harness/adapters/midas_small.py @@ -0,0 +1,19 @@ +"""Official MiDaS v2.1 small: relative inverse depth at static 256x256.""" +from pathlib import Path + +from .midas_common import (EXPORTER_VERSION, INPUT_SPECS, MODEL_SPECS, OPSET, PREPROCESSES, + RESTORE_ALIGN_CORNERS, RESTORE_INTERPOLATION, build_variant, + checkpoint_info, preprocess_for) + +CHECKPOINT = checkpoint_info("small") +INPUT_SPEC = INPUT_SPECS["small"] +MODEL_SPEC = MODEL_SPECS["small"] +PREPROCESS = PREPROCESSES["small"] + + +def build(weights: Path, input_spec: str): + return build_variant("small", weights, input_spec) + + +def preprocess(image, input_spec: str = INPUT_SPEC): + return preprocess_for(image, input_spec, "small") diff --git a/harness/adapters/mmdet_dense_common.py b/harness/adapters/mmdet_dense_common.py new file mode 100644 index 0000000..02b4886 --- /dev/null +++ b/harness/adapters/mmdet_dense_common.py @@ -0,0 +1,222 @@ +"""Pinned MMDetection dense detectors at the CSV's static 640 deployment size. + +Only deployment geometry differs from the upstream 1333x800 test pipeline: +keep-ratio resize to 640, then normalized zero padding at bottom/right to 640. +Weights, normalization, raw heads and official postprocessing are unchanged. +Heavy imports remain inside runtime functions for lightweight recipe discovery. +""" +from functools import lru_cache +from types import SimpleNamespace + +from .openmmlab import build as build_official, get_recipe + +RUNTIME_PYTHON = "/home/ming01.wang/outputs/full-coverage-20260917/openmmlab/venv/bin/python" +INPUT_SPEC = "1x3x640x640" +MODEL_SPEC = {"source": "custom", "dataset": "coco2017", "input_spec": INPUT_SPEC} +MODELS = ("retinanet.r50.fpn", "retinanet.r101.fpn", "fcos.r50.fpn", + "atss.r50.fpn", "gfl.r50.fpn", "vfnet.r50.fpn", + "foveabox.r50.fpn", "fsaf.r50.fpn", "reppoints.r50.fpn") +HEADS = {"retinanet": ("RetinaHead", (720, 36)), "fcos": ("FCOSHead", (80, 4, 1)), + "atss": ("ATSSHead", (80, 4, 1)), "gfl": ("GFLHead", (80, 68)), + "vfnet": ("VFNetHead", (80, 4)), "foveabox": ("FoveaHead", (80, 4)), + "fsaf": ("FSAFHead", (80, 4)), "reppoints": ("RepPointsHead", (80, 4))} +STRIDES = (8, 16, 32, 64, 128) +TEST_CFG = {"nms_pre": 1000, "min_bbox_size": 0, "score_thr": .05, + "nms": {"type": "nms", "iou_threshold": .5}, "max_per_img": 100} + + +def test_config(family): + from copy import deepcopy + cfg = deepcopy(TEST_CFG) + if family in ("atss", "gfl", "vfnet"): + cfg["nms"]["iou_threshold"] = .6 + if family == "foveabox": + del cfg["min_bbox_size"] + return cfg + + +def recipe(model): + if model not in MODELS: + raise ValueError(model) + dp = get_recipe(model)["preprocess"]["official_data_preprocessor"] + return {"recipe": "mmdet-dense-static640-v1", "input_spec": INPUT_SPEC, + "color": "RGB" if dp["bgr_to_rgb"] else "BGR", + "mean": dp["mean"], "std": dp["std"], + "resize": {"scale": [640, 640], "keep_ratio": True, + "rounding": "int(value+0.5)", "backend": "cv2", "interpolation": "bilinear"}, + "pad": {"size": [640, 640], "placement": "bottom_right", "value": 0, + "stage": "after_normalization"}, "layout": "NCHW", "dtype": "float32"} + + +def geometry(original_size): + w, h = map(int, original_size) + if min(w, h) <= 0: + raise ValueError(original_size) + scale = 640 / max(w, h) + nw, nh = int(w * scale + .5), int(h * scale + .5) + if min(nw, nh) < 1: + raise ValueError("Image collapses under official keep-ratio resize") + # Clipping is to resized image geometry, not the normalized padded canvas. + return (nw, nh), (nw / w, nh / h), (nh, nw) + + +def preprocess_variant(model, image, input_spec=INPUT_SPEC): + if input_spec != INPUT_SPEC: + raise ValueError(f"Expected {INPUT_SPEC}, got {input_spec}") + import mmcv + import numpy as np + meta = recipe(model) + bgr = np.asarray(image.convert("RGB"))[:, :, ::-1].copy() + x = mmcv.imrescale(bgr, (640, 640), interpolation="bilinear", backend="cv2") + if meta["color"] == "RGB": + x = x[:, :, ::-1] + x = np.ascontiguousarray(x.transpose(2, 0, 1), dtype=np.float32) + x = (x - np.asarray(meta["mean"], dtype=np.float32)[:, None, None]) / np.asarray(meta["std"], dtype=np.float32)[:, None, None] + out = np.zeros((1, 3, 640, 640), dtype=np.float32) + out[0, :, :x.shape[1], :x.shape[2]] = x + return out + + +@lru_cache(maxsize=8) +def _decoder(family): + # The official BaseDenseHead owns filtering, centerness, clipping, rescaling + # and class-aware NMS. This context has no learned layers or checkpoint. + from mmengine.config import ConfigDict + from mmdet.models.dense_heads.base_dense_head import BaseDenseHead + from mmdet.models.task_modules.prior_generators import AnchorGenerator, MlvlPointGenerator + from mmdet.models.task_modules.coders import DeltaXYWHBBoxCoder, DistancePointBBoxCoder, TBLRBBoxCoder + + class Decoder(BaseDenseHead): + def loss_by_feat(self, **kwargs): + raise NotImplementedError("Inference-only decoder") + + if family == "gfl": + from mmdet.models.dense_heads.gfl_head import GFLHead, Integral + Decoder._predict_by_feat_single = GFLHead._predict_by_feat_single + Decoder.anchor_center = GFLHead.anchor_center + elif family in ("foveabox", "reppoints"): + from mmdet.models.dense_heads import FoveaHead, RepPointsHead + official = FoveaHead if family == "foveabox" else RepPointsHead + Decoder._predict_by_feat_single = official._predict_by_feat_single + Decoder._bbox_decode = official._bbox_decode + decoder = Decoder() + if family in ("fcos", "vfnet", "foveabox", "reppoints"): + decoder.prior_generator = MlvlPointGenerator(STRIDES, + offset=.5 if family in ("fcos", "foveabox") else 0.) + else: + retina = family == "retinanet" + decoder.prior_generator = AnchorGenerator(strides=STRIDES, + ratios=[.5, 1., 2.] if retina else [1.], + octave_base_scale=4 if retina else (1 if family == "fsaf" else 8), + scales_per_octave=3 if retina else 1) + if family in ("retinanet", "atss"): + decoder.bbox_coder = DeltaXYWHBBoxCoder(target_means=[0., 0., 0., 0.], + target_stds=[.1, .1, .2, .2] if family == "atss" else [1., 1., 1., 1.]) + elif family == "fsaf": + decoder.bbox_coder = TBLRBBoxCoder(normalizer=4.0) + else: + decoder.bbox_coder = DistancePointBBoxCoder() + if family == "gfl": + decoder.integral = Integral(16) + elif family == "foveabox": + decoder.strides = list(STRIDES) + decoder.base_edge_list = [16, 32, 64, 128, 256] + elif family == "reppoints": + decoder.point_strides = list(STRIDES) + decoder.cls_out_channels = 80 + decoder.use_sigmoid_cls = True + decoder.loss_cls = SimpleNamespace(custom_cls_channels=False) + decoder.test_cfg = ConfigDict(test_config(family)) + return decoder + + +def output_names(model): + groups = ("cls", "bbox", "centerness") if model.split(".")[0] in ("fcos", "atss") else ("cls", "bbox") + return [f"{group}_s{s}" for group in groups for s in STRIDES] + + +def decode_variant(model, outputs, original_size): + import numpy as np + import torch + family = model.split(".")[0] + channels = HEADS[family][1] + if len(outputs) != len(channels) * 5: + raise ValueError(f"{model}: wrong raw head count {len(outputs)}") + raw = [] + for i, value in enumerate(outputs): + expected = (1, channels[i // 5], 640 // STRIDES[i % 5], 640 // STRIDES[i % 5]) + array = np.asarray(value) + if array.shape != expected or not np.isfinite(array).all(): + raise ValueError(f"{model}: invalid raw head {i}: {array.shape}, expected {expected}") + raw.append(torch.as_tensor(np.ascontiguousarray(array), dtype=torch.float32)) + _, scales, img_shape = geometry(original_size) + meta = dict(img_shape=img_shape, ori_shape=tuple(reversed(original_size)), + pad_shape=(640, 640), batch_input_shape=(640, 640), scale_factor=scales) + with torch.inference_mode(): + result = _decoder(family).predict_by_feat( + raw[:5], raw[5:10], score_factors=raw[10:] if len(channels) == 3 else None, + batch_img_metas=[meta], rescale=True)[0] + return result.bboxes.numpy(), result.scores.numpy(), result.labels.numpy() + + +def build_variant(model, weights, input_spec): + import torch + raw = build_official(model, weights, input_spec) + cfg, head = raw.cfg, raw.upstream.bbox_head + meta = recipe(model) + dp = cfg.model.data_preprocessor + family = model.split(".")[0] + if (dict(cfg.model.test_cfg) != test_config(family) or head.num_classes != 80 + or head.__class__.__name__ != HEADS[family][0] + or not head.use_sigmoid_cls or head.cls_out_channels != 80 + or list(dp.mean) != meta["mean"] or list(dp.std) != meta["std"] + or dp.bgr_to_rgb != (meta["color"] == "RGB") or dp.pad_size_divisor != 32): + raise ValueError("Pinned official dense detector contract changed") + decoder = _decoder(family) + # Compare actual upstream geometry helpers, including defaults not in config. + for name in ("strides", "num_base_priors"): + if getattr(head.prior_generator, name) != getattr(decoder.prior_generator, name): + raise ValueError(f"Official prior {name} differs") + sizes = [(640 // s, 640 // s) for s in STRIDES] + for expected, actual in zip(head.prior_generator.grid_priors(sizes, device="cpu"), + decoder.prior_generator.grid_priors(sizes, device="cpu")): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + if head.bbox_coder.__dict__ != decoder.bbox_coder.__dict__: + raise ValueError("Official bbox coder differs") + + if family == "gfl": + torch.testing.assert_close(head.integral.project, decoder.integral.project, rtol=0, atol=0) + elif family == "foveabox": + if (list(head.strides) != decoder.strides or list(head.base_edge_list) != decoder.base_edge_list + or head.with_deform): + raise ValueError("Official Fovea geometry differs") + elif family == "reppoints": + if (head.point_strides != decoder.point_strides or head.num_points != 9 + or head.transform_method != "moment"): + raise ValueError("Official RepPoints geometry differs") + + if family in ("vfnet", "reppoints"): + # VFNet's upstream head overrides its state-load hook; verify every + # learned parameter against the checkpoint before wrapping export ops. + state = torch.load(str(weights), map_location="cpu")["state_dict"] + for name, parameter in raw.upstream.named_parameters(): + torch.testing.assert_close(parameter, state[name], rtol=0, atol=0) + del state + from .vfnet_deformconv import enable_export + if family == "reppoints": + enable_export(head, ("reppoints_cls_conv", "reppoints_pts_refine_conv")) + else: + enable_export(head) + + class RawHeads(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = raw.upstream + self.upstream.cfg = cfg + self.cfg = cfg + self.checkpoint_sha256 = raw.checkpoint_sha256 + + def forward(self, x): + return tuple(value for level in self.upstream._forward(x) for value in level) + + return RawHeads().eval() diff --git a/harness/adapters/mmdet_detr_common.py b/harness/adapters/mmdet_detr_common.py new file mode 100644 index 0000000..63bc7e5 --- /dev/null +++ b/harness/adapters/mmdet_detr_common.py @@ -0,0 +1,141 @@ +"""Exact pinned MMDetection DETR models, including their complete transformers. + +Static640 follows the CSV deployment protocol. Padding masks are derived inside +the graph from normalized-zero padding, never frozen to the export image shape. +Only parameter-free official box selection and geometry run in host decode. +""" +from functools import lru_cache +from types import SimpleNamespace + +from .openmmlab import build as build_official, get_recipe +from .mmdet_dense_common import RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, geometry + +MODELS = ("detr.r50", "deformable.detr.r50", "dab.detr.r50") +OPSET = 16 # Standard ONNX GridSample for genuine multi-scale deformable attention. +CONTRACTS = {"detr.r50": ("DETR", "DETRHead", 100, 81, 100), + "deformable.detr.r50": ("DeformableDETR", "DeformableDETRHead", 300, 80, 100), + "dab.detr.r50": ("DABDETR", "DABDETRHead", 300, 80, 300)} + + +def recipe(model): + if model not in MODELS: + raise ValueError(model) + from .mmdet_dense_common import recipe as image_recipe + meta = image_recipe("retinanet.r50.fpn") + dp = get_recipe(model)["preprocess"]["official_data_preprocessor"] + if (dp["mean"] != meta["mean"] or dp["std"] != meta["std"] + or not dp["bgr_to_rgb"] or dp["pad_size_divisor"] != 1): + raise ValueError("Official DETR image normalization changed") + # Same image transformation permits existing calibration cache reuse. + return meta + + +def preprocess_variant(model, image, input_spec=INPUT_SPEC): + from .mmdet_dense_common import preprocess_variant as image_preprocess + recipe(model) # verify exact equality before reusing the image-only helper + return image_preprocess("retinanet.r50.fpn", image, input_spec) + + +def output_names(model): + if model not in MODELS: + raise ValueError(model) + return ["class_logits", "boxes_cxcywh"] + + +def padding_mask(x): + # Resized RGB pixels are uint8 and all official channel means are fractional: + # a real pixel cannot be normalized zero. Zero is exclusively padding. + return (x == 0).all(dim=1) + + +def transformer_inputs(network, features, x, deformable): + """Official pre_transformer math with the real per-image mask retained.""" + import torch + from torch.nn import functional as F + mask = padding_mask(x).to(x.dtype) + masks = [F.interpolate(mask[:, None], size=f.shape[-2:]).to(torch.bool)[:, 0] + for f in features] + positions = [network.positional_encoding(m) for m in masks] + flatten = lambda f: f.flatten(2).transpose(1, 2) + if not deformable: + feat, pos, mask = flatten(features[-1]), flatten(positions[-1]), masks[-1].flatten(1) + return dict(feat=feat, feat_mask=mask, feat_pos=pos), dict(memory_mask=mask, memory_pos=pos) + shapes = torch.tensor([list(f.shape[-2:]) for f in features], dtype=torch.long, device=x.device) + starts = torch.cat((shapes.new_zeros(1), shapes.prod(1).cumsum(0)[:-1])) + valid = torch.stack([network.get_valid_ratio(m) for m in masks], 1) + mask = torch.cat([m.flatten(1) for m in masks], 1) + pos = torch.cat([flatten(p) + network.level_embed[i].view(1, 1, -1) + for i, p in enumerate(positions)], 1) + extra = dict(spatial_shapes=shapes, level_start_index=starts, valid_ratios=valid) + return (dict(feat=torch.cat([flatten(f) for f in features], 1), feat_mask=mask, feat_pos=pos, **extra), + dict(memory_mask=mask, **extra)) + + +@lru_cache(maxsize=3) +def _decoder(model): + from mmdet.models.dense_heads.detr_head import DETRHead + _, _, _, channels, topk = CONTRACTS[model] + context = SimpleNamespace(test_cfg={"max_per_img": topk}, num_classes=80, + loss_cls=SimpleNamespace(use_sigmoid=channels == 80)) + from functools import partial + return partial(DETRHead._predict_by_feat_single, context) + + +def decode_variant(model, outputs, original_size): + import numpy as np + import torch + _, _, queries, channels, _ = CONTRACTS[model] + if len(outputs) != 2: + raise ValueError("Expected final query logits and cxcywh tensors") + tensors = [] + for value, shape in zip(outputs, ((1, queries, channels), (1, queries, 4))): + value = np.asarray(value) + if value.shape != shape or not np.isfinite(value).all(): + raise ValueError(f"Invalid {model} raw output: {value.shape}, expected {shape}") + tensors.append(torch.tensor(value, dtype=torch.float32)[0]) + _, scales, img_shape = geometry(original_size) + with torch.inference_mode(): + result = _decoder(model)(*tensors, dict(img_shape=img_shape, scale_factor=scales), rescale=True) + return result.bboxes.numpy(), result.scores.numpy(), result.labels.numpy() + + +def build_variant(model, weights, input_spec): + import torch + raw = build_official(model, weights, input_spec) + network, cfg = raw.upstream, raw.cfg + kind, head, queries, channels, topk = CONTRACTS[model] + meta = recipe(model) + dp = cfg.model.data_preprocessor + if (type(network).__name__ != kind or type(network.bbox_head).__name__ != head + or network.num_queries != queries or network.bbox_head.cls_out_channels != channels + or network.bbox_head.num_classes != 80 or dict(cfg.model.test_cfg) != {"max_per_img": topk} + or list(dp.mean) != meta["mean"] or list(dp.std) != meta["std"] + or not dp.bgr_to_rgb or dp.pad_size_divisor != 1): + raise ValueError("Pinned official DETR contract changed") + deformable = kind == "DeformableDETR" + if deformable and (network.with_box_refine or network.as_two_stage or network.num_feature_levels != 4): + raise ValueError("Wrong Deformable-DETR variant") + state = torch.load(str(weights), map_location="cpu")["state_dict"] + for name, parameter in network.named_parameters(): + torch.testing.assert_close(parameter, state[name], rtol=0, atol=0) + del state + + class RawQueries(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = network + self.upstream.cfg = cfg + self.cfg = cfg + self.checkpoint_sha256 = raw.checkpoint_sha256 + + def forward(self, x): + features = self.upstream.extract_feat(x) + encoder, decoder = transformer_inputs(self.upstream, features, x, deformable) + memory = self.upstream.forward_encoder(**encoder) + query, head_inputs = self.upstream.pre_decoder(**memory) + decoder.update(query) + head_inputs.update(self.upstream.forward_decoder(**decoder)) + scores, boxes = self.upstream.bbox_head(**head_inputs) + return scores[-1], boxes[-1] + + return RawQueries().eval() diff --git a/harness/adapters/mmdet_iseg_common.py b/harness/adapters/mmdet_iseg_common.py new file mode 100644 index 0000000..c3f2ec8 --- /dev/null +++ b/harness/adapters/mmdet_iseg_common.py @@ -0,0 +1,290 @@ +"""Pinned MMDetection query-based instance segmentation at the CSV static 640 size. + +Mask2Former COCO instance checkpoints: the graph returns the final decoder +layer's query classification logits and per-query mask logits over the +stride-4 mask feature map. The official MaskFormerHead.predict upsampling and +the parameter-free MaskFormerFusionHead.instance_postprocess run in host decode +on those two raw tensors, exactly like the dense-detector decoders. + +Deployment geometry follows mmdet_dense_common (upstream 1333x800 keep-ratio +test resize pinned to 640, normalized zero padding at bottom/right), so the +image recipe and calibration digests stay shared. Heavy imports stay inside +runtime functions for lightweight recipe discovery. +""" +from functools import lru_cache + +from .openmmlab import get_recipe, load_config, verify_checkpoint +from .mmdet_dense_common import RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, geometry + +MODELS = ("mask2former.swin.t.instance", "mask2former.swin.s.instance", + "mask2former.swin.b.instance") +# stage-3 blocks, patch-embed channels, queries, classes, stuff classes and +# the official test_cfg flavor, per resolved official config. +CONTRACTS = {"mask2former.swin.t.instance": (6, 96, 100, 80, 0, "instance"), + "mask2former.swin.s.instance": (18, 96, 100, 80, 0, "instance"), + "mask2former.swin.b.instance": (18, 128, 100, 133, 53, "panoptic")} +TEST_CFGS = {"instance": {"panoptic_on": False, "semantic_on": False, "instance_on": True, + "max_per_image": 100, "iou_thr": 0.8, "filter_low_score": True}, + "panoptic": {"panoptic_on": True, "semantic_on": False, "instance_on": True, + "max_per_image": 100, "iou_thr": 0.8, "filter_low_score": True}} +# The COCO instance configs load to float32 before resizing; the swin-b +# panoptic config's test pipeline keeps the default uint8 load. +F32_LOAD = {"mask2former.swin.t.instance", "mask2former.swin.s.instance"} + + +def recipe(model): + """Image recipe variant: instance configs load to float32 before resizing.""" + if model not in MODELS: + raise ValueError(model) + from .mmdet_dense_common import recipe as image_recipe + meta = image_recipe("retinanet.r50.fpn") + dp = get_recipe(model)["preprocess"]["official_data_preprocessor"] + if (dp["mean"] != meta["mean"] or dp["std"] != meta["std"] + or not dp["bgr_to_rgb"] or dp["pad_size_divisor"] != 32): + raise ValueError("Official Mask2Former image normalization changed") + if model in F32_LOAD: + return {**meta, "recipe": "mmdet-mask2former-static640-f32-v1", + "load_dtype": "float32_before_resize"} + return meta + + +def preprocess_variant(model, image, input_spec=INPUT_SPEC): + if input_spec != INPUT_SPEC: + raise ValueError(f"Expected {INPUT_SPEC}, got {input_spec}") + meta = recipe(model) + if model not in F32_LOAD: + from .mmdet_dense_common import preprocess_variant as image_preprocess + return image_preprocess("retinanet.r50.fpn", image, input_spec) + import mmcv + import numpy as np + # Official pipeline: LoadImageFromFile(to_float32=True) -> Resize -> normalize. + bgr = np.asarray(image.convert("RGB"))[:, :, ::-1].copy().astype(np.float32) + x = mmcv.imrescale(bgr, (640, 640), interpolation="bilinear", backend="cv2") + if meta["color"] == "RGB": + x = x[:, :, ::-1] + x = np.ascontiguousarray(x.transpose(2, 0, 1), dtype=np.float32) + x = (x - np.asarray(meta["mean"], dtype=np.float32)[:, None, None]) / np.asarray(meta["std"], dtype=np.float32)[:, None, None] + out = np.zeros((1, 3, 640, 640), dtype=np.float32) + out[0, :, :x.shape[1], :x.shape[2]] = x + return out + + +def output_names(model): + if model not in MODELS: + raise ValueError(model) + return ["cls_logits", "mask_logits"] + + +DEBUG_OUTPUT_NAMES = ["feat3", "mask_features", "msm0", "msm1", "msm2", "cls_logits", "mask_logits"] + + +def build_debug_variant(model, weights, input_spec): + """Same official forward with pre-decoder tensors exposed for the probe. + + The Mask2Former transformer decoder derives boolean attention masks from + interpolated mask logits (sigmoid < 0.5). Torch/ONNX fp reassociation + (~1e-4) can flip borderline bits, and each flip changes decoder outputs + discretely (measured: flipping 0.001% of the bits moves class logits by + 5.2 and mask logits by 112). Raw-output cosine therefore only gates the + continuous backbone/pixel-decoder segment; decoder equivalence is judged + on decoded predictions and the COCO metric. + """ + network = build_variant(model, weights, input_spec) + upstream, head = network.upstream, network.upstream.panoptic_head + import torch + + class DebugTaps(torch.nn.Module): + def __init__(self): + super().__init__() + self.raw = network + + def forward(self, x): + feats = upstream.extract_feat(x) + mask_features, multi_scale_memorys = head.pixel_decoder(feats) + batch_size = x.shape[0] + decoder_inputs, decoder_positional_encodings = [], [] + for i in range(head.num_transformer_feat_level): + decoder_input = head.decoder_input_projs[i](multi_scale_memorys[i]) + decoder_input = decoder_input.flatten(2).permute(0, 2, 1) + decoder_input = decoder_input + head.level_embed.weight[i].view(1, 1, -1) + mask = decoder_input.new_zeros( + (batch_size,) + multi_scale_memorys[i].shape[-2:], dtype=torch.bool) + positional = head.decoder_positional_encoding(mask).flatten(2).permute(0, 2, 1) + decoder_inputs.append(decoder_input) + decoder_positional_encodings.append(positional) + query_feat = head.query_feat.weight.unsqueeze(0).repeat((batch_size, 1, 1)) + query_embed = head.query_embed.weight.unsqueeze(0).repeat((batch_size, 1, 1)) + cls_pred, mask_pred, attn_mask = head._forward_head( + query_feat, mask_features, multi_scale_memorys[0].shape[-2:]) + for i in range(head.num_transformer_decoder_layers): + level_idx = i % head.num_transformer_feat_level + mask_sum = (attn_mask.sum(-1) != attn_mask.shape[-1]).unsqueeze(-1) + attn_mask = attn_mask & mask_sum + layer = head.transformer_decoder.layers[i] + query_feat = layer( + query=query_feat, key=decoder_inputs[level_idx], + value=decoder_inputs[level_idx], query_pos=query_embed, + key_pos=decoder_positional_encodings[level_idx], + cross_attn_mask=attn_mask, query_key_padding_mask=None, + key_padding_mask=None) + cls_pred, mask_pred, attn_mask = head._forward_head( + query_feat, mask_features, + multi_scale_memorys[(i + 1) % head.num_transformer_feat_level].shape[-2:]) + return (feats[-1], mask_features, multi_scale_memorys[0], + multi_scale_memorys[1], multi_scale_memorys[2], cls_pred, mask_pred) + + return DebugTaps().eval() + + +@lru_cache(maxsize=3) +def _decoder(model): + """Official parameter-free instance postprocess; no learned weights here.""" + from mmdet.models.seg_heads.panoptic_fusion_heads.maskformer_fusion_head import MaskFormerFusionHead + from mmengine.config import ConfigDict + _, _, _, classes, stuff, flavor = CONTRACTS[model] + decoder = MaskFormerFusionHead.__new__(MaskFormerFusionHead) + decoder.num_classes = classes + decoder.num_things_classes = classes - stuff + decoder.num_stuff_classes = stuff + decoder.test_cfg = ConfigDict(TEST_CFGS[flavor]) + return decoder + + +def decode_variant(model, outputs, original_size): + """Official predict path: upsample -> crop -> rescale -> instance_postprocess.""" + import numpy as np + import torch + from torch.nn import functional as F + if model not in MODELS: + raise ValueError(model) + queries, classes = CONTRACTS[model][2], CONTRACTS[model][3] + if len(outputs) != 2: + raise ValueError(f"{model}: expected query logits and mask logits") + cls_logits, mask_logits = outputs + cls_logits = np.asarray(cls_logits) + mask_logits = np.asarray(mask_logits) + if cls_logits.shape != (1, queries, classes + 1) or not np.isfinite(cls_logits).all(): + raise ValueError(f"{model}: invalid cls logits {cls_logits.shape}") + if (mask_logits.ndim != 4 or mask_logits.shape[:2] != (1, queries) + or not np.isfinite(mask_logits).all()): + raise ValueError(f"{model}: invalid mask logits {mask_logits.shape}") + (nw, nh), scales, img_shape = geometry(original_size) + size = int(INPUT_SPEC.split("x")[-1]) + with torch.inference_mode(): + # MaskFormerHead.predict: upsample to batch_input_shape first. + masks = F.interpolate(torch.from_numpy(mask_logits), size=(size, size), + mode="bilinear", align_corners=False) + # MaskFormerFusionHead.predict: remove padding, then rescale. + masks = masks[0, ..., :nh, :nw] + ori_h, ori_w = int(original_size[1]), int(original_size[0]) + masks = F.interpolate(masks[:, None], size=(ori_h, ori_w), + mode="bilinear", align_corners=False)[:, 0] + result = _decoder(model).instance_postprocess( + torch.from_numpy(cls_logits)[0], masks) + return (result.bboxes.numpy(), result.scores.numpy(), result.labels.numpy(), + result.masks.numpy()) + + +def g2_input_variant(model, t1, item, input_spec): + """Diagnostic-input hook for harness/eval/run.py g2_inputs.""" + return preprocess_variant(model, t1.image(item), input_spec) + + +def predict_variant(model, runner, t1, input_spec, pose_evidence=None): + """Shared harness/eval/coco_extra.py hook: COCO segm RLE predictions.""" + import numpy as np + from pycocotools import mask as mask_utils + from ..eval.common import run_batched + if pose_evidence is not None: + raise ValueError(f"{model}: iseg predict takes no pose evidence") + cat_map = t1.manifest["cat_map"] + predictions = [] + for offset in range(0, len(t1.items), 16): + items = t1.items[offset:offset + 16] + images = [t1.image(it) for it in items] + xs = [preprocess_variant(model, image, input_spec) for image in images] + outputs = run_batched(runner, xs) + for it, image, outs in zip(items, images, outputs): + _, scores, labels, masks = decode_variant(model, outs, image.size) + for mask, score, label in zip(masks, scores, labels): + rle = mask_utils.encode(np.asfortranarray(mask.astype(np.uint8))) + rle["counts"] = rle["counts"].decode("ascii") + predictions.append({"image_id": it["id"], + "category_id": cat_map[str(int(label))], + "score": float(score), "segmentation": rle}) + return predictions + + +def build_variant(model, weights, input_spec): + """Construct the official Mask2Former and strictly load its checkpoint. + + MaskFormer has no bbox_head, so the shared openmmlab tensor route cannot + serve it; this builder uses the same public recipe/config/checkpoint + helpers and the same strict-load rules as openmmlab.build. + """ + import importlib + import torch + from mmengine.registry import init_default_scope + from mmengine.runner import load_checkpoint + + if model not in MODELS or input_spec != INPUT_SPEC: + raise ValueError(f"{model}: expected {INPUT_SPEC}") + model_recipe = get_recipe(model) + if model_recipe["resolution_status"] != "resolved_catalog": + raise ValueError(f"Unresolved recipe {model}: {model_recipe['blockers']}") + if input_spec != model_recipe["input_spec"]: + raise ValueError(f"{model}: recipe input {model_recipe['input_spec']}, got {input_spec}") + sha = verify_checkpoint(model_recipe, weights) + cfg = load_config(model) + importlib.import_module("mmdet.utils").register_all_modules(init_default_scope=False) + init_default_scope("mmdet") + registry = importlib.import_module("mmdet.registry") + network = registry.MODELS.build(cfg.model) + # MMEngine marks normalization buffers non-persistent; official checkpoints + # keep them. Preserve every key and verify before restoring persistence. + state = torch.load(str(weights), map_location="cpu")["state_dict"] + preprocessor = network.data_preprocessor + for name in ("mean", "std"): + key = f"data_preprocessor.{name}" + if key in state and name in preprocessor._non_persistent_buffers_set: + expected = getattr(preprocessor, name) + if not torch.equal(state[key], expected): + raise ValueError(f"Checkpoint {key} disagrees with the official config") + preprocessor.register_buffer(name, expected, persistent=True) + load_checkpoint(network, str(weights), map_location="cpu", strict=True) + network.cpu().eval() + + depth3, embed, queries, classes, stuff, flavor = CONTRACTS[model] + head = network.panoptic_head + dp = cfg.model.data_preprocessor + meta = recipe(model) + if (type(network).__name__ != "Mask2Former" or type(head).__name__ != "Mask2FormerHead" + or type(network.backbone).__name__ != "SwinTransformer" + or [len(stage.blocks) for stage in network.backbone.stages] != [2, 2, depth3, 2] + or network.backbone.patch_embed.projection.out_channels != embed + or head.num_queries != queries or head.num_classes != classes + or head.num_things_classes != classes - stuff or head.num_stuff_classes != stuff + or dict(cfg.model.test_cfg) != TEST_CFGS[flavor] + or list(dp.mean) != meta["mean"] or list(dp.std) != meta["std"] + or not dp.bgr_to_rgb or dp.pad_size_divisor != 32): + raise ValueError("Pinned official Mask2Former contract changed") + # Verify every learned parameter against the checkpoint bytes. + for name, parameter in network.named_parameters(): + torch.testing.assert_close(parameter, state[name], rtol=0, atol=0) + del state + + class RawQueries(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = network + self.upstream.cfg = cfg + self.cfg = cfg + self.checkpoint_sha256 = sha + + def forward(self, x): + # Mask2FormerHead.forward is metadata-free: batch size from x. + cls_pred_list, mask_pred_list = self.upstream.panoptic_head( + self.upstream.extract_feat(x), None) + return cls_pred_list[-1], mask_pred_list[-1] + + return RawQueries().eval() diff --git a/harness/adapters/mmdet_solo_common.py b/harness/adapters/mmdet_solo_common.py new file mode 100644 index 0000000..3ae582a --- /dev/null +++ b/harness/adapters/mmdet_solo_common.py @@ -0,0 +1,209 @@ +"""Pinned MMDetection SOLO/SOLOv2 instance segmentation at static 640. + +Both heads emit dense static tensors (per-grid mask/kernel predictions, class +scores and a unified mask feature map); the official matrix-NMS decode and the +dynamic mask convolutions are parameter-free and run in host decode, exactly +like the dense-detector decoders. SOLO result bboxes are upstream placeholders +(all zeros); masks carry the real geometry. + +The test pipeline keeps the default uint8 load (no to_float32), so the image +recipe and calibration digest stay shared with the dense detectors. +""" +from functools import lru_cache + +from .openmmlab import build as build_official, get_recipe +from .mmdet_dense_common import RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, geometry + +MODELS = ("solo.r50", "solov2.r50") +GRIDS = (40, 36, 24, 16, 12) +STRIDES = (8, 8, 16, 32, 32) +TEST_CFG = {"nms_pre": 500, "score_thr": .1, "mask_thr": .5, "filter_thr": .05, + "kernel": "gaussian", "sigma": 2.0, "max_per_img": 100} + + +def recipe(model): + """Same image recipe as the dense detectors once normalization is verified.""" + if model not in MODELS: + raise ValueError(model) + from .mmdet_dense_common import recipe as image_recipe + meta = image_recipe("retinanet.r50.fpn") + dp = get_recipe(model)["preprocess"]["official_data_preprocessor"] + if (dp["mean"] != meta["mean"] or dp["std"] != meta["std"] + or not dp["bgr_to_rgb"] or dp["pad_size_divisor"] != 32): + raise ValueError("Official SOLO image normalization changed") + return meta + + +def preprocess_variant(model, image, input_spec=INPUT_SPEC): + from .mmdet_dense_common import preprocess_variant as image_preprocess + recipe(model) # verify exact equality before reusing the image-only helper + return image_preprocess("retinanet.r50.fpn", image, input_spec) + + +def output_names(model): + if model not in MODELS: + raise ValueError(model) + first = "mask" if model == "solo.r50" else "kernel" + names = [f"{first}_g{g}" for g in GRIDS] + [f"cls_g{g}" for g in GRIDS] + if model == "solov2.r50": + names.append("mask_feats") + return names + + +DEBUG_OUTPUT_NAMES = [f"fpn{i}" for i in range(5)] + + +def build_debug_variant(model, weights, input_spec): + """Backbone+FPN taps: the continuous segment gated at .9999 cosine. + + The SOLO heads stack seven conv+GroupNorm+ReLU blocks over generated + coordinate grids; fp reassociation across frameworks flips near-zero + pre-activations and shifts head logits by a few 1e-3 (measured, identical + at opset 11 and 16). Head equivalence is therefore judged on decoded + predictions and the COCO metric, not on raw logits. + """ + network = build_variant(model, weights, input_spec) + import torch + + class NeckTaps(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = network.upstream + + def forward(self, x): + return tuple(self.upstream.extract_feat(x)) + + return NeckTaps().eval() + + +@lru_cache(maxsize=2) +def _decoder(model): + """Official parameter-free SOLO decode context; no learned weights.""" + from mmengine.config import ConfigDict + if model == "solo.r50": + from mmdet.models.dense_heads.solo_head import SOLOHead + head = SOLOHead.__new__(SOLOHead) + head.cls_out_channels = 80 + else: + from mmdet.models.dense_heads.solov2_head import SOLOV2Head + head = SOLOV2Head.__new__(SOLOV2Head) + head.cls_out_channels = 80 + head.kernel_out_channels = 256 + head.dynamic_conv_size = 1 + head.mask_stride = 4 + head.num_grids = list(GRIDS) + head.num_levels = len(GRIDS) + head.strides = list(STRIDES) + head.test_cfg = ConfigDict(TEST_CFG) + return head + + +def decode_variant(model, outputs, original_size): + """Official predict_by_feat on the raw dense head tensors.""" + import numpy as np + import torch + if model not in MODELS: + raise ValueError(model) + count = 11 if model == "solov2.r50" else 10 + if len(outputs) != count: + raise ValueError(f"{model}: wrong raw head count {len(outputs)}") + tensors = [] + for index, value in enumerate(outputs): + array = np.asarray(value) + if array.ndim != 4 or not np.isfinite(array).all(): + raise ValueError(f"{model}: invalid raw head {index}: {array.shape}") + tensors.append(torch.as_tensor(np.ascontiguousarray(array), dtype=torch.float32)) + for level, grid in enumerate(GRIDS): + if tensors[5 + level].shape[1] != 80 or tensors[5 + level].shape[2:] != (grid, grid): + raise ValueError(f"{model}: unexpected class head {level}: {tensors[5 + level].shape}") + (nw, nh), scales, img_shape = geometry(original_size) + meta = dict(img_shape=img_shape, ori_shape=(int(original_size[1]), int(original_size[0])), + pad_shape=(640, 640), batch_input_shape=(640, 640), scale_factor=scales) + with torch.inference_mode(): + if model == "solo.r50": + result = _decoder(model).predict_by_feat(tensors[:5], tensors[5:10], [meta])[0] + else: + result = _decoder(model).predict_by_feat(tensors[:5], tensors[5:10], tensors[10], [meta])[0] + return (result.bboxes.numpy(), result.scores.numpy(), result.labels.numpy(), + result.masks.numpy()) + + +def g2_input_variant(model, t1, item, input_spec): + """Diagnostic-input hook for harness/eval/run.py g2_inputs.""" + return preprocess_variant(model, t1.image(item), input_spec) + + +def predict_variant(model, runner, t1, input_spec, pose_evidence=None): + """Shared harness/eval/coco_extra.py hook: COCO segm RLE predictions.""" + import numpy as np + from pycocotools import mask as mask_utils + from ..eval.common import run_batched + if pose_evidence is not None: + raise ValueError(f"{model}: iseg predict takes no pose evidence") + cat_map = t1.manifest["cat_map"] + predictions = [] + for offset in range(0, len(t1.items), 16): + items = t1.items[offset:offset + 16] + images = [t1.image(it) for it in items] + xs = [preprocess_variant(model, image, input_spec) for image in images] + outputs = run_batched(runner, xs) + for it, image, outs in zip(items, images, outputs): + _, scores, labels, masks = decode_variant(model, outs, image.size) + for mask, score, label in zip(masks, scores, labels): + rle = mask_utils.encode(np.asfortranarray(mask.astype(np.uint8))) + rle["counts"] = rle["counts"].decode("ascii") + predictions.append({"image_id": it["id"], + "category_id": cat_map[str(int(label))], + "score": float(score), "segmentation": rle}) + return predictions + + +def build_variant(model, weights, input_spec): + """Construct the official SOLO model and strictly load its checkpoint.""" + import torch + + if model not in MODELS or input_spec != INPUT_SPEC: + raise ValueError(f"{model}: expected {INPUT_SPEC}") + raw = build_official(model, weights, input_spec) + network, cfg = raw.upstream, raw.cfg + meta = recipe(model) + dp = cfg.model.data_preprocessor + kind, head_kind = (("SOLO", "SOLOHead") if model == "solo.r50" + else ("SOLOv2", "SOLOV2Head")) + head = network.mask_head + if (type(network).__name__ != kind or type(head).__name__ != head_kind + or network.with_bbox or head.num_classes != 80 + or list(head.strides) != list(STRIDES) or list(head.num_grids) != list(GRIDS) + or dict(head.test_cfg) != TEST_CFG + or list(dp.mean) != meta["mean"] or list(dp.std) != meta["std"] + or not dp.bgr_to_rgb or dp.pad_size_divisor != 32): + raise ValueError("Pinned official SOLO contract changed") + if model == "solov2.r50" and (head.kernel_out_channels != 256 + or head.dynamic_conv_size != 1 or head.mask_stride != 4): + raise ValueError("Pinned official SOLOv2 mask geometry changed") + # Verify every learned parameter against the checkpoint bytes. + state = torch.load(str(weights), map_location="cpu")["state_dict"] + for name, parameter in network.named_parameters(): + torch.testing.assert_close(parameter, state[name], rtol=0, atol=0) + del state + + class RawHeads(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = network + self.upstream.cfg = cfg + self.cfg = cfg + self.checkpoint_sha256 = raw.checkpoint_sha256 + + def forward(self, x): + outs = self.upstream.extract_feat(x) + results = self.upstream.mask_head.forward(outs) + flat = [] + for group in results: + if isinstance(group, (list, tuple)): + flat.extend(group) + else: + flat.append(group) + return tuple(flat) + + return RawHeads().eval() diff --git a/harness/adapters/mmseg_common.py b/harness/adapters/mmseg_common.py new file mode 100644 index 0000000..862a00b --- /dev/null +++ b/harness/adapters/mmseg_common.py @@ -0,0 +1,83 @@ +"""Reuse pinned MMSeg models with an explicit static-image benchmark recipe.""" +from pathlib import Path + +from .openmmlab import build as official_build, get_recipe + +RUNTIME_PYTHON = "/home/ming01.wang/outputs/full-coverage-20260917/openmmlab/venv/bin/python" +EXPORTER_VERSION = "mmseg-static-raw-logits-v1" + + +def lower_static_pools(network): + """Export exact adaptive-pooling bins for the fixed deployment shape.""" + import torch + + class StaticPool(torch.nn.AdaptiveAvgPool2d): + def forward(self, x): + if not torch.onnx.is_in_onnx_export(): + return super().forward(x) + oh, ow = (self.output_size, self.output_size) if isinstance(self.output_size, int) else self.output_size + h, w = map(int, x.shape[-2:]) + if h % oh == 0 and w % ow == 0: + return super().forward(x) + rows = [] + for i in range(oh): + top, bottom = i * h // oh, ((i + 1) * h + oh - 1) // oh + columns = [] + for j in range(ow): + left, right = j * w // ow, ((j + 1) * w + ow - 1) // ow + columns.append(x[..., top:bottom, left:right].mean((-2, -1), keepdim=True)) + rows.append(torch.cat(columns, dim=-1)) + return torch.cat(rows, dim=-2) + + for name, child in list(network.named_children()): + if isinstance(child, torch.nn.AdaptiveAvgPool2d): + setattr(network, name, StaticPool(child.output_size)) + else: + lower_static_pools(child) + + +def metadata(model: str): + recipe = get_recipe(model) + if recipe["task"] != "seg" or recipe["source"]["checkout"] != "mmsegmentation": + raise ValueError(f"Not an MMSeg semantic segmentation recipe: {model}") + data = recipe["preprocess"]["official_data_preprocessor"] + _, _, h, w = map(int, recipe["input_spec"].split("x")) + checkpoint = dict(recipe["checkpoint"], classes={"ade20k": 150, "cityscapes": 19}[recipe["dataset"]]) + spec = {"source": "custom", "dataset": recipe["dataset"], "input_spec": recipe["input_spec"]} + preprocess = {"recipe": "mmseg-static-resize-v1", "input_spec": recipe["input_spec"], + "dataset": recipe["dataset"], "color": "RGB" if data.get("bgr_to_rgb") else "BGR", + "mean": data["mean"], "std": data["std"], "scale": 1.0, + "layout": "NCHW", "dtype": "float32", "resize": [h, w], + "interpolation": "opencv_inter_linear", "logits_resize": "bilinear", + "align_corners": False, "logits_intermediate_size": [h, w], + "protocol": "bench static resize; whole-image logits; original label coordinates"} + return checkpoint, spec, preprocess + + +def preprocess_image(recipe: dict, image, input_spec: str): + import cv2 + import numpy as np + + if input_spec != recipe["input_spec"]: + raise ValueError(f"MMSeg input differs from verified recipe: {input_spec}") + h, w = recipe["resize"] + x = np.asarray(image.convert("RGB")) + if x.shape[:2] != (h, w): + x = cv2.resize(x, (w, h), interpolation=cv2.INTER_LINEAR) + if recipe["color"] == "BGR": + x = x[:, :, ::-1] + x = (x.astype(np.float32) - np.asarray(recipe["mean"], np.float32)) / np.asarray(recipe["std"], np.float32) + return np.ascontiguousarray(x.transpose(2, 0, 1)[None]) + + +def build_model(model: str, weights: Path, input_spec: str): + network = official_build(model, weights, input_spec) + head = network.upstream.decode_head + checkpoint, _, recipe = metadata(model) + if head.align_corners != recipe["align_corners"] or head.num_classes != checkpoint["classes"]: + raise ValueError("Official head differs from the frozen label/resize recipe") + if network.upstream.test_cfg.mode != "whole": + raise ValueError("This adapter requires official whole-image inference") + if model.startswith(("pspnet.", "upernet.")): + lower_static_pools(network.upstream.decode_head) + return network diff --git a/harness/adapters/ocrnet_hrnet_w18.py b/harness/adapters/ocrnet_hrnet_w18.py new file mode 100644 index 0000000..e88808b --- /dev/null +++ b/harness/adapters/ocrnet_hrnet_w18.py @@ -0,0 +1,17 @@ +"""Official ADE20K OCRNet-HRNet18 with both learned cascade heads.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, RUNTIME_PYTHON, EXPORTER_VERSION +from .openmmlab import build as official_build + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("ocrnet.hrnet.w18") +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = official_build("ocrnet.hrnet.w18", weights, input_spec) + heads = network.upstream.decode_head + if (len(heads) != 2 or any(head.align_corners is not False or + head.num_classes != CHECKPOINT["classes"] for head in heads) + or network.upstream.test_cfg.mode != "whole"): + raise ValueError("Official OCRNet cascade differs from its frozen geometry") + return network diff --git a/harness/adapters/ocrnet_hrnet_w48.py b/harness/adapters/ocrnet_hrnet_w48.py new file mode 100644 index 0000000..972121f --- /dev/null +++ b/harness/adapters/ocrnet_hrnet_w48.py @@ -0,0 +1,17 @@ +"""Official ADE20K OCRNet-HRNet48 with both learned cascade heads.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, RUNTIME_PYTHON, EXPORTER_VERSION +from .openmmlab import build as official_build + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("ocrnet.hrnet.w48") +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = official_build("ocrnet.hrnet.w48", weights, input_spec) + heads = network.upstream.decode_head + if (len(heads) != 2 or any(head.align_corners is not False or + head.num_classes != CHECKPOINT["classes"] for head in heads) + or network.upstream.test_cfg.mode != "whole"): + raise ValueError("Official OCRNet cascade differs from its frozen geometry") + return network diff --git a/harness/adapters/openmmlab.py b/harness/adapters/openmmlab.py new file mode 100644 index 0000000..b335507 --- /dev/null +++ b/harness/adapters/openmmlab.py @@ -0,0 +1,184 @@ +"""Pinned official OpenMMLab construction; catalog resolution is not admission. + +The existing harness may call get_recipe(model) without importing PyTorch or +OpenMMLab. build() runs only in an explicitly provisioned isolated environment. +It never downloads, installs, substitutes architectures, or changes the registry. +The returned module consumes ALREADY PREPROCESSED float NCHW tensors and returns +upstream raw head tensors. Official geometry/decoding and board evaluation remain +separate gates; a successful build/export does not satisfy them. +""" +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +from functools import lru_cache +from copy import deepcopy +import yaml +from pathlib import Path +import subprocess + +_RECIPES = Path(__file__).with_name("openmmlab_recipes.yaml") +_PACKAGES = {"mmdetection": "mmdet", "mmsegmentation": "mmseg", "mmpose": "mmpose"} + + +@lru_cache(maxsize=1) +def _catalog() -> dict: + return yaml.safe_load(_RECIPES.read_text())["recipes"] + + +def get_recipe(model: str) -> dict: + """Return a detached recipe from the once-loaded catalog.""" + return deepcopy(_catalog()[model]) + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def _source(recipe: dict, source_root: Path | None = None) -> tuple[Path, str]: + source = recipe["source"] + checkout = source["checkout"] + if checkout not in _PACKAGES: + raise RuntimeError(f"{checkout} requires its separately recorded legacy environment/adapter") + package = _PACKAGES[checkout] + # Use the actual imported official package, or the caller's pinned checkout. + spec = importlib.util.find_spec(package) + if spec is None or spec.origin is None: + raise RuntimeError(f"Missing {package}; provision the isolated recipe environment first") + installed_root = Path(spec.origin).resolve().parent.parent + root = Path(source_root).resolve() if source_root else installed_root + if installed_root != root: + raise RuntimeError(f"Imported {package} comes from {installed_root}, not requested {root}") + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + dirty = subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=no"], cwd=root, text=True + ).strip() + if revision != source["revision"] or dirty: + raise RuntimeError(f"Unverified {package} checkout: expected clean {source['revision']}; got {revision}") + config = root / source["config"] + if _sha256(config) != source["config_sha256"]: + raise RuntimeError(f"Config hash mismatch: {config}") + return config, package + + +def verify_checkpoint(recipe: dict, weights: Path) -> str: + """Require the full digest recorded from an official-download receipt. + + Filename tokens are provenance only: some official MMPose files retain + an old token after checkpoint conversion and do not match their SHA256. + """ + weights = Path(weights) + if not weights.is_file(): + raise FileNotFoundError(f"Official checkpoint required: {recipe['checkpoint']['url']}") + sha = _sha256(weights) + info = recipe["checkpoint"] + expected = info.get("sha256") + if not expected or len(expected) != 64: + raise RuntimeError("Recipe lacks a verifiable checkpoint digest; record the official download hash first") + if sha != expected: + raise ValueError(f"Checkpoint identity mismatch: expected {expected}, got {sha}") + return sha + + +def load_config(model: str, *, source_root: Path | None = None): + """Parse official config and scoped bases from the verified source checkout. + + Editable installs can lack the wheel-only .mim resources. Keep MMEngine's + parser and scope handling; resolve same-package files in the pinned repo. + """ + config_path, package = _source(get_recipe(model), source_root) + from mmengine.config import Config + from unittest.mock import patch + + config_root = Path(importlib.util.find_spec(package).origin).resolve().parents[1] / "configs" + original = Config._get_cfg_path + + def resolve(base, filename): + if base.startswith(f"{package}::"): + candidate = (config_root / base.split("::", 1)[1]).resolve() + if candidate.is_relative_to(config_root) and candidate.is_file(): + return str(candidate), package + return original(base, filename) + + with patch.object(Config, "_get_cfg_path", staticmethod(resolve)): + return Config.fromfile(str(config_path)) + + +def build(model: str, weights: Path, input_spec: str, *, source_root: Path | None = None): + """Construct the exact official model and strictly load its complete weights. + + No tensor/key remapping or non-strict fallback is provided. Upstream load + hooks handle their own historical format migrations; MMEngine's documented + DDP ``module.`` prefix removal is retained. Evidence records strict=True. + Input normalization, affine geometry and output decoding are NOT implicit. + """ + recipe = get_recipe(model) + if recipe["resolution_status"] != "resolved_catalog": + raise ValueError(f"Unresolved recipe {model}: {recipe['blockers']}") + sha = verify_checkpoint(recipe, weights) + shape = tuple(int(v) for v in input_spec.split("x")) + if len(shape) != 4 or shape[0:2] != (1, 3) or min(shape) <= 0: + raise ValueError(f"Expected static batch-one NCHW RGB/BGR input, got {input_spec}") + expected = recipe.get("input_spec") + if not expected: + raise RuntimeError(f"{model}: static probe input contract has not been resolved") + if input_spec != expected: + raise ValueError(f"{model}: recipe input {expected}, got {input_spec}; no silent shape substitution") + + import torch + from mmengine.runner import load_checkpoint + from mmengine.registry import init_default_scope + + cfg = load_config(model, source_root=source_root) + package = _PACKAGES[recipe["source"]["checkout"]] + utils = importlib.import_module(f"{package}.utils") + utils.register_all_modules(init_default_scope=False) + init_default_scope(package) + registry = importlib.import_module(f"{package}.registry") + network = registry.MODELS.build(cfg.model) + # MMEngine now marks normalization buffers non-persistent; older official + # checkpoints saved them. Preserve every key and strictly compare its value + # with this recipe before restoring persistence. Never drop learned keys. + state = torch.load(str(weights), map_location="cpu")["state_dict"] + preprocessor = network.data_preprocessor + for name in ("mean", "std"): + key = f"data_preprocessor.{name}" + # MMPose's official migration hook removes legacy non-learned norm keys. + if package != "mmpose" and key in state and name in preprocessor._non_persistent_buffers_set: + expected_buffer = getattr(preprocessor, name) + if not torch.equal(state[key], expected_buffer): + raise ValueError(f"Checkpoint {key} disagrees with the official config") + preprocessor.register_buffer(name, expected_buffer, persistent=True) + del state + # Do not call init_weights(): the complete trained checkpoint is mandatory. + load_checkpoint(network, str(weights), map_location="cpu", strict=True) + network.cpu().eval() + + class RawTensorModel(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = network + self.openmmlab_recipe = recipe + self.checkpoint_sha256 = sha + self.cfg = cfg + + def forward(self, x): + # Two-stage detector/transformer heads require image metadata even + # in tensor mode. It describes this already-preprocessed tensor, + # NOT a claim about original-image resize or coordinate recovery. + samples = None + if package == "mmdet": + from mmdet.structures import DetDataSample + samples = [DetDataSample(metainfo=dict( + img_shape=shape[-2:], ori_shape=shape[-2:], + pad_shape=shape[-2:], batch_input_shape=shape[-2:], + scale_factor=(1., 1.) + ))] + return self.upstream(x, samples, mode="tensor") + + return RawTensorModel().eval() diff --git a/harness/adapters/openmmlab_recipes.yaml b/harness/adapters/openmmlab_recipes.yaml new file mode 100644 index 0000000..1a7d6a5 --- /dev/null +++ b/harness/adapters/openmmlab_recipes.yaml @@ -0,0 +1,5097 @@ +schema: openmmlab_recipe_catalog.v1 +validated: false +evidence: outputs/full-coverage-20260917/openmmlab/ +recipes: + faster.r.cnn.r50.fpn: + model_id: faster.r.cnn.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py + config_sha256: 0a6f25c90df0393e48f4a4749b54df4ec3c01b26ecf9bb882d13cb0d92b338f4 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth + filename: faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth + sha256: null + official_model_name: faster-rcnn_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + faster.r.cnn.r101.fpn: + model_id: faster.r.cnn.r101.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/faster_rcnn/faster-rcnn_r101_fpn_1x_coco.py + config_sha256: b36044c117465c096b8e4889c2042379c61b15fd26368218dade840d38c84479 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r101_fpn_1x_coco/faster_rcnn_r101_fpn_1x_coco_20200130-f513f705.pth + filename: faster_rcnn_r101_fpn_1x_coco_20200130-f513f705.pth + sha256: null + official_model_name: faster-rcnn_r101_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + cascade.r.cnn.r50.fpn: + model_id: cascade.r.cnn.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py + config_sha256: cc2cb7d9398202e090dacae8b0196176c0992787725a7d99b656c0e5973d2d4d + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_rcnn_r50_fpn_1x_coco/cascade_rcnn_r50_fpn_1x_coco_20200316-3dc56deb.pth + filename: cascade_rcnn_r50_fpn_1x_coco_20200316-3dc56deb.pth + sha256: null + official_model_name: cascade-rcnn_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + cascade.r.cnn.r101.fpn: + model_id: cascade.r.cnn.r101.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/cascade_rcnn/cascade-rcnn_r101_fpn_1x_coco.py + config_sha256: 975f791f66beb7f85596969ba7e968c68ff1ef3f5ac06d0a8afedd8505630f11 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_rcnn_r101_fpn_1x_coco/cascade_rcnn_r101_fpn_1x_coco_20200317-0b6a2fbf.pth + filename: cascade_rcnn_r101_fpn_1x_coco_20200317-0b6a2fbf.pth + sha256: null + official_model_name: cascade-rcnn_r101_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + retinanet.r50.fpn: + model_id: retinanet.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/retinanet/retinanet_r50_fpn_1x_coco.py + config_sha256: ca6a8abfd62f5fdafba16e18ef260cb71df18df716cee8da04c9a4e0cc865552 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_1x_coco/retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth + filename: retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth + sha256: c2398f9ec0843ed9a29e72d7a788741abbf2b4250e64f8f793092eaecaa0ab4f + official_model_name: retinanet_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + retinanet.r101.fpn: + model_id: retinanet.r101.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/retinanet/retinanet_r101_fpn_1x_coco.py + config_sha256: ef6f219cd8807bebac0a2222ce24b5df3ae2df22192c4d3968fe596df697f2bc + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_1x_coco/retinanet_r101_fpn_1x_coco_20200130-7a93545f.pth + filename: retinanet_r101_fpn_1x_coco_20200130-7a93545f.pth + sha256: 7a93545fbb3bfbd4a34f25733bf6ef07888f1d5df639dd04137f85da83460d46 + official_model_name: retinanet_r101_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + fcos.r50.fpn: + model_id: fcos.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/fcos/fcos_r50-caffe_fpn_gn-head_1x_coco.py + config_sha256: a0e63a69104c2106967dfd7d6a4ad3d49041e1922a210ebbe413c628764885f0 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_1x_coco/fcos_r50_caffe_fpn_gn-head_1x_coco-821213aa.pth + filename: fcos_r50_caffe_fpn_gn-head_1x_coco-821213aa.pth + sha256: 821213aad703a60f243b8da533391459d924eddd2c1a89fa11ff23ddf308b6b0 + official_model_name: fcos_r50-caffe_fpn_gn-head_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 102.9801 + - 115.9465 + - 122.7717 + std: + - 1.0 + - 1.0 + - 1.0 + bgr_to_rgb: false + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + atss.r50.fpn: + model_id: atss.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/atss/atss_r50_fpn_1x_coco.py + config_sha256: ee1d42765e3f3fe97d5feb5611309a22eea64fa7a5dd93d26435a2de0f40da99 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r50_fpn_1x_coco/atss_r50_fpn_1x_coco_20200209-985f7bd0.pth + filename: atss_r50_fpn_1x_coco_20200209-985f7bd0.pth + sha256: 985f7bd0daaafc240ec848d2480b686dd23f7114c6a28b78796247ffa6d70da8 + official_model_name: atss_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + gfl.r50.fpn: + model_id: gfl.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/gfl/gfl_r50_fpn_1x_coco.py + config_sha256: 2170bee8162089c3834b1c1807a773296837aac7b9f58ae4f00abc72bceb7f6d + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_1x_coco/gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth + filename: gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth + sha256: 259442872724852384cf6dc6a453a6b03325f5e0730a5b4105efb180469490c3 + official_model_name: gfl_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + vfnet.r50.fpn: + model_id: vfnet.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/vfnet/vfnet_r50_fpn_1x_coco.py + config_sha256: 54b98dfef80ab5609a28a98466468d4b53ebb1d5bc743833adb4399c0255048c + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/vfnet/vfnet_r50_fpn_1x_coco/vfnet_r50_fpn_1x_coco_20201027-38db6f58.pth + filename: vfnet_r50_fpn_1x_coco_20201027-38db6f58.pth + sha256: 38db6f58bc9d56e1f7a0353c71ca9c00298a5b924437968eb261b38eececf9c2 + official_model_name: vfnet_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + foveabox.r50.fpn: + model_id: foveabox.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/foveabox/fovea_r50_fpn_4xb4-1x_coco.py + config_sha256: d822bed3edc4c87cc6f2d82c921e09ae78a4c2610b975b93913bd58f76031b7a + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/foveabox/fovea_r50_fpn_4x4_1x_coco/fovea_r50_fpn_4x4_1x_coco_20200219-ee4d5303.pth + filename: fovea_r50_fpn_4x4_1x_coco_20200219-ee4d5303.pth + sha256: ee4d53031ff155a5ef66289b9e0cabe2aef5c5dc52af83a1e89230fb1d2189eb + official_model_name: fovea_r50_fpn_4xb4-1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + fsaf.r50.fpn: + model_id: fsaf.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/fsaf/fsaf_r50_fpn_1x_coco.py + config_sha256: 6d604ac26276f56be618221e63c9c98a3352f837f36758fb14c7005b376948fc + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/fsaf/fsaf_r50_fpn_1x_coco/fsaf_r50_fpn_1x_coco-94ccc51f.pth + filename: fsaf_r50_fpn_1x_coco-94ccc51f.pth + sha256: 94ccc51fcd324371754818b43e724fad95eaaa8153a24a9fd40fb5868c1c5c06 + official_model_name: fsaf_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + reppoints.r50.fpn: + model_id: reppoints.r50.fpn + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/reppoints/reppoints-moment_r50_fpn_1x_coco.py + config_sha256: e71ede4bb92f98b52d7a692b7bede6c72a73e16712f04b448b3ba96ed73b1e5d + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/reppoints/reppoints_moment_r50_fpn_1x_coco/reppoints_moment_r50_fpn_1x_coco_20200330-b73db8d1.pth + filename: reppoints_moment_r50_fpn_1x_coco_20200330-b73db8d1.pth + sha256: b73db8d10a98dd7a182eaddfb51451822a612f632419c08f48eab2b452972b1c + official_model_name: reppoints-moment_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + cornernet.hourglass.104: + model_id: cornernet.hourglass.104 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py + config_sha256: bb495ccab79fbdbf1abffcc6cf8bd8571823d88b12e5fa9f86c02d0fdeab7727 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/cornernet/cornernet_hourglass104_mstest_8x6_210e_coco/cornernet_hourglass104_mstest_8x6_210e_coco_20200825_150618-79b44c30.pth + filename: cornernet_hourglass104_mstest_8x6_210e_coco_20200825_150618-79b44c30.pth + sha256: null + official_model_name: cornernet_hourglass104_8xb6-210e-mstest_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + ssd300.vgg16: + model_id: ssd300.vgg16 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x300x300 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/ssd/ssd300_coco.py + config_sha256: 7fb559c523ea2423229807992e9ca6e82c21590e44d237041c0cfb6ab1bcbd8c + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/ssd/ssd300_coco/ssd300_coco_20210803_015428-d231a06e.pth + filename: ssd300_coco_20210803_015428-d231a06e.pth + sha256: null + official_model_name: ssd300_coco + preprocess: + input_spec: 1x3x300x300 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 1 + - 1 + - 1 + bgr_to_rgb: true + pad_size_divisor: 1 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x300x300; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + ssdlite320.mobilenetv3.large: + model_id: ssdlite320.mobilenetv3.large + task: det + resolution_status: mainline_exact_source_correction + input_spec: null + dataset: coco2017 + validated: false + model_spec: + source: torchvision + dataset: coco2017 + external_source: + documentation: https://docs.pytorch.org/vision/stable/models/generated/torchvision.models.detection.ssdlite320_mobilenet_v3_large.html + factory: torchvision.models.detection.ssdlite320_mobilenet_v3_large + weights_enum: SSDLite320_MobileNet_V3_Large_Weights.COCO_V1 + owner: mainline + blockers: + - code: source_entry_correction + owner: public_code + detail: Exactly one CSV row. The exact SSDLite320-MobileNetV3-Large COCO model is officially available + in torchvision; mainline owns the explicit source override. This is an entry correction, not an architecture + substitution or user material gap. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + yolov3.darknet53: + model_id: yolov3.darknet53 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/yolo/yolov3_d53_8xb8-ms-608-273e_coco.py + config_sha256: ffa39f0aea4d0c0e86b6959cc91c6336a9555288d2355faac66695ca70440c00 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/yolo/yolov3_d53_mstrain-608_273e_coco/yolov3_d53_mstrain-608_273e_coco_20210518_115020-a2c3acb8.pth + filename: yolov3_d53_mstrain-608_273e_coco_20210518_115020-a2c3acb8.pth + sha256: null + official_model_name: yolov3_d53_mstrain-608_273e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 0 + - 0 + - 0 + std: + - 255.0 + - 255.0 + - 255.0 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + rtmdet.tiny: + model_id: rtmdet.tiny + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/rtmdet/rtmdet_tiny_8xb32-300e_coco.py + config_sha256: 873923f8221c2e5ae6501dae248eddf26743daeccb82ae9055cb6435fcabc992 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet_tiny_8xb32-300e_coco/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth + filename: rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth + sha256: 78e30dcce0c6f594eaff0d6977b84b4103688b4aff0ad1aa16008a8cc854a7fb + official_model_name: rtmdet_tiny_8xb32-300e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 103.53 + - 116.28 + - 123.675 + std: + - 57.375 + - 57.12 + - 58.395 + bgr_to_rgb: false + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + probe: + status: raw_fp32_export_probe_pass + strict_load: true + real_image_forward: true + onnx_export: true + onnxruntime_parity: true + evidence: outputs/full-coverage-20260917/openmmlab/probes/rtmdet.tiny/probe.json + rtmdet.s: + model_id: rtmdet.s + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/rtmdet/rtmdet_s_8xb32-300e_coco.py + config_sha256: 4be51e72681ced00447b4a4f84bb801b780bb261924516799048e51825325607 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet_s_8xb32-300e_coco/rtmdet_s_8xb32-300e_coco_20220905_161602-387a891e.pth + filename: rtmdet_s_8xb32-300e_coco_20220905_161602-387a891e.pth + sha256: 387a891e157cf0ab57d76b3ffc17bf77247089d672532427930b3140f9e789d6 + official_model_name: rtmdet_s_8xb32-300e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 103.53 + - 116.28 + - 123.675 + std: + - 57.375 + - 57.12 + - 58.395 + bgr_to_rgb: false + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + rtmdet.m: + model_id: rtmdet.m + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/rtmdet/rtmdet_m_8xb32-300e_coco.py + config_sha256: c04a67ac0fbb48df14ada534f16186e1e4eb6cd5bb5ad0c32b1c64835f4e39e4 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet_m_8xb32-300e_coco/rtmdet_m_8xb32-300e_coco_20220719_112220-229f527c.pth + filename: rtmdet_m_8xb32-300e_coco_20220719_112220-229f527c.pth + sha256: 229f527ca88498e8894a778a62a878a322b4a3ea2cae09ea537d34b7e907792b + official_model_name: rtmdet_m_8xb32-300e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 103.53 + - 116.28 + - 123.675 + std: + - 57.375 + - 57.12 + - 58.395 + bgr_to_rgb: false + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + rtmdet.l: + model_id: rtmdet.l + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/rtmdet/rtmdet_l_8xb32-300e_coco.py + config_sha256: 71ba745a1d08f734d94797a98e5ee2be0bcf30dc98b3b2a89aa1ce211174d274 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet_l_8xb32-300e_coco/rtmdet_l_8xb32-300e_coco_20220719_112030-5a0be7c4.pth + filename: rtmdet_l_8xb32-300e_coco_20220719_112030-5a0be7c4.pth + sha256: 5a0be7c4a8123d1e634ce09b08682af373f503de2040e1532d254791f9332cec + official_model_name: rtmdet_l_8xb32-300e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 103.53 + - 116.28 + - 123.675 + std: + - 57.375 + - 57.12 + - 58.395 + bgr_to_rgb: false + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + rtmdet.x: + model_id: rtmdet.x + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/rtmdet/rtmdet_x_8xb32-300e_coco.py + config_sha256: 00f19ecd7e8ab72a19098481f5ef68b30fc667972bd81cbc29ca859b1176c7a7 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet_x_8xb32-300e_coco/rtmdet_x_8xb32-300e_coco_20220715_230555-cc79b9ae.pth + filename: rtmdet_x_8xb32-300e_coco_20220715_230555-cc79b9ae.pth + sha256: cc79b9ae0970f7f597e2aa5f81663129f3aa24e5612549b4f1a383da7fff825d + official_model_name: rtmdet_x_8xb32-300e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 103.53 + - 116.28 + - 123.675 + std: + - 57.375 + - 57.12 + - 58.395 + bgr_to_rgb: false + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + detr.r50: + model_id: detr.r50 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/detr/detr_r50_8xb2-150e_coco.py + config_sha256: 9474db908e12cd0cf27d8fc0925929ed224f077b25d35dfb70df240e3edb1863 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/detr/detr_r50_8xb2-150e_coco/detr_r50_8xb2-150e_coco_20221023_153551-436d03e8.pth + filename: detr_r50_8xb2-150e_coco_20221023_153551-436d03e8.pth + sha256: 436d03e8871a50b012994c43782c02662c791110f1f71e023db7f15538f52f5f + official_model_name: detr_r50_8xb2-150e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 1 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + deformable.detr.r50: + model_id: deformable.detr.r50 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/deformable_detr/deformable-detr_r50_16xb2-50e_coco.py + config_sha256: ac8a01033c7327a028423dd30f5e7c2b33661f018c1503e24ab85ccd924a4160 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/deformable_detr/deformable-detr_r50_16xb2-50e_coco/deformable-detr_r50_16xb2-50e_coco_20221029_210934-6bc7d21b.pth + filename: deformable-detr_r50_16xb2-50e_coco_20221029_210934-6bc7d21b.pth + sha256: 6bc7d21b4139f652065fb1df11fed6557bf8f7e77e60188cecfde2accc850381 + official_model_name: deformable-detr_r50_16xb2-50e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 1 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + conditional.detr.r50: + model_id: conditional.detr.r50 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/conditional_detr/conditional-detr_r50_8xb2-50e_coco.py + config_sha256: 48b07d312c2df3a3ecd5304292df48d82c09e7ee4932a62b8b3b7ce8f06348a3 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/conditional_detr/conditional-detr_r50_8xb2-50e_coco/conditional-detr_r50_8xb2-50e_coco_20221121_180202-c83a1dc0.pth + filename: conditional-detr_r50_8xb2-50e_coco_20221121_180202-c83a1dc0.pth + sha256: c83a1dc0c9d8a4d9df8dcf1a422d491724d024218ef877f16a0debf5793bb4ad + official_model_name: conditional-detr_r50_8xb2-50e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 1 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + dab.detr.r50: + model_id: dab.detr.r50 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/dab_detr/dab-detr_r50_8xb2-50e_coco.py + config_sha256: af14452a85be7587519bb3d7e1d8656613dddf36558ced7b29c5c8f1b8e94314 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/dab_detr/dab-detr_r50_8xb2-50e_coco/dab-detr_r50_8xb2-50e_coco_20221122_120837-c1035c8c.pth + filename: dab-detr_r50_8xb2-50e_coco_20221122_120837-c1035c8c.pth + sha256: c1035c8c92465c7a3f0cebf08950ba7460384c44e5fd6b1dadd17b2efb320978 + official_model_name: dab-detr_r50_8xb2-50e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 1 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + dino.r50: + model_id: dino.r50 + task: det + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/dino/dino-4scale_r50_8xb2-12e_coco.py + config_sha256: 60f902024cb992151569237f2b4cf0c8133f195e431592e08851d08e1e4b5179 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/dino/dino-4scale_r50_8xb2-12e_coco/dino-4scale_r50_8xb2-12e_coco_20221202_182705-55b2bba2.pth + filename: dino-4scale_r50_8xb2-12e_coco_20221202_182705-55b2bba2.pth + sha256: null + official_model_name: dino-4scale_r50_8xb2-12e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 1 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize/pad and family-specific raw-head box decoding/NMS plus COCO + mapping; current YOLO decoder is not compatible. + fcn.resnet50: + model_id: fcn.resnet50 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/fcn/fcn_r50-d8_4xb4-160k_ade20k-512x512.py + config_sha256: 0bba61160a47e7d3a0646a4e6b9a09c07261267d4d97edecf86b3c3281bd3e5f + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/fcn/fcn_r50-d8_512x512_160k_ade20k/fcn_r50-d8_512x512_160k_ade20k_20200615_100713-4edbc3b4.pth + filename: fcn_r50-d8_512x512_160k_ade20k_20200615_100713-4edbc3b4.pth + sha256: 4edbc3b4095c42d3f327cfcdc330cf516f2639a2fdde9937cf007b6219b239b7 + official_model_name: fcn_r50-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + probe: + status: raw_fp32_export_probe_pass + strict_load: true + real_image_forward: true + onnx_export: true + onnxruntime_parity: true + evidence: outputs/full-coverage-20260917/openmmlab/probes/fcn.resnet50/probe.json + fcn.resnet101: + model_id: fcn.resnet101 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/fcn/fcn_r101-d8_4xb4-160k_ade20k-512x512.py + config_sha256: 0592e8a0289e7b931ba5e4003bc8b6e55341b691cd3a418028f7838e53ff2730 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/fcn/fcn_r101-d8_512x512_160k_ade20k/fcn_r101-d8_512x512_160k_ade20k_20200615_105816-fd192bd5.pth + filename: fcn_r101-d8_512x512_160k_ade20k_20200615_105816-fd192bd5.pth + sha256: fd192bd5eced800e4f1d80dec4092295b8b68846debfff6c98549be029f862a2 + official_model_name: fcn_r101-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + deeplabv3.resnet50: + model_id: deeplabv3.resnet50 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/deeplabv3/deeplabv3_r50-d8_4xb4-160k_ade20k-512x512.py + config_sha256: d5aeafdad68c2025013896ebcdc330fec4e7c60bba1aa1c9851e2ef1ceeb626b + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k/deeplabv3_r50-d8_512x512_160k_ade20k_20200615_123227-5d0ee427.pth + filename: deeplabv3_r50-d8_512x512_160k_ade20k_20200615_123227-5d0ee427.pth + sha256: 5d0ee427caff2e8f96e3573bc2466cb5753a90090f55abfe7b5c834d2e8f5e4c + official_model_name: deeplabv3_r50-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + deeplabv3.resnet101: + model_id: deeplabv3.resnet101 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/deeplabv3/deeplabv3_r101-d8_4xb4-160k_ade20k-512x512.py + config_sha256: 11e300fb9a3ca988b0e92b27e5ca21e68161fa1d283138c33099ffde17a21f34 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/deeplabv3/deeplabv3_r101-d8_512x512_160k_ade20k/deeplabv3_r101-d8_512x512_160k_ade20k_20200615_105816-b1f72b3b.pth + filename: deeplabv3_r101-d8_512x512_160k_ade20k_20200615_105816-b1f72b3b.pth + sha256: b1f72b3ba583af703ed52297a86171647a0acf608cc7499a93548105beb3f918 + official_model_name: deeplabv3_r101-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + deeplabv3.mobilenetv3.large: + model_id: deeplabv3.mobilenetv3.large + task: seg + resolution_status: unresolved + input_spec: null + dataset: ade20k + validated: false + blockers: + - code: identity_not_in_official_repository + owner: user_decision + detail: MMSeg MobileNetV3 catalog publishes LR-ASPP Cityscapes, not DeepLabV3 MobileNetV3-Large ADE20K. + Need intended source/config/checkpoint or explicit correction. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + deeplabv3plus.resnet50: + model_id: deeplabv3plus.resnet50 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/deeplabv3plus/deeplabv3plus_r50-d8_4xb4-160k_ade20k-512x512.py + config_sha256: e3df3d88bf7493a3793dc75d3c7ee7b2b88478c1440148e7c83c58e8677acf63 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k/deeplabv3plus_r50-d8_512x512_160k_ade20k_20200615_124504-6135c7e0.pth + filename: deeplabv3plus_r50-d8_512x512_160k_ade20k_20200615_124504-6135c7e0.pth + sha256: 6135c7e0b6c11df116bb793dcfd6d88fadbb0512a0810858f7f51d36a0ad85c4 + official_model_name: deeplabv3plus_r50-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + deeplabv3plus.resnet101: + model_id: deeplabv3plus.resnet101 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/deeplabv3plus/deeplabv3plus_r101-d8_4xb4-160k_ade20k-512x512.py + config_sha256: 057302ac8e7ef4783889c52258fe74d770b6dc47f7f91c96dab12d3a88b8db23 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/deeplabv3plus/deeplabv3plus_r101-d8_512x512_160k_ade20k/deeplabv3plus_r101-d8_512x512_160k_ade20k_20200615_123232-38ed86bb.pth + filename: deeplabv3plus_r101-d8_512x512_160k_ade20k_20200615_123232-38ed86bb.pth + sha256: 38ed86bbd4684eae5e4a0fe7578f8fa6adfecc334409c97b84fe19b678305758 + official_model_name: deeplabv3plus_r101-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + pspnet.resnet50: + model_id: pspnet.resnet50 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/pspnet/pspnet_r50-d8_4xb4-160k_ade20k-512x512.py + config_sha256: 0481afa1eef35e983551b9e16f26e7ca8708e0776beeb8f42f6a14452d8afa4c + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/pspnet/pspnet_r50-d8_512x512_160k_ade20k/pspnet_r50-d8_512x512_160k_ade20k_20200615_184358-1890b0bd.pth + filename: pspnet_r50-d8_512x512_160k_ade20k_20200615_184358-1890b0bd.pth + sha256: 1890b0bd680a1fbe840bc5066757f4a97dc4c5e7c2925df208e1584ce4d35bae + official_model_name: pspnet_r50-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + pspnet.resnet101: + model_id: pspnet.resnet101 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/pspnet/pspnet_r101-d8_4xb4-160k_ade20k-512x512.py + config_sha256: 097febd04a504c93edfceec9b09306627c57ab9beeb0ac23abd29560727dae27 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/pspnet/pspnet_r101-d8_512x512_160k_ade20k/pspnet_r101-d8_512x512_160k_ade20k_20200615_100650-967c316f.pth + filename: pspnet_r101-d8_512x512_160k_ade20k_20200615_100650-967c316f.pth + sha256: 967c316f8cd513d6310d31282f6735775e19ed6ad25f79e7f21ef15ca77dd9ea + official_model_name: pspnet_r101-d8_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + upernet.resnet50: + model_id: upernet.resnet50 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/upernet/upernet_r50_4xb4-160k_ade20k-512x512.py + config_sha256: fa16597599a3dc9540761bfb76717408f35d5a39785b34e14439ca6ea040fd55 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/upernet/upernet_r50_512x512_160k_ade20k/upernet_r50_512x512_160k_ade20k_20200615_184328-8534de8d.pth + filename: upernet_r50_512x512_160k_ade20k_20200615_184328-8534de8d.pth + sha256: 8534de8df30fa733f597a8ebf64dc475a933399ba22d098390130d77d3dbbb55 + official_model_name: upernet_r50_4xb4-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + upernet.swin.t: + model_id: upernet.swin.t + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/swin/swin-tiny-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512.py + config_sha256: bb2e4ed42414fe8e0451e3c1f6c0f27df5868cbbf42cbbdfc4a5ed38d269bf83 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth + filename: upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth + sha256: e380ad3e5d94060d89e4b62b5d393cdcc7f1f3406b1d46bcab547d3c276b6064 + official_model_name: swin-tiny-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + upernet.swin.s: + model_id: upernet.swin.s + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/swin/swin-small-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512.py + config_sha256: bcaab230b7c0be3e070732d9ca43b822f5b4fb11cb389ddfb8e14493659d3e37 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_small_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K/upernet_swin_small_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210526_192015-ee2fff1c.pth + filename: upernet_swin_small_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210526_192015-ee2fff1c.pth + sha256: ee2fff1c4fce3440474ef18f8ba4ff3890ff86fafb42b198b00f835fb425ae4b + official_model_name: swin-small-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + upernet.swin.b: + model_id: upernet.swin.b + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/swin/swin-base-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512.py + config_sha256: d219a26ab7bb11cba307667118627e7f2ceeeb7ec5366e55391634488c06ef6a + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/swin/upernet_swin_base_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K/upernet_swin_base_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210526_192340-593b0e13.pth + filename: upernet_swin_base_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210526_192340-593b0e13.pth + sha256: 593b0e13d13c69ae093fb21074e32d51dc021168fef4f6cdc87c75396760f851 + official_model_name: swin-base-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segformer.b0: + model_id: segformer.b0 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segformer/segformer_mit-b0_8xb2-160k_ade20k-512x512.py + config_sha256: 566262a0a22e81243e6221505fde426dbb2f402860ddbbdacd20778c375343d3 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segformer/segformer_mit-b0_512x512_160k_ade20k/segformer_mit-b0_512x512_160k_ade20k_20210726_101530-8ffa8fda.pth + filename: segformer_mit-b0_512x512_160k_ade20k_20210726_101530-8ffa8fda.pth + sha256: 8ffa8fdac140ffaef9119150aca65e9fcb4e7a165e60b80be0102cc805d73a9c + official_model_name: segformer_mit-b0_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segformer.b1: + model_id: segformer.b1 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segformer/segformer_mit-b1_8xb2-160k_ade20k-512x512.py + config_sha256: 6e5eb644f7ad4d852f5b407f8d2fe4fda9eb53c885f22c10b82d1e22715868ef + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segformer/segformer_mit-b1_512x512_160k_ade20k/segformer_mit-b1_512x512_160k_ade20k_20210726_112106-d70e859d.pth + filename: segformer_mit-b1_512x512_160k_ade20k_20210726_112106-d70e859d.pth + sha256: d70e859d8a80f6f4f7b58db1c03ac149006be79351590c74060b7e243aa62566 + official_model_name: segformer_mit-b1_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segformer.b2: + model_id: segformer.b2 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segformer/segformer_mit-b2_8xb2-160k_ade20k-512x512.py + config_sha256: d88bc5bc9edc2ddef78455f7f9013cdff0219479f97073ad59e8cc7ebb2a578f + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segformer/segformer_mit-b2_512x512_160k_ade20k/segformer_mit-b2_512x512_160k_ade20k_20210726_112103-cbd414ac.pth + filename: segformer_mit-b2_512x512_160k_ade20k_20210726_112103-cbd414ac.pth + sha256: cbd414ac055c8436207914ac3a8e08c69879e8ae9e70f1a57cf0f7e17f19025a + official_model_name: segformer_mit-b2_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segformer.b3: + model_id: segformer.b3 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segformer/segformer_mit-b3_8xb2-160k_ade20k-512x512.py + config_sha256: 5555e749939a316ba20864186fa908603486cf394673e747a662b2202603096e + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segformer/segformer_mit-b3_512x512_160k_ade20k/segformer_mit-b3_512x512_160k_ade20k_20210726_081410-962b98d2.pth + filename: segformer_mit-b3_512x512_160k_ade20k_20210726_081410-962b98d2.pth + sha256: 962b98d28ad11a2feb0d4fadfb9679d44683be02ee75436cc962876f796613ef + official_model_name: segformer_mit-b3_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segformer.b4: + model_id: segformer.b4 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segformer/segformer_mit-b4_8xb2-160k_ade20k-512x512.py + config_sha256: 3d5548febd082e6c21d008440a9b127cac6e3f74bcda0581aefff97bc631f8eb + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segformer/segformer_mit-b4_512x512_160k_ade20k/segformer_mit-b4_512x512_160k_ade20k_20210728_183055-7f509d7d.pth + filename: segformer_mit-b4_512x512_160k_ade20k_20210728_183055-7f509d7d.pth + sha256: 7f509d7d2a035072fbbbf41af6707558111d48a9828d6e4e798cddbd3423a0eb + official_model_name: segformer_mit-b4_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segformer.b5: + model_id: segformer.b5 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py + config_sha256: 0ab05510534c52c2a32f58fda0b0ad7a8ed84d016f561542f9e7a16beaff474f + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segformer/segformer_mit-b5_512x512_160k_ade20k/segformer_mit-b5_512x512_160k_ade20k_20210726_145235-94cedf59.pth + filename: segformer_mit-b5_512x512_160k_ade20k_20210726_145235-94cedf59.pth + sha256: 94cedf59b7e73be6b695e4781cb0fcf000ede08208ded2a27c24b475d5574766 + official_model_name: segformer_mit-b5_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + mask2former.swin.t: + model_id: mask2former.swin.t + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/mask2former/mask2former_swin-t_8xb2-160k_ade20k-512x512.py + config_sha256: d914bac9d1cfdcb31b8ad9298e3c38401bcdacd0b8e4608e512276b931666e19 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/mask2former/mask2former_swin-t_8xb2-160k_ade20k-512x512/mask2former_swin-t_8xb2-160k_ade20k-512x512_20221203_234230-7d64e5dd.pth + filename: mask2former_swin-t_8xb2-160k_ade20k-512x512_20221203_234230-7d64e5dd.pth + sha256: null + official_model_name: mask2former_swin-t_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + mask2former.swin.s: + model_id: mask2former.swin.s + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/mask2former/mask2former_swin-s_8xb2-160k_ade20k-512x512.py + config_sha256: 5ae4e3c6645b8b7d0534de6bfb830f11b4047b6149b5a6abefb74cd36d786cbb + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/mask2former/mask2former_swin-s_8xb2-160k_ade20k-512x512/mask2former_swin-s_8xb2-160k_ade20k-512x512_20221204_143905-e715144e.pth + filename: mask2former_swin-s_8xb2-160k_ade20k-512x512_20221204_143905-e715144e.pth + sha256: null + official_model_name: mask2former_swin-s_8xb2-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + mask2former.swin.b: + model_id: mask2former.swin.b + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/mask2former/mask2former_swin-b-in1k-384x384-pre_8xb2-160k_ade20k-640x640.py + config_sha256: 5a0b5b8793bfeb07b062be5b8f3862e54e883d787b65f6be8a89acdbe6188a96 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/mask2former/mask2former_swin-b-in1k-384x384-pre_8xb2-160k_ade20k-640x640/mask2former_swin-b-in1k-384x384-pre_8xb2-160k_ade20k-640x640_20221129_125118-a4a086d2.pth + filename: mask2former_swin-b-in1k-384x384-pre_8xb2-160k_ade20k-640x640_20221129_125118-a4a086d2.pth + sha256: null + official_model_name: mask2former_swin-b-in1k-384x384-pre_8xb2-160k_ade20k-640x640 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + ddrnet.23.slim: + model_id: ddrnet.23.slim + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/ddrnet/ddrnet_23-slim_in1k-pre_2xb6-120k_cityscapes-1024x1024.py + config_sha256: 2b3bdcbc68a8d44c473a12083118685e96f2ce4b40a887cbbf728caf2b64f230 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/ddrnet/ddrnet_23-slim_in1k-pre_2xb6-120k_cityscapes-1024x1024/ddrnet_23-slim_in1k-pre_2xb6-120k_cityscapes-1024x1024_20230426_145312-6a5e5174.pth + filename: ddrnet_23-slim_in1k-pre_2xb6-120k_cityscapes-1024x1024_20230426_145312-6a5e5174.pth + sha256: d823ec1336cec3750b36e63d8e894b8b3b40e0af05f70752dce9f955ed161a8d + official_model_name: ddrnet_23-slim_in1k-pre_2xb6-120k_cityscapes-1024x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + ddrnet.23: + model_id: ddrnet.23 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/ddrnet/ddrnet_23_in1k-pre_2xb6-120k_cityscapes-1024x1024.py + config_sha256: b77017c60d72c42f09c72c319c8005fa9d857750d1f28d58a152a7e0151b9e2a + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/ddrnet/ddrnet_23_in1k-pre_2xb6-120k_cityscapes-1024x1024/ddrnet_23_in1k-pre_2xb6-120k_cityscapes-1024x1024_20230425_162633-81601db0.pth + filename: ddrnet_23_in1k-pre_2xb6-120k_cityscapes-1024x1024_20230425_162633-81601db0.pth + sha256: a8215ec949fa23ab7fca96d68ad5c9885c2c8d610f92ee36898bff551322fd7f + official_model_name: ddrnet_23_in1k-pre_2xb6-120k_cityscapes-1024x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + ddrnet.39: + model_id: ddrnet.39 + task: seg + resolution_status: resolved_external_catalog + input_spec: null + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + source: custom + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + source: + csv: openmmlab + recipe: super-gradients + reason: Author-linked SuperGradients publishes the exact DDRNet-39 Cityscapes implementation/checkpoint; + no network substitution. + external_builder_options: + architecture: ddrnet_39 + num_classes: 19 + source: + repository: https://github.com/Deci-AI/super-gradients + checkout: super-gradients + revision: 63de22c404d5740f34f7706c302b37fce3c8fe5d + config: src/super_gradients/recipes/cityscapes_ddrnet.yaml + config_sha256: 2759bea1c483d75d8f0813289d880f587e53c916a76ae4ec53e18156fd62b8c3 + checkpoint: + url: https://d2gjn4b69gu75n.cloudfront.net/models/ddrnet_39_cityscapes.pth + filename: ddrnet_39_cityscapes.pth + sha256: null + official_model_name: DDRNet39 / ddrnet_39 + identity_note: Author DDRNet README links SuperGradients. Its catalog publishes ddrnet_39_cityscapes; source + is a named downstream official implementation, not an author-original Cityscapes-39 checkpoint. Do not + load it into a different implementation without strict verification. + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: external_official_builder_not_implemented + owner: public_code + detail: Exact DDRNet-39 Cityscapes code/config/checkpoint are public. Download/hash bytes and implement + the official SuperGradients factory with strict loading, then verify export and Cityscapes preprocessing/decoding. + Current OpenMMLab generic builder deliberately does not call another framework. + pidnet.s: + model_id: pidnet.s + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/pidnet/pidnet-s_2xb6-120k_1024x1024-cityscapes.py + config_sha256: 0c307a671eeb5686ab3cc05f65f6fa7b8629c857a0568d3fcfc62772b86a0c33 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/pidnet/pidnet-s_2xb6-120k_1024x1024-cityscapes/pidnet-s_2xb6-120k_1024x1024-cityscapes_20230302_191700-bb8e3bcc.pth + filename: pidnet-s_2xb6-120k_1024x1024-cityscapes_20230302_191700-bb8e3bcc.pth + sha256: bb8e3bccdda7aa8e35362422363ce2190c68a09b46d0df261d438e7516adaf5c + official_model_name: pidnet-s_2xb6-120k_1024x1024-cityscapes + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + pidnet.m: + model_id: pidnet.m + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/pidnet/pidnet-m_2xb6-120k_1024x1024-cityscapes.py + config_sha256: 2c86703a38b4be1ed1796660eb77a529306dd806fa6cfdb8008a67ff8cef3283 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/pidnet/pidnet-m_2xb6-120k_1024x1024-cityscapes/pidnet-m_2xb6-120k_1024x1024-cityscapes_20230301_143452-f9bcdbf3.pth + filename: pidnet-m_2xb6-120k_1024x1024-cityscapes_20230301_143452-f9bcdbf3.pth + sha256: f9bcdbf3ca339e6991722b6c3a7ec7480be97f6677a345043df56ea931301134 + official_model_name: pidnet-m_2xb6-120k_1024x1024-cityscapes + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + pidnet.l: + model_id: pidnet.l + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/pidnet/pidnet-l_2xb6-120k_1024x1024-cityscapes.py + config_sha256: 3a80127e12db7325741d0398e35182f91a92673d0e0da7822b3c7183ac7d917c + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/pidnet/pidnet-l_2xb6-120k_1024x1024-cityscapes/pidnet-l_2xb6-120k_1024x1024-cityscapes_20230303_114514-0783ca6b.pth + filename: pidnet-l_2xb6-120k_1024x1024-cityscapes_20230303_114514-0783ca6b.pth + sha256: 0783ca6bb5a68a6992daf07a6ece908a236ef9dd17313d53e95467f13d810d2b + official_model_name: pidnet-l_2xb6-120k_1024x1024-cityscapes + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + bisenetv1.r18: + model_id: bisenetv1.r18 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/bisenetv1/bisenetv1_r18-d32-in1k-pre_4xb4-160k_cityscapes-1024x1024.py + config_sha256: 39ed36906b111c6f6dba624cd20b731eeecb47d94051d854a668bd388c02ba3d + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/bisenetv1/bisenetv1_r18-d32_in1k-pre_4x4_1024x1024_160k_cityscapes/bisenetv1_r18-d32_in1k-pre_4x4_1024x1024_160k_cityscapes_20210905_220251-8ba80eff.pth + filename: bisenetv1_r18-d32_in1k-pre_4x4_1024x1024_160k_cityscapes_20210905_220251-8ba80eff.pth + sha256: 8ba80effec0006553f247bdc9cab3dcbca5629b878cfd615ab802be61109bc56 + official_model_name: bisenetv1_r18-d32-in1k-pre_4xb4-160k_cityscapes-1024x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + bisenetv2: + model_id: bisenetv2 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/bisenetv2/bisenetv2_fcn_4xb4-160k_cityscapes-1024x1024.py + config_sha256: 683334b2aba9203f0b53ad7ace4c442bb932e2a39e8ecf6d86c7096542153bc8 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/bisenetv2/bisenetv2_fcn_4x4_1024x1024_160k_cityscapes/bisenetv2_fcn_4x4_1024x1024_160k_cityscapes_20210902_015551-bcf10f09.pth + filename: bisenetv2_fcn_4x4_1024x1024_160k_cityscapes_20210902_015551-bcf10f09.pth + sha256: bcf10f09267bf3b7d024531a33d515ae51ccd278c4180791f884d037a39e9f44 + official_model_name: bisenetv2_fcn_4xb4-160k_cityscapes-1024x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + stdc1.seg: + model_id: stdc1.seg + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/stdc/stdc1_in1k-pre_4xb12-80k_cityscapes-512x1024.py + config_sha256: 726253c26f713da7a12b47d543ae85959cb537b6b07e6d5b5dda38691c41b581 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/stdc/stdc1_in1k-pre_512x1024_80k_cityscapes/stdc1_in1k-pre_512x1024_80k_cityscapes_20220224_141648-3d4c2981.pth + filename: stdc1_in1k-pre_512x1024_80k_cityscapes_20220224_141648-3d4c2981.pth + sha256: 3d4c298177ecc873557905a7a6b09e9300f40aaf07d27193db2ca1cd0dfd3875 + official_model_name: stdc1_in1k-pre_4xb12-80k_cityscapes-512x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + stdc2.seg: + model_id: stdc2.seg + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/stdc/stdc2_in1k-pre_4xb12-80k_cityscapes-512x1024.py + config_sha256: 8367f881f97f8b684e68a319d69c5eb51a68af11a6e9770c10f703c26c78989e + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/stdc/stdc2_in1k-pre_512x1024_80k_cityscapes/stdc2_in1k-pre_512x1024_80k_cityscapes_20220224_073048-1f8f0f6c.pth + filename: stdc2_in1k-pre_512x1024_80k_cityscapes_20220224_073048-1f8f0f6c.pth + sha256: 1f8f0f6c3531dd8f78179c6a144f733e56be5e85a90b4a5e733eff05c450d3c7 + official_model_name: stdc2_in1k-pre_4xb12-80k_cityscapes-512x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + fast.scnn: + model_id: fast.scnn + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/fastscnn/fast_scnn_8xb4-160k_cityscapes-512x1024.py + config_sha256: c8c739b68843b3021d4f56ae6b01ac1dfb5a13b7d15904fc734abf3d38ec858a + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/fast_scnn/fast_scnn_lr0.12_8x4_160k_cityscapes/fast_scnn_lr0.12_8x4_160k_cityscapes_20210630_164853-0cec9937.pth + filename: fast_scnn_lr0.12_8x4_160k_cityscapes_20210630_164853-0cec9937.pth + sha256: 0cec993755ce92b50d12d7325655691316ab9e6745bfec61dc8cac1f57604944 + official_model_name: fast_scnn_8xb4-160k_cityscapes-512x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + hrnet.w18: + model_id: hrnet.w18 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/hrnet/fcn_hr18_4xb2-80k_cityscapes-512x1024.py + config_sha256: ec36cc5e1243b7246031e8af5f917b93302104502bf3a276a168db5dd6282fee + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/hrnet/fcn_hr18_512x1024_80k_cityscapes/fcn_hr18_512x1024_80k_cityscapes_20200601_223255-4e7b345e.pth + filename: fcn_hr18_512x1024_80k_cityscapes_20200601_223255-4e7b345e.pth + sha256: 4e7b345ed24e29ed044d148a510da64b67cf45e146f9702f86e355d86a6c2a89 + official_model_name: fcn_hr18_4xb2-80k_cityscapes-512x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + hrnet.w48: + model_id: hrnet.w48 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: cityscapes + validated: false + model_spec: + dataset: cityscapes + csv_discrepancy: + dataset: + csv: ade20k + recipe: cityscapes + reason: Same architecture official checkpoint is Cityscapes; mainline confirmed existing gates allow + this domain and Cityscapes 500 images are ingested. + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/hrnet/fcn_hr48_4xb2-80k_cityscapes-512x1024.py + config_sha256: be4deef05954e505ec63d9be3e1e3216a981bff3eef365af69841e1e77170f1a + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/hrnet/fcn_hr48_512x1024_80k_cityscapes/fcn_hr48_512x1024_80k_cityscapes_20200601_202606-58ea95d6.pth + filename: fcn_hr48_512x1024_80k_cityscapes_20200601_202606-58ea95d6.pth + sha256: 58ea95d642a106c535bf6f0b65818a19427b1cb9de30b397d451608a9dbd4af1 + official_model_name: fcn_hr48_4xb2-80k_cityscapes-512x1024 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and Cityscapes trainId mapping/ignore-index + to mIoU; reuse the ingested Cityscapes pool and generate matching calibration. + ocrnet.hrnet.w18: + model_id: ocrnet.hrnet.w18 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/ocrnet/ocrnet_hr18_4xb4-80k_ade20k-512x512.py + config_sha256: cfb967303856fce6582fdbac4eddeb98b45455931ba2a077ec41c640b4198ccc + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/ocrnet/ocrnet_hr18_512x512_80k_ade20k/ocrnet_hr18_512x512_80k_ade20k_20200615_053157-d173d83b.pth + filename: ocrnet_hr18_512x512_80k_ade20k_20200615_053157-d173d83b.pth + sha256: d173d83b6dde312c15025d3761264b706ad1cc32200837d876329dc91e30d21a + official_model_name: ocrnet_hr18_4xb4-80k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + ocrnet.hrnet.w48: + model_id: ocrnet.hrnet.w48 + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/ocrnet/ocrnet_hr48_4xb4-80k_ade20k-512x512.py + config_sha256: 8017f60c9f9a9b90464e33de7167f1353dd5318c8c2da439ae48c47a7b706705 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/ocrnet/ocrnet_hr48_512x512_80k_ade20k/ocrnet_hr48_512x512_80k_ade20k_20200615_021518-d168c2d1.pth + filename: ocrnet_hr48_512x512_80k_ade20k_20200615_021518-d168c2d1.pth + sha256: d168c2d1802a70bb5c1a857e306f3211526e823bfcf4d0fcca77d0fab4dab687 + official_model_name: ocrnet_hr48_4xb4-80k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segnext.tiny: + model_id: segnext.tiny + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segnext/segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512.py + config_sha256: 50d86f03c8f4aabd6601566ab13e12f7010d702414213e254d96445a25f6504a + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segnext/segnext_mscan-t_1x16_512x512_adamw_160k_ade20k/segnext_mscan-t_1x16_512x512_adamw_160k_ade20k_20230210_140244-05bd8466.pth + filename: segnext_mscan-t_1x16_512x512_adamw_160k_ade20k_20230210_140244-05bd8466.pth + sha256: 226138b438730509f8e1af8df54e2499f53f21cbab0ed2b0098fd1c44dbb347e + official_model_name: segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segnext.small: + model_id: segnext.small + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segnext/segnext_mscan-s_1xb16-adamw-160k_ade20k-512x512.py + config_sha256: 7cfe19978253b77503800544f2d82fc501722d5328bcfeb5ebf02d31e0febf2d + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segnext/segnext_mscan-s_1x16_512x512_adamw_160k_ade20k/segnext_mscan-s_1x16_512x512_adamw_160k_ade20k_20230214_113014-43013668.pth + filename: segnext_mscan-s_1x16_512x512_adamw_160k_ade20k_20230214_113014-43013668.pth + sha256: null + official_model_name: segnext_mscan-s_1xb16-adamw-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segnext.base: + model_id: segnext.base + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segnext/segnext_mscan-b_1xb16-adamw-160k_ade20k-512x512.py + config_sha256: 33dcb71bcbf41bbdd6e2da2cfa399f84587b4edffb8de39c147d3e58178bc74b + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segnext/segnext_mscan-b_1x16_512x512_adamw_160k_ade20k/segnext_mscan-b_1x16_512x512_adamw_160k_ade20k_20230209_172053-b6f6c70c.pth + filename: segnext_mscan-b_1x16_512x512_adamw_160k_ade20k_20230209_172053-b6f6c70c.pth + sha256: null + official_model_name: segnext_mscan-b_1xb16-adamw-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + segnext.large: + model_id: segnext.large + task: seg + resolution_status: resolved_catalog + input_spec: 1x3x512x1024 + dataset: ade20k + validated: false + source: + repository: https://github.com/open-mmlab/mmsegmentation + checkout: mmsegmentation + revision: b040e147adfa027bbc071b624bedf0ae84dfc922 + config: configs/segnext/segnext_mscan-l_1xb16-adamw-160k_ade20k-512x512.py + config_sha256: c2bb54cc67d52ed25c91a95e3f8d6b9160173fb1606d54d2e8ee5ab219560715 + checkpoint: + url: https://download.openmmlab.com/mmsegmentation/v0.5/segnext/segnext_mscan-l_1x16_512x512_adamw_160k_ade20k/segnext_mscan-l_1x16_512x512_adamw_160k_ade20k_20230209_172055-19b14b63.pth + filename: segnext_mscan-l_1x16_512x512_adamw_160k_ade20k_20230209_172055-19b14b63.pth + sha256: null + official_model_name: segnext_mscan-l_1xb16-adamw-160k_ade20k-512x512 + preprocess: + input_spec: 1x3x512x1024 + official_data_preprocessor: + type: SegDataPreProcessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_val: 0 + seg_pad_val: 255 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/resize, logits upsampling and ADE20K label mapping/ignore-index to + mIoU; ingest real ADE20K annotations/calibration. + mask.r.cnn.r50.fpn: + model_id: mask.r.cnn.r50.fpn + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py + config_sha256: d6713165d805ed830ab79e5fa808824453588e45e6b0d9259ddf11e8c24dbf3f + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_1x_coco/mask_rcnn_r50_fpn_1x_coco_20200205-d4b0c5d6.pth + filename: mask_rcnn_r50_fpn_1x_coco_20200205-d4b0c5d6.pth + sha256: null + official_model_name: mask-rcnn_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + mask.r.cnn.r101.fpn: + model_id: mask.r.cnn.r101.fpn + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/mask_rcnn/mask-rcnn_r101_fpn_1x_coco.py + config_sha256: 3097b9291030e38e351b589abf4193378250d9fe87532b49f9e8bf7ab74edcea + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/mask_rcnn/mask_rcnn_r101_fpn_1x_coco/mask_rcnn_r101_fpn_1x_coco_20200204-1efe0ed5.pth + filename: mask_rcnn_r101_fpn_1x_coco_20200204-1efe0ed5.pth + sha256: null + official_model_name: mask-rcnn_r101_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + cascade.mask.r.cnn.r50.fpn: + model_id: cascade.mask.r.cnn.r50.fpn + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py + config_sha256: aa8e94807d862058404dc68e1f063f2b6539fb173923abd17aa7724d120ef3f3 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_mask_rcnn_r50_fpn_1x_coco/cascade_mask_rcnn_r50_fpn_1x_coco_20200203-9d4dcb24.pth + filename: cascade_mask_rcnn_r50_fpn_1x_coco_20200203-9d4dcb24.pth + sha256: null + official_model_name: cascade-mask-rcnn_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + cascade.mask.r.cnn.r101.fpn: + model_id: cascade.mask.r.cnn.r101.fpn + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/cascade_rcnn/cascade-mask-rcnn_r101_fpn_1x_coco.py + config_sha256: a2cf36f6eebb2fee0f90184c99538306be02967283e9c004f127d28d1a34d261 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_mask_rcnn_r101_fpn_1x_coco/cascade_mask_rcnn_r101_fpn_1x_coco_20200203-befdf6ee.pth + filename: cascade_mask_rcnn_r101_fpn_1x_coco_20200203-befdf6ee.pth + sha256: null + official_model_name: cascade-mask-rcnn_r101_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + htc.r50.fpn: + model_id: htc.r50.fpn + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/htc/htc_r50_fpn_1x_coco.py + config_sha256: 3153f1f7cd8dfcd564831c76c0adccfed255b4c73f01fef1a5766f0815f91d1d + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/htc/htc_r50_fpn_1x_coco/htc_r50_fpn_1x_coco_20200317-7332cf16.pth + filename: htc_r50_fpn_1x_coco_20200317-7332cf16.pth + sha256: null + official_model_name: htc_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + htc.r101.fpn: + model_id: htc.r101.fpn + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/htc/htc_r101_fpn_20e_coco.py + config_sha256: ed01ec699b203d51ec8a8b603c2ba5627feb76c6806a3eb10201de2a92645402 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/htc/htc_r101_fpn_20e_coco/htc_r101_fpn_20e_coco_20200317-9b41b48f.pth + filename: htc_r101_fpn_20e_coco_20200317-9b41b48f.pth + sha256: null + official_model_name: htc_r101_fpn_20e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + solo.r50: + model_id: solo.r50 + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/solo/solo_r50_fpn_1x_coco.py + config_sha256: ba7429707d06416634b064e1288e9e6eefa8299e6e87db9f8ae4dc63826e31ec + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/solo/solo_r50_fpn_1x_coco/solo_r50_fpn_1x_coco_20210821_035055-2290a6b8.pth + filename: solo_r50_fpn_1x_coco_20210821_035055-2290a6b8.pth + sha256: 2290a6b867d29fbe1a1a1c7d79803e396b6ed1bba212cdffcf8abfe916389434 + official_model_name: solo_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + solov2.r50: + model_id: solov2.r50 + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/solov2/solov2_r50_fpn_1x_coco.py + config_sha256: c8076b03febe80bdb091b39657146c54c64c1c33917549e73fe42b4ba7fa2e70 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/solov2/solov2_r50_fpn_1x_coco/solov2_r50_fpn_1x_coco_20220512_125858-a357fa23.pth + filename: solov2_r50_fpn_1x_coco_20220512_125858-a357fa23.pth + sha256: a357fa231fdc664fb78662fb31469a6d15bd6ac644a3efbb069e2cb21bd89073 + official_model_name: solov2_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + condinst.r50: + model_id: condinst.r50 + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/condinst/condinst_r50_fpn_ms-poly-90k_coco_instance.py + config_sha256: 71f78948ff16e2a37b0450cad284dd2560fa25f55d8626bbe2a0ad9edb3b63ac + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/condinst/condinst_r50_fpn_ms-poly-90k_coco_instance/condinst_r50_fpn_ms-poly-90k_coco_instance_20221129_125223-4c186406.pth + filename: condinst_r50_fpn_ms-poly-90k_coco_instance_20221129_125223-4c186406.pth + sha256: null + official_model_name: condinst_r50_fpn_ms-poly-90k_coco_instance + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + queryinst.r50: + model_id: queryinst.r50 + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/queryinst/queryinst_r50_fpn_1x_coco.py + config_sha256: 34ed818cbdcac65a71f1ef6f0391197cd94bc0669864fabaf8187901634585fb + checkpoint: + url: https://download.openmmlab.com/mmdetection/v2.0/queryinst/queryinst_r50_fpn_1x_coco/queryinst_r50_fpn_1x_coco_20210907_084916-5a8f1998.pth + filename: queryinst_r50_fpn_1x_coco_20210907_084916-5a8f1998.pth + sha256: null + official_model_name: queryinst_r50_fpn_1x_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + mask2former.swin.t.instance: + model_id: mask2former.swin.t.instance + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/mask2former/mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco.py + config_sha256: 3d3e01e6a387fb4bfcb9c19e22a32f14054a888162ebb1e1f83925e4d597e782 + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/mask2former/mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco/mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco_20220508_091649-01b0f990.pth + filename: mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco_20220508_091649-01b0f990.pth + sha256: 01b0f990130474da40281286f01d1bdb5899c6ac57b38555d81d36dd95a43e78 + official_model_name: mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + mask2former.swin.s.instance: + model_id: mask2former.swin.s.instance + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/mask2former/mask2former_swin-s-p4-w7-224_8xb2-lsj-50e_coco.py + config_sha256: 32aa1d25f67e7c3d3e43dbe720ea4b1c009e235815eb33198818a900ffbc04af + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/mask2former/mask2former_swin-s-p4-w7-224_8xb2-lsj-50e_coco/mask2former_swin-s-p4-w7-224_8xb2-lsj-50e_coco_20220504_001756-c9d0c4f2.pth + filename: mask2former_swin-s-p4-w7-224_8xb2-lsj-50e_coco_20220504_001756-c9d0c4f2.pth + sha256: c9d0c4f21c47ecdfaf7595ff1edfa42158d70cbd8c79e93e1c76d5fcad58cd47 + official_model_name: mask2former_swin-s-p4-w7-224_8xb2-lsj-50e_coco + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + mask2former.swin.b.instance: + model_id: mask2former.swin.b.instance + task: iseg + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmdetection + checkout: mmdetection + revision: cfd5d3a985b0249de009b67d04f37263e11cdf3d + config: configs/mask2former/mask2former_swin-b-p4-w12-384_8xb2-lsj-50e_coco-panoptic.py + config_sha256: f2cc9cd7de37c877af87d4e617fc03262baabb5e9fba7f9f1c815b6de27ddfae + checkpoint: + url: https://download.openmmlab.com/mmdetection/v3.0/mask2former/mask2former_swin-b-p4-w12-384_8xb2-lsj-50e_coco-panoptic/mask2former_swin-b-p4-w12-384_8xb2-lsj-50e_coco-panoptic_20220331_002244-8a651d82.pth + filename: mask2former_swin-b-p4-w12-384_8xb2-lsj-50e_coco-panoptic_20220331_002244-8a651d82.pth + sha256: 8a651d821a61c0fa0b98fa5f77206228e85b070370018139dde8132ee16cc6ba + official_model_name: mask2former_swin-b-p4-w12-384_8xb2-lsj-50e_coco-panoptic + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: DetDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + identity_note: Official Swin-B panoptic config enables instance_on=True and publishes COCO mask AP 44.9. + Retain the full 80 thing + 53 stuff class head; evaluate the official thing-only instance_postprocess + branch. This is not an instance-only-trained checkpoint and not the Swin-T/S 80-class head. + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official normalization/geometry and family-specific boxes/masks decoding to COCO segmentation + AP; raw tensor export is not an evaluator. + simplebaseline.r50: + model_id: simplebaseline.r50 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_res50_8xb64-210e_coco-256x192.py + config_sha256: d02397812e759c76fd7e158647777f8e83e7cc2adf5ce72e4cbbfb783c9565a5 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_res50_8xb64-210e_coco-256x192-04af38ce_20220923.pth + filename: td-hm_res50_8xb64-210e_coco-256x192-04af38ce_20220923.pth + sha256: null + official_model_name: td-hm_res50_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + simplebaseline.r101: + model_id: simplebaseline.r101 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_res101_8xb64-210e_coco-256x192.py + config_sha256: 9ef8b14dab484535bf4bb8cfee913015385a189baf513ceb38a30ea616b3134e + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_res101_8xb64-210e_coco-256x192-065d3625_20220926.pth + filename: td-hm_res101_8xb64-210e_coco-256x192-065d3625_20220926.pth + sha256: null + official_model_name: td-hm_res101_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + simplebaseline.r152: + model_id: simplebaseline.r152 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_res152_8xb32-210e_coco-256x192.py + config_sha256: 13e7f1487f734cf579c90728623ef8fad4e505b87a98a8087ef98f9adb255eef + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_res152_8xb32-210e_coco-256x192-0345f330_20220928.pth + filename: td-hm_res152_8xb32-210e_coco-256x192-0345f330_20220928.pth + sha256: null + official_model_name: td-hm_res152_8xb32-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + hrnet.w32: + model_id: hrnet.w32 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_hrnet-w32_8xb64-210e_coco-256x192.py + config_sha256: cf343ebf5cbdb727bbfe5e61ee32225a670aa03c58465432db03050a760dace3 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_hrnet-w32_8xb64-210e_coco-256x192-81c58e40_20220909.pth + filename: td-hm_hrnet-w32_8xb64-210e_coco-256x192-81c58e40_20220909.pth + sha256: 81c58e403a1ba13a741eb02ba790b5b78bff2a45a9390feebf875f43b8502c8f + official_model_name: td-hm_hrnet-w32_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + hrnet.w48.2: + model_id: hrnet.w48.2 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_hrnet-w48_8xb32-210e_coco-256x192.py + config_sha256: b4a3daa66d21e600d2f9bf429fe259194c16d2cb451ec547458e2dba8adbd173 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_hrnet-w48_8xb32-210e_coco-256x192-0e67c616_20220913.pth + filename: td-hm_hrnet-w48_8xb32-210e_coco-256x192-0e67c616_20220913.pth + sha256: 0e67c6167d6a10fe8f27e3da1e9a415b57289d5820dcca2b42bd8079df4b7a3a + official_model_name: td-hm_hrnet-w48_8xb32-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + litehrnet.18: + model_id: litehrnet.18 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_litehrnet-18_8xb64-210e_coco-256x192.py + config_sha256: a5903c265b8048bd5f4863e83745f9312042a37b22a63417d59c7acacc121f3c + checkpoint: + url: https://download.openmmlab.com/mmpose/top_down/litehrnet/litehrnet18_coco_256x192-6bace359_20211230.pth + filename: litehrnet18_coco_256x192-6bace359_20211230.pth + sha256: null + official_model_name: td-hm_litehrnet-18_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + litehrnet.30: + model_id: litehrnet.30 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_litehrnet-30_8xb64-210e_coco-256x192.py + config_sha256: 7444227187e463e07ac83768142d7a6099996f0c37308a53f5be928157a2a6b4 + checkpoint: + url: https://download.openmmlab.com/mmpose/top_down/litehrnet/litehrnet30_coco_256x192-4176555b_20210626.pth + filename: litehrnet30_coco_256x192-4176555b_20210626.pth + sha256: null + official_model_name: td-hm_litehrnet-30_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + rtmpose.tiny: + model_id: rtmpose.tiny + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/rtmpose/coco/rtmpose-t_8xb256-420e_coco-256x192.py + config_sha256: 7a8e5057017515237274334b367c1fb87b2af6ea03ac77e8a852715794c923fa + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/rtmpose-tiny_simcc-coco_pt-aic-coco_420e-256x192-e613ba3f_20230127.pth + filename: rtmpose-tiny_simcc-coco_pt-aic-coco_420e-256x192-e613ba3f_20230127.pth + sha256: 77ffc7e802acf10951c353e8bc68b4f05218121177ceaea163aa124436ba6fb7 + official_model_name: rtmpose-t_8xb256-420e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + probe: + status: raw_fp32_export_probe_pass + strict_load: true + real_image_forward: true + onnx_export: true + onnxruntime_parity: true + evidence: outputs/full-coverage-20260917/openmmlab/probes/rtmpose.tiny/probe.json + rtmpose.s: + model_id: rtmpose.s + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/rtmpose/coco/rtmpose-s_8xb256-420e_coco-256x192.py + config_sha256: 6b92f84c54d39c00126a8bd7dbc137705d652574e3c7b3018d6322550791f99d + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/rtmpose-s_simcc-coco_pt-aic-coco_420e-256x192-8edcf0d7_20230127.pth + filename: rtmpose-s_simcc-coco_pt-aic-coco_420e-256x192-8edcf0d7_20230127.pth + sha256: 29dcacbb5c5f3ab2f03a67fedcb58cf7287f93b9a8a9d893f42416b63fc304ba + official_model_name: rtmpose-s_8xb256-420e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + rtmpose.m: + model_id: rtmpose.m + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/rtmpose/coco/rtmpose-m_8xb256-420e_coco-256x192.py + config_sha256: f27da50a81c3d3b5c426262118b9a030d15c9dbddccda7100b8cc0bcd07f6fff + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/rtmpose-m_simcc-coco_pt-aic-coco_420e-256x192-d8dd5ca4_20230127.pth + filename: rtmpose-m_simcc-coco_pt-aic-coco_420e-256x192-d8dd5ca4_20230127.pth + sha256: 929f00f2d84764aa0297decf13a84556642e43cbf5fca8cfb16e7326d92dfd97 + official_model_name: rtmpose-m_8xb256-420e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + rtmpose.l: + model_id: rtmpose.l + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/rtmpose/coco/rtmpose-l_8xb256-420e_coco-256x192.py + config_sha256: 053c95c96a1185e43f7a62a5ebeedd64a41121517b3299b25d3d748154fcbca5 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/rtmpose-l_simcc-coco_pt-aic-coco_420e-256x192-1352a4d2_20230127.pth + filename: rtmpose-l_simcc-coco_pt-aic-coco_420e-256x192-1352a4d2_20230127.pth + sha256: bdacdf0fd7026877d9c4d8823b510057e4e3922271f93041f44322cd03fe094d + official_model_name: rtmpose-l_8xb256-420e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + rtmpose.x: + model_id: rtmpose.x + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x384x288 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: projects/rtmpose/rtmpose/body_2d_keypoint/rtmpose-x_8xb256-700e_coco-384x288.py + config_sha256: 6d1c4cb00c36ac1928e404ecc10e1a83744b90a545a995a74904b29bb1c39657 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/rtmpose-x_simcc-body7_pt-body7_700e-384x288-71d7b7e9_20230629.pth + filename: rtmpose-x_simcc-body7_pt-body7_700e-384x288-71d7b7e9_20230629.pth + sha256: 71d7b7e957646a52c504bdaff511dfa37813f99b7725378c5530ba6eeaaab200 + official_model_name: rtmpose-x_8xb256-700e_coco-384x288 + identity_note: Official project README weights are body7-pretrained, COCO-17 evaluation. The body8-halpe26 + and wholebody variants were not selected. + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_config_parse_failed + owner: public_code + detail: '[Errno 2] No such file or directory: ''/home/ming01.wang/outputs/full-coverage-20260917/openmmlab/venv/lib/python3.10/site-packages/mmpose/.mim/model-index.yml''' + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + vitpose.s: + model_id: vitpose.s + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-small_8xb64-210e_coco-256x192.py + config_sha256: db3c5c5acdb216e6679f9dfcda977144ac7188fa3e6888958a05cd5d1ba4d3f3 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-small_8xb64-210e_coco-256x192-62d7a712_20230314.pth + filename: td-hm_ViTPose-small_8xb64-210e_coco-256x192-62d7a712_20230314.pth + sha256: null + official_model_name: td-hm_ViTPose-small_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + vitpose.b: + model_id: vitpose.b + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-base_8xb64-210e_coco-256x192.py + config_sha256: 8c4bef3f65ab7e3ce854ec6b4c484af7bc1889ed75685f1a0933f5d8ffe3c97b + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-base_8xb64-210e_coco-256x192-216eae50_20230314.pth + filename: td-hm_ViTPose-base_8xb64-210e_coco-256x192-216eae50_20230314.pth + sha256: null + official_model_name: td-hm_ViTPose-base_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + vitpose.l: + model_id: vitpose.l + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-large_8xb64-210e_coco-256x192.py + config_sha256: eec6e80c77425130cf75a38e0cb2bf3e32ed5ed7bca856c962bf8d3bcb075a1d + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-large_8xb64-210e_coco-256x192-53609f55_20230314.pth + filename: td-hm_ViTPose-large_8xb64-210e_coco-256x192-53609f55_20230314.pth + sha256: null + official_model_name: td-hm_ViTPose-large_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + vitpose.h: + model_id: vitpose.h + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-huge_8xb64-210e_coco-256x192.py + config_sha256: 72fcd88a4483742869867a1da2aa6e2af533155950185e524bf4ed24e7c15d36 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/topdown_heatmap/coco/td-hm_ViTPose-huge_8xb64-210e_coco-256x192-e32adcd4_20230314.pth + filename: td-hm_ViTPose-huge_8xb64-210e_coco-256x192-e32adcd4_20230314.pth + sha256: null + official_model_name: td-hm_ViTPose-huge_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + mspn.r50: + model_id: mspn.r50 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/topdown_heatmap/coco/td-hm_mspn50_8xb32-210e_coco-256x192.py + config_sha256: f3be7ebc74ea530da33d7b113d07363306f9c0b537b52b2a40c60282154bb218 + checkpoint: + url: https://download.openmmlab.com/mmpose/top_down/mspn/mspn50_coco_256x192-8fbfb5d0_20201123.pth + filename: mspn50_coco_256x192-8fbfb5d0_20201123.pth + sha256: null + official_model_name: td-hm_mspn50_8xb32-210e_coco-256x192 + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + mspn.r101: + model_id: mspn.r101 + task: pose + resolution_status: unresolved + input_spec: null + dataset: coco2017 + validated: false + blockers: + - code: identity_not_in_official_repository + owner: user_decision + detail: Official MMPose MSPN is MSPN-50 (one to four stages); no MSPN-R101 config/checkpoint. Need intended + exact variant or own config/weights. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + higherhrnet.w32: + model_id: higherhrnet.w32 + task: pose + resolution_status: resolved_catalog + input_spec: null + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose-0x + revision: 0d9a00c1d64b68999e4b2a4ee5ed0d5ac2c3e296 + config: configs/body/2d_kpt_sview_rgb_img/associative_embedding/coco/higherhrnet_w32_coco_512x512.py + config_sha256: 5aff886ddf2420e692be8ae814adc52dc489c4400e78fff11e7d7cc1dbf8c302 + checkpoint: + url: https://download.openmmlab.com/mmpose/bottom_up/higher_hrnet32_coco_512x512-8ae85183_20200713.pth + filename: higher_hrnet32_coco_512x512-8ae85183_20200713.pth + sha256: null + official_model_name: associative_embedding_higherhrnet_w32_coco_512x512 + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: legacy_mmpose_0x_environment + owner: public_code + detail: Pinned HigherHRNet needs isolated MMPose 0.x + MMCV 1.x and legacy forward/decode/export; current + generic adapter deliberately rejects it. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + higherhrnet.w48: + model_id: higherhrnet.w48 + task: pose + resolution_status: resolved_catalog + input_spec: null + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose-0x + revision: 0d9a00c1d64b68999e4b2a4ee5ed0d5ac2c3e296 + config: configs/body/2d_kpt_sview_rgb_img/associative_embedding/coco/higherhrnet_w48_coco_512x512.py + config_sha256: 96f138d1f9c0fd0abb5c480409015d391abaa1ea337e98faaf29950055922e69 + checkpoint: + url: https://download.openmmlab.com/mmpose/bottom_up/higher_hrnet48_coco_512x512-60fedcbc_20200712.pth + filename: higher_hrnet48_coco_512x512-60fedcbc_20200712.pth + sha256: null + official_model_name: associative_embedding_higherhrnet_w48_coco_512x512 + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: legacy_mmpose_0x_environment + owner: public_code + detail: Pinned HigherHRNet needs isolated MMPose 0.x + MMCV 1.x and legacy forward/decode/export; current + generic adapter deliberately rejects it. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + dekr.w32: + model_id: dekr.w32 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/dekr/coco/dekr_hrnet-w32_8xb10-140e_coco-512x512.py + config_sha256: 1f23e980816ef407533591e706f0e86c5c17df2f7d3109ff81599493229fd1a1 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/dekr/coco/dekr_hrnet-w32_8xb10-140e_coco-512x512_ac7c17bf-20221228.pth + filename: dekr_hrnet-w32_8xb10-140e_coco-512x512_ac7c17bf-20221228.pth + sha256: null + official_model_name: dekr_hrnet-w32_8xb10-140e_coco-512x512 + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + dekr.w48: + model_id: dekr.w48 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/dekr/coco/dekr_hrnet-w48_8xb10-140e_coco-640x640.py + config_sha256: 7ab06cd6f7347963916ad72a8116877c0c75acb78ff538874be86b384f0e383e + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/dekr/coco/dekr_hrnet-w48_8xb10-140e_coco-640x640_74796c32-20230124.pth + filename: dekr_hrnet-w48_8xb10-140e_coco-640x640_74796c32-20230124.pth + sha256: null + official_model_name: dekr_hrnet-w48_8xb10-140e_coco-640x640 + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + simcc.resnet50: + model_id: simcc.resnet50 + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x256x192 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/simcc/coco/simcc_res50_8xb64-210e_coco-256x192.py + config_sha256: 2ca765be597e99f28cafd108db850ec70bf8c16f76fe9dd8e4add6c2adfeb1ef + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/body_2d_keypoint/simcc/coco/simcc_res50_8xb64-210e_coco-256x192-8e0f5b59_20220919.pth + filename: simcc_res50_8xb64-210e_coco-256x192-8e0f5b59_20220919.pth + sha256: a6be8208dafc3f5a6edc0789fdad2b5b576f48dbaf79131fee7b4a6e261bce43 + official_model_name: simcc_res50_8xb64-210e_coco-256x192 + preprocess: + input_spec: 1x3x256x192 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 123.675 + - 116.28 + - 103.53 + std: + - 58.395 + - 57.12 + - 57.375 + bgr_to_rgb: true + geometry_validated: false + output_kind: official_raw_head_tensors + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: registry_shape_conflicts_with_official_fixed_input + owner: public_code + detail: CSV 1x3x640x640 versus official 1x3x256x192; freeze explicit recipe input and implement matching + geometry/decoder for new jobs only. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + rtmo.s: + model_id: rtmo.s + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/rtmo/coco/rtmo-s_8xb32-600e_coco-640x640.py + config_sha256: 72e1f36e39d649de368da5af4b0fc18ec5b5dfabcd9e52ca01a537c694c98080 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmo/rtmo-s_8xb32-600e_coco-640x640-8db55a59_20231211.pth + filename: rtmo-s_8xb32-600e_coco-640x640-8db55a59_20231211.pth + sha256: null + official_model_name: rtmo-s_8xb32-600e_coco-640x640 + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 0 + - 0 + - 0 + std: + - 1 + - 1 + - 1 + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + identity_note: Official rtmo_coco.yml labels CrowdPose incorrectly; selected config test_dataloader is + CocoDataset/17 keypoints, corroborated by official RTMO COCO README. Keep original YAML Results above + as provenance, not a claimed reproduced metric. + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + rtmo.m: + model_id: rtmo.m + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/rtmo/coco/rtmo-m_16xb16-600e_coco-640x640.py + config_sha256: a5421115ea0a7542f174b954d9a83eb663a0915bdcdf3ea571d41a938a3a25ae + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmo/rtmo-m_16xb16-600e_coco-640x640-6f4e0306_20231211.pth + filename: rtmo-m_16xb16-600e_coco-640x640-6f4e0306_20231211.pth + sha256: null + official_model_name: rtmo-m_16xb16-600e_coco-640x640 + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 0 + - 0 + - 0 + std: + - 1 + - 1 + - 1 + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + identity_note: Official rtmo_coco.yml labels CrowdPose incorrectly; selected config test_dataloader is + CocoDataset/17 keypoints, corroborated by official RTMO COCO README. Keep original YAML Results above + as provenance, not a claimed reproduced metric. + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. + rtmo.l: + model_id: rtmo.l + task: pose + resolution_status: resolved_catalog + input_spec: 1x3x640x640 + dataset: coco2017 + validated: false + source: + repository: https://github.com/open-mmlab/mmpose + checkout: mmpose + revision: 759b39c13fea6ba094afc1fa932f51dc1b11cbf9 + config: configs/body_2d_keypoint/rtmo/coco/rtmo-l_16xb16-600e_coco-640x640.py + config_sha256: ba86fe88e35f604f742d4059fe9fa3fec9dc06956bd04e8aebd52f43adab49f8 + checkpoint: + url: https://download.openmmlab.com/mmpose/v1/projects/rtmo/rtmo-l_16xb16-600e_coco-640x640-516a421f_20231211.pth + filename: rtmo-l_16xb16-600e_coco-640x640-516a421f_20231211.pth + sha256: null + official_model_name: rtmo-l_16xb16-600e_coco-640x640 + preprocess: + input_spec: 1x3x640x640 + official_data_preprocessor: + type: PoseDataPreprocessor + mean: + - 0 + - 0 + - 0 + std: + - 1 + - 1 + - 1 + pad_size_divisor: 32 + geometry_validated: false + output_kind: official_raw_head_tensors + identity_note: Official rtmo_coco.yml labels CrowdPose incorrectly; selected config test_dataloader is + CocoDataset/17 keypoints, corroborated by official RTMO COCO README. Keep original YAML Results above + as provenance, not a claimed reproduced metric. + blockers: + - code: checkpoint_bytes_not_verified + owner: public_code + detail: Official checkpoint URL resolved; download exact bytes and record full SHA256 before strict build. + Filename tokens are not a digest guarantee. + - code: official_environment_and_strict_load + owner: public_code + detail: Provision pinned source/MMEngine/MMCV in an isolated environment, download and hash the official + checkpoint, strictly load every learned parameter and run actual input. + - code: preprocess_export_decode_evaluation + owner: public_code + detail: Wire official affine person crops or bottom-up geometry and official codec to COCO-17 OKS; current + 640-square YOLO-pose decoder cannot interpret heatmaps/SimCC. diff --git a/harness/adapters/pidnet_l.py b/harness/adapters/pidnet_l.py new file mode 100644 index 0000000..f9f3991 --- /dev/null +++ b/harness/adapters/pidnet_l.py @@ -0,0 +1,17 @@ +"""Official Cityscapes PIDNet-L preserving its align_corners=True geometry.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, RUNTIME_PYTHON, EXPORTER_VERSION +from .openmmlab import build as official_build + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("pidnet.l") +PREPROCESS["align_corners"] = True +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = official_build("pidnet.l", weights, input_spec) + head = network.upstream.decode_head + if (head.align_corners is not True or head.num_classes != CHECKPOINT["classes"] + or network.upstream.test_cfg.mode != "whole"): + raise ValueError("Official PIDNet-L head differs from its frozen geometry") + return network diff --git a/harness/adapters/pidnet_m.py b/harness/adapters/pidnet_m.py new file mode 100644 index 0000000..2811a9f --- /dev/null +++ b/harness/adapters/pidnet_m.py @@ -0,0 +1,17 @@ +"""Official Cityscapes PIDNet-M preserving its align_corners=True geometry.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, RUNTIME_PYTHON, EXPORTER_VERSION +from .openmmlab import build as official_build + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("pidnet.m") +PREPROCESS["align_corners"] = True +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = official_build("pidnet.m", weights, input_spec) + head = network.upstream.decode_head + if (head.align_corners is not True or head.num_classes != CHECKPOINT["classes"] + or network.upstream.test_cfg.mode != "whole"): + raise ValueError("Official PIDNet-M head differs from its frozen geometry") + return network diff --git a/harness/adapters/pidnet_s.py b/harness/adapters/pidnet_s.py new file mode 100644 index 0000000..07cbd21 --- /dev/null +++ b/harness/adapters/pidnet_s.py @@ -0,0 +1,17 @@ +"""Official Cityscapes PIDNet-S preserving its align_corners=True geometry.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, RUNTIME_PYTHON, EXPORTER_VERSION +from .openmmlab import build as official_build + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("pidnet.s") +PREPROCESS["align_corners"] = True +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = official_build("pidnet.s", weights, input_spec) + head = network.upstream.decode_head + if (head.align_corners is not True or head.num_classes != CHECKPOINT["classes"] + or network.upstream.test_cfg.mode != "whole"): + raise ValueError("Official PIDNet-S head differs from its frozen geometry") + return network diff --git a/harness/adapters/pspnet_resnet101.py b/harness/adapters/pspnet_resnet101.py new file mode 100644 index 0000000..ab4afd4 --- /dev/null +++ b/harness/adapters/pspnet_resnet101.py @@ -0,0 +1,7 @@ +"""Official ADE20K pspnet.resnet101; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("pspnet.resnet101") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "pspnet.resnet101") diff --git a/harness/adapters/pspnet_resnet50.py b/harness/adapters/pspnet_resnet50.py new file mode 100644 index 0000000..fe595b9 --- /dev/null +++ b/harness/adapters/pspnet_resnet50.py @@ -0,0 +1,7 @@ +"""Official ADE20K pspnet.resnet50; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("pspnet.resnet50") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "pspnet.resnet50") diff --git a/harness/adapters/r2d2.py b/harness/adapters/r2d2.py new file mode 100644 index 0000000..46efe40 --- /dev/null +++ b/harness/adapters/r2d2.py @@ -0,0 +1,76 @@ +"""Official R2D2 WASF_N16 identity, single-scale at the fixed feature protocol size.""" +from functools import lru_cache +from . import feature_sources as sources + +INPUT_SPEC = '1x3x480x640' +MODEL_SPEC = {'input_spec': INPUT_SPEC, 'dataset': 'hpatches', 'input_override_reason': 'Official R2D2 uses ImageNet-normalized RGB.'} +EXPORTER_VERSION = 'r2d2-static-1' +OPSET = 17 +CHECKPOINT = {'url': 'https://raw.githubusercontent.com/naver/r2d2/0ff8f6afcbea91f19613d0cb7d93143a977830f5/models/r2d2_WASF_N16.pt', + 'sha256': '9ae90e02a9a133d100ca7aeaa32f4d4d7736a6dd222a530a25c8f7da5e508528', + 'filename': 'r2d2_WASF_N16.pt', 'official_relative_path': str(sources.ROOT/'r2d2/models/r2d2_WASF_N16.pt')} +PREPROCESS = {'input_spec': INPUT_SPEC, 'color': 'RGB', 'scale': 1/255, + 'mean': [.485,.456,.406], 'std': [.229,.224,.225], 'resize': 'opencv_linear_640x480', + 'output_layout': 'NCHW', 'protocol': 'r2d2-official-rgb-single-scale-v1'} + + +@lru_cache(None) +def _functions(): + import torchvision.transforms as tvf + ns = sources.definitions('r2d2/tools/dataloader.py', ['RGB_mean','RGB_std','norm_RGB'], {'tvf': tvf}) + return sources.definitions('r2d2/extract.py', ['NonMaxSuppression','extract_multiscale'], ns) + + +def preprocess(image, input_spec=INPUT_SPEC): + import cv2 + import numpy as np + from PIL import Image + if input_spec != INPUT_SPEC: raise ValueError(input_spec) + rgb = cv2.resize(np.asarray(image.convert('RGB')), (640,480), interpolation=cv2.INTER_LINEAR) + return _functions()['norm_RGB'](Image.fromarray(rgb))[None].numpy() + + +def official_model(weights): + import ast + import torch + sources.verify_weights(weights, CHECKPOINT['sha256']) + checkpoint = torch.load(weights, map_location='cpu', weights_only=False) + # Preserve the checkpoint's exact constructor without executing arbitrary eval. + expr = ast.parse(checkpoint['net'], mode='eval').body + assert isinstance(expr, ast.Call) and isinstance(expr.func, ast.Name) and not expr.args + assert expr.func.id == 'Quad_L2Net_ConfCFS', checkpoint['net'] + kwargs = {k.arg: ast.literal_eval(k.value) for k in expr.keywords} + module = sources.module('r2d2/nets/patchnet.py') + m = getattr(module, expr.func.id)(**kwargs) + m.load_state_dict({k.replace('module.',''):v for k,v in checkpoint['state_dict'].items()}, strict=True) + return m.eval() + + +def official_features(m, x): + import torch + funcs = _functions(); detector = funcs['NonMaxSuppression'](rel_thr=.7, rep_thr=.7) + k,d,s = funcs['extract_multiscale'](m, x, detector, min_scale=1., max_scale=1., min_size=0, max_size=1024) + out = sources.top_features(k.cpu().numpy(), s.cpu().numpy(), d.cpu().numpy()) + return tuple(torch.from_numpy(out[key]) for key in ('keypoints','scores','descriptors')) + + +def build(weights, input_spec): + import torch + if input_spec != INPUT_SPEC: raise ValueError(input_spec) + class Graph(torch.nn.Module): + def __init__(self): super().__init__(); self.net = official_model(weights) + def forward(self, x): + out = self.net(imgs=[x]) + return out['descriptors'][0], out['reliability'][0], out['repeatability'][0] + return Graph().eval() + + +def decode(outputs): + import torch + maps = sources.tensors(outputs) + assert [tuple(v.shape) for v in maps] == [(1,128,480,640),(1,1,480,640),(1,1,480,640)] + class DenseMaps(torch.nn.Module): + def forward(self, imgs): + return {name:[value] for name,value in zip(('descriptors','reliability','repeatability'),maps)} + with torch.no_grad(): k,s,d = official_features(DenseMaps(), torch.zeros(1,3,480,640)) + return {'keypoints': k.numpy(), 'scores': s.numpy(), 'descriptors': d.numpy()} diff --git a/harness/adapters/raft_common.py b/harness/adapters/raft_common.py new file mode 100644 index 0000000..232dcf3 --- /dev/null +++ b/harness/adapters/raft_common.py @@ -0,0 +1,88 @@ +"""Official torchvision RAFT with one fixed, packed two-frame input. + +No architecture changes: retain 12 recurrent updates and return the final flow. +Channels 0:3 are RGB at t; 3:6 are RGB at t+1. Output is (u, v), in +input-image pixels, mapping the first frame to the second frame. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +INPUT_SPEC = "1x6x384x1024" +MODEL_SPEC = {"source": "custom", "input_spec": INPUT_SPEC, "resize": 0} +OPSET = 16 # The official correlation lookup uses ONNX GridSample (>=16). +EXPORTER_VERSION = "raft-pair-v1-opset16-updates12" +PREPROCESS = { + "id": "raft-rgb-pair-v1", + "layout": "NCHW", + "dtype": "float32", + "channels": ["t.R", "t.G", "t.B", "t+1.R", "t+1.G", "t+1.B"], + "resize": "bilinear, align_corners=False, antialias=False; fixed H,W", + "normalize": "RGB uint8 / 255 * 2 - 1", + "output": "last of 12 updates; N,2,H,W; horizontal u, vertical v; input pixels", + "evaluation": "resize flow to original image; multiply u by W_orig/W and v by H_orig/H", +} + +_CHECKPOINTS = { + "small": ("C_T_V2", "raft_small_C_T_V2-01064c6d.pth", "01064c6d"), + "large": ("C_T_SKHT_V2", "raft_large_C_T_SKHT_V2-ff5fadd5.pth", "ff5fadd5"), +} + + +def checkpoint_info(variant: str) -> dict: + enum, filename, prefix = _CHECKPOINTS[variant] + return {"url": f"https://download.pytorch.org/models/{filename}", + "enum": enum, "sha256_prefix": prefix, "source": "torchvision.models.optical_flow"} + + +def input_shape(input_spec: str) -> tuple[int, int, int, int]: + shape = tuple(int(v) for v in input_spec.split("x")) + if len(shape) != 4 or shape[:2] != (1, 6): + raise ValueError(f"RAFT requires packed RGB frame pairs, 1x6xHxW; got {input_spec}") + if any(v < 128 or v % 8 for v in shape[-2:]): + raise ValueError("RAFT H,W must be >=128 and divisible by 8 (four correlation levels)") + return shape + + +def preprocess(image1, image2, input_spec: str = INPUT_SPEC): + """Shared FP32, G2 and calibration recipe; two PIL RGB images -> numpy NCHW.""" + import torch + from torchvision.transforms import InterpolationMode + from torchvision.transforms import functional as F + + shape = input_shape(input_spec) + if image1.size != image2.size: + raise ValueError(f"frame dimensions differ: {image1.size} != {image2.size}") + frames = [] + for image in (image1, image2): + x = F.convert_image_dtype(F.pil_to_tensor(image.convert("RGB")), torch.float32) + x = F.resize(x, list(shape[-2:]), interpolation=InterpolationMode.BILINEAR, antialias=False) + frames.append(F.normalize(x, mean=[0.5] * 3, std=[0.5] * 3)) + return torch.cat(frames, dim=0).unsqueeze(0).contiguous().numpy() + + +def build_variant(variant: str, weights: Path, input_spec: str): + import torch + from torchvision.models import optical_flow + + shape = input_shape(input_spec) + checkpoint = checkpoint_info(variant) + digest = hashlib.sha256(Path(weights).read_bytes()).hexdigest() + if not digest.startswith(checkpoint["sha256_prefix"]): + raise ValueError(f"wrong official RAFT-{variant} checkpoint: sha256={digest}") + core = getattr(optical_flow, f"raft_{variant}")(weights=None, progress=False) + # Strict load; no dropped/missing keys, replacement layers or random fallback. + core.load_state_dict(torch.load(weights, map_location="cpu", weights_only=True), strict=True) + + class PackedRaft(torch.nn.Module): + def __init__(self): + super().__init__() + self.core = core + + def forward(self, pair): + if not torch.jit.is_tracing() and tuple(pair.shape) != shape: + raise ValueError(f"expected fixed packed RAFT input {shape}, got {tuple(pair.shape)}") + return self.core(pair[:, :3], pair[:, 3:6], num_flow_updates=12)[-1] + + return PackedRaft().eval() diff --git a/harness/adapters/raft_large.py b/harness/adapters/raft_large.py new file mode 100644 index 0000000..c6cac3c --- /dev/null +++ b/harness/adapters/raft_large.py @@ -0,0 +1,11 @@ +"""RAFT-large, official torchvision C_T_SKHT_V2 checkpoint.""" +from pathlib import Path + +from .raft_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("large") + + +def build(weights: Path, input_spec: str): + return build_variant("large", weights, input_spec) diff --git a/harness/adapters/raft_small.py b/harness/adapters/raft_small.py new file mode 100644 index 0000000..3328f59 --- /dev/null +++ b/harness/adapters/raft_small.py @@ -0,0 +1,11 @@ +"""RAFT-small, official torchvision C_T_V2 checkpoint.""" +from pathlib import Path + +from .raft_common import (EXPORTER_VERSION, INPUT_SPEC, MODEL_SPEC, OPSET, PREPROCESS, + build_variant, checkpoint_info, preprocess) + +CHECKPOINT = checkpoint_info("small") + + +def build(weights: Path, input_spec: str): + return build_variant("small", weights, input_spec) diff --git a/harness/adapters/reppoints_r50_fpn.py b/harness/adapters/reppoints_r50_fpn.py new file mode 100644 index 0000000..78d1306 --- /dev/null +++ b/harness/adapters/reppoints_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection reppoints.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "reppoints.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-reppoints.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/retinanet_r101_fpn.py b/harness/adapters/retinanet_r101_fpn.py new file mode 100644 index 0000000..ee82ad0 --- /dev/null +++ b/harness/adapters/retinanet_r101_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection retinanet.r101.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "retinanet.r101.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-retinanet.r101.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/retinanet_r50_fpn.py b/harness/adapters/retinanet_r50_fpn.py new file mode 100644 index 0000000..ef86b05 --- /dev/null +++ b/harness/adapters/retinanet_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection retinanet.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "retinanet.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-retinanet.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/rtmdet_common.py b/harness/adapters/rtmdet_common.py new file mode 100644 index 0000000..68ed44e --- /dev/null +++ b/harness/adapters/rtmdet_common.py @@ -0,0 +1,146 @@ +"""Official MMDetection RTMDet COCO recipe; heavy imports are runtime-only. + +preprocess consumes PIL RGB and returns normalized BGR float32 NCHW. +decode consumes six raw head arrays and PIL image.size (width, height), returning +original-image xyxy boxes, scores and COCO category indices 0..79. Bottom/right +padding and independent rounded width/height scales exactly follow MMCV. +""" +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from .openmmlab import get_recipe, build as build_official + +RUNTIME_PYTHON = "/home/ming01.wang/outputs/full-coverage-20260917/openmmlab/venv/bin/python" +INPUT_SPEC = "1x3x640x640" +MODEL_SPEC = {"dataset": "coco2017", "input_spec": INPUT_SPEC} +PREPROCESS = { + "recipe": "mmdet-rtmdet-coco-v1", + "input_spec": INPUT_SPEC, + "color": "BGR", + "resize": {"scale": [640, 640], "keep_ratio": True, "rounding": "int(value+0.5)", + "backend": "cv2", "interpolation": "bilinear"}, + "pad": {"size": [640, 640], "placement": "bottom_right", "value": 114}, + "mean": [103.53, 116.28, 123.675], + "std": [57.375, 57.12, 58.395], + "layout": "NCHW", "dtype": "float32", +} +TEST_CFG = {"nms_pre": 30000, "min_bbox_size": 0, "score_thr": 0.001, + "nms": {"type": "nms", "iou_threshold": 0.65}, "max_per_img": 300} + + +def checkpoint_info(variant): + if variant not in ("tiny", "s", "m", "l", "x"): + raise ValueError(f"Unsupported RTMDet variant: {variant}") + return get_recipe(f"rtmdet.{variant}")["checkpoint"] + + +def _check_shape(input_spec): + if input_spec != INPUT_SPEC: + raise ValueError(f"RTMDet fixed recipe requires {INPUT_SPEC}, got {input_spec}") + + +def geometry(original_size): + """Return official resized (w,h), exact axis scales, and padded image shape.""" + w, h = map(int, original_size) + if min(w, h) <= 0: + raise ValueError(f"Invalid original image size: {original_size}") + scale = 640 / max(w, h) + nw, nh = int(w * scale + .5), int(h * scale + .5) + if min(nw, nh) < 1: + raise ValueError("Image aspect ratio collapses under official MMCV resize") + return (nw, nh), (nw / w, nh / h), (640, 640) + + +def preprocess(image, input_spec=INPUT_SPEC): + _check_shape(input_spec) + import numpy as np + import mmcv + bgr = np.asarray(image.convert("RGB"))[:, :, ::-1].copy() + resized = mmcv.imrescale(bgr, (640, 640), interpolation="bilinear", backend="cv2") + padded = mmcv.impad(resized, shape=(640, 640), pad_val=(114, 114, 114)) + x = np.ascontiguousarray(padded.transpose(2, 0, 1)[None], dtype=np.float32) + mean = np.asarray(PREPROCESS["mean"], dtype=np.float32)[None, :, None, None] + std = np.asarray(PREPROCESS["std"], dtype=np.float32)[None, :, None, None] + return np.ascontiguousarray((x - mean) / std) + + +@lru_cache(maxsize=1) +def _decode_tools(): + # Reuse upstream prior generation, filtering, distance decoding and NMS. + # No model construction or weight loading occurs here, even on first use. + from mmdet.models.task_modules.prior_generators import MlvlPointGenerator + from mmdet.models.task_modules.coders import DistancePointBBoxCoder + from mmdet.models.utils import filter_scores_and_topk + from mmcv.ops import batched_nms + priors = MlvlPointGenerator(strides=[8, 16, 32], offset=0).grid_priors( + [(80, 80), (40, 40), (20, 20)], device="cpu") + return priors, DistancePointBBoxCoder(), filter_scores_and_topk, batched_nms + + +def decode(outputs, original_size): + import numpy as np + import torch + if len(outputs) != 6: + raise ValueError(f"RTMDet expects cls8/16/32 then distances8/16/32; got {len(outputs)} outputs") + priors, coder, filter_scores, nms = _decode_tools() + _, scales, padded_shape = geometry(original_size) + box_parts, score_parts, label_parts = [], [], [] + for i, side in enumerate((80, 40, 20)): + cls = torch.as_tensor(np.ascontiguousarray(outputs[i]), dtype=torch.float32) + reg = torch.as_tensor(np.ascontiguousarray(outputs[i + 3]), dtype=torch.float32) + if tuple(cls.shape) != (1, 80, side, side) or tuple(reg.shape) != (1, 4, side, side): + raise ValueError(f"Wrong RTMDet level {i} shapes: {cls.shape}/{reg.shape}") + if not torch.isfinite(cls).all() or not torch.isfinite(reg).all(): + raise ValueError("Nonfinite RTMDet head output") + scores = cls[0].permute(1, 2, 0).reshape(-1, 80).sigmoid() + distances = reg[0].permute(1, 2, 0).reshape(-1, 4) + scores, labels, _, kept = filter_scores( + scores, TEST_CFG["score_thr"], TEST_CFG["nms_pre"], + {"bbox_pred": distances, "priors": priors[i]}) + box_parts.append(coder.decode(kept["priors"], kept["bbox_pred"], max_shape=padded_shape)) + score_parts.append(scores); label_parts.append(labels) + boxes, scores, labels = torch.cat(box_parts), torch.cat(score_parts), torch.cat(label_parts) + # MMCV Pad updates img_shape to 640x640. Upstream clips there, then rescales; + # it does not add a second original-image clip after removing bottom/right pad. + boxes = boxes * boxes.new_tensor([1 / scales[0], 1 / scales[1]] * 2) + wh = boxes[:, 2:] - boxes[:, :2] + valid = (wh[:, 0] > TEST_CFG["min_bbox_size"]) & (wh[:, 1] > TEST_CFG["min_bbox_size"]) + boxes, scores, labels = boxes[valid], scores[valid], labels[valid] + if boxes.numel(): + dets, keep = nms(boxes, scores, labels, TEST_CFG["nms"]) + keep = keep[:TEST_CFG["max_per_img"]] + boxes, scores, labels = boxes[keep], dets[:len(keep), -1], labels[keep] + return boxes.numpy(), scores.numpy(), labels.numpy() + + +def build_variant(variant, weights: Path, input_spec: str): + _check_shape(input_spec) + import torch + raw = build_official(f"rtmdet.{variant}", weights, input_spec) + cfg = raw.cfg + if dict(cfg.model.test_cfg) != TEST_CFG: + raise ValueError("Official RTMDet postprocess config changed") + head = cfg.model.bbox_head + expected_prior = {"type": "MlvlPointGenerator", "offset": 0, "strides": [8, 16, 32]} + dp = cfg.model.data_preprocessor + if (head.type != "RTMDetSepBNHead" or head.num_classes != 80 or head.with_objectness + or dict(head.anchor_generator) != expected_prior + or head.bbox_coder.type != "DistancePointBBoxCoder" + or not head.loss_cls.use_sigmoid or dp.bgr_to_rgb + or list(dp.mean) != PREPROCESS["mean"] or list(dp.std) != PREPROCESS["std"]): + raise ValueError("Official RTMDet tensor recipe changed") + + class RawHeads(torch.nn.Module): + def __init__(self): + super().__init__() + self.upstream = raw.upstream + self.upstream.cfg = cfg + self.cfg = cfg + self.checkpoint_sha256 = raw.checkpoint_sha256 + + def forward(self, x): + cls, distances = self.upstream._forward(x) + return tuple(cls) + tuple(distances) + + return RawHeads().eval() diff --git a/harness/adapters/rtmdet_l.py b/harness/adapters/rtmdet_l.py new file mode 100644 index 0000000..d5e787f --- /dev/null +++ b/harness/adapters/rtmdet_l.py @@ -0,0 +1,7 @@ +"""RTMDet-l: official COCO checkpoint and shared verified tensor contract.""" +from .rtmdet_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, PREPROCESS, + checkpoint_info, preprocess, decode, build_variant) +CHECKPOINT = checkpoint_info("l") + +def build(weights, input_spec): + return build_variant("l", weights, input_spec) diff --git a/harness/adapters/rtmdet_m.py b/harness/adapters/rtmdet_m.py new file mode 100644 index 0000000..a30f6d5 --- /dev/null +++ b/harness/adapters/rtmdet_m.py @@ -0,0 +1,7 @@ +"""RTMDet-m: official COCO checkpoint and shared verified tensor contract.""" +from .rtmdet_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, PREPROCESS, + checkpoint_info, preprocess, decode, build_variant) +CHECKPOINT = checkpoint_info("m") + +def build(weights, input_spec): + return build_variant("m", weights, input_spec) diff --git a/harness/adapters/rtmdet_s.py b/harness/adapters/rtmdet_s.py new file mode 100644 index 0000000..fa923b9 --- /dev/null +++ b/harness/adapters/rtmdet_s.py @@ -0,0 +1,7 @@ +"""RTMDet-s: official COCO checkpoint and shared verified tensor contract.""" +from .rtmdet_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, PREPROCESS, + checkpoint_info, preprocess, decode, build_variant) +CHECKPOINT = checkpoint_info("s") + +def build(weights, input_spec): + return build_variant("s", weights, input_spec) diff --git a/harness/adapters/rtmdet_tiny.py b/harness/adapters/rtmdet_tiny.py new file mode 100644 index 0000000..c51ef94 --- /dev/null +++ b/harness/adapters/rtmdet_tiny.py @@ -0,0 +1,7 @@ +"""RTMDet-tiny: official COCO checkpoint and shared verified tensor contract.""" +from .rtmdet_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, PREPROCESS, + checkpoint_info, preprocess, decode, build_variant) +CHECKPOINT = checkpoint_info("tiny") + +def build(weights, input_spec): + return build_variant("tiny", weights, input_spec) diff --git a/harness/adapters/rtmdet_x.py b/harness/adapters/rtmdet_x.py new file mode 100644 index 0000000..57f9280 --- /dev/null +++ b/harness/adapters/rtmdet_x.py @@ -0,0 +1,7 @@ +"""RTMDet-x: official COCO checkpoint and shared verified tensor contract.""" +from .rtmdet_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, PREPROCESS, + checkpoint_info, preprocess, decode, build_variant) +CHECKPOINT = checkpoint_info("x") + +def build(weights, input_spec): + return build_variant("x", weights, input_spec) diff --git a/harness/adapters/rtmpose_common.py b/harness/adapters/rtmpose_common.py new file mode 100644 index 0000000..f814238 --- /dev/null +++ b/harness/adapters/rtmpose_common.py @@ -0,0 +1,228 @@ +"""Pinned RTMPose COCO person crops, official SimCC decoding and OKS NMS. + +The 256x192 configs comment out bbox_file; X enables it. Both use the pinned +HRNet val2017 person detections linked by MMPose's official dataset guide. +No GT boxes, detector substitution, score cutoff or full-image fallback is used +for evaluation. Calibration uses separately recorded real COCO train boxes. +""" +from __future__ import annotations + +from functools import lru_cache +import hashlib +import json +from pathlib import Path +import warnings + +from .openmmlab import build as official_build, get_recipe + +RUNTIME_PYTHON = "/home/ming01.wang/outputs/full-coverage-20260917/openmmlab/venv/bin/python" +EXPORTER_VERSION = "rtmpose-official-simcc-v1" +INPUT_SPEC = "1x3x256x192" +LARGE_INPUT_SPEC = "1x3x384x288" +BBOX_FILE = Path("/home/ming01.wang/outputs/pose/COCO_val2017_detections_AP_H_56_person.json") +BBOX_SHA256 = "53ba0ad8d0fd461c5a000cd90797fa8c39cd8c38cd125125c0412626ff592d59" +SOURCE_COMMIT = "759b39c13fea6ba094afc1fa932f51dc1b11cbf9" +PREPROCESS = { + "recipe": "rtmpose-coco-official-topdown-affine-v1", "dataset": "coco2017", + "input_spec": INPUT_SPEC, "color": "RGB", "mean": [123.675, 116.28, 103.53], + "std": [58.395, 57.12, 57.375], "scale": 1.0, "dtype": "float32", "layout": "NCHW", + "bbox_padding": 1.25, "rotation": 0, "use_udp": False, "interpolation": "opencv_inter_linear", + "border": "constant_zero_before_normalization", "simcc_split_ratio": 2.0, + "flip_test": True, "use_dark": False, "keypoint_score_thr": 0.2, + "score_mode": "bbox_keypoint", "nms_mode": "oks_nms", "nms_thr": 0.9, + "evaluation_boxes": BBOX_FILE.name, "evaluation_boxes_sha256": BBOX_SHA256, + "evaluation_boxes_source": "MMPose official dataset guide / HRNet person_detection_results", + "crop": "real person bbox_xyxy; official GetBBoxCenterScale then TopdownAffine", + "calibration_boxes": "COCO person_keypoints_train2017 annotations; 128 distinct images", + "calibration_selection": "seed20260914; largest noncrowd labeled person per image, area then annotation ID", +} + + +def input_size(input_spec): + if input_spec not in (INPUT_SPEC, LARGE_INPUT_SPEC): + raise ValueError(f"Unverified RTMPose input: {input_spec}") + _, _, height, width = (int(v) for v in input_spec.split("x")) + return width, height + + +def preprocess_recipe(model): + return ({**PREPROCESS, "input_spec": LARGE_INPUT_SPEC, "codec_sigma": [6., 6.93]} + if model == "rtmpose.x" else dict(PREPROCESS)) + + +def metadata(model: str): + recipe = get_recipe(model) + if model not in ("rtmpose.tiny", "rtmpose.s", "rtmpose.m", "rtmpose.l", "rtmpose.x") or recipe["source"]["revision"] != SOURCE_COMMIT: + raise ValueError(f"Unverified RTMPose recipe: {model}") + spec = LARGE_INPUT_SPEC if model == "rtmpose.x" else INPUT_SPEC + return dict(recipe["checkpoint"]), { + "source": "custom", "dataset": "coco2017", "input_spec": spec, + "input_override_reason": f"CSV 640 square conflicts with official {spec.removeprefix('1x3x')} person affine input", + "bbox_override_reason": ("Official X config explicitly selects the same pinned HRNet val2017 detection file" + if model == "rtmpose.x" else + "Enable documented HRNet val2017 detections; config bbox_file is commented out"), + } + + +@lru_cache(maxsize=2) +def components(input_spec=INPUT_SPEC): + from .openmmlab import load_config + from mmpose.utils import register_all_modules + from mmpose.datasets.transforms import GetBBoxCenterScale, TopdownAffine + from mmpose.apis.inference import dataset_meta_from_config + from mmpose.registry import KEYPOINT_CODECS + + cfg = load_config("rtmpose.x" if input_spec == LARGE_INPUT_SPEC else "rtmpose.tiny") + register_all_modules() + codec = KEYPOINT_CODECS.build(cfg.codec) + meta = dataset_meta_from_config(cfg) + return GetBBoxCenterScale(padding=1.25), TopdownAffine(input_size=input_size(input_spec), use_udp=False), codec, meta + + +def build_model(model: str, weights: Path, input_spec: str): + expected = LARGE_INPUT_SPEC if model == "rtmpose.x" else INPUT_SPEC + if input_spec != expected: + raise ValueError(f"RTMPose requires official {expected}, got {input_spec}") + network = official_build(model, weights, input_spec) + cfg = network.cfg + if (tuple(cfg.codec.input_size) != input_size(input_spec) or cfg.codec.simcc_split_ratio != 2 + or cfg.codec.use_dark or not cfg.model.test_cfg.flip_test + or cfg.model.data_preprocessor.mean != PREPROCESS["mean"] + or cfg.model.data_preprocessor.std != PREPROCESS["std"] + or not cfg.model.data_preprocessor.bgr_to_rgb): + raise ValueError("Official RTMPose config differs from the pinned geometry/codec recipe") + return network + + +def crop(image, input_spec: str, bbox): + """Return normalized crop plus the actual official affine/center/scale.""" + import numpy as np + from mmpose.structures.bbox import get_warp_matrix + + size = input_size(input_spec) + if bbox is None: + raise ValueError("RTMPose needs explicit real person bbox_xyxy") + box = np.asarray(bbox, dtype=np.float32) + if box.shape != (4,) or not np.isfinite(box).all() or (box[2:] <= box[:2]).any(): + raise ValueError("Invalid person bbox_xyxy") + center_scale, affine, _, _ = components(input_spec) + data = center_scale({"img": np.asarray(image.convert("RGB")), "bbox": box[None].copy()}) + data = affine(data) + matrix = get_warp_matrix(data["input_center"], data["input_scale"], 0, size) + x = (data["img"].astype(np.float32) - np.asarray(PREPROCESS["mean"], np.float32)) / np.asarray(PREPROCESS["std"], np.float32) + return (np.ascontiguousarray(x.transpose(2, 0, 1)[None]), matrix, + data["input_center"].copy(), data["input_scale"].copy()) + + +def preprocess(image, input_spec: str = INPUT_SPEC, bbox=None): + return crop(image, input_spec, bbox)[0] + + +@lru_cache(maxsize=1) +def detection_boxes(): + import numpy as np + + data = BBOX_FILE.read_bytes() + if hashlib.sha256(data).hexdigest() != BBOX_SHA256: + raise ValueError("Official person detection file SHA256 mismatch") + images = {} + for index, detection in enumerate(json.loads(data)): + if detection["category_id"] != 1: + raise ValueError("Pinned person detection file contains a non-person category") + box = np.asarray(detection["bbox"], dtype=np.float32) + if box.shape != (4,) or not np.isfinite(box).all() or min(box[2:]) <= 0: + raise ValueError("Invalid box in official detector result") + box[2:] += box[:2] + score = float(np.float32(detection["score"])) + if not np.isfinite(score): + raise ValueError("Invalid detector score") + images.setdefault(int(detection["image_id"]), []).append( + {"source_index": index, "bbox_xyxy": box, "bbox_score": score}) + return images + + +def boxes_for(t1, item): + if t1.manifest.get("dataset") != "coco2017": + raise ValueError("RTMPose recipe requires COCO2017") + return detection_boxes().get(int(item["id"]), []) + + +def diagnostic_box(t1, item): + boxes = boxes_for(t1, item) + return max(boxes, key=lambda b: (b["bbox_score"], -b["source_index"])) if boxes else None + + +def g2_input(t1, item, input_spec): + box = diagnostic_box(t1, item) + if box is None: + warnings.warn(f"RTMPose G2 image {item['id']}: no fixed detector person bbox; skip, no fallback") + return None + return preprocess(t1.image(item), input_spec, box["bbox_xyxy"]) + + +def decode(outputs, flipped_outputs, center, scale, input_spec=INPUT_SPEC): + import numpy as np + + _, _, codec, meta = components(input_spec) + width, height = input_size(input_spec) + if len(outputs) != 2 or len(flipped_outputs) != 2: + raise ValueError("RTMPose requires exactly two SimCC tensors") + arrays = [np.asarray(v, np.float32) for v in (*outputs, *flipped_outputs)] + for value, shape in zip(arrays, ((1, 17, width * 2), (1, 17, height * 2)) * 2): + if value.shape != shape or not np.isfinite(value).all(): + raise ValueError(f"Invalid SimCC tensor {value.shape}; expected {shape}") + x, y, fx, fy = arrays + indices = meta["flip_indices"] + # The exact RTMCCHead flip_vectors + mean rule, with no softmax/shift. + x = (x + fx[:, indices, ::-1]) * np.float32(0.5) + y = (y + fy[:, indices]) * np.float32(0.5) + points, scores = codec.decode(x, y) + # TopdownPoseEstimator.add_pred_to_datasample: retain unclipped coordinates. + points[0] = points[0] / np.asarray((width, height)) * scale + center - 0.5 * scale + return np.concatenate((points[0], scores[0, :, None]), axis=-1) + + +def predict(runner, t1, input_spec, pose_evidence=None): + import numpy as np + from mmpose.evaluation.functional import oks_nms + from ..eval.common import run_batched + + predictions = [] + _, _, _, meta = components(input_spec) + if pose_evidence is not None: + pose_evidence.update(boxes_file=str(BBOX_FILE), boxes_sha256=BBOX_SHA256, + pose_protocol=preprocess_recipe("rtmpose.x" if input_spec == LARGE_INPUT_SPEC else "rtmpose.tiny")) + for item in t1.items: + image = t1.image(item) + boxes = boxes_for(t1, item) + candidates = [] + for start in range(0, len(boxes), 16): + chunk = boxes[start:start + 16] + crops = [crop(image, input_spec, b["bbox_xyxy"]) for b in chunk] + xs = [value for x, _, _, _ in crops for value in (x, np.ascontiguousarray(x[..., ::-1]))] + outputs = run_batched(runner, xs) + if len(outputs) != len(xs): + raise ValueError("RTMPose runner omitted person/flip outputs") + for j, (box, (_, matrix, center, scale)) in enumerate(zip(chunk, crops)): + points = decode(outputs[j * 2], outputs[j * 2 + 1], center, scale, input_spec) + scores = points[:, 2] + valid = scores[scores > PREPROCESS["keypoint_score_thr"]] + score = box["bbox_score"] * (sum(float(v) for v in valid) / len(valid) if len(valid) else 0.) + candidates.append({"keypoints": points, "score": score, "area": float(np.prod(scale)), + "bbox_xyxy": box["bbox_xyxy"], "affine": matrix, + "source_index": box["source_index"]}) + keep = oks_nms(candidates, PREPROCESS["nms_thr"], sigmas=meta["sigmas"]) if candidates else [] + retained = [candidates[i] for i in keep] + if pose_evidence is not None: + pose_evidence["images"].append({ + "image_id": item["id"], "image_size": list(image.size), "transform": "topdown_affine", + "input_detections": len(boxes), "no_person_bbox": not bool(boxes), + "detections": [{"category_id": 1, "bbox_xyxy": p["bbox_xyxy"].tolist(), + "score": p["score"], "keypoints": p["keypoints"].tolist(), + "affine": p["affine"].tolist(), "source_index": p["source_index"]} + for p in retained]}) + for p in retained: + bbox = p["bbox_xyxy"].copy(); bbox[2:] -= bbox[:2] + predictions.append({"image_id": item["id"], "category_id": 1, "score": p["score"], + "bbox": bbox.tolist(), "keypoints": p["keypoints"].reshape(-1).tolist()}) + return predictions diff --git a/harness/adapters/rtmpose_l.py b/harness/adapters/rtmpose_l.py new file mode 100644 index 0000000..4c8c97c --- /dev/null +++ b/harness/adapters/rtmpose_l.py @@ -0,0 +1,7 @@ +"""Official RTMPose L, COCO-17 at 256x192.""" +from functools import partial +from .rtmpose_common import (metadata, build_model, preprocess, predict, g2_input, + PREPROCESS, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata("rtmpose.l") +build = partial(build_model, "rtmpose.l") diff --git a/harness/adapters/rtmpose_m.py b/harness/adapters/rtmpose_m.py new file mode 100644 index 0000000..82c7eaa --- /dev/null +++ b/harness/adapters/rtmpose_m.py @@ -0,0 +1,7 @@ +"""Official RTMPose m, complete person-crop/SimCC recipe.""" +from functools import partial +from .rtmpose_common import (metadata, build_model, preprocess, predict, g2_input, + PREPROCESS, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata("rtmpose.m") +build = partial(build_model, "rtmpose.m") diff --git a/harness/adapters/rtmpose_s.py b/harness/adapters/rtmpose_s.py new file mode 100644 index 0000000..84e1da9 --- /dev/null +++ b/harness/adapters/rtmpose_s.py @@ -0,0 +1,7 @@ +"""Official RTMPose s, complete person-crop/SimCC recipe.""" +from functools import partial +from .rtmpose_common import (metadata, build_model, preprocess, predict, g2_input, + PREPROCESS, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata("rtmpose.s") +build = partial(build_model, "rtmpose.s") diff --git a/harness/adapters/rtmpose_tiny.py b/harness/adapters/rtmpose_tiny.py new file mode 100644 index 0000000..66dcea4 --- /dev/null +++ b/harness/adapters/rtmpose_tiny.py @@ -0,0 +1,7 @@ +"""Official RTMPose tiny, complete person-crop/SimCC recipe.""" +from functools import partial +from .rtmpose_common import (metadata, build_model, preprocess, predict, g2_input, + PREPROCESS, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata("rtmpose.tiny") +build = partial(build_model, "rtmpose.tiny") diff --git a/harness/adapters/rtmpose_x.py b/harness/adapters/rtmpose_x.py new file mode 100644 index 0000000..01764ac --- /dev/null +++ b/harness/adapters/rtmpose_x.py @@ -0,0 +1,8 @@ +"""Official RTMPose X body7/COCO-17 at 384x288, not Halpe26/wholebody.""" +from functools import partial +from .rtmpose_common import (metadata, build_model, preprocess, predict, g2_input, + preprocess_recipe, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata("rtmpose.x") +PREPROCESS = preprocess_recipe("rtmpose.x") +build = partial(build_model, "rtmpose.x") diff --git a/harness/adapters/segformer_b0.py b/harness/adapters/segformer_b0.py new file mode 100644 index 0000000..bda37a9 --- /dev/null +++ b/harness/adapters/segformer_b0.py @@ -0,0 +1,8 @@ +"""Official ADE20K segformer.b0; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +OPSET = 14 # Official attention export requires unflatten and scaled-dot-product attention. +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segformer.b0") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "segformer.b0") diff --git a/harness/adapters/segformer_b1.py b/harness/adapters/segformer_b1.py new file mode 100644 index 0000000..660cf9e --- /dev/null +++ b/harness/adapters/segformer_b1.py @@ -0,0 +1,8 @@ +"""Official ADE20K segformer.b1; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +OPSET = 14 # Official attention export requires unflatten and scaled-dot-product attention. +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segformer.b1") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "segformer.b1") diff --git a/harness/adapters/segformer_b2.py b/harness/adapters/segformer_b2.py new file mode 100644 index 0000000..20864de --- /dev/null +++ b/harness/adapters/segformer_b2.py @@ -0,0 +1,8 @@ +"""Official ADE20K segformer.b2; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +OPSET = 14 # Official attention export requires unflatten and scaled-dot-product attention. +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segformer.b2") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "segformer.b2") diff --git a/harness/adapters/segformer_b3.py b/harness/adapters/segformer_b3.py new file mode 100644 index 0000000..3d1e800 --- /dev/null +++ b/harness/adapters/segformer_b3.py @@ -0,0 +1,8 @@ +"""Official ADE20K segformer.b3; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +OPSET = 14 # Official attention export requires unflatten and scaled-dot-product attention. +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segformer.b3") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "segformer.b3") diff --git a/harness/adapters/segformer_b4.py b/harness/adapters/segformer_b4.py new file mode 100644 index 0000000..b2c11ea --- /dev/null +++ b/harness/adapters/segformer_b4.py @@ -0,0 +1,8 @@ +"""Official ADE20K segformer.b4; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +OPSET = 14 # Official attention export requires unflatten and scaled-dot-product attention. +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segformer.b4") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "segformer.b4") diff --git a/harness/adapters/segformer_b5.py b/harness/adapters/segformer_b5.py new file mode 100644 index 0000000..d42a140 --- /dev/null +++ b/harness/adapters/segformer_b5.py @@ -0,0 +1,8 @@ +"""Official ADE20K segformer.b5; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +OPSET = 14 # Official attention export requires unflatten and scaled-dot-product attention. +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segformer.b5") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "segformer.b5") diff --git a/harness/adapters/segnext_base.py b/harness/adapters/segnext_base.py new file mode 100644 index 0000000..fd9148c --- /dev/null +++ b/harness/adapters/segnext_base.py @@ -0,0 +1,13 @@ +"""Official ADE20K SegNeXt-Base; each evaluation keeps fresh random NMF bases.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segnext.base") +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = build_model("segnext.base", weights, input_spec) + if network.upstream.decode_head.hamburger.ham.rand_init is not True: + raise ValueError("Official SegNeXt-Base requires per-call random NMF bases") + return network diff --git a/harness/adapters/segnext_large.py b/harness/adapters/segnext_large.py new file mode 100644 index 0000000..384124b --- /dev/null +++ b/harness/adapters/segnext_large.py @@ -0,0 +1,13 @@ +"""Official ADE20K SegNeXt-Large; each evaluation keeps fresh random NMF bases.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segnext.large") +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = build_model("segnext.large", weights, input_spec) + if network.upstream.decode_head.hamburger.ham.rand_init is not True: + raise ValueError("Official SegNeXt-Large requires per-call random NMF bases") + return network diff --git a/harness/adapters/segnext_small.py b/harness/adapters/segnext_small.py new file mode 100644 index 0000000..24a256d --- /dev/null +++ b/harness/adapters/segnext_small.py @@ -0,0 +1,13 @@ +"""Official ADE20K SegNeXt-Small; each evaluation keeps fresh random NMF bases.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segnext.small") +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = build_model("segnext.small", weights, input_spec) + if network.upstream.decode_head.hamburger.ham.rand_init is not True: + raise ValueError("Official SegNeXt-Small requires per-call random NMF bases") + return network diff --git a/harness/adapters/segnext_tiny.py b/harness/adapters/segnext_tiny.py new file mode 100644 index 0000000..d2d0330 --- /dev/null +++ b/harness/adapters/segnext_tiny.py @@ -0,0 +1,13 @@ +"""Official SegNeXt-Tiny; each evaluation keeps fresh random NMF bases.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("segnext.tiny") +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = build_model("segnext.tiny", weights, input_spec) + if network.upstream.decode_head.hamburger.ham.rand_init is not True: + raise ValueError("Official SegNeXt-Tiny requires per-call random NMF bases") + return network diff --git a/harness/adapters/simcc_resnet50.py b/harness/adapters/simcc_resnet50.py new file mode 100644 index 0000000..2a0480f --- /dev/null +++ b/harness/adapters/simcc_resnet50.py @@ -0,0 +1,9 @@ +"""Official simcc.resnet50 COCO17 at 256x192.""" +from functools import partial +from .topdown_pose_common import (metadata, build_model, predict as _predict, decode as _decode, + preprocess, g2_input, PREPROCESS, RUNTIME_PYTHON, EXPORTER_VERSION) + +CHECKPOINT, MODEL_SPEC = metadata('simcc.resnet50') +build = partial(build_model, 'simcc.resnet50') +predict = partial(_predict, 'simcc.resnet50') +decode = partial(_decode, 'simcc.resnet50') diff --git a/harness/adapters/solo_r50.py b/harness/adapters/solo_r50.py new file mode 100644 index 0000000..a567365 --- /dev/null +++ b/harness/adapters/solo_r50.py @@ -0,0 +1,16 @@ +"""Exact MMDetection solo.r50; see mmdet_solo_common.""" +from functools import partial + +from .mmdet_solo_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, + build_variant, predict_variant, g2_input_variant) + +MODEL_ID = "solo.r50" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-solo-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) +predict = partial(predict_variant, MODEL_ID) +g2_input = partial(g2_input_variant, MODEL_ID) diff --git a/harness/adapters/solov2_r50.py b/harness/adapters/solov2_r50.py new file mode 100644 index 0000000..9bc1743 --- /dev/null +++ b/harness/adapters/solov2_r50.py @@ -0,0 +1,16 @@ +"""Exact MMDetection solov2.r50; see mmdet_solo_common.""" +from functools import partial + +from .mmdet_solo_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, + build_variant, predict_variant, g2_input_variant) + +MODEL_ID = "solov2.r50" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-solov2-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) +predict = partial(predict_variant, MODEL_ID) +g2_input = partial(g2_input_variant, MODEL_ID) diff --git a/harness/adapters/ssd300_vgg16.py b/harness/adapters/ssd300_vgg16.py new file mode 100644 index 0000000..b6b8f6b --- /dev/null +++ b/harness/adapters/ssd300_vgg16.py @@ -0,0 +1,108 @@ +"""Torchvision's official SSD300 VGG16 COCO_V1, static head and host NMS.""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +from .ssdlite_common import COCO_CATEGORY_IDS + +INPUT_SPEC = "1x3x300x300" +MODEL_SPEC = {"source": "custom", "input_spec": INPUT_SPEC} +CHECKPOINT = { + "url": "https://download.pytorch.org/models/ssd300_vgg16_coco-b556d3b4.pth", + "filename": "ssd300_vgg16_coco-b556d3b4.pth", + "sha256": "b556d3b43ab6c3f63d81bfb8835fe8756ac22da664357da100dccf96b6a6b42d", +} +PREPROCESS = { + "recipe": "torchvision-ssd300-vgg16-coco-v1", "color": "RGB", + "layout": "NCHW", "dtype": "float32", "scale": 1 / 255, + "mean": [0.48235, 0.45882, 0.40784], "std": [1 / 255] * 3, + "resize": [300, 300], "normalize_before_resize": True, + "interpolation": "torch-bilinear-align_corners_false", "antialias": False, +} +EXPORTER_VERSION = "ssd300-vgg16-coco-v1-static-1" +OPSET = 11 +OUTPUT_SHAPE = (1, 8732, 95) + + +def preprocess(image, input_spec=INPUT_SPEC): + import torch + from torchvision.transforms.functional import pil_to_tensor + if input_spec != INPUT_SPEC: + raise ValueError(f"SSD300 requires {INPUT_SPEC}") + x = pil_to_tensor(image.convert("RGB")).float() / 255 + x = (x - x.new_tensor(PREPROCESS["mean"])[:, None, None]) / x.new_tensor(PREPROCESS["std"])[:, None, None] + return torch.nn.functional.interpolate(x[None], size=(300, 300), mode="bilinear", align_corners=False).numpy() + + +def reference_model(weights): + import torch + from torchvision.models.detection import ssd300_vgg16 + if hashlib.sha256(Path(weights).read_bytes()).hexdigest() != CHECKPOINT["sha256"]: + raise ValueError("SSD300 official checkpoint SHA256 mismatch") + model = ssd300_vgg16(weights=None, weights_backbone=None, num_classes=91) + model.load_state_dict(torch.load(weights, map_location="cpu", weights_only=True), strict=True) + return model.eval() + + +def build(weights, input_spec): + import torch + from torchvision.models.detection.image_list import ImageList + if input_spec != INPUT_SPEC: + raise ValueError(f"SSD300 requires {INPUT_SPEC}") + model = reference_model(weights) + + class StaticSSD(torch.nn.Module): + def __init__(self): + super().__init__() + self.backbone, self.head, self.box_coder = model.backbone, model.head, model.box_coder + with torch.no_grad(): + x = torch.zeros(1, 3, 300, 300) + anchors = model.anchor_generator(ImageList(x, [(300, 300)]), list(self.backbone(x).values()))[0] + self.register_buffer("anchors", anchors) + + def forward(self, x): + heads = self.head(list(self.backbone(x).values())) + boxes = self.box_coder.decode_single(heads["bbox_regression"][0], self.anchors) + return torch.cat((boxes[None], torch.softmax(heads["cls_logits"], dim=-1)), dim=-1) + + return StaticSSD().eval() + + +def decode(outputs, original_size): + import numpy as np + import torch + from torchvision.ops import batched_nms, clip_boxes_to_image + if len(outputs) != 1 or np.shape(outputs[0]) != OUTPUT_SHAPE or not np.isfinite(outputs[0]).all(): + raise ValueError("SSD300 expects one finite [1,8732,95] tensor") + tensor = torch.from_numpy(np.asarray(outputs[0], dtype=np.float32))[0] + boxes = clip_boxes_to_image(tensor[:, :4], (300, 300)) + selected_boxes, selected_scores, selected_classes = [], [], [] + for cls, cid in enumerate(COCO_CATEGORY_IDS): + score = tensor[:, 4 + cid] + keep = score > .01 + score, box = score[keep], boxes[keep] + score, order = score.topk(min(len(score), 400)) + selected_boxes.append(box[order]); selected_scores.append(score) + selected_classes.append(torch.full_like(score, cid, dtype=torch.int64)) + boxes, scores, classes = torch.cat(selected_boxes), torch.cat(selected_scores), torch.cat(selected_classes) + keep = batched_nms(boxes, scores, classes, .45)[:200] + w, h = original_size + boxes = boxes[keep] * boxes.new_tensor([w / 300, h / 300, w / 300, h / 300]) + category_to_class = torch.full((91,), -1, dtype=torch.int64) + category_to_class[list(COCO_CATEGORY_IDS)] = torch.arange(80) + return boxes.numpy(), scores[keep].numpy(), category_to_class[classes[keep]].numpy() + + +def reference_predict(model, image): + """Probe-only: the independent full official model, including transform/NMS.""" + import numpy as np + import torch + from torchvision.transforms.functional import pil_to_tensor + x = pil_to_tensor(image.convert("RGB")).float() / 255 + with torch.inference_mode(): + prepared = model.transform([x])[0].tensors.numpy() + result = model([x])[0] + mapping = {cid: i for i, cid in enumerate(COCO_CATEGORY_IDS)} + return prepared, (result["boxes"].numpy(), result["scores"].numpy(), + np.asarray([mapping[int(c)] for c in result["labels"]], dtype=np.int64)) diff --git a/harness/adapters/ssdlite320_mobilenetv3_large.py b/harness/adapters/ssdlite320_mobilenetv3_large.py new file mode 100644 index 0000000..a1c40a4 --- /dev/null +++ b/harness/adapters/ssdlite320_mobilenetv3_large.py @@ -0,0 +1,19 @@ +"""Exact Torchvision SSDLite320 MobileNetV3-Large COCO_V1 for the CSV row.""" +from pathlib import Path + +from .ssdlite_common import PREPROCESS, INPUT_SPEC, decode, official_model, preprocess, static_model + +MODEL_SPEC = {"source": "custom", "input_spec": INPUT_SPEC} +CHECKPOINT = { + "url": "https://download.pytorch.org/models/ssdlite320_mobilenet_v3_large_coco-a79551df.pth", + "filename": "ssdlite320_mobilenet_v3_large_coco-a79551df.pth", + "sha256": "a79551df90c79834bcd3bb3845ef9d966b5449a3a9b2833ae8404778ca5d65d2", +} +EXPORTER_VERSION = "ssdlite320-coco-v1-static-1" +OPSET = 11 + + +def build(weights: Path, input_spec: str): + if input_spec != INPUT_SPEC: + raise ValueError(f"SSDLite320 requires {INPUT_SPEC}, got {input_spec}") + return static_model(official_model(weights, CHECKPOINT["sha256"])) diff --git a/harness/adapters/ssdlite_common.py b/harness/adapters/ssdlite_common.py new file mode 100644 index 0000000..377873b --- /dev/null +++ b/harness/adapters/ssdlite_common.py @@ -0,0 +1,123 @@ +"""Torchvision SSD input, static head export, and official host postprocessing. + +Only image resizing/normalization and NMS are outside the compiled graph. +The 91-slot head vocabulary is mapped to contiguous 80-class indices for harness. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path + +INPUT_SPEC = "1x3x320x320" +PREPROCESS = { + "recipe": "torchvision-ssdlite320-coco-v1", + "color": "RGB", "layout": "NCHW", "dtype": "float32", + "scale": 1 / 255, "mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5], + "resize": [320, 320], "interpolation": "torch-bilinear-align_corners_false", + "normalize_before_resize": True, "antialias": False, +} +COCO_CATEGORY_IDS = ( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + 59, 60, 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, +) +SCORE_THRESH, NMS_THRESH, TOPK_CANDIDATES, DETECTIONS_PER_IMG = .001, .55, 300, 300 + + +def preprocess(image, input_spec: str = INPUT_SPEC) -> np.ndarray: + """Exactly SSD.transform: normalize RGB [0,1], then stretch to 320x320.""" + import numpy as np + import torch + from torchvision.transforms.functional import pil_to_tensor + + if input_spec != INPUT_SPEC: + raise ValueError(f"SSDLite320 requires {INPUT_SPEC}, got {input_spec}") + x = pil_to_tensor(image.convert("RGB")).to(dtype=torch.float32) / 255 + x = (x - .5) / .5 + x = torch.nn.functional.interpolate(x[None], size=(320, 320), mode="bilinear", align_corners=False) + return x.contiguous().numpy() + + +def official_model(weights: Path, expected_sha256: str): + """Construct the exact COCO architecture, with no implicit backbone download.""" + import torch + from torchvision.models.detection import ssdlite320_mobilenet_v3_large + + if hashlib.sha256(Path(weights).read_bytes()).hexdigest() != expected_sha256: + raise ValueError("SSDLite checkpoint SHA256 does not match official COCO_V1 weights") + model = ssdlite320_mobilenet_v3_large(weights=None, weights_backbone=None, num_classes=91) + model.load_state_dict(torch.load(weights, map_location="cpu", weights_only=True), strict=True) + return model.eval() + + +def static_model(model): + """Freeze official anchors; emit [1,3234,95]: decoded xyxy + 91 probabilities. + + The official BoxCoder and head are retained. Thresholding, TopK, NMS, and + restoring the original image size run only in decode(), on the host. + """ + import torch + from torchvision.models.detection.image_list import ImageList + + class StaticSSDLite(torch.nn.Module): + def __init__(self, detector): + super().__init__() + self.backbone, self.head = detector.backbone, detector.head + self.box_coder = detector.box_coder + with torch.no_grad(): + x = torch.zeros(1, 3, 320, 320) + features = list(self.backbone(x).values()) + anchors = detector.anchor_generator(ImageList(x, [(320, 320)]), features)[0] + self.register_buffer("anchors", anchors) + + def forward(self, images): + heads = self.head(list(self.backbone(images).values())) + boxes = self.box_coder.decode_single(heads["bbox_regression"][0], self.anchors) + scores = torch.softmax(heads["cls_logits"], dim=-1) + return torch.cat((boxes.unsqueeze(0), scores), dim=-1) + + return StaticSSDLite(model).eval() + + +def decode(outputs, original_size: tuple[int, int]): + """Return original xyxy, scores, 0..79 classes; size is PIL (width,height). + + Harness applies its standard COCO cat_map, but must not undo letterboxing. + """ + import numpy as np + import torch + from torchvision.ops import batched_nms, clip_boxes_to_image + + if isinstance(outputs, (list, tuple)): + if len(outputs) != 1: + raise ValueError("SSDLite expects one packed boxes/scores tensor") + outputs = outputs[0] + packed = np.asarray(outputs, dtype=np.float32) + if packed.shape != (1, 3234, 95) or not np.isfinite(packed).all(): + raise ValueError(f"invalid SSDLite output: {packed.shape}") + width, height = original_size + if width <= 0 or height <= 0: + raise ValueError("original image dimensions must be positive") + tensor = torch.from_numpy(packed[0]) + boxes = clip_boxes_to_image(tensor[:, :4], (320, 320)) + scores = tensor[:, 4:] + selected_boxes, selected_scores, labels = [], [], [] + # The pretrained 91 slots are category IDs, not a contiguous 80-class map. + for label in COCO_CATEGORY_IDS: + score = scores[:, label] + keep = score > SCORE_THRESH + score, box = score[keep], boxes[keep] + score, order = score.topk(min(len(score), TOPK_CANDIDATES)) + selected_boxes.append(box[order]) + selected_scores.append(score) + labels.append(torch.full_like(score, label, dtype=torch.int64)) + boxes, scores, labels = torch.cat(selected_boxes), torch.cat(selected_scores), torch.cat(labels) + keep = batched_nms(boxes, scores, labels, NMS_THRESH)[:DETECTIONS_PER_IMG] + boxes, scores, labels = boxes[keep], scores[keep], labels[keep] + # Match torchvision.transform.resize_boxes, separately on x and y. + boxes = boxes * boxes.new_tensor([width / 320, height / 320, width / 320, height / 320]) + category_to_class = torch.full((91,), -1, dtype=torch.int64) + category_to_class[list(COCO_CATEGORY_IDS)] = torch.arange(80) + return boxes.numpy(), scores.numpy(), category_to_class[labels].numpy() diff --git a/harness/adapters/stdc1_seg.py b/harness/adapters/stdc1_seg.py new file mode 100644 index 0000000..d9898b1 --- /dev/null +++ b/harness/adapters/stdc1_seg.py @@ -0,0 +1,17 @@ +"""Official Cityscapes STDC1 preserving its align_corners=True geometry.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, RUNTIME_PYTHON, EXPORTER_VERSION +from .openmmlab import build as official_build + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("stdc1.seg") +PREPROCESS["align_corners"] = True +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = official_build("stdc1.seg", weights, input_spec) + head = network.upstream.decode_head + if (head.align_corners is not True or head.num_classes != CHECKPOINT["classes"] + or network.upstream.test_cfg.mode != "whole"): + raise ValueError("Official STDC1 head differs from its frozen geometry") + return network diff --git a/harness/adapters/stdc2_seg.py b/harness/adapters/stdc2_seg.py new file mode 100644 index 0000000..67f690d --- /dev/null +++ b/harness/adapters/stdc2_seg.py @@ -0,0 +1,17 @@ +"""Official Cityscapes STDC2 preserving its align_corners=True geometry.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, RUNTIME_PYTHON, EXPORTER_VERSION +from .openmmlab import build as official_build + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("stdc2.seg") +PREPROCESS["align_corners"] = True +preprocess = partial(preprocess_image, PREPROCESS) + + +def build(weights, input_spec): + network = official_build("stdc2.seg", weights, input_spec) + head = network.upstream.decode_head + if (head.align_corners is not True or head.num_classes != CHECKPOINT["classes"] + or network.upstream.test_cfg.mode != "whole"): + raise ValueError("Official STDC2 head differs from its frozen geometry") + return network diff --git a/harness/adapters/superpoint.py b/harness/adapters/superpoint.py new file mode 100644 index 0000000..b5c2c99 --- /dev/null +++ b/harness/adapters/superpoint.py @@ -0,0 +1,12 @@ +"""Original MagicLeap SuperPoint v1; dense graph, official sparse host decoder.""" +from . import feature_common as common +EXPORTER_VERSION = common.EXPORTER_VERSION +INPUT_SPEC = common.input_spec('superpoint') +MODEL_SPEC = {'input_spec': INPUT_SPEC, 'dataset': 'hpatches'} +CHECKPOINT = common.checkpoint_info('superpoint') +PREPROCESS = common.recipe('superpoint') +OPSET = 17 +preprocess = common.preprocess + +def build(weights, input_spec): + return common.build('superpoint', weights, input_spec) diff --git a/harness/adapters/topdown_pose_common.py b/harness/adapters/topdown_pose_common.py new file mode 100644 index 0000000..f7b7160 --- /dev/null +++ b/harness/adapters/topdown_pose_common.py @@ -0,0 +1,149 @@ +"""Official COCO HRNet / SimCC heads on the existing verified person crops.""" +from functools import lru_cache +from pathlib import Path + +from .openmmlab import build as official_build, get_recipe, load_config +from .rtmpose_common import ( + BBOX_FILE, BBOX_SHA256, INPUT_SPEC, RUNTIME_PYTHON, SOURCE_COMMIT, + PREPROCESS as CROP_RECIPE, boxes_for, crop, diagnostic_box, g2_input, preprocess, +) + +MODELS = ("hrnet.w32", "hrnet.w48.2", "simcc.resnet50") +EXPORTER_VERSION = "mmpose-topdown-heatmap-simcc-v1" +# These three models share image geometry; decoding is verified separately. +PREPROCESS = {k: v for k, v in CROP_RECIPE.items() if k not in ( + "simcc_split_ratio", "flip_test", "use_dark", "keypoint_score_thr", + "score_mode", "nms_mode", "nms_thr")} +PREPROCESS["recipe"] = "mmpose-coco-official-topdown-affine-v1" + + +def metadata(model): + recipe = get_recipe(model) + if model not in MODELS or recipe["source"]["revision"] != SOURCE_COMMIT: + raise ValueError(f"Unverified topdown recipe: {model}") + if recipe["input_spec"] != INPUT_SPEC or recipe["dataset"] != "coco2017": + raise ValueError("Official model identity disagrees with COCO256x192") + return dict(recipe["checkpoint"]), { + "source": "custom", "dataset": "coco2017", "input_spec": INPUT_SPEC, + "input_override_reason": "CSV 640 square conflicts with official 256x192 person affine input", + "bbox_source": "Official config selects pinned HRNet val2017 person detections", + } + + +@lru_cache(maxsize=3) +def components(model): + from mmpose.apis.inference import dataset_meta_from_config + from mmpose.registry import KEYPOINT_CODECS + from mmpose.utils import register_all_modules + + metadata(model) + cfg = load_config(model) + register_all_modules() + processor = cfg.model.data_preprocessor + pipeline = cfg.test_dataloader.dataset.pipeline + if (tuple(cfg.codec.input_size) != (192, 256) + or processor.mean != CROP_RECIPE["mean"] or processor.std != CROP_RECIPE["std"] + or not processor.bgr_to_rgb + or [p.type for p in pipeline] != ["LoadImage", "GetBBoxCenterScale", "TopdownAffine", "PackPoseInputs"] + or pipeline[1].get("padding", 1.25) != 1.25 + or tuple(pipeline[2].input_size) != (192, 256) or pipeline[2].get("use_udp", False) + or Path(cfg.test_dataloader.dataset.bbox_file).name != BBOX_FILE.name): + raise ValueError("Official config differs from the verified person crop recipe") + if model.startswith("hrnet."): + if (cfg.model.head.type != "HeatmapHead" or cfg.codec.type != "MSRAHeatmap" + or tuple(cfg.codec.heatmap_size) != (48, 64) or cfg.codec.get("unbiased", False) + or dict(cfg.model.test_cfg) != {"flip_test": True, "flip_mode": "heatmap", "shift_heatmap": True}): + raise ValueError("Unsupported HRNet heatmap/flip contract") + elif (cfg.model.head.type != "SimCCHead" or cfg.codec.type != "SimCCLabel" + or cfg.codec.simcc_split_ratio != 2 or cfg.codec.get("use_dark", False) + or dict(cfg.model.test_cfg) != {"flip_test": True}): + raise ValueError("Unsupported SimCC contract") + return cfg, KEYPOINT_CODECS.build(cfg.codec), dataset_meta_from_config(cfg) + + +def build_model(model, weights, input_spec): + if input_spec != INPUT_SPEC: + raise ValueError(f"Official topdown input must be {INPUT_SPEC}") + components(model) + return official_build(model, weights, input_spec) + + + +def output_shapes(model): + return ((1, 17, 64, 48),) if model.startswith("hrnet.") else ((1, 17, 384), (1, 17, 512)) + + +def decode(model, outputs, flipped_outputs, center, scale, input_spec=INPUT_SPEC): + import numpy as np + import torch + from mmpose.models.utils.tta import flip_heatmaps, flip_vectors + + if input_spec != INPUT_SPEC: + raise ValueError("Incorrect topdown decode input size") + cfg, codec, meta = components(model) + shapes = output_shapes(model) + if len(outputs) != len(shapes) or len(flipped_outputs) != len(shapes): + raise ValueError("Raw official head output count mismatch") + arrays = [np.asarray(x, np.float32) for x in (*outputs, *flipped_outputs)] + for x, shape in zip(arrays, shapes * 2): + if x.shape != shape or not np.isfinite(x).all(): + raise ValueError(f"Invalid official head tensor: {x.shape}, expected {shape}") + values = [torch.from_numpy(x) for x in arrays] + if model.startswith("hrnet."): + original, flipped = values + flipped = flip_heatmaps(flipped, flip_indices=meta["flip_indices"], + flip_mode=cfg.model.test_cfg.flip_mode, + shift_heatmap=cfg.model.test_cfg.shift_heatmap) + points, scores = codec.decode(((original + flipped) * .5).numpy()[0]) + else: + x, y, fx, fy = values + fx, fy = flip_vectors(fx, fy, flip_indices=meta["flip_indices"]) + points, scores = codec.decode(((x + fx) * .5).numpy(), ((y + fy) * .5).numpy()) + # Exact TopdownPoseEstimator input-to-original-image transform, unclipped. + points[0] = points[0] / np.asarray((192, 256)) * scale + center - .5 * scale + return np.concatenate((points[0], scores[0, :, None]), axis=-1) + + +def predict(model, runner, t1, input_spec, pose_evidence=None): + import numpy as np + from mmpose.evaluation.functional import oks_nms + from ..eval.common import run_batched + + _, _, meta = components(model) + predictions = [] + if pose_evidence is not None: + pose_evidence.update(boxes_file=str(BBOX_FILE), boxes_sha256=BBOX_SHA256, + pose_protocol=PREPROCESS, official_model=model) + for item in t1.items: + image = t1.image(item) + boxes = boxes_for(t1, item) + candidates = [] + for start in range(0, len(boxes), 16): + chunk = boxes[start:start + 16] + crops = [crop(image, input_spec, b["bbox_xyxy"]) for b in chunk] + xs = [value for x, _, _, _ in crops for value in (x, np.ascontiguousarray(x[..., ::-1]))] + outputs = run_batched(runner, xs) + if len(outputs) != len(xs): + raise ValueError("Runner omitted a person or flip output") + for j, (box, (_, matrix, center, scale)) in enumerate(zip(chunk, crops)): + points = decode(model, outputs[2*j], outputs[2*j+1], center, scale, input_spec) + valid = points[:, 2][points[:, 2] > .2] + score = box["bbox_score"] * (sum(float(v) for v in valid) / len(valid) if len(valid) else 0.) + candidates.append({"keypoints": points, "score": score, "area": float(np.prod(scale)), + "bbox_xyxy": box["bbox_xyxy"], "affine": matrix, + "source_index": box["source_index"]}) + keep = oks_nms(candidates, .9, sigmas=meta["sigmas"]) if candidates else [] + retained = [candidates[i] for i in keep] + if pose_evidence is not None: + pose_evidence["images"].append({ + "image_id": item["id"], "image_size": list(image.size), "transform": "topdown_affine", + "input_detections": len(boxes), "no_person_bbox": not bool(boxes), + "detections": [{"category_id": 1, "bbox_xyxy": p["bbox_xyxy"].tolist(), + "score": p["score"], "keypoints": p["keypoints"].tolist(), + "affine": p["affine"].tolist(), "source_index": p["source_index"]} + for p in retained]}) + for p in retained: + bbox = p["bbox_xyxy"].copy(); bbox[2:] -= bbox[:2] + predictions.append({"image_id": item["id"], "category_id": 1, "score": p["score"], + "bbox": bbox.tolist(), "keypoints": p["keypoints"].reshape(-1).tolist()}) + return predictions diff --git a/harness/adapters/upernet_resnet50.py b/harness/adapters/upernet_resnet50.py new file mode 100644 index 0000000..e69c839 --- /dev/null +++ b/harness/adapters/upernet_resnet50.py @@ -0,0 +1,7 @@ +"""Official ADE20K upernet.resnet50; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("upernet.resnet50") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "upernet.resnet50") diff --git a/harness/adapters/upernet_swin_b.py b/harness/adapters/upernet_swin_b.py new file mode 100644 index 0000000..d8e84c8 --- /dev/null +++ b/harness/adapters/upernet_swin_b.py @@ -0,0 +1,7 @@ +"""Official ADE20K upernet.swin.b; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("upernet.swin.b") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "upernet.swin.b") diff --git a/harness/adapters/upernet_swin_s.py b/harness/adapters/upernet_swin_s.py new file mode 100644 index 0000000..bb61fae --- /dev/null +++ b/harness/adapters/upernet_swin_s.py @@ -0,0 +1,7 @@ +"""Official ADE20K upernet.swin.s; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("upernet.swin.s") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "upernet.swin.s") diff --git a/harness/adapters/upernet_swin_t.py b/harness/adapters/upernet_swin_t.py new file mode 100644 index 0000000..e638ec2 --- /dev/null +++ b/harness/adapters/upernet_swin_t.py @@ -0,0 +1,7 @@ +"""Official ADE20K upernet.swin.t; admission requires real validation.""" +from functools import partial +from .mmseg_common import metadata, preprocess_image, build_model, RUNTIME_PYTHON, EXPORTER_VERSION + +CHECKPOINT, MODEL_SPEC, PREPROCESS = metadata("upernet.swin.t") +preprocess = partial(preprocess_image, PREPROCESS) +build = partial(build_model, "upernet.swin.t") diff --git a/harness/adapters/vfnet_deformconv.py b/harness/adapters/vfnet_deformconv.py new file mode 100644 index 0000000..9f6ea06 --- /dev/null +++ b/harness/adapters/vfnet_deformconv.py @@ -0,0 +1,64 @@ +"""Exact unmodulated VFNet/RepPoints convolution in standard ONNX ops. + +FP32 retains the original MMCV operation. Export retains dynamic offsets, +zero-outside bilinear sampling and every learned kernel weight; no ordinary +fixed-grid convolution is substituted for the deformable sampling. +""" + + +def sampled_convolution(x, offset, weight): + import torch + from torch.nn import functional as F + n, channels, height, width = map(int, x.shape) + if n != 1 or tuple(offset.shape) != (1, 18, height, width): + raise ValueError("Only the verified static 3x3 deformconv contract is supported") + offsets = offset.reshape(1, 9, 2, height, width) + ky = torch.arange(3, device=x.device, dtype=x.dtype).repeat_interleave(3) - 1 + kx = torch.arange(3, device=x.device, dtype=x.dtype).repeat(3) - 1 + yy = torch.arange(height, device=x.device, dtype=x.dtype)[None, None, :, None] + xx = torch.arange(width, device=x.device, dtype=x.dtype)[None, None, None, :] + y = yy + ky[None, :, None, None] + offsets[:, :, 0] + z = xx + kx[None, :, None, None] + offsets[:, :, 1] + y0, x0 = y.floor(), z.floor() + y1, x1 = y0 + 1, x0 + 1 + low_y, low_x = y - y0, z - x0 + high_y, high_x = 1 - low_y, 1 - low_x + flat = x.reshape(1, channels, height * width) + + def gather(yi, xi): + valid = (yi >= 0) & (yi < height) & (xi >= 0) & (xi < width) + index = (yi.clamp(0, height - 1) * width + xi.clamp(0, width - 1)).long() + index = index.reshape(1, 1, -1).expand(1, channels, -1) + values = flat.gather(2, index).reshape(1, channels, 9, height, width) + return values * valid[:, None].to(x.dtype) + + sampled = (gather(y0, x0) * (high_y * high_x)[:, None] + + gather(y0, x1) * (high_y * low_x)[:, None] + + gather(y1, x0) * (low_y * high_x)[:, None] + + gather(y1, x1) * (low_y * low_x)[:, None]) + return F.conv2d(sampled.reshape(1, channels * 9, height, width), + weight.reshape(int(weight.shape[0]), channels * 9, 1, 1)) + + +def enable_export(head, names=("vfnet_reg_refine_dconv", "vfnet_cls_dconv")): + import torch + from mmcv.ops import DeformConv2d + + class ExportableStar(torch.nn.Module): + def __init__(self, native): + super().__init__() + self.native = native + + def forward(self, x, offset): + if torch.onnx.is_in_onnx_export(): + return sampled_convolution(x, offset, self.native.weight) + return self.native(x, offset) + + for name in names: + layer = getattr(head, name) + if (type(layer) is not DeformConv2d or layer.kernel_size != (3, 3) + or layer.stride != (1, 1) or layer.padding != (1, 1) + or layer.dilation != (1, 1) or layer.groups != 1 + or layer.deform_groups != 1 or getattr(layer, "bias", None) is not None): + raise ValueError(f"Unverified deformable convolution contract: {name}") + setattr(head, name, ExportableStar(layer)) diff --git a/harness/adapters/vfnet_r50_fpn.py b/harness/adapters/vfnet_r50_fpn.py new file mode 100644 index 0000000..ddce123 --- /dev/null +++ b/harness/adapters/vfnet_r50_fpn.py @@ -0,0 +1,12 @@ +"""Exact MMDetection vfnet.r50.fpn; see mmdet_dense_common for static deployment geometry.""" +from functools import partial +from .mmdet_dense_common import (RUNTIME_PYTHON, INPUT_SPEC, MODEL_SPEC, recipe, + get_recipe, preprocess_variant, decode_variant, build_variant) + +MODEL_ID = "vfnet.r50.fpn" +CHECKPOINT = get_recipe(MODEL_ID)["checkpoint"] +PREPROCESS = recipe(MODEL_ID) +EXPORTER_VERSION = "mmdet-vfnet.r50.fpn-640-raw-v1" +preprocess = partial(preprocess_variant, MODEL_ID) +decode = partial(decode_variant, MODEL_ID) +build = partial(build_variant, MODEL_ID) diff --git a/harness/adapters/xfeat.py b/harness/adapters/xfeat.py new file mode 100644 index 0000000..ab18957 --- /dev/null +++ b/harness/adapters/xfeat.py @@ -0,0 +1,70 @@ +"""Official XFeat sparse extractor, without LighterGlue or dense fine matching.""" +from functools import lru_cache +from . import feature_sources as sources +from .feature_common import MAX_KEYPOINTS + +INPUT_SPEC = '1x1x480x640' +MODEL_SPEC = {'input_spec': INPUT_SPEC, 'dataset': 'hpatches'} +EXPORTER_VERSION = 'xfeat-static-1' +OPSET = 17 +CHECKPOINT = {'url': 'https://raw.githubusercontent.com/verlab/accelerated_features/e92685f57f8318b18725c5c8c0bd28c7fe188d9a/weights/xfeat.pt', + 'sha256': '0f5187fd7bedd26c7fe6acc9685444493a165a35ecc087b33c2db3627f3ea10b', + 'filename': 'xfeat.pt', 'official_relative_path': str(sources.ROOT/'xfeat/weights/xfeat.pt')} +PREPROCESS = {'input_spec': INPUT_SPEC, 'color': 'RGB_arithmetic_mean', 'scale': 1/255, + 'resize': 'opencv_linear_640x480', 'output_layout': 'NCHW', + 'protocol': 'xfeat-official-channel-mean-static-v1'} + + +def preprocess(image, input_spec=INPUT_SPEC): + import cv2 + import numpy as np + import torch + if input_spec != INPUT_SPEC: raise ValueError(input_spec) + rgb = cv2.resize(np.asarray(image.convert('RGB')), (640,480), interpolation=cv2.INTER_LINEAR) + x = torch.from_numpy(rgb.copy()).permute(2,0,1).float().div(255).mean(0,keepdim=True) + return x[None].numpy() + + +def official_module(): + return sources.module('xfeat/modules/xfeat.py') + + +def official_model(weights): + import torch + sources.verify_weights(weights, CHECKPOINT['sha256']) + m = official_module().XFeat(weights=None, top_k=MAX_KEYPOINTS, detection_threshold=.05) + m.dev = torch.device('cpu'); m.net.cpu() + m.net.load_state_dict(torch.load(weights, map_location='cpu', weights_only=False), strict=True) + return m.eval() + + +def official_features(m, x): + result = m.detectAndCompute(x, top_k=MAX_KEYPOINTS, detection_threshold=.05)[0] + return result['keypoints'], result['scores'], result['descriptors'] + + +def build(weights, input_spec): + if input_spec != INPUT_SPEC: raise ValueError(input_spec) + return official_model(weights).net + + +@lru_cache(None) +def _decoder(): + import torch + m = official_module().XFeat(weights=None, top_k=MAX_KEYPOINTS, detection_threshold=.05) + m.dev = torch.device('cpu') + # Only the official non-learned sparse decoder will be used below. + m.net = torch.nn.Identity() + return m + + +def decode(outputs): + import torch + maps = sources.tensors(outputs) + assert [tuple(v.shape) for v in maps] == [(1,64,60,80),(1,65,60,80),(1,1,60,80)] + class DenseMaps(torch.nn.Module): + def forward(self, x): return tuple(maps) + m = _decoder(); m.net = DenseMaps() + # Official postprocess uses the image only for its shape after net(x). + with torch.no_grad(): k,s,d = official_features(m, torch.zeros(1,1,480,640)) + return {'keypoints': k.numpy(), 'scores': s.numpy(), 'descriptors': d.numpy()} diff --git a/harness/adapters/yolox_common.py b/harness/adapters/yolox_common.py new file mode 100644 index 0000000..ec6b557 --- /dev/null +++ b/harness/adapters/yolox_common.py @@ -0,0 +1,153 @@ +"""Pinned, unmodified Megvii YOLOX factories, transforms, and postprocess. + +Optional packages are isolated beside upstream source; nothing is installed or +downloaded by this module. Metadata imports use only the standard library. +""" +from __future__ import annotations + +from functools import lru_cache +import hashlib +import importlib +from pathlib import Path +import sys + +UPSTREAM_COMMIT = "419778480ab6ec0590e5d3831b3afb3b46ab2aa3" +SOURCE_SHA256 = "7730105dd1633d7b9cee677a1acae3797908e45ef732bbec725b29e62de7c08e" +RUNTIME_ROOT = Path.home() / "bench-cache/sources/ssd-yolox" +SIZES = {"tiny": 416, "s": 640, "m": 640, "l": 640, "x": 640} +WEIGHT_SHAS = { + "tiny": "9de513de589ac98bb92d3bca53b5af7b9acfa9b0bacb831f7999d0f7afaee8f0", + "s": "f55ded7181e1b0c13285c56e7790b8f0e8f8db590fe4edb37f0b7f345c913a30", + "m": "60076992b32da82951c90cfa7bd6ab70eba9eda243e08b940a396f60ac2d19b6", + "l": "1e6b7fa6240375370b2a8a8eab9066b3cdd43fd1d0bfa8d2027fd3a51def2917", + "x": "5652330b6ae860043f091b8f550a60c10e1129f416edfdb65c259be6caf355cf", +} + + +def checkpoint(variant): + filename = f"yolox_{variant}.pth" + return {"url": f"https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/{filename}", + "filename": filename, "sha256": WEIGHT_SHAS[variant]} + + +def recipe(variant): + return {"recipe": "megvii-yolox-0.3.0-coco", "source_commit": UPSTREAM_COMMIT, + "color": "BGR", "range": [0, 255], "layout": "NCHW", "dtype": "float32", + "resize": {"size": [SIZES[variant]] * 2, "keep_ratio": True, + "rounding": "int", "interpolation": "cv2.INTER_LINEAR"}, + "pad": {"value": 114, "placement": "bottom_right"}, "legacy": False} + + +@lru_cache(maxsize=1) +def upstream(): + source = RUNTIME_ROOT / f"YOLOX-{UPSTREAM_COMMIT}" + digest = hashlib.sha256() + for path in sorted(source.rglob("*.py")): + digest.update(str(path.relative_to(source)).encode() + b"\0" + path.read_bytes()) + if digest.hexdigest() != SOURCE_SHA256: + raise RuntimeError(f"YOLOX official source differs from pinned commit: {source}") + original_path = sys.path[:] + try: + sys.path[:0] = [str(source), str(RUNTIME_ROOT / "deps")] + package = importlib.import_module("yolox") + if Path(package.__file__).resolve() != (source / "yolox/__init__.py").resolve(): + raise RuntimeError("A different YOLOX package is already loaded") + from yolox.exp import get_exp + from yolox.data import ValTransform + from yolox.utils import postprocess + return source, get_exp, ValTransform, postprocess + finally: + sys.path[:] = original_path + + +def preprocess(image, input_spec, variant): + import numpy as np + size = SIZES[variant] + if input_spec != f"1x3x{size}x{size}": + raise ValueError(f"YOLOX-{variant} requires official {size}x{size} input") + _, _, transform, _ = upstream() + bgr = np.asarray(image.convert("RGB"))[:, :, ::-1].copy() + x, _ = transform(legacy=False)(bgr, None, (size, size)) + return x[None] + + +def reference_model(weights, variant): + import torch + if hashlib.sha256(Path(weights).read_bytes()).hexdigest() != WEIGHT_SHAS[variant]: + raise ValueError(f"YOLOX-{variant} official checkpoint SHA256 mismatch") + source, get_exp, _, _ = upstream() + original_path = sys.path[:] + try: + exp = get_exp(str(source / f"exps/default/yolox_{variant}.py"), None) + model = exp.get_model() + finally: + sys.path[:] = original_path + model.load_state_dict(torch.load(weights, map_location="cpu", weights_only=True)["model"], strict=True) + return model.eval() + + +def build(weights, input_spec, variant): + import torch + size = SIZES[variant] + if input_spec != f"1x3x{size}x{size}": + raise ValueError(f"YOLOX-{variant} requires official {size}x{size} input") + model = reference_model(weights, variant) + model.head.decode_in_inference = False + + class StaticYOLOX(torch.nn.Module): + def __init__(self): + super().__init__() + self.model = model + with torch.no_grad(): + model(torch.zeros(1, 3, size, size)) + grids, strides = [], [] + for (h, w), stride in zip(model.head.hw, model.head.strides): + y, x = torch.meshgrid(torch.arange(h), torch.arange(w), indexing="ij") + grid = torch.stack((x, y), 2).reshape(1, -1, 2).float() + grids.append(grid); strides.append(torch.full((1, h * w, 1), float(stride))) + self.register_buffer("grid", torch.cat(grids, 1)) + self.register_buffer("strides", torch.cat(strides, 1)) + + def forward(self, x): + raw = self.model(x) + return torch.cat(((raw[..., :2] + self.grid) * self.strides, + torch.exp(raw[..., 2:4]) * self.strides, raw[..., 4:]), -1) + + return StaticYOLOX().eval() + + +def decode(outputs, original_size, variant): + import numpy as np + import torch + _, _, _, postprocess = upstream() + size = SIZES[variant] + count = sum((size // stride) ** 2 for stride in (8, 16, 32)) + if len(outputs) != 1 or np.shape(outputs[0]) != (1, count, 85) or not np.isfinite(outputs[0]).all(): + raise ValueError("invalid YOLOX output tensor") + # Upstream modifies xywh in place, so preserve the caller's raw G2 tensor. + prediction = postprocess(torch.from_numpy(np.asarray(outputs[0], dtype=np.float32).copy()), 80, .01, .65)[0] + if prediction is None: + return np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int64) + w, h = original_size + boxes = prediction[:, :4] / min(size / h, size / w) + return boxes.numpy(), (prediction[:, 4] * prediction[:, 5]).numpy(), prediction[:, 6].long().numpy() + + +def reference_predict(model, image, variant): + """Probe-only, unwrapped official model plus official COCO evaluation geometry.""" + import numpy as np + import torch + _, _, transform, postprocess = upstream() + size = SIZES[variant] + bgr = np.asarray(image.convert("RGB"))[:, :, ::-1].copy() + x, _ = transform(legacy=False)(bgr, None, (size, size)) + with torch.inference_mode(): + raw = model(torch.from_numpy(x[None])) + pred = postprocess(raw, 80, .01, .65)[0] + if pred is None: + prediction = np.empty((0, 4), np.float32), np.empty(0, np.float32), np.empty(0, np.int64) + else: + w, h = image.size + prediction = ((pred[:, :4] / min(size / h, size / w)).numpy(), + (pred[:, 4] * pred[:, 5]).numpy(), pred[:, 6].long().numpy()) + return x[None], prediction diff --git a/harness/adapters/yolox_l.py b/harness/adapters/yolox_l.py new file mode 100644 index 0000000..67407c2 --- /dev/null +++ b/harness/adapters/yolox_l.py @@ -0,0 +1,29 @@ +"""Megvii YOLOX-L, official 640px COCO checkpoint.""" +from . import yolox_common as common + +MODEL_SPEC = {"source": "custom", "input_spec": "1x3x640x640"} +CHECKPOINT = common.checkpoint("l") +PREPROCESS = common.recipe("l") +EXPORTER_VERSION = "megvii-yolox-l-0.3.0-static-1" +OPSET = 11 +OUTPUT_SHAPE = (1, 8400, 85) + + +def build(weights, input_spec): + return common.build(weights, input_spec, "l") + + +def preprocess(image, input_spec): + return common.preprocess(image, input_spec, "l") + + +def decode(outputs, original_size): + return common.decode(outputs, original_size, "l") + + +def reference_model(weights): + return common.reference_model(weights, "l") + + +def reference_predict(model, image): + return common.reference_predict(model, image, "l") diff --git a/harness/adapters/yolox_m.py b/harness/adapters/yolox_m.py new file mode 100644 index 0000000..f61b868 --- /dev/null +++ b/harness/adapters/yolox_m.py @@ -0,0 +1,29 @@ +"""Megvii YOLOX-M, official 640px COCO checkpoint.""" +from . import yolox_common as common + +MODEL_SPEC = {"source": "custom", "input_spec": "1x3x640x640"} +CHECKPOINT = common.checkpoint("m") +PREPROCESS = common.recipe("m") +EXPORTER_VERSION = "megvii-yolox-m-0.3.0-static-1" +OPSET = 11 +OUTPUT_SHAPE = (1, 8400, 85) + + +def build(weights, input_spec): + return common.build(weights, input_spec, "m") + + +def preprocess(image, input_spec): + return common.preprocess(image, input_spec, "m") + + +def decode(outputs, original_size): + return common.decode(outputs, original_size, "m") + + +def reference_model(weights): + return common.reference_model(weights, "m") + + +def reference_predict(model, image): + return common.reference_predict(model, image, "m") diff --git a/harness/adapters/yolox_s.py b/harness/adapters/yolox_s.py new file mode 100644 index 0000000..3dbfd2c --- /dev/null +++ b/harness/adapters/yolox_s.py @@ -0,0 +1,29 @@ +"""Megvii YOLOX-S, official 640px COCO checkpoint.""" +from . import yolox_common as common + +MODEL_SPEC = {"source": "custom", "input_spec": "1x3x640x640"} +CHECKPOINT = common.checkpoint("s") +PREPROCESS = common.recipe("s") +EXPORTER_VERSION = "megvii-yolox-s-0.3.0-static-1" +OPSET = 11 +OUTPUT_SHAPE = (1, 8400, 85) + + +def build(weights, input_spec): + return common.build(weights, input_spec, "s") + + +def preprocess(image, input_spec): + return common.preprocess(image, input_spec, "s") + + +def decode(outputs, original_size): + return common.decode(outputs, original_size, "s") + + +def reference_model(weights): + return common.reference_model(weights, "s") + + +def reference_predict(model, image): + return common.reference_predict(model, image, "s") diff --git a/harness/adapters/yolox_tiny.py b/harness/adapters/yolox_tiny.py new file mode 100644 index 0000000..fb467c7 --- /dev/null +++ b/harness/adapters/yolox_tiny.py @@ -0,0 +1,29 @@ +"""Megvii YOLOX-Tiny, official 416px COCO checkpoint.""" +from . import yolox_common as common + +MODEL_SPEC = {"source": "custom", "input_spec": "1x3x416x416"} +CHECKPOINT = common.checkpoint("tiny") +PREPROCESS = common.recipe("tiny") +EXPORTER_VERSION = "megvii-yolox-tiny-0.3.0-static-1" +OPSET = 11 +OUTPUT_SHAPE = (1, 3549, 85) + + +def build(weights, input_spec): + return common.build(weights, input_spec, "tiny") + + +def preprocess(image, input_spec): + return common.preprocess(image, input_spec, "tiny") + + +def decode(outputs, original_size): + return common.decode(outputs, original_size, "tiny") + + +def reference_model(weights): + return common.reference_model(weights, "tiny") + + +def reference_predict(model, image): + return common.reference_predict(model, image, "tiny") diff --git a/harness/adapters/yolox_x.py b/harness/adapters/yolox_x.py new file mode 100644 index 0000000..9c8d04c --- /dev/null +++ b/harness/adapters/yolox_x.py @@ -0,0 +1,29 @@ +"""Megvii YOLOX-X, official 640px COCO checkpoint.""" +from . import yolox_common as common + +MODEL_SPEC = {"source": "custom", "input_spec": "1x3x640x640"} +CHECKPOINT = common.checkpoint("x") +PREPROCESS = common.recipe("x") +EXPORTER_VERSION = "megvii-yolox-x-0.3.0-static-1" +OPSET = 11 +OUTPUT_SHAPE = (1, 8400, 85) + + +def build(weights, input_spec): + return common.build(weights, input_spec, "x") + + +def preprocess(image, input_spec): + return common.preprocess(image, input_spec, "x") + + +def decode(outputs, original_size): + return common.decode(outputs, original_size, "x") + + +def reference_model(weights): + return common.reference_model(weights, "x") + + +def reference_predict(model, image): + return common.reference_predict(model, image, "x") diff --git a/harness/backends/ptq.py b/harness/backends/ptq.py index 3fc073d..8ec12c7 100644 --- a/harness/backends/ptq.py +++ b/harness/backends/ptq.py @@ -31,7 +31,7 @@ def tool(name: str) -> str: def export_cmd(job: dict, onnx_out: Path, weights: Path) -> list[str]: - return [str(paths.venv_python("bench")), "-m", "harness.tools.export_onnx", # 导 ONNX 用 bench venv(torch+torchvision+ultralytics) + return [str(paths.model_python(job)), "-m", "harness.tools.export_onnx", "--model", job["model"], "--source", job["model_spec"]["source"], "--weights", str(weights), "--input-spec", job["model_spec"]["input_spec"], "--out", str(onnx_out)] diff --git a/harness/cli.py b/harness/cli.py index d12ccbb..4bbe7c9 100644 --- a/harness/cli.py +++ b/harness/cli.py @@ -51,7 +51,8 @@ def cmd_expand(a) -> int: from .pipeline import expand with db.connect() as con: rid = _run_id(con, a.run) - print(json.dumps({"run_id": rid, **expand(rid, a.modes.split(",") if a.modes else None, a.marches.split(",") if a.marches else None)})) + print(json.dumps({"run_id": rid, **expand(rid, a.modes.split(",") if a.modes else None, a.marches.split(",") if a.marches else None, + model_ids=a.model or None, eval_manifests=a.t1 or None)})) return 0 @@ -59,7 +60,7 @@ def cmd_run(a) -> int: from .pipeline import run_oracle with db.connect() as con: rid = _run_id(con, a.run) - s = run_oracle(rid, a.job or None, dry_run=a.dry_run, parallel=a.parallel) + s = run_oracle(rid, a.job or None, dry_run=a.dry_run, parallel=a.parallel, prepare_only=not a.oracle) print(json.dumps({"run_id": rid, **s})) return 0 @@ -95,10 +96,23 @@ def cmd_status(a) -> int: if manifest.exists(): m = json.loads(manifest.read_text()) print(f" dataset: {manifest.parent} stats={json.dumps(m.get('stats', {}), ensure_ascii=False)}") + # 导出没被触发是静默的(collector 被 database is locked 打死过,manifest 就停在那一刻)。 + # 拿 manifest 里记的可导出条数和现在的比一比,落后就提示补导。复用上面的 ro,零新增查询。 + skip = {"RUNNING", "QUEUED", "VERIFYING", "TRANSFERRED"} # 与 to_agentrl.scan() 的口径一致 + now_n = sum(r["n"] for r in ro if r["state"] not in skip) + was = (m.get("coverage", {}).get(rid) or {}).get("agent") or {} + was_n = sum(n for s, n in was.items() if s not in skip) + if now_n > was_n: + # collector / export --follow 活着时导出已经在排队了(pipeline.py 的 export_dirty), + # 这时的落后是正常在途,不该催人补导——否则 --parallel 12 下会常态刷屏。 + from .pipeline import scheduler_holders + busy = bool(scheduler_holders(rid)) + tail = "(导出在途)" if busy else f":bench export --run {rid}" + print(f" {'' if busy else '⚠ '}dataset 落后 {now_n - was_n} 条(manifest {was_n} / 现在 {now_n}){tail}") from .scheduler import pool_usage, pool_size from .registry import status_counts print(" pools: " + " ".join(f"{p}={n}/{pool_size(p)}" for p, n in pool_usage().items())) - print(" registry: " + " ".join(f"{k}={v}" for k, v in status_counts().items()) + " (只有 ready 会被展开)") + print(" registry: " + " ".join(f"{k}={v}" for k, v in status_counts().items()) + " (原始标签;显式批次进度见 coverage --all-runs)") return 0 @@ -147,6 +161,10 @@ def cmd_report(a) -> int: def cmd_coverage(a) -> int: from .coverage import render, to_csv + if a.all_runs: + from .coverage import write_master + print(write_master(a.csv or str(paths.REPORTS / "model-coverage.csv"))) + return 0 with db.connect() as con: rid = _run_id(con, a.run) if a.csv: @@ -156,15 +174,33 @@ def cmd_coverage(a) -> int: def cmd_drain(a) -> int: - from .pipeline import drain_file + from .pipeline import drain_file, collect_drain_file with db.connect() as con: rid = _run_id(con, a.run) drain_file(rid).touch() - print(f"drain 已置位:{drain_file(rid)}。正在跑的 job 会跑完;新 job 不再认领。之后 scripts/start_run.sh --resume 用新代码续跑。") + if a.all: + collect_drain_file(rid).touch() + print(f"drain 已置位:{rid},范围={'流水线和 Agent' if a.all else '流水线'}。等已领取的任务结束后再交接。") print("注意:2026-09-15 之前启动的旧调度进程(代码在内存里)不认这个文件,只能等它跑完或 kill + reset-stale。") return 0 +def cmd_continue(a) -> int: + from . import continuation + try: + if a.op == "plan": + result = continuation.plan(a.run, a.t1, a.to) + elif a.op == "apply": + result = continuation.apply(json.loads(Path(a.plan).read_text())) + else: + result = continuation.check(a.run) + except (ValueError, OSError) as exc: + print(f"continue: {exc}", file=sys.stderr) + return 2 + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 + + def cmd_heartbeat(a) -> int: from .heartbeat import heartbeat return heartbeat(a.run) @@ -231,7 +267,7 @@ def cmd_collect(a) -> int: rid = _run_id(con, a.run) s = collect_rollouts(rid, a.job or None, a.modes.split(",") if a.modes else None, parallel=a.parallel, dry_run=a.dry_run, limit=a.limit, - coverage=a.coverage, watch=a.watch) + coverage=a.coverage, watch=a.watch, first_pass=a.first_pass) print(json.dumps({"run_id": rid, **s}, ensure_ascii=False)) return 0 @@ -243,7 +279,6 @@ def cmd_export(a) -> int: import fcntl import os from .pipeline import refresh_run_dataset - from .verify import verify_rollout with db.readonly() as con: rid = _run_id(con, a.run) lock = None @@ -264,14 +299,7 @@ def cmd_export(a) -> int: signature = hashlib.sha256(json.dumps([list(map(tuple, jobs)), list(map(tuple, rollouts))]).encode()).hexdigest() if signature != previous: try: - # 老调度进程还在内存中使用旧 verifier;只重核已结束项,不碰正在跑的任务。 - for row in rollouts: - if row["state"] in ("QUEUED", "RUNNING", "VERIFYING") or not row["verdict"]: - continue - verdict = json.loads(row["verdict"]) - if verdict.get("schema") != "bench.verdict/2": - rd = paths.RUNS / rid / row["job_id"] / "rollouts" / f"{row['kk']:02d}" - verify_rollout(rd) + # 导出器只读重核旧结果;历史 verdict 和数据库保持原样。 manifest = refresh_run_dataset(rid, strict=True) except Exception as exc: if not a.follow: @@ -284,7 +312,7 @@ def cmd_export(a) -> int: previous = signature if not a.follow: return 0 - # 队列暂空不代表 producer 已结束:最后一个 oracle 之后仍可能生成 repair。 + # 队列暂空不代表在途 Agent 已结束。 if a.until_pid: try: os.kill(a.until_pid, 0) @@ -331,8 +359,12 @@ def main(argv=None) -> int: x = sp.add_parser("preflight"); x.add_argument("--modes"); x.add_argument("--oracle-only", action="store_true", help="只跑 benchmark 那一半时放行 agent 运行时缺失"); x.set_defaults(f=cmd_preflight) x = sp.add_parser("new-run"); x.add_argument("--note"); x.set_defaults(f=cmd_new_run) - x = sp.add_parser("expand"); x.add_argument("--run"); x.add_argument("--modes"); x.add_argument("--marches", help="只展开这些 march,逗号分隔"); x.set_defaults(f=cmd_expand) + x = sp.add_parser("expand"); x.add_argument("--run"); x.add_argument("--modes"); x.add_argument("--marches", help="只展开这些 march,逗号分隔") + x.add_argument("--model", action="append", help="显式分批接入;核查真实前置,保留原 march/backend/policy/k,不改 CSV 标签") + x.add_argument("--t1", action="append", help="新批次直接使用已验证的 T1 manifest;每个数据集一份,不改旧任务") + x.set_defaults(f=cmd_expand) x = sp.add_parser("run"); x.add_argument("--run"); x.add_argument("--job", action="append") + x.add_argument("--oracle", action="store_true", help="显式运行完整01–08对照;默认只准备01–03输入") x.add_argument("--dry-run", action="store_true"); x.add_argument("--parallel", type=int); x.set_defaults(f=cmd_run) x = sp.add_parser("status"); x.add_argument("--run"); x.add_argument("--fail", action="store_true"); x.set_defaults(f=cmd_status) x = sp.add_parser("show"); x.add_argument("job_id"); x.add_argument("--run"); x.set_defaults(f=cmd_show) @@ -345,18 +377,27 @@ def main(argv=None) -> int: x.add_argument("--rollout", action="append", metavar="JOB_ID[:KK]", help="补测 rollout(可重复),板卡故障导致 UNVERIFIED 的轨迹用这个回收") x.set_defaults(f=cmd_board_backfill) x = sp.add_parser("coverage"); x.add_argument("--run"); x.add_argument("--task"); x.add_argument("--missing", action="store_true") + x.add_argument("--all-runs", action="store_true", help="338 模型总表及逐任务/阶段/真实轨迹 JSON") x.add_argument("--detail", action="store_true"); x.add_argument("--csv"); x.set_defaults(f=cmd_coverage) x = sp.add_parser("heartbeat"); x.add_argument("--run", required=True); x.set_defaults(f=cmd_heartbeat) x = sp.add_parser("watch"); x.add_argument("--run"); x.add_argument("--interval", type=int, default=30); x.add_argument("--once", action="store_true"); x.set_defaults(f=cmd_watch) - x = sp.add_parser("reset-stale"); x.add_argument("--run"); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_reset_stale) + # --run 必填:默认取最新 run,而最新 run 往往正是有活 rollout 的那个,误打代价太大 + x = sp.add_parser("reset-stale"); x.add_argument("--run", required=True); x.add_argument("--force", action="store_true"); x.set_defaults(f=cmd_reset_stale) x = sp.add_parser("drain", help="让 oracle 调度进程跑完手里的 job 后不再认领新 job(换代码/配置前用,不用 kill)") - x.add_argument("--run"); x.set_defaults(f=cmd_drain) + x.add_argument("--run"); x.add_argument("--all", action="store_true", help="同时停止 Agent 领取"); x.set_defaults(f=cmd_drain) + x = sp.add_parser("continue", help="预览、应用和核对剩余任务交接;不全量展开") + cs = x.add_subparsers(dest="op", required=True) + y = cs.add_parser("plan"); y.add_argument("--run", required=True); y.add_argument("--to") + y.add_argument("--t1", action="append", required=True, help="新 T1 的 manifest.json,每个数据集一份"); y.set_defaults(f=cmd_continue) + y = cs.add_parser("apply"); y.add_argument("plan", help="已核对的 JSON 清单"); y.set_defaults(f=cmd_continue) + y = cs.add_parser("check"); y.add_argument("--run", required=True); y.set_defaults(f=cmd_continue) x = sp.add_parser("collect"); x.add_argument("--run"); x.add_argument("--job", action="append") x.add_argument("--modes", help="e2e,repair"); x.add_argument("--parallel", type=int) x.add_argument("--limit", type=int, help="先采 N 条试水"); x.add_argument("--dry-run", action="store_true") x.add_argument("--coverage", choices=["matrix", "all"], default="matrix", help="all 为本 run 的 oracle 模型补齐 e2e 计划,只放开 priority 限制") - x.add_argument("--watch", action="store_true", help="跟随 oracle 新结果,直到流程与采集队列结束") + x.add_argument("--watch", action="store_true", help="等待输入准备与在途任务,持续采集") + x.add_argument("--first-pass", action="store_true", help="每个 job 只领取首条,保留后续采样编号和原 k") x.set_defaults(f=cmd_collect) x = sp.add_parser("export", help="导出本 run 已完成的流程与 Agent 数据;--follow 跟随当前批次更新同一目录") x.add_argument("--run"); x.add_argument("--follow", action="store_true") diff --git a/harness/continuation.py b/harness/continuation.py new file mode 100644 index 0000000..4e32e43 --- /dev/null +++ b/harness/continuation.py @@ -0,0 +1,305 @@ +"""A remaining-work handoff: inspect a JSON plan, then apply it after draining. + +No matrix expansion, execution-history copying, or second scheduler. The plan is +also the receipt used to check exact job/rollout identities after the handoff. +""" +from __future__ import annotations + +from collections import Counter +from copy import deepcopy +from contextlib import contextmanager +import hashlib +import json +from pathlib import Path +import re +import tarfile +import time + +from . import db, paths, registry +from .eval import EVALUATOR_VERSION + + +def _sha(value) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def _read(path: Path): + return json.loads(path.read_text()) + + +def _write_json(path: Path, value: dict, *, readonly: bool = False) -> None: + temporary = path.with_name(f".{path.name}.tmp") + temporary.unlink(missing_ok=True) + temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False)) + if readonly: + temporary.chmod(0o444) + temporary.replace(path) + + +def load_eval_set(manifest: str | Path) -> tuple[str, dict]: + """Validate the actual package, not just an advertised sample count.""" + manifest = Path(manifest).resolve() + m = _read(manifest) + tar = manifest.parent / m["tar"] + if Path(m["tar"]).name != m["tar"]: + raise ValueError("T1 manifest.tar must be a filename") + with tar.open("rb") as stream: + digest = hashlib.sha256() + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + h = digest.hexdigest() + if h != m["sha256"]: + raise ValueError(f"T1 package hash mismatch: {tar}") + with tarfile.open(tar) as archive: + inner = json.load(archive.extractfile("manifest.json")) + items = inner["items"] + names = set(archive.getnames()) + if (not items or len(items) != m["n"] or inner["n"] != m["n"] + or len({str(i["id"]) for i in items}) != len(items) + or any(i["file"] not in names for i in items) + or inner["dataset"] != m["dataset"] + or inner["sha256"] != m["content_sha256"]): + raise ValueError(f"T1 manifest/items mismatch: {tar}") + return m["dataset"], { + "path": str(tar), "n": len(items), "seed": m["seed"], + "manifest_sha": h, "content_sha": m["content_sha256"], + "eval_set_id": m.get("eval_set_id") or f"{m['dataset']}-n{len(items)}-{m['content_sha256'][:12]}", + } + + +def _snapshot(con, source: str) -> dict: + run = db.get_run(con, source) + if not run: + raise ValueError(f"unknown source run: {source}") + jobs = [dict(r) for r in con.execute("SELECT * FROM jobs WHERE run_id=? ORDER BY job_id,mode,policy", (source,))] + rollouts = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=? ORDER BY job_id,kk", (source,))] + stages = [dict(r) for r in con.execute("SELECT * FROM stages WHERE run_id=? ORDER BY job_id,rollout,stage,attempt", (source,))] + specs = {jid: _read(paths.RUNS / source / jid / "job.json") + for jid in sorted({j["job_id"] for j in jobs} | {r["job_id"] for r in rollouts})} + return {"run": run, "jobs": jobs, "rollouts": rollouts, "stages": stages, "specs": specs, + "registry_sha": registry.registry_fingerprint(include_board_resources=False)} + + +def plan(source: str, manifests: list[str], target: str | None = None) -> dict: + replacements = dict(load_eval_set(p) for p in manifests) + if len(replacements) != len(manifests): + raise ValueError("one T1 manifest per dataset is required") + with db.readonly() as con: + con.execute("BEGIN") + snapshot = _snapshot(con, source) + target = target or db.next_run_id(con, time.strftime("%Y%m%d")) + if db.get_run(con, target): + raise ValueError(f"target run already exists: {target}") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", target) or target == source: + raise ValueError("target must be a new, simple run id") + old_manifest = json.loads(snapshot["run"]["manifest"]) + if old_manifest.get("continuation") or old_manifest.get("continued_to"): + raise ValueError("source has already been continued; use its saved handoff receipt") + expected = old_manifest.get("registry_task_fingerprint") + matches = (expected == snapshot["registry_sha"] if expected else + old_manifest.get("registry_fingerprint") == registry.registry_fingerprint()) + initial_boards = paths.RUNS / source / "board_registry.initial.yaml" + if not matches and not expected and initial_boards.exists(): + # Legacy runs froze the whole registry, including the board inventory. + # Match that original inventory and the current static task rules. + initial = initial_boards.read_bytes() + matches = (registry.registry_fingerprint(board_bytes=initial) == old_manifest.get("registry_fingerprint") + and registry.registry_fingerprint(include_board_resources=False, board_bytes=initial) + == snapshot["registry_sha"]) + if not matches: + raise ValueError("source registry changed; restore its task rules before planning") + jobs, blockers = [], [] + oracle_ids = {j["job_id"] for j in snapshot["jobs"] if j["mode"] == "oracle"} + if any(r["job_id"] not in oracle_ids and r["state"] in ("QUEUED", "RUNNING", "VERIFYING") for r in snapshot["rollouts"]): + blockers.append("rollout has no source Oracle task") + for row in snapshot["jobs"]: + if row["mode"] != "oracle": + continue + jid = row["job_id"] + jd = paths.RUNS / source / jid + existing = [r for r in snapshot["rollouts"] if r["job_id"] == jid] + oracle = row["state"] == "PENDING" + if row["state"] == "ORACLE_RUNNING": + blockers.append(f"oracle still running: {jid}") + if oracle and (row["started_at"] is not None or row["finished_at"] is not None + or row["result"] is not None or any(s["job_id"] == jid and s["rollout"] == -1 for s in snapshot["stages"]) + or any((jd / "oracle" / "stages").glob("*/a*")) or (jd / "oracle" / "result.json").exists()): + blockers.append(f"PENDING has execution history: {jid}") + queued = [] + for r in existing: + if r["state"] in ("RUNNING", "VERIFYING"): + blockers.append(f"rollout still active: {jid}#{r['kk']}") + if r["state"] != "QUEUED": + continue + if (r["started_at"] is not None or r["finished_at"] is not None or r["verdict"] is not None + or (jd / "rollouts" / f"{r['kk']:02d}").exists() + or any(s["job_id"] == jid and s["rollout"] == r["kk"] for s in snapshot["stages"])): + blockers.append(f"QUEUED has execution history: {jid}#{r['kk']}") + queued.append({"kk": r["kk"], "mode": r["mode"], "policy": r["policy"], "source_exists": True}) + job = deepcopy(snapshot["specs"][jid]) + if not oracle and not queued: + continue + if not oracle and row["state"] not in ("PREPARED", "PASS", "FAIL", "UNVERIFIED"): + blockers.append(f"oracle dependency not finished: {jid} ({row['state']})") + ds = job["data"]["dataset"] + if job["data"]["t1"].get("n") not in (None, 0): + if ds in replacements: + job["data"]["t1"] = deepcopy(replacements[ds]) + elif job["data"]["t1"].get("eval_set_id"): + # Newly admitted jobs may already freeze their intended subset. + # Preserve it after checking its bytes; no extra handoff required. + frozen = job["data"]["t1"] + actual_ds, actual = load_eval_set(Path(frozen["path"]).parent / "manifest.json") + if actual_ds != ds or actual != frozen: + raise ValueError(f"frozen explicit T1 changed: {jid}") + else: + raise ValueError(f"missing replacement T1 manifest for {ds} ({jid})") + job["run_id"] = target + job["evaluator_version"] = EVALUATOR_VERSION + job["continuation"] = {"source_run_id": source, "source_job_id": jid} + if not oracle: + job["oracle_ref"] = {"run_id": source, "job_id": jid} + job["rollout_plans"] = [{"mode": mode, "policy": policy, "k": n} + for (mode, policy), n in Counter((r["mode"], r["policy"]) for r in queued).items()] + jobs.append({"job": job, "oracle": oracle, "source_oracle_state": row["state"], + "source_error_class": row["error_class"], "estimate_s": row["estimate_s"], "rollouts": queued}) + if any(s["state"] == "RUNNING" for s in snapshot["stages"]): + blockers.append("stages still running") + counts = {"oracle": sum(j["oracle"] for j in jobs), + "agent_queued": sum(r["source_exists"] for j in jobs for r in j["rollouts"]), + "agent_due_repairs": 0, "conditional_repairs": 0} + result = {"schema": "bench.continuation/1", "source_run_id": source, "target_run_id": target, + "source_sha256": _sha(snapshot), "eval_manifests": [str(Path(p).resolve()) for p in manifests], + "jobs": jobs, "conditional_repairs": [], "counts": counts, "blockers": blockers} + result["plan_id"] = _sha(result) + return result + + +def receipt(run_id: str) -> dict | None: + p = paths.RUNS / run_id / "continuation.json" + return _read(p) if p.exists() else None + + +@contextmanager +def _drained(con, source: str): + import fcntl + from .pipeline import drain_file, collect_drain_file, scheduler_alive + if not drain_file(source).exists() or not collect_drain_file(source).exists(): + raise ValueError("drain both pipeline and collector before applying") + if scheduler_alive(): + raise ValueError("pipeline scheduler still running; wait for it to exit") + p = paths.STATE / "locks" / f"collect-{source}.lock" + p.parent.mkdir(parents=True, exist_ok=True) + with p.open("a") as f: + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ValueError("collector still running; wait for it to exit") from None + active = con.execute("SELECT 1 FROM jobs WHERE run_id=? AND state='ORACLE_RUNNING' UNION ALL " + "SELECT 1 FROM rollouts WHERE run_id=? AND state IN ('RUNNING','VERIFYING') UNION ALL " + "SELECT 1 FROM stages WHERE run_id=? AND state='RUNNING'", (source, source, source)).fetchone() + if active: + raise ValueError("source still has active work") + yield + + +def apply(handoff: dict) -> dict: + from .pipeline import build_manifest + if handoff.get("plan_id") != _sha({k: v for k, v in handoff.items() if k != "plan_id"}): + raise ValueError("plan was edited; regenerate it instead") + source, target = handoff["source_run_id"], handoff["target_run_id"] + with db.readonly() as con: + previous = db.get_run(con, target) + if previous: + if json.loads(previous["manifest"]).get("continuation", {}).get("plan_id") != handoff["plan_id"]: + raise ValueError("target belongs to another plan") + return check(target) + if handoff["blockers"] or not handoff["jobs"]: + raise ValueError("plan is blocked or empty; regenerate after draining") + # Rebuild the read-only plan: catches changed packages, configuration and state. + fresh = plan(source, handoff["eval_manifests"], target) + if fresh != handoff: + raise ValueError("source changed since preview; regenerate the plan") + manifest = build_manifest() + manifest["continuation"] = {"source_run_id": source, "plan_id": handoff["plan_id"]} + rd = paths.RUNS / target + with db.connect() as con, _drained(con, source): + con.execute("BEGIN IMMEDIATE") + # Recheck inside the write transaction as well as under the collector lock. + if con.execute("SELECT 1 FROM jobs WHERE run_id=? AND state='ORACLE_RUNNING' UNION ALL " + "SELECT 1 FROM rollouts WHERE run_id=? AND state IN ('RUNNING','VERIFYING') UNION ALL " + "SELECT 1 FROM stages WHERE run_id=? AND state='RUNNING'", (source, source, source)).fetchone(): + raise ValueError("source still has active work") + if _sha(_snapshot(con, source)) != handoff["source_sha256"]: + raise ValueError("source changed during apply; regenerate the plan") + if rd.exists(): + saved = receipt(target) + if saved != handoff and (saved is not None or any(p.name != ".continuation.json.tmp" for p in rd.iterdir())): + raise ValueError(f"target directory already exists: {rd}") + rd.mkdir(parents=True, exist_ok=True) + # Files precede the DB commit. An interrupted preparation is retryable + # only with the same receipt; no schedulable run exists until commit. + _write_json(rd / "continuation.json", handoff) + _write_json(rd / "manifest.json", manifest) + for entry in handoff["jobs"]: + job = entry["job"] + jid = job["job_id"] + jd = rd / jid + jd.mkdir(exist_ok=True) + jp = jd / "job.json" + if jp.exists(): + if _read(jp) != job: + raise ValueError(f"prepared job differs: {jp}") + else: + _write_json(jp, job, readonly=True) + if not (jd / "data").exists() and not (jd / "data").is_symlink(): + (jd / "data").symlink_to(paths.DATA) + if entry["oracle"]: + (jd / "oracle" / "stages").mkdir(parents=True, exist_ok=True) + (jd / "oracle" / "artifacts").mkdir(exist_ok=True) + db.upsert_job(con, target, {**job, "mode": "oracle"}, entry["estimate_s"]) + con.execute("UPDATE jobs SET state='TRANSFERRED' WHERE run_id=? AND job_id=? AND mode='oracle'", (source, jid)) + for r in entry["rollouts"]: + db.rollout_upsert(con, target, jid, r["kk"], r["mode"], r["policy"]) + con.execute("UPDATE rollouts SET state='TRANSFERRED' WHERE run_id=? AND job_id=? AND kk=?", (source, jid, r["kk"])) + db.create_run(con, target, manifest, f"remaining work from {source}") + old = json.loads(db.get_run(con, source)["manifest"]) + old["continued_to"] = target + con.execute("UPDATE runs SET manifest=? WHERE run_id=?", (json.dumps(old), source)) + con.commit() # release the collector lock only after the ownership change + return check(target) + + +def enqueue_repairs(run_id: str, handoff: dict, job_ids=None, modes=None) -> int: + """历史交接回执兼容入口;旧 conditional slots 不再激活。""" + with db.connect() as con: + db.assert_run_open(con, run_id) + return 0 + + +def check(run_id: str) -> dict: + handoff = receipt(run_id) + if not handoff: + raise ValueError("run has no continuation receipt") + if handoff["plan_id"] != _sha({k: v for k, v in handoff.items() if k != "plan_id"}): + raise ValueError("continuation receipt was edited") + for entry in handoff["jobs"]: + jp = paths.RUNS / run_id / entry["job"]["job_id"] / "job.json" + if not jp.exists() or _read(jp) != entry["job"]: + raise ValueError(f"handoff job snapshot missing or changed: {jp}") + expected_jobs = {j["job"]["job_id"] for j in handoff["jobs"] if j["oracle"]} + expected = {(j["job"]["job_id"], r["kk"], r["mode"], r["policy"]) for j in handoff["jobs"] for r in j["rollouts"]} + conditional = {(j["job_id"], r["kk"], r["mode"], r["policy"]) for j in handoff["conditional_repairs"] for r in j["rollouts"]} + with db.readonly() as con: + jobs = db.list_jobs(con, run_id, mode="oracle") + rows = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE run_id=?", (run_id,))] + source = handoff["source_run_id"] + moved_jobs = {r[0] for r in con.execute("SELECT job_id FROM jobs WHERE run_id=? AND mode='oracle' AND state='TRANSFERRED'", (source,))} + moved = {(r[0], r[1], r[2], r[3]) for r in con.execute("SELECT job_id,kk,mode,policy FROM rollouts WHERE run_id=? AND state='TRANSFERRED'", (source,))} + actual = {(r["job_id"], r["kk"], r["mode"], r["policy"]) for r in rows} + actual_jobs = {j["job_id"] for j in jobs} + if actual_jobs != expected_jobs or moved_jobs != expected_jobs or moved != expected or not expected <= actual or actual - expected - conditional: + raise ValueError("handoff identity mismatch: missing, additional, or untransferred tasks") + return {"run_id": run_id, "source_run_id": source, "plan_id": handoff["plan_id"], "identities_match": True, + "counts": handoff["counts"], "oracle_states": dict(Counter(j["state"] for j in jobs)), + "agent_states": dict(Counter(r["state"] for r in rows)), "activated_repairs": len(actual - expected)} diff --git a/harness/coverage.py b/harness/coverage.py index 9d721e6..d2b98d9 100644 --- a/harness/coverage.py +++ b/harness/coverage.py @@ -14,6 +14,7 @@ from __future__ import annotations import csv as csvmod +import json import time from collections import Counter, defaultdict from pathlib import Path @@ -113,10 +114,16 @@ def collect(run_id: str | None) -> dict: d["planned_rollouts"] += planned_rollouts.get(j["job_id"], 0) for r in ro: jrow = next((x for x in jobs if x["job_id"] == r["job_id"]), None) - model = jrow["model"] if jrow else None + if jrow: + model = jrow["model"] + else: + jp = paths.RUNS / run_id / r["job_id"] / "job.json" + model = json.loads(jp.read_text()).get("model") if jp.exists() else None d = per_model.get(model) if d is None: continue + if not jrow: + d["planned_rollouts"] += 1 d["rollout"][r["state"]] += 1 if (run_id, r["job_id"], r["kk"]) in exported: d["exported"] += 1 @@ -214,3 +221,239 @@ def to_csv(run_id: str, out: str) -> str: "rollout_planned": d["planned_rollouts"], "rollout_done": sum(r[k] for k in ROLLOUT_DONE), "rollout_pass": r["PASS"], "rollout_queued": r["QUEUED"], "exported": d["exported"]}) return out + + +def model_blockers(row: dict, cache: dict | None = None, evaluation: dict | None = None) -> list[str]: + """Check concrete local prerequisites, independently of CSV status labels. + + This is admission checking, not a claim of successful conversion or accuracy. + The latter can only come from stages and verdicts. + """ + from .eval import TASK_MODULES + from .registry import datasets + from .adapters import model_spec, calibration_dir + row = model_spec(row) + cache = {} if cache is None else cache + def once(key, read): + if key not in cache: + cache[key] = read() + return cache[key] + def exists(p): + return once(("file", str(p)), p.is_file) + def read_json(p): + return once(("json", str(p)), lambda: json.loads(p.read_text())) + task, mid, source = row["task"], row["model_id"], row.get("source") + if source == "opencv": + return ["non_neural_algorithm: no neural weights/PTQ target; retain in inventory"] + reasons = [] + uri = row.get("weights", "") + if uri.startswith(("https://github.com/", "https://docs.", "https://huggingface.co/")) and not any( + part in uri for part in ("/releases/download/", "/resolve/")): + reasons.append("weights_is_catalog_page: concrete checkpoint/revision missing") + elif not uri.startswith(("torchvision://", "ultralytics://", "hf://", "file://", "https://", "http://")): + reasons.append("weights_uri_unsupported_or_missing") + adapter = paths.BENCH_ROOT / "harness" / "adapters" / (mid.replace(".", "_").replace("-", "_") + ".py") + adapters = once("adapter_files", lambda: {p.name for p in adapter.parent.glob("*.py")}) + if source not in ("torchvision", "ultralytics") and adapter.name not in adapters: + reasons.append(f"model_builder_export_adapter_missing: {adapter.relative_to(paths.BENCH_ROOT)}") + if mid.startswith("fastsam."): + reasons.append("metric_semantics: FastSAM class-agnostic masks cannot use 80-class COCO mask AP") + if task not in TASK_MODULES: + reasons.append(f"task_evaluator_missing: {task}") + if source == "ultralytics" and task in ("iseg", "pose") and not mid.startswith("fastsam."): + probe = paths.CACHE / "adapter-probes" / mid / "probe.json" + proof = read_json(probe) if exists(probe) else {} + if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid: + reasons.append(f"real_checkpoint_export_probe_missing_or_failed: {probe}") + if task == "pose": + probe = paths.CACHE / "adapter-probes" / mid / "probe.json" + proof = read_json(probe) if exists(probe) else {} + if (proof.get("pose_export_shift", {}).get("status") != "measured" + or proof["pose_export_shift"]["keypoint_mean_shift_px"] > 2): + reasons.append("auxiliary_gate_probe_missing: actual FP32/ONNX keypoint correspondence and <=2px shift required") + evidence = proof.get("pose_export_evidence") or {} + if not evidence.get("sha256") or not exists(Path(evidence.get("path", "__missing__"))): + reasons.append("auxiliary_gate_raw_evidence_missing: replayable matched keypoint coordinates required") + if source not in ("torchvision", "ultralytics", "opencv") and adapter.name in adapters: + probe = paths.CACHE / "adapter-probes" / mid / "probe.json" + proof = read_json(probe) if exists(probe) else {} + if proof.get("status") != "validated_fp32_export_only" or proof.get("model") != mid: + reasons.append("custom_adapter_unvalidated: builder present; environment/preprocessing/real forward/export proof still required") + if row.get("preprocess") and json.dumps(proof.get("preprocess"), sort_keys=True) != json.dumps(row["preprocess"], sort_keys=True): + reasons.append("preprocessing_recipe_not_validated: probe must use the current model recipe") + if task in ("stereo", "flow", "feat", "occ"): + probe = paths.CACHE / "adapter-probes" / mid / "probe.json" + proof = read_json(probe) if exists(probe) else {} + contract = proof.get("input_contract") or {} + if (proof.get("status") != "validated_fp32_export_only" or contract.get("status") != "validated" + or contract.get("input_spec") != row.get("input_spec")): + reasons.append("input_contract_unverified: real pair/multi-input/camera metadata and export proof required") + ds = row.get("dataset", "") + ds_spec = datasets().get("datasets", {}).get(ds, {}) + if task not in ds_spec.get("tasks", []): + reasons.append(f"dataset_task_mismatch: {ds} is not registered for {task}") + if evaluation is None: + probe_path = paths.CACHE / "adapter-probes" / mid / "probe.json" + proof = read_json(probe_path) if exists(probe_path) else {} + if proof.get("status") == "validated_fp32_export_only": + evaluation = proof.get("eval_set") + manifest = (Path(evaluation["path"]).parent / "manifest.json" if evaluation else + paths.DATA / "t1" / ds / "manifest.json") + if not exists(manifest): + reasons.append(f"t1_missing: {manifest}") + else: + m = read_json(manifest) + if not m.get("sha256") or not exists(manifest.parent / m.get("tar", "__missing__")): + reasons.append("t1_package_or_identity_missing") + elif evaluation and (m.get("dataset") != ds or m.get("n") != evaluation.get("n") + or m.get("sha256") != evaluation.get("manifest_sha") + or m.get("content_sha256") != evaluation.get("content_sha")): + reasons.append("t1_explicit_identity_mismatch") + elif task in ("iseg", "pose") and ds == "coco2017": + import tarfile + def annotations(): + with tarfile.open(manifest.parent / m["tar"]) as archive: + return json.load(archive.extractfile("annotations.json")) + ann = once(("annotations", ds), annotations) + if task == "pose" and not ann.get("keypoints", {}).get("annotations"): + reasons.append("person_keypoint_annotations_missing") + if task == "iseg" and not any(a.get("segmentation") for a in ann.get("annotations", [])): + reasons.append("instance_mask_annotations_missing") + calib = calibration_dir(row) + if not exists(calib / "manifest.json"): + reasons.append(f"calibration_missing: {calib}") + else: + cm = read_json(calib / "manifest.json") + count = once(("calibration_count", str(calib)), lambda: len(list((calib / cm.get("npy_dir", "npy_f32")).glob("*.npy")))) + if not cm.get("sha256") or count < cm.get("count", 128) or not count: + reasons.append("calibration_samples_or_identity_missing") + return reasons + + +def master_inventory() -> dict: + """All CSV models, all runs, with evidence joins. Never changes task state.""" + from .verify import completion_summary + with db.readonly() as con: + jobs = [dict(r) for r in con.execute("SELECT * FROM jobs WHERE mode='oracle'")] + rollouts = [dict(r) for r in con.execute("SELECT * FROM rollouts")] + stages = [dict(r) for r in con.execute("SELECT * FROM stages")] + by_job = {(j["run_id"], j["job_id"]): j for j in jobs} + exports = {} + for f in sorted(paths.DATASETS.glob("*/teacher.jsonl")): + if (f.parent / "VOID").exists(): + continue + for line in f.open(): + record = json.loads(line) + if record.get("source") == "agent_events" and record.get("messages"): + key = (record["run_id"], record["job_id"], record["rollout"]) + exports[key] = {"path": str(f), "id": record["id"], "model": record.get("model"), + "sft_eligible": record.get("sft_eligible", False)} + per_model, prerequisite_cache = {}, {} + for row in models(): + from .adapters import model_spec + row = model_spec(row) + mid = row["model_id"] + per_model[mid] = {"model": mid, "name": row.get("name"), "task": row["task"], + "priority": row["priority"], "registry_status": row.get("status"), + "source": row.get("source"), "weights": row.get("weights"), + "blockers": model_blockers(row, prerequisite_cache), "jobs": [], "rounds": []} + probe = paths.CACHE / "adapter-probes" / mid / "probe.json" + per_model[mid]["adapter_probe"] = {"path": str(probe), **json.loads(probe.read_text())} if probe.is_file() else None + for j in jobs: + d = per_model.get(j["model"]) + if d is None: + continue + result = json.loads(j["result"]) if j.get("result") else {} + directory = paths.RUNS / j["run_id"] / j["job_id"] + d["jobs"].append({"run_id": j["run_id"], "job_id": j["job_id"], "state": j["state"], + "error_class": j["error_class"], "stage_failed": j["stage_failed"], + "path": str(directory / "oracle"), + "completion": result.get("completion") or completion_summary(result.get("gates", {}), result.get("stages", {})), + "gates": result.get("gates", {}), "eval_set": result.get("eval_set"), + "stages": [s for s in stages if s["run_id"] == j["run_id"] and s["job_id"] == j["job_id"] and s["rollout"] == -1]}) + for r in rollouts: + job = by_job.get((r["run_id"], r["job_id"])) + mid = job["model"] if job else r["job_id"].split("--")[1] + if mid not in per_model: + continue + rd = paths.RUNS / r["run_id"] / r["job_id"] / "rollouts" / f"{r['kk']:02d}" + # Hundreds of QUEUED rows have no directory yet. Avoid FUSE lookups for + # each absent file; pending plans are not execution evidence. + started = r["state"] != "QUEUED" or r.get("started_at") is not None + meta = json.loads((rd / "rollout.json").read_text()) if started and (rd / "rollout.json").is_file() else {} + verdict = json.loads(r["verdict"]) if r.get("verdict") else {} + raw = {str(p): p.stat().st_size for p in (rd / "agent" / "events.jsonl", rd / "agent" / "session.jsonl") if started and p.is_file()} + observed_models = set() + events = rd / "agent" / "events.jsonl" + if str(events) in raw: + with events.open() as stream: + for line in stream: + try: + event = json.loads(line) + except json.JSONDecodeError: # an active writer's last partial line + continue + message = event.get("message") or {} + if (event.get("type") == "message_end" and isinstance(message, dict) + and message.get("role") == "assistant" and message.get("model") and message.get("content")): + observed_models.add(message["model"]) + break # first real response identity; raw full stream stays linked + exported = exports.get((r["run_id"], r["job_id"], r["kk"])) + per_model[mid]["rounds"].append({"run_id": r["run_id"], "job_id": r["job_id"], "kk": r["kk"], + "state": r["state"], "mode": r["mode"], "policy": r["policy"], "model": meta.get("model"), + "raw_traces": raw, "observed_models": sorted(observed_models), "path": str(rd), "export": exported, + "completion": verdict.get("completion") or completion_summary(verdict.get("gates", {}), verdict.get("stages", {})), + "gates": verdict.get("gates", {}), + "eval_set": verdict.get("eval_set"), "error_class": verdict.get("error_class")}) + return {"schema": "bench.coverage/2", "captured_at": time.time(), "models": list(per_model.values())} + + +def write_master(out: str) -> str: + """CSV summary + adjacent JSON containing every run/job/round/evidence link.""" + import os + import tempfile + inventory = master_inventory() + destination = Path(out) + destination.parent.mkdir(parents=True, exist_ok=True) + fields = ["model", "name", "task", "priority", "registry_status", "source", "weights", "admission", + "blockers", "adapter_validation", "adapter_evidence", "job_ids", "latest_run", "latest_oracle_state", "latest_stage", "latest_error", + "oracle_states", "calibration", "quantization", "compile", "hbm_generated", "board", "accuracy", "accuracy_passed", + "first_glm53_trace", "glm53_exported", "sft_eligible", "round_states", "runtime_errors", "evidence_json"] + fd, tmp = tempfile.mkstemp(dir=destination.parent, prefix=destination.name + ".") + try: + with os.fdopen(fd, "w") as stream: + writer = csvmod.DictWriter(stream, fieldnames=fields) + writer.writeheader() + for d in inventory["models"]: + all_results = d["jobs"] + d["rounds"] + def done(key): + return sum(bool(x.get("completion", {}).get(key)) for x in all_results) + glm = [r for r in d["rounds"] if r["model"] == "glm-5.3" and "glm-5.3" in r["observed_models"]] + exported = [r for r in glm if r["export"] and r["export"]["model"] == "glm-5.3"] + latest = max(d["jobs"], key=lambda j: j["run_id"], default={}) + latest_stage = max(latest.get("stages", []), key=lambda s: s.get("started_at") or 0, default={}) + writer.writerow({**{k: d.get(k) for k in fields if k in d and k != "blockers"}, + "admission": "blocked" if d["blockers"] else "prerequisites_present", + "blockers": " | ".join(d["blockers"]), + "adapter_validation": (d.get("adapter_probe") or {}).get("status", ""), + "adapter_evidence": (d.get("adapter_probe") or {}).get("path", ""), + "job_ids": " | ".join(j["run_id"] + "/" + j["job_id"] for j in d["jobs"]), + "latest_run": latest.get("run_id", ""), "latest_oracle_state": latest.get("state", "not_queued"), + "latest_stage": latest_stage.get("stage", ""), "latest_error": latest.get("error_class", ""), + "oracle_states": json.dumps(dict(Counter(j["state"] for j in d["jobs"]))), + "calibration": sum(any(s["stage"] == "04-calibrate" and s["state"] in ("OK", "SKIPPED") for s in j["stages"]) for j in d["jobs"]), + "quantization": done("compiled"), "compile": done("compiled"), "hbm_generated": done("hbm_generated"), + "board": done("board_completed"), "accuracy": done("accuracy_evaluated"), "accuracy_passed": done("accuracy_passed"), + "first_glm53_trace": glm[0]["path"] if glm else "", "glm53_exported": len(exported), + "sft_eligible": sum(r["export"]["sft_eligible"] for r in exported), + "round_states": json.dumps(dict(Counter(r["state"] for r in d["rounds"]))), + "runtime_errors": " | ".join(sorted({str(x.get("stage_failed") or "verdict") + ":" + x["error_class"] for x in all_results if x.get("error_class")})), + "evidence_json": str(destination.with_suffix(".json"))}) + os.replace(tmp, destination) + fdj, tmpj = tempfile.mkstemp(dir=destination.parent, prefix=destination.stem + ".json.") + with os.fdopen(fdj, "w") as stream: + json.dump(inventory, stream, ensure_ascii=False, indent=2) + os.replace(tmpj, destination.with_suffix(".json")) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + return str(destination) diff --git a/harness/db.py b/harness/db.py index 9fc07ef..4aae1de 100644 --- a/harness/db.py +++ b/harness/db.py @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS jobs ( mode TEXT NOT NULL, -- oracle | e2e | repair policy TEXT, -- teacher | student | NULL k INTEGER DEFAULT 1, - state TEXT NOT NULL, -- PENDING ORACLE_RUNNING PASS FAIL UNVERIFIED + state TEXT NOT NULL, -- PENDING ORACLE_RUNNING PREPARED PASS FAIL UNVERIFIED error_class TEXT, stage_failed TEXT, estimate_s REAL, @@ -114,6 +114,13 @@ def get_run(con, run_id: str) -> dict | None: return dict(r) if r else None +def assert_run_open(con, run_id: str) -> None: + run = get_run(con, run_id) + target = json.loads(run["manifest"]).get("continued_to") if run else None + if target: + raise ValueError(f"run {run_id} handed off its remaining work to {target}; execution is closed") + + def latest_run_id(con) -> str | None: r = con.execute("SELECT run_id FROM runs ORDER BY created_at DESC LIMIT 1").fetchone() return r[0] if r else None @@ -121,7 +128,7 @@ def latest_run_id(con) -> str | None: def next_run_id(con, date_str: str) -> str: rows = con.execute("SELECT run_id FROM runs WHERE run_id LIKE ?", (f"r{date_str}-%",)).fetchall() - seq = 1 + max([int(r[0].rsplit("-", 1)[1]) for r in rows] or [0]) + seq = 1 + max([int(r[0].rsplit("-", 1)[1]) for r in rows if r[0].rsplit("-", 1)[1].isdigit()] or [0]) return f"r{date_str}-{seq:02d}" @@ -141,10 +148,15 @@ def upsert_job(con, run_id: str, spec_row: dict, estimate_s: float | None) -> bo return True -def claim_job(con, run_id: str, job_id: str, mode: str = "oracle", force: bool = False) -> bool: - """原子认领:只有 PENDING 的 job 能被置成 ORACLE_RUNNING。两个 `bench run` 进程同时挑到同一个 job 时只有一个赢。 - force=True 允许重跑已结束的 job(只给 --job 显式指定时用;结果仍是原地覆盖 result.json,慎用)。""" - allowed = "('PENDING')" if not force else "('PENDING','PASS','FAIL','UNVERIFIED')" +def claim_job(con, run_id: str, job_id: str, mode: str = "oracle", force: bool = False, + prepare_only: bool = False) -> bool: + """原子认领:准备只取 PENDING;完整 Oracle 也可取 PREPARED,同一 job 只有一个进程赢。 + force=True 只允许完整 Oracle 重跑已结束的 job(显式 --oracle --job,原地覆盖 result.json)。""" + con.execute("BEGIN IMMEDIATE") + assert_run_open(con, run_id) + allowed = "('PENDING','PREPARED')" if not force else "('PENDING','PREPARED','PASS','FAIL','UNVERIFIED')" + if prepare_only: + allowed = "('PENDING')" # Preparation never overwrites a finished job, even with --job. cur = con.execute(f"UPDATE jobs SET state='ORACLE_RUNNING', started_at=? WHERE run_id=? AND job_id=? AND mode=? AND state IN {allowed}", (time.time(), run_id, job_id, mode)) con.commit() @@ -221,6 +233,7 @@ def rollout_upsert(con, run_id, job_id, kk, mode, policy, state="QUEUED") -> Non def claim_rollout(con, run_id: str, job_id: str, kk: int, max_active: int | None = None) -> bool: """原子认领一条 QUEUED;同一 job 只允许一条运行,跨 collector 也不重复。""" con.execute("BEGIN IMMEDIATE") + assert_run_open(con, run_id) if max_active is not None: active = con.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0] if active >= max_active: diff --git a/harness/errors.py b/harness/errors.py index 48cdecb..9ec8dfc 100644 --- a/harness/errors.py +++ b/harness/errors.py @@ -9,11 +9,12 @@ import re DETERMINISTIC = ["unsupported_op", "shape_infer", "export_fail", "calib_nan", "verifier_mismatch", "config_error", # 门失败(gates.classify_gate_failure) "accuracy_collapse", "accuracy_drop", "numeric_mismatch", "march_mismatch", "perf_over_budget"] -TRANSIENT = ["oom", "board_offline", "stalled"] +TRANSIENT = ["oom", "board_offline", "board_busy", "stalled"] ALL = DETERMINISTIC + TRANSIENT + ["unknown"] # (error_class, compiled regex)。写规则时对着真实 stderr 写,别猜。 RULES: list[tuple[str, re.Pattern]] = [ + ("board_busy", re.compile(r'"event":\s*"board_busy"|board busy after .*refusing contended', re.I)), ("config_error", re.compile(r"Key '[^']+' error:|Wrong key '|Missing key|yaml.*(invalid|error)|schema.*error", re.I)), # hb_compile yaml 校验不过:harness/agent 配置写错 ("oom", re.compile(r"CUDA out of memory|OutOfMemoryError|std::bad_alloc|Cannot allocate memory|MemoryError|Killed process .* out of memory", re.I)), ("board_offline", re.compile(r"Connection (refused|timed out|reset by peer)|No route to host|ssh: connect to host|board .*offline|hrt_.*: failed to connect", re.I)), diff --git a/harness/eval/__init__.py b/harness/eval/__init__.py index ba1211f..d2c4e46 100644 --- a/harness/eval/__init__.py +++ b/harness/eval/__init__.py @@ -9,5 +9,12 @@ COMPATIBLE_BASELINE_VERSIONS = {"0.4.0", EVALUATOR_VERSION} TASK_MODULES = { "cls": "harness.eval.cls", "det": "harness.eval.det", - # P4+:seg iseg pose mde stereo flow feat occ + # Additive task support; cls/det preprocessing and metrics remain 0.5-compatible. + "iseg": "harness.eval.coco_extra", + "pose": "harness.eval.coco_extra", + "flow": "harness.eval.flow", + "seg": "harness.eval.seg", + "mde": "harness.eval.mde", + "feat": "harness.eval.feat", + # P4+:stereo occ } diff --git a/harness/eval/cls.py b/harness/eval/cls.py index 3ff2de1..8a931e1 100644 --- a/harness/eval/cls.py +++ b/harness/eval/cls.py @@ -14,7 +14,8 @@ def _size(input_spec: str) -> int: return int(input_spec.split("x")[-1]) -def predict(runner, t1: T1, input_spec: str, items: list[dict] | None = None, resize: int = 0) -> tuple[np.ndarray, np.ndarray]: +def predict(runner, t1: T1, input_spec: str, items: list[dict] | None = None, resize: int = 0, + model: str = "") -> tuple[np.ndarray, np.ndarray]: """返回 (top5_pred[n,5], labels[n])。resize=0 → size*256/224(经典 224/256);torchvision 各权重的 resize 由 models.csv 给。""" items = items or t1.items size = _size(input_spec) @@ -22,7 +23,7 @@ def predict(runner, t1: T1, input_spec: str, items: list[dict] | None = None, re preds, labels = [], [] for i in range(0, len(items), BATCH): chunk = items[i:i + BATCH] - xs = [preprocess_cls(t1.image(it), size=size, resize=resize) for it in chunk] + xs = [preprocess_cls(t1.image(it), size=size, resize=resize, model=model) for it in chunk] for it, outs in zip(chunk, run_batched(runner, xs)): logits = outs[0].reshape(-1) preds.append(np.argsort(-logits)[:5]) @@ -43,13 +44,13 @@ def paired_se(top5_a: np.ndarray, top5_b: np.ndarray, labels: np.ndarray) -> flo def run_fp32(runner, t1: T1, input_spec: str, out: dict, resize: int = 0) -> None: - top5, labels = predict(runner, t1, input_spec, resize=resize) + top5, labels = predict(runner, t1, input_spec, resize=resize, model=out.get("model", "")) out["metrics"] = metrics(top5, labels) out["_pred"] = top5.tolist() def run_quant(runner, t1: T1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None: - top5, labels = predict(runner, t1, input_spec, resize=resize) + top5, labels = predict(runner, t1, input_spec, resize=resize, model=out.get("model", "")) m = metrics(top5, labels) se = paired_se(np.asarray(baseline["_pred"]), top5, labels) if baseline.get("_pred") else None out["metrics"] = m diff --git a/harness/eval/coco_extra.py b/harness/eval/coco_extra.py new file mode 100644 index 0000000..32da2cd --- /dev/null +++ b/harness/eval/coco_extra.py @@ -0,0 +1,292 @@ +"""COCO mask AP / keypoint OKS AP for the existing Ultralytics exporters. + +Both FP32 and board outputs use exactly the same letterbox, decoding and COCOeval. +FastSAM is deliberately excluded: its class-agnostic masks are not COCO classes. +""" +from __future__ import annotations + +import json +import numpy as np + +from .common import letterbox, run_batched +from .det import CONF, IOU, MAX_DET, _nms + +METRICS = {"iseg": "mask_ap", "pose": "oks_ap"} +POSE_SHIFT_SCHEMA = "coco-pose-bbox-iou-v1" +POSE_MATCH_IOU = 0.5 + + +def decode(outputs, task): + nc, extra = (80, 32) if task == "iseg" else (1, 51) + candidates = [o for o in outputs if o.ndim == 3 and o.shape[1] == 4 + nc + extra] + if len(candidates) != 1: + raise ValueError(f"{task}: expected COCO YOLO head [1,{4+nc+extra},N], got {[o.shape for o in outputs]}") + p = candidates[0][0].T + labels = p[:, 4:4 + nc].argmax(1) + scores = p[:, 4:4 + nc].max(1) + selected = scores > CONF + p, labels, scores = p[selected], labels[selected], scores[selected] + boxes = np.concatenate((p[:, :2] - p[:, 2:4] / 2, p[:, :2] + p[:, 2:4] / 2), axis=1) + keep = (np.empty(0, dtype=np.intp) if task == "pose" and not len(scores) + else _nms(boxes + labels[:, None] * 4096, scores, IOU)[:MAX_DET]) + proto = None + if task == "iseg": + protos = [o for o in outputs if o.ndim == 4 and o.shape[:2] == (1, extra)] + if len(protos) != 1: + raise ValueError("segmentation requires exactly one 32-channel prototype output") + proto = protos[0][0] + return boxes[keep], scores[keep], labels[keep], p[keep, 4 + nc:], proto + + +def masks_to_original(proto, coeff, boxes, size, image_size, scale, pad): + """Undo our actual integer letterbox offsets, including asymmetric rounding.""" + import torch + from torch.nn.functional import interpolate + from ultralytics.utils.ops import crop_mask + w, h = image_size + px, py = pad + nw, nh = round(w * scale), round(h * scale) + # Bound peak memory for 300 detections at original image resolution. + for i in range(0, len(coeff), 16): + logits = torch.from_numpy(coeff[i:i + 16]) @ torch.from_numpy(proto).reshape(proto.shape[0], -1) + logits = logits.reshape(-1, *proto.shape[1:]) + logits = interpolate(logits[:, None], (size, size), mode="bilinear", align_corners=False) + logits = logits[:, :, py:py + nh, px:px + nw] + masks = interpolate(logits, (h, w), mode="bilinear", align_corners=False)[:, 0] > 0 + yield from crop_mask(masks.byte(), torch.from_numpy(boxes[i:i + 16]).float()).numpy() + + +def predict(runner, t1, input_spec, task, pose_evidence=None, model=""): + if model: + from ..adapters import get + custom = getattr(get(model), "predict", None) + if custom is not None: + return custom(runner, t1, input_spec, pose_evidence=pose_evidence) + if task == "iseg": + from pycocotools import mask as mask_utils + size = int(input_spec.split("x")[-1]) + predictions = [] + for offset in range(0, len(t1.items), 16): + items = t1.items[offset:offset + 16] + images = [t1.image(it) for it in items] + pre = [letterbox(im, size) for im in images] + outputs = run_batched(runner, [v[0] for v in pre]) + if task == "pose" and len(outputs) != len(items): + raise ValueError("pose: missing image outputs") + for it, im, (_, scale, pad), outs in zip(items, images, pre, outputs): + boxes, scores, labels, extra, proto = decode(outs, task) + if task == "pose" and pose_evidence is not None: + # Keep unclipped original-image geometry: clipping at image edges + # can hide a real shift. These records never enter COCOeval. + raw_boxes = (boxes.astype(np.float64) - np.array([*pad, *pad])) / scale + raw_points = extra.astype(np.float64).reshape(-1, 17, 3).copy() + raw_points[:, :, :2] = (raw_points[:, :, :2] - np.array(pad)) / scale + pose_evidence["images"].append({ + "image_id": it["id"], "image_size": list(im.size), "scale": scale, "pad": list(pad), + "detections": [{"category_id": 1, "bbox_xyxy": box.tolist(), "score": float(score), + "keypoints": points.tolist()} + for box, score, points in zip(raw_boxes, scores, raw_points)], + }) + boxes = (boxes - np.array([*pad, *pad], dtype=np.float32)) / scale + boxes[:, [0, 2]] = boxes[:, [0, 2]].clip(0, im.width) + boxes[:, [1, 3]] = boxes[:, [1, 3]].clip(0, im.height) + if task == "iseg": + masks = masks_to_original(proto, extra, boxes, size, im.size, scale, pad) + for mask, score, label in zip(masks, scores, labels): + rle = mask_utils.encode(np.asfortranarray(mask.astype(np.uint8))) + rle["counts"] = rle["counts"].decode("ascii") + predictions.append({"image_id": it["id"], "category_id": t1.manifest["cat_map"][str(int(label))], + "score": float(score), "segmentation": rle}) + else: + points = extra.reshape(-1, 17, 3).copy() + points[:, :, :2] = (points[:, :, :2] - np.array(pad)) / scale + points[:, :, 0] = points[:, :, 0].clip(0, im.width) + points[:, :, 1] = points[:, :, 1].clip(0, im.height) + for points_one, score in zip(points, scores): + predictions.append({"image_id": it["id"], "category_id": 1, "score": float(score), + "keypoints": points_one.reshape(-1).tolist()}) + return predictions + + +def pose_snapshot(t1, input_spec): + """Stored in the existing metrics.json/eval.json; no separate cache or runner.""" + return {"schema": POSE_SHIFT_SCHEMA, "input_spec": input_spec, + "t1_manifest_sha": t1.manifest.get("sha256"), + "image_ids": [it["id"] for it in t1.items], "images": []} + + +def measure_pose_shift(fp32, quant): + """Mean L2 distance in original pixels over matched post-NMS poses, all 17 joints. + + Match only within an image/category, greedily by descending box IoU >= 0.5; + ties use the saved post-NMS indices. Each detection is used at most once. + Neither keypoint positions nor keypoint confidence select pairs/joints. + Unmatched detections do not contribute zeros; their indices and coverage are + retained. OKS AP separately judges missed/extra people. No pairs => unverified. + """ + result = {"status": "unverified", "method": POSE_SHIFT_SCHEMA, + "coordinate_space": "original_image_pixels_unclipped", "match_iou_min": POSE_MATCH_IOU, + "joint_policy": "all_17_no_confidence_filter", "keypoint_mean_shift_px": None, + "distance_sum_px": None, + "n_fp32": 0, "n_quant": 0, "n_matched": 0, "n_keypoints": 0, + "matches": [], "unmatched_fp32": [], "unmatched_quant": []} + if not isinstance(fp32, dict) or not isinstance(quant, dict): + return {**result, "reason": "pose reference or measurement missing; rerun FP32 to record geometry"} + for key in ("schema", "input_spec", "t1_manifest_sha", "image_ids"): + if key not in fp32 or key not in quant or fp32[key] != quant[key]: + return {**result, "reason": f"pose evidence {key} mismatch or missing"} + if fp32["schema"] != POSE_SHIFT_SCHEMA: + return {**result, "reason": "unsupported pose evidence schema"} + if not isinstance(fp32["t1_manifest_sha"], str) or not fp32["t1_manifest_sha"]: + return {**result, "reason": "pose evidence T1 identity missing"} + + def checked_images(snapshot): + ids = snapshot["image_ids"] + if not isinstance(ids, list) or len(set(ids)) != len(ids): + raise ValueError("duplicate or invalid image IDs") + images = snapshot["images"] + if [im["image_id"] for im in images] != ids: + raise ValueError("missing, reordered or extra image evidence") + for im in images: + size = np.asarray(im["image_size"], dtype=float) + if size.shape != (2,) or not np.isfinite(size).all() or (size <= 0).any(): + raise ValueError("invalid image geometry") + if im.get("transform") != "topdown_affine": + pad = np.asarray(im["pad"], dtype=float) + if (pad.shape != (2,) or not np.isfinite(pad).all() + or not np.isfinite(im["scale"]) or im["scale"] <= 0): + raise ValueError("invalid letterbox geometry") + for d in im["detections"]: + box, points = np.asarray(d["bbox_xyxy"], dtype=float), np.asarray(d["keypoints"], dtype=float) + if (box.shape != (4,) or points.shape != (17, 3) or not np.isfinite(box).all() + or not np.isfinite(points).all() or not np.isfinite(d["score"]) + or (box[2:] <= box[:2]).any() or d["category_id"] != 1): + raise ValueError("invalid/non-finite COCO pose detection") + if im.get("transform") == "topdown_affine": + affine = np.asarray(d["affine"], dtype=float) + if (affine.shape != (2, 3) or not np.isfinite(affine).all() + or abs(np.linalg.det(affine[:, :2])) < 1e-12): + raise ValueError("invalid measured top-down affine geometry") + return images + + try: + refs, measured = checked_images(fp32), checked_images(quant) + for ref, cur in zip(refs, measured): + if ref.get("transform") != cur.get("transform"): + raise ValueError("FP32/quant image transform mismatch") + keys = ("image_size",) if ref.get("transform") == "topdown_affine" else ("image_size", "scale", "pad") + if any(ref[k] != cur[k] for k in keys): + raise ValueError("FP32/quant image geometry mismatch") + except (KeyError, TypeError, ValueError, OverflowError) as e: + return {**result, "reason": str(e)} + + for ref, cur in zip(refs, measured): + a, b = ref["detections"], cur["detections"] + result["n_fp32"] += len(a) + result["n_quant"] += len(b) + candidates = [] + if a and b: + ab = np.asarray([d["bbox_xyxy"] for d in a], dtype=np.float64) + bb = np.asarray([d["bbox_xyxy"] for d in b], dtype=np.float64) + wh = np.maximum(0, np.minimum(ab[:, None, 2:], bb[None, :, 2:]) + - np.maximum(ab[:, None, :2], bb[None, :, :2])) + inter = wh.prod(axis=2) + union = (ab[:, 2:] - ab[:, :2]).prod(axis=1)[:, None] + (bb[:, 2:] - bb[:, :2]).prod(axis=1) - inter + ious = inter / union + if not np.isfinite(ious).all(): + return {**result, "reason": "non-finite box IoU"} + candidates = sorted((-float(ious[i, j]), int(i), int(j)) + for i, j in np.argwhere(ious >= POSE_MATCH_IOU)) + used_a, used_b = set(), set() + for neg_iou, i, j in candidates: + if i in used_a or j in used_b: + continue + if ref.get("transform") == "topdown_affine" and a[i]["affine"] != b[j]["affine"]: + return {**result, "reason": "FP32/quant matched crop affine mismatch"} + used_a.add(i) + used_b.add(j) + delta = (np.asarray(b[j]["keypoints"], dtype=np.float64)[:, :2] + - np.asarray(a[i]["keypoints"], dtype=np.float64)[:, :2]) + distances = np.hypot(delta[:, 0], delta[:, 1]) + if not np.isfinite(distances).all(): + return {**result, "reason": "non-finite keypoint distance"} + result["matches"].append({"image_id": ref["image_id"], "fp32_index": i, "quant_index": j, + "bbox_iou": -neg_iou, "fp32": a[i], "quant": b[j], + "distances_px": distances.tolist()}) + for key, dets, used in (("unmatched_fp32", a, used_a), ("unmatched_quant", b, used_b)): + result[key].extend({"image_id": ref["image_id"], "index": i} for i in range(len(dets)) if i not in used) + result["n_matched"] = len(result["matches"]) + result["n_keypoints"] = 17 * result["n_matched"] + if not result["n_matched"]: + return {**result, "reason": "no matched FP32/quant poses"} + total = float(np.sum([d for pair in result["matches"] for d in pair["distances_px"]])) + mean = total / result["n_keypoints"] + if not np.isfinite(mean): + return {**result, "reason": "non-finite mean keypoint shift"} + return {**result, "status": "measured", "keypoint_mean_shift_px": mean, "distance_sum_px": total} + + +def pose_shift_summary(evidence): + # run.py prints G3 into stage logs: keep full per-pair geometry in eval.json + # at the top level, so logs/verdicts do not duplicate thousands of poses. + return {**{k: v for k, v in evidence.items() if k not in ("matches", "unmatched_fp32", "unmatched_quant")}, + "n_unmatched_fp32": len(evidence["unmatched_fp32"]), + "n_unmatched_quant": len(evidence["unmatched_quant"]), + "evidence": "eval.json#/pose_shift_evidence"} + + +def evaluate(t1, predictions, task): + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + ann = json.load(t1.tar.extractfile("annotations.json")) + if task == "pose": + keypoints = ann.get("keypoints") + if not keypoints or not keypoints.get("annotations"): + raise ValueError("COCO T1 is missing person keypoint annotations") + ann = {**ann, "annotations": keypoints["annotations"], "categories": keypoints["categories"]} + elif not any(a.get("segmentation") for a in ann["annotations"]): + raise ValueError("COCO T1 is missing instance masks") + gt = COCO() + gt.dataset = ann + gt.createIndex() + metric = METRICS[task] + if not predictions: + return {metric: 0.0, "ap50": 0.0, "n": len(t1.items)} + ev = COCOeval(gt, gt.loadRes(predictions), "segm" if task == "iseg" else "keypoints") + ev.params.imgIds = [it["id"] for it in t1.items] + if task == "pose": + ev.params.catIds = [1] + ev.evaluate() + ev.accumulate() + ev.summarize() + if ev.stats[0] < 0: + raise ValueError("no evaluable ground truth for this COCO subset") + return {metric: float(ev.stats[0] * 100), "ap50": float(ev.stats[1] * 100), "n": len(t1.items)} + + +def run_fp32(runner, t1, input_spec, out, resize=0): + task = out["task"] + evidence = pose_snapshot(t1, input_spec) if task == "pose" else None + predictions = predict(runner, t1, input_spec, task, pose_evidence=evidence, model=out.get("model", "")) + if evidence is not None: + out["pose_reference"] = evidence + out["metrics"] = evaluate(t1, predictions, task) + + +def run_quant(runner, t1, input_spec, baseline, out, resize=0): + task = out["task"] + metric = METRICS[task] + evidence = pose_snapshot(t1, input_spec) if task == "pose" else None + predictions = predict(runner, t1, input_spec, task, pose_evidence=evidence, model=out.get("model", "")) + measured = evaluate(t1, predictions, task) + out["metrics"] = measured + out["G3"] = {"metric": metric, "fp32": baseline["metrics"][metric], "quant": measured[metric], + "tier": "t1", "n": measured["n"], "se": None, "also": {"ap50": measured["ap50"]}} + if evidence is not None: + out["pose_reference"] = evidence + shift = measure_pose_shift(baseline.get("pose_reference"), evidence) + if baseline.get("model") != out.get("model"): + shift.update(status="unverified", keypoint_mean_shift_px=None, reason="FP32/quant model mismatch") + out["pose_shift_evidence"] = shift + out["G3"]["extra"] = {"keypoint_mean_shift_px": shift["keypoint_mean_shift_px"], + "keypoint_shift": pose_shift_summary(shift)} diff --git a/harness/eval/common.py b/harness/eval/common.py index 4002a47..23497d9 100644 --- a/harness/eval/common.py +++ b/harness/eval/common.py @@ -50,7 +50,12 @@ class T1: # ---------------------------------------------------------------- 预处理(口径钉死,fp32 与 quant 同一份) -def preprocess_cls(img, size: int = 224, resize: int = 256) -> np.ndarray: +def preprocess_cls(img, size: int = 224, resize: int = 256, model: str = "") -> np.ndarray: + if model: + from ..adapters import get + adapter = get(model) + if transform := getattr(adapter, "preprocess", None): + return transform(img, f"1x3x{size}x{size}") from PIL import Image w, h = img.size s = resize / min(w, h) diff --git a/harness/eval/det.py b/harness/eval/det.py index e822f49..c65f447 100644 --- a/harness/eval/det.py +++ b/harness/eval/det.py @@ -54,17 +54,27 @@ def decode_yolov8(out: np.ndarray) -> tuple[np.ndarray, np.ndarray, np.ndarray]: BATCH = 64 -def predict(runner, t1: T1, input_spec: str, decode=decode_yolov8) -> list[dict]: +def predict(runner, t1: T1, input_spec: str, decode=decode_yolov8, model: str = "") -> list[dict]: + from ..adapters import get + adapter = get(model) if model else None + custom = callable(getattr(adapter, "decode", None)) and callable(getattr(adapter, "preprocess", None)) size = int(input_spec.split("x")[-1]) cat_map = t1.manifest.get("cat_map") # 模型类别 idx → coco category_id dets = [] items = t1.items for i in range(0, len(items), BATCH): chunk = items[i:i + BATCH] - pre = [letterbox(t1.image(it), size) for it in chunk] - for it, (x, s, (px, py)), outs in zip(chunk, pre, run_batched(runner, [p[0] for p in pre])): - boxes, scores, cls = decode(outs[0]) - boxes = (boxes - np.array([px, py, px, py])) / s + images = [t1.image(it) for it in chunk] + if custom: + pre = [(adapter.preprocess(image, input_spec), 1., (0., 0.)) for image in images] + else: + pre = [letterbox(image, size) for image in images] + for it, image, (x, s, (px, py)), outs in zip(chunk, images, pre, run_batched(runner, [p[0] for p in pre])): + if custom: + boxes, scores, cls = adapter.decode(outs, image.size) + else: + boxes, scores, cls = decode(outs[0]) + boxes = (boxes - np.array([px, py, px, py])) / s for b, sc, c in zip(boxes, scores, cls): cid = cat_map[str(int(c))] if cat_map else int(c) dets.append({"image_id": it["id"], "category_id": cid, "score": float(sc), @@ -91,11 +101,11 @@ def coco_eval(t1: T1, dets: list[dict]) -> dict: def run_fp32(runner, t1: T1, input_spec: str, out: dict, resize: int = 0) -> None: - out["metrics"] = coco_eval(t1, predict(runner, t1, input_spec)) + out["metrics"] = coco_eval(t1, predict(runner, t1, input_spec, model=out.get("model", ""))) def run_quant(runner, t1: T1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None: - m = coco_eval(t1, predict(runner, t1, input_spec)) + m = coco_eval(t1, predict(runner, t1, input_spec, model=out.get("model", ""))) out["metrics"] = m out["G3"] = {"metric": METRIC, "fp32": baseline["metrics"]["map_50_95"], "quant": m["map_50_95"], "tier": "t1", "n": m["n"], "se": None, "also": {"ap50": m["ap50"], "fp32_ap50": baseline["metrics"]["ap50"]}} diff --git a/harness/eval/feat.py b/harness/eval/feat.py new file mode 100644 index 0000000..49d23d2 --- /dev/null +++ b/harness/eval/feat.py @@ -0,0 +1,127 @@ +"""HPatches homography AUC. Single-image inference, paired geometric evaluation. + +Reference: cvg/glue-factory 2d17e3b, HPatchesPipeline / AUCMetric / +homography_corner_error / OpenCVHomographyEstimator. Descriptor similarities +only establish mutual nearest neighbours; the gate always uses geometry. +""" +import hashlib +import json + +import numpy as np + +from ..adapters import feature_common as common + +METRIC = 'hpatches_homography_auc' +PROTOCOL = { + 'version': 'hpatches-static640x480-mnn-opencv-v1', 'max_keypoints': 1024, + 'network_image_size': [640, 480], 'network_resize': 'opencv INTER_LINEAR; anisotropic fixed NCHW', + 'evaluation_coordinates': 'official short-side=480 preserving aspect; integer floor; diagonal scale', + 'matcher': 'mutual nearest neighbour on L2-normalized descriptors; no ratio/distance threshold', + 'homography_estimator': 'OpenCV RANSAC', 'ransac_threshold_px': 1.0, + 'ransac_max_iters': 3000, 'ransac_confidence': .995, 'ransac_seed': 20260914, + 'error': 'mean transformed source-image corner distance; corners (0,0),(W,0),(W,H),(0,H)', + 'auc_thresholds_px': [1, 3, 5], 'scalar': 'mean normalized AUC@1/3/5; range 0..1', + 'failure': 'infinite corner error; counted in recall denominator', + 'reference': 'https://github.com/cvg/glue-factory/tree/2d17e3b3bd7d30f0c828d4c4d3eac4ecefbf283d', + 'boundary': 'bench static-input protocol; not a reproduction of published variable-aspect network results', +} +PROTOCOL_SHA = hashlib.sha256(json.dumps(PROTOCOL, sort_keys=True).encode()).hexdigest() + + +def inputs(t1, item, input_spec, model): + if input_spec != common.input_spec(model): raise ValueError('feature input recipe mismatch') + return common.preprocess(t1.image(item), input_spec, model) + + +def mutual_matches(a, b): + import torch + if len(a) == 0 or len(b) == 0: return np.empty((0, 2), np.int64) + a, b = torch.from_numpy(a), torch.from_numpy(b) + sim = a @ b.T + # topk tie behaviour agrees with the official nearest-neighbour matcher. + ab = sim.topk(1, dim=1).indices[:, 0]; ba = sim.T.topk(1, dim=1).indices[:, 0] + ids = torch.arange(len(a)); keep = ba[ab] == ids + return torch.stack((ids[keep], ab[keep]), -1).numpy() + + +def evaluation_size(size): + w, h = size + return (int(480 * w / h), 480) if w >= h else (480, int(480 * h / w)) + + +def corner_error(H, ground_truth, size): + w, h = size + corners = np.asarray([[0, 0, 1], [w, 0, 1], [w, h, 1], [0, h, 1]], np.float64) + def transform(T): + v = corners @ T.T + if np.any(abs(v[:, 2]) < 1e-12): return None + return v[:, :2] / v[:, 2:] + pred, truth = transform(H), transform(ground_truth) + if pred is None or truth is None: return float('inf') + error = float(np.linalg.norm(pred - truth, axis=1).mean()) + return error if np.isfinite(error) else float('inf') + + +def pair_metrics(first, second, H, size0, size1): + import cv2 + matches = mutual_matches(first['descriptors'], second['descriptors']) + sizes = [evaluation_size(s) for s in (size0, size1)] + scales = [np.diag([e[0] / o[0], e[1] / o[1], 1.]) for e, o in zip(sizes, (size0, size1))] + truth = scales[1] @ np.asarray(H, np.float64) @ np.linalg.inv(scales[0]) + k0 = first['keypoints'][matches[:, 0]].astype(np.float64) * (np.asarray(sizes[0]) / [640, 480]) + k1 = second['keypoints'][matches[:, 1]].astype(np.float64) * (np.asarray(sizes[1]) / [640, 480]) + estimate = None; mask = None + if len(matches) >= 4: + cv2.setRNGSeed(PROTOCOL['ransac_seed']) + estimate, mask = cv2.findHomography(k0, k1, cv2.RANSAC, PROTOCOL['ransac_threshold_px'], + maxIters=3000, confidence=.995) + error = corner_error(estimate, truth, sizes[0]) if estimate is not None else float('inf') + return {'matches': len(matches), 'inliers': int(mask.sum()) if mask is not None else 0, + 'keypoints0': len(first['keypoints']), 'keypoints1': len(second['keypoints']), + 'corner_error_px': error if np.isfinite(error) else None, + 'success': bool(np.isfinite(error)), 'estimated_H': estimate.tolist() if estimate is not None else None} + + +def auc(errors, thresholds=(1, 3, 5)): + if not errors: raise ValueError('no geometric samples') + values = np.sort(np.asarray([float('inf') if e is None else e for e in errors])) + recall = (np.arange(len(values)) + 1.) / len(values) + values = np.r_[0., values]; recall = np.r_[0., recall] + results = [] + for t in thresholds: + last = np.searchsorted(values, t) + e = np.r_[values[:last], t]; r = np.r_[recall[:last], recall[last - 1]] + results.append(float(np.trapz(r, x=e) / t)) + return results + + +def evaluate(runner, t1, input_spec, model): + if t1.manifest.get('dataset') != 'hpatches': raise ValueError('HPatches T1 required') + cache = {}; sizes = {}; samples = [] + for item in t1.items: + for field in ('file', 'file2'): + name = item[field] + if name not in cache: + image = t1.image({'file': name}); sizes[name] = image.size + cache[name] = common.decode(model, runner(common.preprocess(image, input_spec, model))) + samples.append({'id': item['id'], **pair_metrics(cache[item['file']], cache[item['file2']], + item['H_0to1'], sizes[item['file']], sizes[item['file2']])}) + values = auc([s['corner_error_px'] for s in samples]) + metrics = {METRIC: float(np.mean(values)), 'auc_1px': values[0], 'auc_3px': values[1], 'auc_5px': values[2], + 'n': len(samples), 'unique_images': len(cache), 'homography_failures': sum(not s['success'] for s in samples)} + return metrics, samples + + +def run_fp32(runner, t1, input_spec, out, resize=0): + out['metrics'], out['_feature_samples'] = evaluate(runner, t1, input_spec, out['model']) + out['feature_protocol'] = PROTOCOL; out['feature_protocol_sha256'] = PROTOCOL_SHA + + +def run_quant(runner, t1, input_spec, baseline, out, resize=0): + if baseline.get('model') != out['model'] or baseline.get('feature_protocol_sha256') != PROTOCOL_SHA: + raise ValueError('feature model/protocol differs from baseline') + run_fp32(runner, t1, input_spec, out, resize) + if [s['id'] for s in baseline['_feature_samples']] != [s['id'] for s in out['_feature_samples']]: + raise ValueError('feature pair identities differ from baseline') + out['G3'] = {'metric': METRIC, 'fp32': baseline['metrics'][METRIC], 'quant': out['metrics'][METRIC], + 'n': out['metrics']['n'], 'tier': 't1', 'se': None} diff --git a/harness/eval/flow.py b/harness/eval/flow.py new file mode 100644 index 0000000..69fd772 --- /dev/null +++ b/harness/eval/flow.py @@ -0,0 +1,121 @@ +"""KITTI2015 flow_occ evaluation in original image coordinates. + +T1 items contain id, file (image_2/*_10.png), file2 (*_11.png), and flow +(16-bit RGB PNG: u*64+32768, v*64+32768, valid). Calibration only needs +file/file2. EPE2D and Fl-all are weighted by valid pixels across images; +per_image_epe is also recorded, matching torchvision's reference metrics. +""" +from __future__ import annotations + +import numpy as np + +from ..adapters.raft_common import input_shape, preprocess + +METRIC = "epe2d" +SUPPORTED_MODELS = {"raft.small", "raft.large"} + + +def inputs(t1, item: dict, input_spec: str, model: str) -> np.ndarray: + """One source of paired inputs for FP32, G2, board evaluation and calibration.""" + if model not in SUPPORTED_MODELS: + raise ValueError(f"no verified flow input recipe for {model}") + return preprocess(t1.image(item), t1.image({"file": item["file2"]}), input_spec) + + +def read_kitti_flow(data: bytes) -> tuple[np.ndarray, np.ndarray]: + """Decode RGB uint16 without the 8-bit truncation of PIL RGB conversion.""" + import torch + from torchvision.io import decode_png + + if data[:8] != b"\x89PNG\r\n\x1a\n" or data[24:26] != bytes((16, 2)): + raise ValueError("KITTI flow must be a 16-bit RGB PNG") + value = decode_png(torch.from_numpy(np.frombuffer(data, dtype=np.uint8).copy())) + if value.dtype != torch.uint16 or value.ndim != 3 or value.shape[0] != 3: + raise ValueError(f"invalid KITTI flow encoding: {value.dtype} {tuple(value.shape)}") + rgb = value.to(torch.float32).numpy() + if not np.isin(rgb[2], (0, 1)).all(): + raise ValueError("KITTI validity channel must contain 0 or 1") + return (rgb[:2] - 32768.0) / 64.0, rgb[2].astype(bool) + + +def ground_truth(t1, item: dict) -> tuple[np.ndarray, np.ndarray]: + if t1.manifest.get("dataset") != "kitti2015": + raise ValueError("this evaluator currently requires real KITTI2015 flow_occ labels") + if item.get("flow_format") != "kitti_png16": + raise ValueError("T1 item must explicitly declare flow_format=kitti_png16") + with t1.tar.extractfile(item["flow"]) as stream: + truth, valid = read_kitti_flow(stream.read()) + if truth.shape[-2:] != t1.image(item).size[::-1]: + raise ValueError(f"flow/image dimensions differ for {item['id']}") + return truth, valid + + +def restore_flow(outputs, input_spec: str, original_hw: tuple[int, int]) -> np.ndarray: + """Reverse image resizing, including the displacement-vector scale.""" + import torch + import torch.nn.functional as F + + shape = input_shape(input_spec) + if len(outputs) != 1: + raise ValueError(f"RAFT must expose one final flow, got {len(outputs)} outputs") + flow = np.asarray(outputs[0], dtype=np.float32) + if flow.shape != (1, 2, *shape[-2:]) or not np.isfinite(flow).all(): + raise ValueError(f"invalid final flow output: {flow.shape}") + h, w = original_hw + restored = F.interpolate(torch.from_numpy(flow), size=(h, w), mode="bilinear", align_corners=False)[0].numpy() + restored[0] *= w / shape[-1] + restored[1] *= h / shape[-2] + return restored + + +def sample_metrics(pred: np.ndarray, truth: np.ndarray, valid: np.ndarray) -> dict: + if pred.shape != truth.shape or valid.shape != truth.shape[-2:] or not valid.any(): + raise ValueError("invalid or empty ground-truth flow/mask") + if not np.isfinite(pred).all() or not np.isfinite(truth[:, valid]).all(): + raise ValueError("non-finite flow") + epe = np.linalg.norm(pred.astype(np.float64) - truth.astype(np.float64), axis=0)[valid] + magnitude = np.linalg.norm(truth.astype(np.float64), axis=0)[valid] + outlier = (epe > 3.0) & (epe > .05 * magnitude) + return {"valid_pixels": int(valid.sum()), "epe_sum": float(epe.sum()), + "outlier_pixels": int(outlier.sum()), "epe2d": float(epe.mean()), + "fl_all": float(outlier.mean() * 100)} + + +def aggregate(samples: list[dict]) -> dict: + count = sum(s["valid_pixels"] for s in samples) + if not samples or count == 0: + raise ValueError("no flow samples with real valid labels") + return {"epe2d": sum(s["epe_sum"] for s in samples) / count, + "fl_all": sum(s["outlier_pixels"] for s in samples) / count * 100, + "per_image_epe": float(np.mean([s["epe2d"] for s in samples])), + "valid_pixels": count, "n": len(samples)} + + +def evaluate(runner, t1, input_spec: str, model: str) -> tuple[dict, list[dict]]: + samples = [] + for item in t1.items: + truth, valid = ground_truth(t1, item) + out = runner(inputs(t1, item, input_spec, model)) + restored = restore_flow(out, input_spec, truth.shape[-2:]) + samples.append({"id": item["id"], **sample_metrics(restored, truth, valid)}) + return aggregate(samples), samples + + +def run_fp32(runner, t1, input_spec: str, out: dict, resize: int = 0) -> None: + out["metrics"], out["_flow_samples"] = evaluate(runner, t1, input_spec, out["model"]) + out["flow_protocol"] = "kitti2015-flow_occ; original pixels; pixel-weighted EPE/Fl-all; >3px AND >5%" + + +def run_quant(runner, t1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None: + if baseline.get("model") != out["model"]: + raise ValueError("flow baseline model mismatch") + run_fp32(runner, t1, input_spec, out, resize=resize) + if [s["id"] for s in baseline.get("_flow_samples", [])] != [s["id"] for s in out["_flow_samples"]]: + raise ValueError("flow baseline and measured sample identities differ") + out["G3"] = {"metric": METRIC, "fp32": baseline["metrics"][METRIC], + "quant": out["metrics"][METRIC], "tier": "t1", "n": out["metrics"]["n"], + "se": None, "also": {"fl_all": out["metrics"]["fl_all"], + "fp32_fl_all": baseline["metrics"]["fl_all"]}, + "per_metric": {name: {"fp32": baseline["metrics"][name], "quant": out["metrics"][name]} + for name in ("epe2d", "fl_all")}} + # execution/source=board is supplied by the existing trusted eval entrypoint. diff --git a/harness/eval/mde.py b/harness/eval/mde.py new file mode 100644 index 0000000..d899e86 --- /dev/null +++ b/harness/eval/mde.py @@ -0,0 +1,167 @@ +"""NYUv2 relative inverse-depth evaluation with an explicit alignment protocol. + +This is a paired bench protocol, not a claim to reproduce a paper's scores. +GT is original NYU filled depth in metres. Model outputs are never treated as +metric predictions without the recorded per-image scale/shift alignment. +""" +from __future__ import annotations + +import io +import numpy as np + +from ..adapters import get +from ..adapters.depth_anything_v2_common import INPUT_SPEC + +PROTOCOL = { + "id": "nyuv2-relative-inverse-ls-v1", + "prediction": "affine_invariant_inverse_depth", + "alignment": "per_image_least_squares_scale_and_shift_in_inverse_depth", + "alignment_mask": "same as score mask; independent identical fit rule for FP32 and quant", + "crop_yxyx": [45, 41, 471, 601], + "min_depth_m": 0.001, "max_depth_m": 10.0, + "valid_mask": "finite GT strictly greater than min and strictly less than max, inside crop", + "aggregation": "arithmetic mean of per-image metrics", + "delta1_unit": "percent", "metric_depth_prediction": False, +} +SUPPORTED_MODELS = {f"depth.anything.{version}.{variant}" + for version in ("v1", "v2") for variant in ("small", "base", "large")} +# Additive families (batch 7): pinned official MiDaS relative models and the +# metric Depth Anything V2 checkpoints from the same pinned upstream tree. +# Depth-Anything v1/v2 relative models keep their exact code paths below. +SUPPORTED_MODELS |= {"midas.small", "dpt.hybrid", "dpt.large"} +SUPPORTED_MODELS |= {f"depth.anything.v2.metric.{domain}.{variant}" + for domain in ("hypersim", "vkitti") for variant in ("small", "base", "large")} +# New families evaluate at their own pinned static input resolution; the +# frozen Depth-Anything recipe stays 1x3x518x518. +FAMILY_INPUT_SPECS = { + "midas.small": "1x3x256x256", + "dpt.hybrid": "1x3x384x384", + "dpt.large": "1x3x384x384", + **{f"depth.anything.v2.metric.{domain}.{variant}": "1x3x518x518" + for domain in ("hypersim", "vkitti") for variant in ("small", "base", "large")}, +} + + +def inputs(t1, item: dict, input_spec: str, model: str) -> np.ndarray: + if model not in SUPPORTED_MODELS: + raise ValueError(f"no verified MDE input recipe for {model}") + return get(model).preprocess(t1.image(item), input_spec) + + +def ground_truth(t1, item: dict) -> tuple[np.ndarray, np.ndarray]: + if t1.manifest.get("dataset") != "nyuv2" or item.get("depth_unit") != "m": + raise ValueError("expected NYUv2 original metre-depth labels") + if item.get("depth_format") != "npy_float32_m": + raise ValueError("depth labels must explicitly declare npy_float32_m") + with t1.tar.extractfile(item["depth"]) as stream: + depth = np.load(io.BytesIO(stream.read()), allow_pickle=False) + if depth.dtype != np.float32 or depth.shape != (480, 640) or t1.image(item).size != (640, 480): + raise ValueError("expected aligned original NYUv2 RGB/depth at 640x480") + mask = np.zeros(depth.shape, dtype=bool) + y1, x1, y2, x2 = PROTOCOL["crop_yxyx"] + mask[y1:y2, x1:x2] = True + mask &= np.isfinite(depth) & (depth > PROTOCOL["min_depth_m"]) & (depth < PROTOCOL["max_depth_m"]) + if not mask.any(): + raise ValueError("no valid NYUv2 depth pixels") + return depth, mask + + +def restore_prediction(outputs, original_hw: tuple[int, int], model: str | None = None) -> np.ndarray: + import torch + import torch.nn.functional as F + + if len(outputs) != 1: + raise ValueError("expected one relative inverse-depth output") + prediction = np.asarray(outputs[0], dtype=np.float32) + # Frozen Depth-Anything shape check stays (1, 518, 518); additive families + # declare their own pinned static resolution. + family_spec = FAMILY_INPUT_SPECS.get(model) if model is not None else None + expected_shape = (1, *(int(v) for v in (family_spec or INPUT_SPEC).split("x")[2:])) + if prediction.shape != expected_shape or not np.isfinite(prediction).all(): + raise ValueError(f"invalid relative inverse-depth output: {prediction.shape}") + if np.any(prediction < 0): + raise ValueError("official Depth Anything forward must return nonnegative values") + if model is not None and model not in SUPPORTED_MODELS: + raise ValueError(f"no verified MDE restore recipe for {model}") + align_corners = getattr(get(model), "RESTORE_ALIGN_CORNERS", True) if model else True + mode = getattr(get(model), "RESTORE_INTERPOLATION", "bilinear") if model else "bilinear" + # Preserve each author's interpolation; the two-argument V2 API stays True. + return F.interpolate(torch.from_numpy(prediction[:, None]), size=original_hw, + mode=mode, align_corners=align_corners)[0, 0].numpy() + + +def aligned_metrics(prediction: np.ndarray, depth_m: np.ndarray, mask: np.ndarray) -> dict: + if prediction.shape != depth_m.shape or mask.shape != depth_m.shape or not mask.any(): + raise ValueError("prediction/GT/mask mismatch or no valid pixels") + if not np.isfinite(prediction).all() or not np.isfinite(depth_m[mask]).all() or np.any(depth_m[mask] <= 0): + raise ValueError("non-finite prediction or invalid metric GT") + x = prediction[mask].astype(np.float64) + gt = depth_m[mask].astype(np.float64) + inverse_gt = 1.0 / gt + dx = x - x.mean() + variance_sum = float(dx @ dx) + if variance_sum <= np.finfo(np.float64).eps * max(1.0, float(x @ x)): + raise ValueError("relative-depth alignment is singular (constant/degenerate prediction)") + scale = float(dx @ (inverse_gt - inverse_gt.mean()) / variance_sum) + shift = float(inverse_gt.mean() - scale * x.mean()) + # Unconstrained affine least squares has a signed solution. Positive + # definiteness applies to the normal matrix, not the fitted coefficient. + if not np.isfinite([scale, shift]).all(): + raise ValueError(f"relative inverse-depth alignment is nonfinite: scale={scale}, shift={shift}") + inverse = scale * x + shift + near, far = PROTOCOL["min_depth_m"], PROTOCOL["max_depth_m"] + clipped = np.clip(inverse, 1.0 / far, 1.0 / near) + aligned = 1.0 / clipped + ratio = np.maximum(aligned / gt, gt / aligned) + return {"absrel": float(np.mean(np.abs(aligned - gt) / gt)), + "rmse": float(np.sqrt(np.mean((aligned - gt) ** 2))), + "delta1": float(np.mean(ratio < 1.25) * 100), + "valid_pixels": int(mask.sum()), "scale": scale, "shift": shift, + "clipped_inverse_pixels": int(np.count_nonzero(clipped != inverse))} + + +def evaluate(runner, t1, input_spec: str, model: str) -> tuple[dict, list[dict]]: + if input_spec != INPUT_SPEC and input_spec != FAMILY_INPUT_SPECS.get(model): + raise ValueError("MDE evaluation input recipe mismatch") + if t1.manifest.get("evaluation_protocol") != PROTOCOL: + raise ValueError("T1 must freeze the explicit relative-depth evaluation protocol") + samples = [] + for item in t1.items: + depth, mask = ground_truth(t1, item) + prediction = restore_prediction(runner(inputs(t1, item, input_spec, model)), depth.shape, model) + samples.append({"id": item["id"], **aligned_metrics(prediction, depth, mask)}) + if not samples: + raise ValueError("empty depth evaluation") + metrics = {key: float(np.mean([item[key] for item in samples])) for key in ("absrel", "rmse", "delta1")} + metrics.update(n=len(samples), valid_pixels=sum(item["valid_pixels"] for item in samples)) + return metrics, samples + + +def run_fp32(runner, t1, input_spec: str, out: dict, resize: int = 0) -> None: + out["metrics"], out["_depth_samples"] = evaluate(runner, t1, input_spec, out["model"]) + out["depth_protocol"] = PROTOCOL + + +def run_quant(runner, t1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None: + if baseline.get("model") != out["model"] or baseline.get("depth_protocol") != PROTOCOL: + raise ValueError("MDE baseline model/protocol mismatch") + run_fp32(runner, t1, input_spec, out, resize) + if [s["id"] for s in baseline.get("_depth_samples", [])] != [s["id"] for s in out["_depth_samples"]]: + raise ValueError("MDE baseline and measured sample identities differ") + fp, measured = baseline["metrics"], out["metrics"] + alignment_samples = [ + {"id": quant["id"], "fp32_scale": reference["scale"], "fp32_shift": reference["shift"], + "quant_scale": quant["scale"], "quant_shift": quant["shift"]} + for reference, quant in zip(baseline["_depth_samples"], out["_depth_samples"]) + ] + out["G3"] = {"metric": "absrel", "fp32": fp["absrel"], "quant": measured["absrel"], + "tier": "t1", "n": measured["n"], "se": None, + "also": {"rmse": measured["rmse"], "fp32_rmse": fp["rmse"], + "delta1": measured["delta1"], "fp32_delta1": fp["delta1"]}, + "extra": {"scale_alignment": {"status": "measured", "protocol": PROTOCOL, + "n": measured["n"], "space": "inverse_depth", + "samples": alignment_samples}, + "delta1": {"status": "measured", "unit": "percent", + "fp32": fp["delta1"], "quant": measured["delta1"], + "delta_pp": measured["delta1"] - fp["delta1"]}}} + # Trusted eval.run attaches execution provenance (board/sim); this module does not. diff --git a/harness/eval/run.py b/harness/eval/run.py index ae90fb9..8d8b70a 100644 --- a/harness/eval/run.py +++ b/harness/eval/run.py @@ -17,13 +17,35 @@ from . import EVALUATOR_VERSION, COMPATIBLE_BASELINE_VERSIONS, TASK_MODULES from .common import T1, TorchRunner, HbmRunner, cosine, preprocess_cls, letterbox, run_batched -def g2_inputs(task: str, t1: T1, input_spec: str, n: int, resize: int = 0) -> dict[str, np.ndarray]: +def g2_inputs(task: str, t1: T1, input_spec: str, n: int, resize: int = 0, + model: str = "") -> dict[str, np.ndarray]: size = int(input_spec.split("x")[-1]) xs = {} + if model: + from ..adapters import get + custom = getattr(get(model), "g2_input", None) + if custom is not None: + # A top-down model cannot infer an image with no detector crop. + # Select from real eligible inputs; never insert None or a GT box. + eligible = [value for item in t1.items + if (value := custom(t1, item, input_spec)) is not None] + count = min(n, len(t1.items)) + if len(eligible) < count: + raise ValueError(f"only {len(eligible)} valid {model} diagnostic inputs; require {count}") + selected = sorted(np.random.default_rng(20260914).permutation(len(eligible))[:count]) + return {f"s{i:03d}": eligible[index] for i, index in enumerate(selected)} for i, it in enumerate(t1.g2_subset(n)): + if task in ("flow", "seg", "mde", "feat"): + module = importlib.import_module(TASK_MODULES[task]) + xs[f"s{i:03d}"] = module.inputs(t1, it, input_spec, model) + continue img = t1.image(it) if task == "cls": - xs[f"s{i:03d}"] = preprocess_cls(img, size=size, resize=resize or round(size * 256 / 224)) + xs[f"s{i:03d}"] = preprocess_cls(img, size=size, resize=resize or round(size * 256 / 224), model=model) + elif task == "det" and model: + from ..adapters import get + transform = getattr(get(model), "preprocess", None) + xs[f"s{i:03d}"] = transform(img, input_spec) if transform else letterbox(img, size)[0] else: xs[f"s{i:03d}"] = letterbox(img, size)[0] return xs @@ -67,7 +89,7 @@ def main() -> int: if a.mode == "fp32": runner = TorchRunner(a.model, a.source, a.weights, a.input_spec) od = Path(a.out); od.mkdir(parents=True, exist_ok=True) - xs = g2_inputs(a.task, t1, a.input_spec, a.g2_samples, a.resize) + xs = g2_inputs(a.task, t1, a.input_spec, a.g2_samples, a.resize, a.model) np.savez(od / "g2_inputs.npz", **xs) np.savez(od / "g2_fp32.npz", **{f"{k}/out{i}": o for k, x in xs.items() for i, o in enumerate(runner(x))}) mod.run_fp32(runner, t1, a.input_spec, out, resize=a.resize) diff --git a/harness/eval/seg.py b/harness/eval/seg.py new file mode 100644 index 0000000..30b34db --- /dev/null +++ b/harness/eval/seg.py @@ -0,0 +1,152 @@ +"""Semantic segmentation in original label coordinates, shared by every runner. + +T1 items declare label_file and label_format. Raw Cityscapes labelIds map to +19 trainIds; ADE20K 0 is ignored and 1..150 map to 0..149. Resize logits with +bicubic/align_corners=False before argmax, as in the official EfficientViT +evaluation. Aggregate pixel intersections/unions over the whole subset, then +average class IoUs with nonzero union (including false-positive-only classes). +""" +from __future__ import annotations + +import io +import numpy as np + +from ..adapters import get + +METRIC = "miou" +PROTOCOL = "seg-original-labels-bicubic-logits-v1" +_CITY_IDS = (7, 8, 11, 12, 13, 17, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 31, 32, 33) +_DATASETS = {"cityscapes": (19, "cityscapes_labelIds"), "ade20k": (150, "ade20k_labelIds")} + + +def _recipe(t1, model: str): + adapter = get(model) + if adapter is None or not callable(getattr(adapter, "preprocess", None)): + raise ValueError(f"no verified segmentation input recipe for {model}") + dataset = t1.manifest.get("dataset") + if dataset not in _DATASETS or dataset != adapter.MODEL_SPEC["dataset"]: + raise ValueError(f"segmentation dataset/model mismatch: {dataset} / {model}") + return adapter, dataset, _DATASETS[dataset][0] + + +def inputs(t1, item: dict, input_spec: str, model: str) -> np.ndarray: + adapter, _, _ = _recipe(t1, model) + return adapter.preprocess(t1.image(item), input_spec) + + +def map_labels(raw: np.ndarray, dataset: str, label_format: str) -> np.ndarray: + if dataset not in _DATASETS or label_format != _DATASETS[dataset][1]: + raise ValueError(f"unsupported segmentation label format: {dataset}/{label_format}") + if raw.ndim != 2 or not np.issubdtype(raw.dtype, np.integer): + raise ValueError("segmentation labels must be a single-channel integer mask, not RGB") + if dataset == "cityscapes": + # Cityscapes license plate id=-1 is stored as uint8 255; other void ids + # remain ignored. Unexpected ids must not silently disappear from IoU. + if not np.isin(raw, (*range(34), 255)).all(): + raise ValueError("invalid Cityscapes labelIds (expected 0..33 or 255)") + lookup = np.full(256, -1, dtype=np.int16) + lookup[list(_CITY_IDS)] = np.arange(19) + return lookup[raw] + if (raw < 0).any() or (raw > 150).any(): + raise ValueError("invalid ADE20K labelIds (expected 0 ignore, 1..150 classes)") + return raw.astype(np.int16) - 1 + + +def ground_truth(t1, item: dict) -> np.ndarray: + from PIL import Image + + with t1.tar.extractfile(item["label_file"]) as stream: + with Image.open(io.BytesIO(stream.read())) as mask: + raw = np.array(mask) # Preserve P/L label indices; never convert RGB. + truth = map_labels(raw, t1.manifest.get("dataset"), item.get("label_format")) + if truth.shape != t1.image(item).size[::-1]: + raise ValueError(f"segmentation image/label dimensions differ: {item['id']}") + return truth + + +def restore_prediction(outputs, classes: int, original_hw: tuple[int, int], recipe: dict | None = None) -> np.ndarray: + import torch + import torch.nn.functional as F + + if len(outputs) != 1: + raise ValueError("segmentation model must expose one NCHW logits tensor") + logits = np.asarray(outputs[0], dtype=np.float32) + if (logits.ndim != 4 or logits.shape[:2] != (1, classes) + or min(logits.shape[-2:]) <= 0 or not np.isfinite(logits).all()): + raise ValueError(f"invalid segmentation logits: {logits.shape}; classes={classes}") + recipe = recipe or {} + interpolation = recipe.get("logits_resize", "bicubic") + align_corners = recipe.get("align_corners", False) + if interpolation not in ("bilinear", "bicubic") or not isinstance(align_corners, bool): + raise ValueError("unsupported segmentation logits resize recipe") + tensor = torch.from_numpy(np.ascontiguousarray(logits)) + # MMSeg first restores head logits to the network input, then to the + # original image. Keeping both host operations preserves its predictions. + intermediate = recipe.get("logits_intermediate_size") + if intermediate and tuple(tensor.shape[-2:]) != tuple(intermediate): + tensor = F.interpolate(tensor, size=intermediate, mode=interpolation, align_corners=align_corners) + if tuple(tensor.shape[-2:]) != tuple(original_hw): + tensor = F.interpolate(tensor, size=original_hw, mode=interpolation, align_corners=align_corners) + if not torch.isfinite(tensor).all(): + raise ValueError("non-finite resized segmentation logits") + return tensor.argmax(dim=1)[0].numpy() + + +def sample_counts(pred: np.ndarray, truth: np.ndarray, classes: int) -> dict: + if pred.shape != truth.shape or not np.issubdtype(pred.dtype, np.integer): + raise ValueError("segmentation prediction/label shape or dtype mismatch") + if ((pred < 0) | (pred >= classes)).any() or ((truth < -1) | (truth >= classes)).any(): + raise ValueError("segmentation trainId outside class range") + valid = truth != -1 + encoded = truth[valid].astype(np.int64) * classes + pred[valid] + confusion = np.bincount(encoded, minlength=classes * classes).reshape(classes, classes) + intersection = confusion.diagonal() + union = confusion.sum(0) + confusion.sum(1) - intersection + return {"intersection": intersection.tolist(), "union": union.tolist(), + "valid_pixels": int(valid.sum()), "ignored_pixels": int((~valid).sum())} + + +def aggregate(samples: list[dict]) -> dict: + if not samples or sum(s["valid_pixels"] for s in samples) == 0: + raise ValueError("no segmentation samples with valid ground-truth pixels") + intersection = np.sum([s["intersection"] for s in samples], axis=0, dtype=np.int64) + union = np.sum([s["union"] for s in samples], axis=0, dtype=np.int64) + present = union > 0 + iou = np.divide(intersection * 100.0, union, out=np.zeros_like(union, dtype=float), where=present) + return {"miou": float(iou[present].mean()), + "per_class_iou": [float(v) if valid else None for v, valid in zip(iou, present)], + "class_ids": list(range(len(union))), "present_classes": int(present.sum()), + "intersection": intersection.tolist(), "union": union.tolist(), + "valid_pixels": sum(s["valid_pixels"] for s in samples), + "ignored_pixels": sum(s["ignored_pixels"] for s in samples), "n": len(samples)} + + +def run_fp32(runner, t1, input_spec: str, out: dict, resize: int = 0) -> None: + adapter, dataset, classes = _recipe(t1, out["model"]) + samples = [] + for item in t1.items: + truth = ground_truth(t1, item) + outputs = runner(inputs(t1, item, input_spec, out["model"])) + pred = restore_prediction(outputs, classes, truth.shape, adapter.PREPROCESS) + samples.append({"id": item["id"], **sample_counts(pred, truth, classes)}) + out["metrics"] = aggregate(samples) + out["_seg_samples"] = samples + protocol = "seg-original-labels-mmseg-logits-v1" if adapter.PREPROCESS.get("logits_intermediate_size") else PROTOCOL + out["seg_protocol"] = {"version": protocol, "dataset": dataset, "classes": classes, + "label_format": _DATASETS[dataset][1], "ignore_train_id": -1, + "absent_classes": "exclude_zero_union", "preprocess": dict(adapter.PREPROCESS)} + + +def run_quant(runner, t1, input_spec: str, baseline: dict, out: dict, resize: int = 0) -> None: + if baseline.get("model") != out["model"] or baseline.get("t1_manifest_sha") != out.get("t1_manifest_sha"): + raise ValueError("segmentation baseline model or T1 identity mismatch") + run_fp32(runner, t1, input_spec, out, resize=resize) + if baseline.get("seg_protocol") != out["seg_protocol"]: + raise ValueError("segmentation baseline/evaluation protocols differ") + if [s["id"] for s in baseline.get("_seg_samples", [])] != [s["id"] for s in out["_seg_samples"]]: + raise ValueError("segmentation baseline/evaluation sample identities differ") + out["G3"] = {"metric": METRIC, "fp32": baseline["metrics"][METRIC], + "quant": out["metrics"][METRIC], "tier": "t1", "n": out["metrics"]["n"], + "se": None, "also": {"per_class_iou": out["metrics"]["per_class_iou"], + "fp32_per_class_iou": baseline["metrics"]["per_class_iou"]}} + # Trusted eval.run supplies actual execution/source=board after real inference. diff --git a/harness/gates.py b/harness/gates.py index 38347fd..5c88b71 100644 --- a/harness/gates.py +++ b/harness/gates.py @@ -57,14 +57,117 @@ def g2(m: dict | None, cfg: dict) -> dict: return out +def _pose_shift_gate(m: dict, maximum) -> dict: + """Validate the measured geometry record, not an OKS score or a status alone.""" + result = {"verdict": "unverified", "threshold_px": maximum, + "evidence": "eval.json#/pose_shift_evidence"} + extra = m.get("extra") or {} + if not isinstance(extra, dict) or not isinstance(extra.get("keypoint_shift"), dict): + return {**result, "reason": "keypoint shift measurement missing"} + shift = extra["keypoint_shift"] + result.update({k: shift.get(k) for k in ("n_matched", "n_keypoints", "n_fp32", "n_quant")}) + if shift.get("status") != "measured": + return {**result, "reason": shift.get("reason", "keypoint shift not measured")} + try: + if (shift.get("method") != "coco-pose-bbox-iou-v1" + or shift.get("coordinate_space") != "original_image_pixels_unclipped" + or shift.get("joint_policy") != "all_17_no_confidence_filter" + or shift.get("match_iou_min") != 0.5 or shift.get("evidence") != result["evidence"]): + raise ValueError("unsupported keypoint correspondence/geometry") + mean, limit = extra["keypoint_mean_shift_px"], maximum + if any(isinstance(v, bool) or not isinstance(v, (int, float)) + or not math.isfinite(v) or v < 0 + for v in (mean, limit, shift["keypoint_mean_shift_px"], shift["distance_sum_px"])): + raise ValueError("invalid keypoint shift or threshold") + n = shift["n_matched"] + if type(n) is not int or n <= 0 or shift["n_keypoints"] != 17 * n: + raise ValueError("no matches or inconsistent keypoint counts") + for count, unmatched in (("n_fp32", "n_unmatched_fp32"), ("n_quant", "n_unmatched_quant")): + if (type(shift[count]) is not int or type(shift[unmatched]) is not int + or shift[unmatched] < 0 or shift[count] != n + shift[unmatched]): + raise ValueError("inconsistent pose coverage counts") + result[unmatched] = shift[unmatched] + recomputed = shift["distance_sum_px"] / shift["n_keypoints"] + if not all(math.isclose(v, recomputed, rel_tol=1e-12, abs_tol=1e-9) + for v in (mean, shift["keypoint_mean_shift_px"])): + raise ValueError("keypoint mean disagrees with distance sum/count") + except (KeyError, TypeError, ValueError, OverflowError) as e: + return {**result, "reason": str(e)} + ok = mean <= limit or math.isclose(mean, limit, rel_tol=1e-12, abs_tol=1e-9) + return {**result, "verdict": "pass" if ok else "fail", "mean_px": mean} + + +def _depth_extra_gate(m: dict, cfg: dict) -> dict: + """Enforce the existing MDE alignment and delta1 requirements.""" + extra = m.get("extra") or {} + alignment = extra.get("scale_alignment") or {} + try: + if cfg.get("scale_alignment") == "required": + samples = alignment.get("samples") or [] + if (alignment.get("status") != "measured" or not alignment.get("protocol") + or type(m.get("n")) is not int or m["n"] <= 0 + or alignment.get("n") != m["n"] or len(samples) != m["n"] + or len({s["id"] for s in samples}) != m["n"]): + raise ValueError("paired depth alignment evidence missing or inconsistent") + for sample in samples: + for prefix in ("fp32", "quant"): + scale, shift = sample[f"{prefix}_scale"], sample[f"{prefix}_shift"] + if (any(isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) + for v in (scale, shift))): + raise ValueError("invalid measured depth alignment") + delta1 = extra.get("delta1") or {} + limit = cfg.get("delta1_abs_pp_tol") + if limit is not None: + fp32, quant = delta1.get("fp32"), delta1.get("quant") + if (delta1.get("status") != "measured" or delta1.get("unit") != "percent" + or any(isinstance(v, bool) or not isinstance(v, (int, float)) or not math.isfinite(v) + or not 0 <= v <= 100 for v in (fp32, quant))): + raise ValueError("delta1 measurements missing or invalid") + delta = quant - fp32 + if not math.isclose(delta1["delta_pp"], delta, rel_tol=1e-12, abs_tol=1e-9): + raise ValueError("delta1 change disagrees with paired measurements") + ok = delta >= limit or math.isclose(delta, limit, rel_tol=1e-12, abs_tol=1e-9) + return {"verdict": "pass" if ok else "fail", "delta1_delta_pp": delta, + "delta1_tolerance_pp": limit, "alignment_n": alignment.get("n")} + except (KeyError, TypeError, ValueError, OverflowError) as exc: + return {"verdict": "unverified", "reason": str(exc)} + return {"verdict": "pass", "alignment_n": alignment.get("n")} + + def g3(m: dict | None, cfg: dict) -> dict: if cfg.get("verdict") == "na" or not cfg.get("required", True): return {"verdict": "na", "reason": cfg.get("reason", "gate not required for this task")} + metrics = cfg.get("metric") + if isinstance(metrics, list): + per_metric = (m or {}).get("per_metric") or {} + checks = {} + for name in metrics: + values = per_metric.get(name) + measurement = ({**m, **values, "metric": name, "source": m.get("source")} + if isinstance(values, dict) else None) + checks[name] = g3(measurement, {**cfg, "metric": name}) + # Keep the decisive scalar at the top level for existing reward/export code. + decisive = next((v for v in checks.values() if v["verdict"] == "fail"), None) + decisive = decisive or next((v for v in checks.values() if v["verdict"] == "unverified"), None) + decisive = decisive or next(iter(checks.values()), {"verdict": "unverified", "reason": "empty metric list"}) + return {**decisive, "per_metric": checks} if not m or m.get("fp32") is None or m.get("quant") is None: return {"verdict": "unverified", "reason": "no measurement", "metric": cfg.get("metric")} if cfg.get("require_source") and m.get("source") != cfg["require_source"]: return {"verdict": "unverified", "reason": f"measurement source must be {cfg['require_source']}", "metric": cfg.get("metric"), "source": m.get("source")} + pose_shift = None + if "keypoint_mean_shift_px_max" in (cfg.get("extra") or {}): + pose_shift = _pose_shift_gate(m, cfg["extra"]["keypoint_mean_shift_px_max"]) + if pose_shift["verdict"] == "unverified": + return {"verdict": "unverified", "reason": pose_shift["reason"], + "metric": cfg.get("metric"), "extra": {"keypoint_shift": pose_shift}} + depth_extra = None + if {"scale_alignment", "delta1_abs_pp_tol"} & (cfg.get("extra") or {}).keys(): + depth_extra = _depth_extra_gate(m, cfg["extra"]) + if depth_extra["verdict"] == "unverified": + return {"verdict": "unverified", "reason": depth_extra["reason"], + "metric": cfg.get("metric"), "extra": {"depth": depth_extra}} kind = cfg.get("kind") if kind not in ("abs_pp", "rel_pct"): raise ValueError(f"G3 kind 必须显式 abs_pp|rel_pct,得到 {kind!r}") @@ -94,9 +197,18 @@ def g3(m: dict | None, cfg: dict) -> dict: eff_tol = tol + cfg.get("t1_screen_widen_se", 2) * float(m["se"]) ok = degr <= eff_tol or math.isclose(degr, eff_tol, rel_tol=1e-12, abs_tol=1e-9) verdict = ("pass-screen" if m.get("tier") == "t1" else "pass") if ok else "fail" - return {"verdict": verdict, "metric": metric_name, "kind": kind, "fp32": fp32, "quant": quant, + result = {"verdict": verdict, "metric": metric_name, "kind": kind, "fp32": fp32, "quant": quant, "delta": delta, "tolerance": tol, "effective_tolerance": eff_tol, "tier": m.get("tier"), "n": m.get("n"), "se": m.get("se"), "source": m.get("source")} + if pose_shift is not None: + result["extra"] = {"keypoint_shift": pose_shift} + if pose_shift["verdict"] == "fail": + result.update(verdict="fail", reason="keypoint mean shift exceeds maximum") + if depth_extra is not None: + result.setdefault("extra", {})["depth"] = depth_extra + if depth_extra["verdict"] == "fail": + result.update(verdict="fail", reason="delta1 drop exceeds maximum") + return result def g4(m: dict | None, cfg: dict) -> dict: diff --git a/harness/paths.py b/harness/paths.py index 0a500fa..e02a9f0 100644 --- a/harness/paths.py +++ b/harness/paths.py @@ -60,6 +60,12 @@ def venv_bin(backend: str, tool: str) -> Path: return expand(config()["paths"]["venv"][backend]) / "bin" / tool +def model_python(job: dict) -> Path: + """New model recipes may require an isolated upstream-compatible runtime.""" + runtime = job.get("model_spec", {}).get("runtime_python") + return Path(runtime) if runtime else venv_python("bench") + + @lru_cache(maxsize=1) def load_env_local() -> dict: """读取 bench/.env.local。只给 harness 用;rollout 的 agent 子进程环境由 stagerun/rollout 单独构造,永远不含这些。""" diff --git a/harness/pipeline.py b/harness/pipeline.py index b7d5469..c92c2ea 100644 --- a/harness/pipeline.py +++ b/harness/pipeline.py @@ -1,7 +1,7 @@ -"""oracle 主流程:preflight → new-run → expand → run。 +"""输入准备与可选 Oracle:preflight → new-run → expand → run。 -run:每 job 顺序跑 01–08;每阶段拿对应资源池;瞬态重试;STALLED 进 long 车道; -产出 oracle/result.json;状态写 jobs.sqlite。--dry-run 走完整状态机但不调工具链。 +run 默认只准备 01–03,Agent 负责 04–08;显式 --oracle 才顺序跑完整对照。 +每阶段复用资源池、重试与证据落盘;状态写 jobs.sqlite。 """ from __future__ import annotations @@ -33,7 +33,15 @@ def refresh_run_dataset(run_id: str, *, strict: bool = False) -> dict | None: """同一 run 始终更新同一数据集;失败可补导,不回滚量编上板结果。""" try: from export.to_agentrl import materialize - return materialize([run_id], f"{run_id}-training", refresh=True, purpose="all") + result = materialize([run_id], f"{run_id}-training", refresh=True, purpose="all") + # Optional, explicitly enabled inventory. Failure cannot discard training data. + if (paths.STATE / "coverage-master.enabled").exists(): + try: + from .coverage import write_master + write_master(str(paths.REPORTS / "model-coverage.csv")) + except Exception as exc: + _stamped(f"coverage master refresh failed: {type(exc).__name__}: {exc}") + return result except Exception as exc: if strict: raise @@ -208,8 +216,23 @@ def check_manifest(run_id: str) -> list[str]: return bad -def expand(run_id: str, modes: list[str] | None = None, marches: list[str] | None = None) -> dict: - specs = expand_matrix(modes) +def expand(run_id: str, modes: list[str] | None = None, marches: list[str] | None = None, + model_ids: list[str] | None = None, eval_manifests: list[str] | None = None) -> dict: + from .continuation import receipt, load_eval_set + replacements = dict(load_eval_set(p) for p in (eval_manifests or [])) + if len(replacements) != len(eval_manifests or []): + raise ValueError("one T1 manifest per dataset is required") + with db.connect() as con: + db.assert_run_open(con, run_id) + if receipt(run_id): + raise ValueError("continuation run has an exact task list; expand is not allowed") + if model_ids and modes == ["oracle"]: + from .coverage import model_blockers + blockers = {m: reasons for m in model_ids if (reasons := model_blockers( + model_row(m), evaluation=replacements.get(model_row(m).get("dataset"))))} + if blockers: + raise ValueError(f"explicit model batch has unresolved prerequisites: {blockers}") + specs = expand_matrix(modes, model_ids) if model_ids else expand_matrix(modes) if marches: specs = [s for s in specs if s.march in marches] # 只展开部分 march(如补跑 nash-p),不改 registry # 按 job_id 聚合 rollout 计划 @@ -219,8 +242,22 @@ def expand(run_id: str, modes: list[str] | None = None, marches: list[str] | Non first.setdefault(s.job_id, s) if s.mode != "oracle": plans.setdefault(s.job_id, []).append({"mode": s.mode, "policy": s.policy, "k": s.k}) + # Validate before adding any rows. An explicit eval set cannot rewrite a job. + for jid, s in first.items(): + ds = model_row(s.model).get("dataset") + if replacements and ds not in replacements: + raise ValueError(f"explicit T1 missing for dataset {ds}") + existing = paths.RUNS / run_id / jid / "job.json" + if replacements and existing.exists(): + old = json.loads(existing.read_text())["data"]["t1"] + if old != replacements[ds]: + raise ValueError(f"cannot change an existing job's T1: {jid}") n_new = 0 with db.connect() as con: + con.execute("BEGIN IMMEDIATE") + db.assert_run_open(con, run_id) + if receipt(run_id): + raise ValueError("continuation run has an exact task list; expand is not allowed") for s in specs: est = scheduler.estimate_job(con, s.model, s.task) n_new += db.upsert_job(con, run_id, s.to_row(), est) @@ -235,7 +272,13 @@ def expand(run_id: str, modes: list[str] | None = None, marches: list[str] | Non jd.mkdir(parents=True, exist_ok=True) (jd / "oracle" / "stages").mkdir(parents=True, exist_ok=True) (jd / "oracle" / "artifacts").mkdir(parents=True, exist_ok=True) - jj.write_text(json.dumps(build_job_json(s, run_id, EVALUATOR_VERSION, plans.get(jid)), indent=2, ensure_ascii=False)) + job = build_job_json(s, run_id, EVALUATOR_VERSION, plans.get(jid)) + if replacements: + job["data"]["t1"] = replacements[job["data"]["dataset"]] + if model_ids: + job["admission"] = {"method": "explicit_model_batch", "prerequisites_checked": modes == ["oracle"], + "registry_status_preserved": job["model_spec"].get("status")} + jj.write_text(json.dumps(job, indent=2, ensure_ascii=False)) os.chmod(jj, 0o444) # data -> 软链只是给 agent 引用方便;那条"一个 job 一个目录"约束产出,不约束输入 link = jd / "data" @@ -252,47 +295,19 @@ def expand(run_id: str, modes: list[str] | None = None, marches: list[str] | Non def enqueue_repairs(run_id: str, job_ids: list[str] | None = None, modes: list[str] | None = None) -> int: - """按现有 matrix 规则补齐已失败 oracle 的 repair 计划;重复扫描不重复采样。""" - if modes and "repair" not in modes: - return 0 - mx = matrix() - cfg = (mx.get("modes") or {}).get("repair") or {} - if not cfg.get("enabled", True) or cfg.get("when") != "oracle_fail": - return 0 - marches = mx.get("marches", paths.MARCHES) if cfg.get("marches", "all") == "all" else cfg["marches"] - backends = mx.get("backends", ["ptq"]) if cfg.get("backends", "all") == "all" else cfg["backends"] - error_classes = set(cfg.get("error_classes") or []) - selected = set(job_ids) if job_ids else None - model_rows = {r["model_id"]: r for r in models()} - k = int(cfg.get("k", 1)) - added = 0 + """历史调用兼容入口;Oracle 不再派生新轨迹。""" with db.connect() as con: - # 锁住读取计数和分配 kk,避免两个 collect 同时补出重复的 repair。 - con.execute("BEGIN IMMEDIATE") - failed = con.execute( - "SELECT DISTINCT job_id, model, march, backend, error_class FROM jobs" - " WHERE run_id=? AND mode='oracle' AND state='FAIL' AND finished_at IS NOT NULL", - (run_id,)).fetchall() - for job in failed: - row = model_rows.get(job["model"]) - if (selected is not None and job["job_id"] not in selected - or job["march"] not in marches or job["backend"] not in backends - or error_classes and job["error_class"] not in error_classes - or row is None or not _select(row, cfg.get("select"))): - continue - existing = db.list_rollouts(con, run_id, job["job_id"]) - kk = max((r["kk"] for r in existing), default=-1) + 1 - for policy in dict.fromkeys(cfg.get("policies") or [None]): - count = sum(r["mode"] == "repair" and r["policy"] == policy for r in existing) - for _ in range(max(0, k - count)): - db.rollout_upsert(con, run_id, job["job_id"], kk, "repair", policy) - kk += 1 - added += 1 - return added + db.assert_run_open(con, run_id) + return 0 def enqueue_e2e_coverage(run_id: str, job_ids: list[str] | None = None) -> int: """显式补采已有 run:只放宽 priority,其他 e2e 规则与历史 job.json 保持不变。""" + from .continuation import receipt + with db.connect() as con: + db.assert_run_open(con, run_id) + if receipt(run_id): + raise ValueError("continuation run must not replenish completed samples") mx = matrix() cfg = (mx.get("modes") or {}).get("e2e") or {} if not cfg or not cfg.get("enabled", True): @@ -306,6 +321,9 @@ def enqueue_e2e_coverage(run_id: str, job_ids: list[str] | None = None) -> int: added = 0 with db.connect() as con: con.execute("BEGIN IMMEDIATE") + db.assert_run_open(con, run_id) + if receipt(run_id): + raise ValueError("continuation run must not replenish completed samples") for job in db.list_jobs(con, run_id, mode="oracle"): row = model_rows.get(job["model"]) if (selected is not None and job["job_id"] not in selected @@ -334,39 +352,52 @@ def collect_drain_file(run_id: str) -> Path: return paths.STATE / f"drain-collect-{run_id}" -def _collection_snapshot(run_id: str, job_ids: list[str] | None, modes: list[str] | None, - repair_ecs: set[str]) -> tuple[list[dict], bool]: +def _collection_snapshot(run_id: str, job_ids: list[str] | None, modes: list[str] | None) -> tuple[list[dict], bool]: """每次填槽读当前状态;优先不同模型的第一条轨迹,再补剩余 k。""" with db.connect() as con: rows = [dict(r) for r in con.execute( "SELECT r.*, j.state AS job_state, j.error_class AS job_ec, j.estimate_s," " COALESCE(h.completed,0) AS completed, COALESCE(h.started,0) AS started" - " FROM rollouts r JOIN jobs j ON j.run_id=r.run_id AND j.job_id=r.job_id AND j.mode='oracle'" + " FROM rollouts r LEFT JOIN jobs j ON j.run_id=r.run_id AND j.job_id=r.job_id AND j.mode='oracle'" " LEFT JOIN (SELECT job_id," " SUM(CASE WHEN state IN ('PASS','FAIL','BLOCKED','TIMEOUT')" " OR (state='UNVERIFIED' AND verdict IS NOT NULL) THEN 1 ELSE 0 END) completed," " SUM(CASE WHEN started_at IS NOT NULL THEN 1 ELSE 0 END) started" - " FROM rollouts WHERE run_id=? GROUP BY job_id) h ON h.job_id=r.job_id" + " FROM rollouts WHERE run_id=? AND mode!='repair' GROUP BY job_id) h ON h.job_id=r.job_id" " WHERE r.run_id=? AND r.state='QUEUED'" " AND NOT EXISTS (SELECT 1 FROM rollouts a WHERE a.run_id=r.run_id AND a.job_id=r.job_id" " AND a.state IN ('RUNNING','VERIFYING'))", (run_id, run_id))] pending = [r[0] for r in con.execute( - "SELECT job_id FROM jobs WHERE run_id=? AND mode='oracle' AND state IN ('PENDING','ORACLE_RUNNING')" - " UNION SELECT job_id FROM rollouts WHERE run_id=? AND state IN ('RUNNING','VERIFYING')", - (run_id, run_id))] - todo = [] - for r in rows: - if job_ids and r["job_id"] not in job_ids: - continue - if modes and r["mode"] not in modes: - continue - if r["job_state"] not in ("PASS", "FAIL", "UNVERIFIED"): - continue - if r["mode"] == "repair": - if r["job_state"] != "FAIL" or repair_ecs and r["job_ec"] not in repair_ecs: + "SELECT job_id FROM rollouts WHERE run_id=? AND mode!='repair' AND state IN ('RUNNING','VERIFYING')", + (run_id,))] + # Continue input/Oracle references and first-pass counts from the source. + # A transferred, never-started row does not consume a first attempt. + run = db.get_run(con, run_id) + source = (json.loads(run["manifest"]).get("continuation") or {}).get("source_run_id") if run else None + history = {r["job_id"]: r for r in con.execute( + "SELECT job_id, COUNT(CASE WHEN state IN ('PASS','FAIL','BLOCKED','TIMEOUT')" + " OR (state='UNVERIFIED' AND verdict IS NOT NULL) THEN 1 END) completed," + " COUNT(CASE WHEN started_at IS NOT NULL THEN 1 END) started" + " FROM rollouts WHERE run_id=? AND mode!='repair' GROUP BY job_id", (source,))} if source else {} + refs = {} + for r in rows: + jid = r["job_id"] + if jid in history: + r["completed"] += history[jid]["completed"] + r["started"] += history[jid]["started"] + if r["job_state"] is not None: continue - todo.append(r) - todo.sort(key=lambda r: (r["completed"] > 0, r["job_state"] != "PASS", r["completed"], + if jid not in refs: + jp = paths.RUNS / run_id / jid / "job.json" + ref = json.loads(jp.read_text()).get("oracle_ref") if jp.exists() else None + refs[jid] = db.get_job(con, ref["run_id"], ref["job_id"]) if ref else None + original = refs[jid] + if original: + r.update(job_state=original["state"], job_ec=original["error_class"], estimate_s=original["estimate_s"]) + todo = [r for r in rows if r["mode"] != "repair" + and (not job_ids or r["job_id"] in job_ids) + and (not modes or r["mode"] in modes)] + todo.sort(key=lambda r: (r["completed"] > 0, r["completed"], r["started"], r["estimate_s"] if r["estimate_s"] is not None else float("inf"), r["mode"] != "e2e", r["kk"], r["job_id"])) return todo, any(not job_ids or jid in job_ids for jid in pending) @@ -374,18 +405,23 @@ def _collection_snapshot(run_id: str, job_ids: list[str] | None, modes: list[str def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[str] | None = None, parallel: int | None = None, dry_run: bool = False, limit: int | None = None, - log=_stamped, coverage: str = "matrix", watch: bool = False) -> dict: - """动态填充 agent 槽位;watch 等待新 oracle 结果,SIGTERM/drain 仅停止继续领取。""" + log=_stamped, coverage: str = "matrix", watch: bool = False, + first_pass: bool = False) -> dict: + """Agent 领取后自行准备输入;SIGTERM/drain 仅停止继续领取。""" import fcntl from .rollout import run_rollout, setup_models_json + from .continuation import receipt + with db.connect() as con: + db.assert_run_open(con, run_id) + handoff = receipt(run_id) + if handoff and coverage == "all": + raise ValueError("continuation run uses its exact queue; omit --coverage all") if coverage not in ("matrix", "all"): raise ValueError(f"unknown coverage: {coverage}") pool_n = int(paths.config()["pools"]["agent"]["workers"]) n = min(int(parallel or pool_n), pool_n) if n < 1: raise ValueError("agent parallel must be positive") - mx = matrix() - repair_ecs = set(((mx.get("modes") or {}).get("repair") or {}).get("error_classes") or []) stop = threading.Event() lockf = None old_handler = None @@ -418,7 +454,7 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[ def one(r: dict) -> dict: try: with scheduler.acquire("agent"): - return run_rollout(run_id, r["job_id"], r["kk"], r["mode"], r["policy"], dry_run=dry_run) + return run_rollout(run_id, r["job_id"], r["kk"], r["mode"], r["policy"], dry_run=dry_run, _claimed=not dry_run) except (SystemExit, Exception) as e: # 前置/运行错误也必须释放认领,不能遗留 RUNNING result = {"terminal": "UNVERIFIED", "reason": repr(e)} if not dry_run: @@ -445,7 +481,7 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[ if coverage == "all" and (not modes or "e2e" in modes): added = enqueue_e2e_coverage(run_id, job_ids) log(f"coverage=all: 补入 {added} 条 e2e 轨迹,原 k 不变") - log(f"collect {run_id}: parallel={n}, coverage={coverage}, watch={watch}") + log(f"collect {run_id}: parallel={n}, coverage={coverage}, watch={watch}, first_pass={first_pass}") with ThreadPoolExecutor(max_workers=n) as ex, ThreadPoolExecutor(max_workers=1) as exporter: futures, export_future, export_dirty = {}, None, False while True: @@ -460,11 +496,11 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[ export_dirty = not dry_run ready, pending = [], False if not stopped() and (limit is None or submitted < limit): - if not dry_run: - added = enqueue_repairs(run_id, job_ids, modes) - if added: - log(f"repair: 按 matrix 补入 {added} 条轨迹") - ready, pending = _collection_snapshot(run_id, job_ids, modes, repair_ecs) + ready, pending = _collection_snapshot(run_id, job_ids, modes) + if first_pass: + # Keep the original k and QUEUED rows. A later ordinary collect + # resumes repeats; a failed attempt never resets kk to zero. + ready = [r for r in ready if not r["started"] and not r["completed"]] active_jobs = {r["job_id"] for r in futures.values()} free = n - len(futures) if not dry_run: @@ -713,21 +749,24 @@ def drain_file(run_id: str) -> Path: return paths.STATE / f"drain-{run_id}" -def run_oracle_job(run_id: str, job_id: str, dry_run: bool = False, log=_stamped, force: bool = False) -> dict: +def run_oracle_job(run_id: str, job_id: str, dry_run: bool = False, log=_stamped, force: bool = False, + prepare_only: bool = False) -> dict: if drain_file(run_id).exists() and not force: return {"terminal": "SKIPPED", "job_id": job_id, "drained": True} job, jd = _load_job(run_id, job_id) cfg = paths.config() with db.connect() as con: - if not db.claim_job(con, run_id, job_id, force=force): + if not db.claim_job(con, run_id, job_id, force=force, prepare_only=prepare_only): log(f"[{job_id}] 已被别的进程认领或已完成,跳过") return {"terminal": "SKIPPED", "job_id": job_id} ctx = StageContext(run_id=run_id, job=job, job_dir=jd, stage_root=jd / "oracle" / "stages", artifacts=jd / "oracle" / "artifacts", rollout=-1, dry_run=dry_run) result = {"schema": "bench.result/1", "run_id": run_id, "job_id": job_id, "round": 1, "stages": {}, "measurements": {}, "started_at": time.time()} + from .verify import eval_set + result["eval_set"] = eval_set(job) terminal, error_class, stage_failed = None, None, None - for S in ORDER: + for S in (ORDER[:3] if prepare_only else ORDER): stage = S() attempt, lane = 0, "normal" while True: @@ -777,6 +816,20 @@ def run_oracle_job(run_id: str, job_id: str, dry_run: bool = False, log=_stamped terminal, error_class, stage_failed = "FAIL", ec, stage.name break + if prepare_only: + # Input readiness is not a passing INT8 evaluation. Keep Oracle's result + # filename absent so verify_rollout treats its comparison as optional. + result.update(schema="bench.preparation/1", terminal=terminal or "PREPARED", + error_class=error_class, stage_failed=stage_failed, finished_at=time.time()) + if dry_run: + result.update(terminal="UNVERIFIED", dry_run=True) + (jd / "oracle" / "preparation.json").write_text(json.dumps(result, indent=2, ensure_ascii=False)) + with db.connect() as con: + db.set_job_state(con, run_id, job_id, result["terminal"], error_class=error_class, + stage_failed=stage_failed, result=result) + log(f"[{job_id}] inputs → {result['terminal']} ec={error_class}") + return result + ev = gates.evaluate(result["measurements"], job["gates"]) if not error_class and (terminal or ev["terminal"]) == "FAIL": error_class = gates.classify_gate_failure(ev) @@ -793,39 +846,78 @@ def run_oracle_job(run_id: str, job_id: str, dry_run: bool = False, log=_stamped return result -def scheduler_alive() -> bool: - """有没有 bench run 进程活着(持有 state/locks/run-*.lock)。""" +def scheduler_holders(run_id: str | None = None) -> list[str]: + """当前被持有的调度类锁名。给了 run_id 就把该 run 的 collector 和 export --follow 一起算上。 + + 只用 flock 试锁,不读文件内容:run-*.lock 的 pid 在文件名里、内容是空的,而且只有正常 + 退出才 unlink —— 死掉的调度进程会一直留着锁文件,按内容或文件名判存活必然误判。 + open 用 "r":flock(2) 不看打开模式,只读就够。用 "w" 会把 collect-*.lock 里活着的 + collector 刚写的 "{pid} {ts}"(pipeline.py:440)截掉;用 "a" 不截断但要写权限, + 锁文件属主不是当前用户时会 PermissionError,被 except OSError 吞掉 → 门误判成 + "没人持有"而放行。只读打开让这条 fail-open 路径消失。 + run-{pid}.lock 的文件名里不含 run_id,所以 oracle 调度器这一路只能是全局语义。 + """ import fcntl - for p in (paths.STATE / "locks").glob("run-*.lock"): - try: - f = open(p, "w") + patterns = ["run-*.lock"] + if run_id: + patterns += [f"collect-{run_id}.lock", f"export-follow-{run_id}.lock"] + held: list[str] = [] + for pattern in patterns: + for p in (paths.STATE / "locks").glob(pattern): try: - fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - fcntl.flock(f, fcntl.LOCK_UN) - except BlockingIOError: - return True - finally: - f.close() - except OSError: - continue - return False + with open(p, "r") as f: + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + fcntl.flock(f, fcntl.LOCK_UN) + except BlockingIOError: + held.append(p.name) + except OSError: + continue + return held + + +def scheduler_alive() -> bool: + """有没有 bench run 进程活着(持有 state/locks/run-*.lock)。语义与签名保持不变: + coverage.py 和 continuation.py 依赖它,且它们问的就是"oracle 调度器活着吗"这个全局问题。""" + return bool(scheduler_holders()) def reset_stale(run_id: str, force: bool = False) -> dict: - """调度进程崩了/被杀后,ORACLE_RUNNING 的 job 回 PENDING,RUNNING 的 stage 记 ABORTED。有活着的调度进程时拒绝(--force 除外)。""" - if scheduler_alive() and not force: - raise SystemExit("还有 bench run 进程活着,拒绝 reset(--force 可强制)") + """调度进程崩了/被杀后,ORACLE_RUNNING 的 job 回 PENDING,终态 rollout 上残留的 RUNNING stage 记 ABORTED。 + 该 run 的调度器 / collector / export --follow 任一活着就拒绝(--force 除外)。 + + 时序要求:改 stages 会让"已生成但尚未 apply"的 continuation plan 立即作废——continuation 把 + stages 全行打进 source_sha256(continuation.py:76),apply 时复核不上就 raise。所以本命令 + 必须排在 `bench continue plan` 之前,绝不能夹在 plan 和 apply 中间。 + """ + holders = scheduler_holders(run_id) + if holders and not force: + raise SystemExit(f"以下锁仍被持有,拒绝 reset:{', '.join(sorted(holders))}(--force 可强制)") with db.connect() as con: + db.assert_run_open(con, run_id) n_jobs = con.execute("UPDATE jobs SET state='PENDING', started_at=NULL WHERE run_id=? AND state='ORACLE_RUNNING'", (run_id,)).rowcount - n_st = con.execute("UPDATE stages SET state='ABORTED', finished_at=? WHERE run_id=? AND state='RUNNING'", (time.time(), run_id)).rowcount + # rollout>=0 的只清宿主已终态的:不加这个条件时,实测这条 UPDATE 会把当时全部 11 个 + # 正在跑的编译 stage 一起标成 ABORTED,heartbeat 从此对它们失明(它只查 state='RUNNING')。 + # --force 仍走一刀切:它是"rollout 卡 RUNNING + stage 也卡住"这种死结的唯一出口, + # 连它一起堵死就没有任何 CLI 手段可解了。注意 rollout=-1 的 oracle 行和上面那条 jobs + # 复位本来就是无条件的,锁门是 oracle 侧唯一的保护。 + guard = "" if force else ( + " AND (rollout=-1 OR EXISTS (SELECT 1 FROM rollouts r WHERE r.run_id=stages.run_id" + " AND r.job_id=stages.job_id AND r.kk=stages.rollout" + " AND r.state NOT IN ('RUNNING','VERIFYING','QUEUED')))") + n_st = con.execute( + "UPDATE stages SET state='ABORTED', finished_at=? WHERE run_id=? AND state='RUNNING'" + guard, + (time.time(), run_id)).rowcount return {"jobs_reset": n_jobs, "stages_aborted": n_st} def run_oracle(run_id: str, job_ids: list[str] | None = None, dry_run: bool = False, - parallel: int | None = None, log=_stamped) -> dict: + parallel: int | None = None, log=_stamped, prepare_only: bool = False) -> dict: import fcntl + with db.connect() as con: + db.assert_run_open(con, run_id) (paths.STATE / "locks").mkdir(parents=True, exist_ok=True) - lockf = open(paths.STATE / "locks" / f"run-{os.getpid()}.lock", "w") + lockf = open(paths.STATE / "locks" / f"run-{os.getpid()}.lock", "a") # 同 scheduler:抢锁前不碰文件 fcntl.flock(lockf, fcntl.LOCK_EX) try: drain_file(run_id).unlink() # 新调度器启动 = 解除 drain @@ -840,17 +932,18 @@ def run_oracle(run_id: str, job_ids: list[str] | None = None, dry_run: bool = Fa if job_ids: jobs = [j for j in jobs if j["job_id"] in job_ids] # 显式指定:允许重跑(force) else: - jobs = [j for j in jobs if j["state"] == "PENDING"] + jobs = [j for j in jobs if j["state"] in (("PENDING",) if prepare_only else ("PENDING", "PREPARED"))] jobs = scheduler.order_jobs(jobs) n = parallel or paths.config()["pools"]["cpu"]["workers"] summary = {"PASS": 0, "FAIL": 0, "UNVERIFIED": 0} - log(f"run {run_id}: {len(jobs)} jobs, parallel={n}") + log(f"run {run_id}: {len(jobs)} jobs, parallel={n}, prepare_only={prepare_only}") with ThreadPoolExecutor(max_workers=n) as ex: - futures = [ex.submit(run_oracle_job, run_id, j["job_id"], dry_run, log, force=bool(job_ids)) for j in jobs] + futures = [ex.submit(run_oracle_job, run_id, j["job_id"], dry_run, log, + force=bool(job_ids), prepare_only=prepare_only) for j in jobs] for future in as_completed(futures): r = future.result() summary[r["terminal"]] = summary.get(r["terminal"], 0) + 1 - if not dry_run and r["terminal"] != "SKIPPED": + if not dry_run and not prepare_only and r["terminal"] != "SKIPPED": refresh_run_dataset(run_id) if drain_file(run_id).exists(): log(f"run {run_id} drained:剩余 PENDING 未认领,`scripts/start_run.sh --resume` 续跑") diff --git a/harness/registry.py b/harness/registry.py index 4d9e836..2319922 100644 --- a/harness/registry.py +++ b/harness/registry.py @@ -203,16 +203,23 @@ def _apply_overrides(row: dict, mx: dict) -> dict: return out -def expand_matrix(modes: list[str] | None = None) -> list[JobSpec]: - """把 registry 展开成 JobSpec 列表。repair 模式的 job 在 oracle FAIL 后由 pipeline 动态追加,这里不展开。""" +def expand_matrix(modes: list[str] | None = None, model_ids: list[str] | None = None) -> list[JobSpec]: + """把 registry 展开成 JobSpec 列表;历史 Oracle repair 模式不再生成。""" mx = matrix() rows = models() + if model_ids: + unknown = set(model_ids) - {r["model_id"] for r in rows} + if unknown: + raise ValueError(f"unknown models: {sorted(unknown)}") + rows = [r for r in rows if r["model_id"] in model_ids] marches_all = mx.get("marches", MARCHES) backends_all = mx.get("backends", ["ptq"]) qcfg_by_march = mx.get("qcfg_by_march", {}) tier = mx.get("data_tier", "t1") out: list[JobSpec] = [] for mode, mcfg in (mx.get("modes") or {}).items(): + if mode == "repair": + continue if modes and mode not in modes: continue if not mcfg.get("enabled", True) or mcfg.get("when") == "oracle_fail": @@ -222,7 +229,7 @@ def expand_matrix(modes: list[str] | None = None) -> list[JobSpec]: policies = mcfg.get("policies") or [None] k = int(mcfg.get("k", 1)) for row in rows: - if not _select(row, mcfg.get("select")): + if not model_ids and not _select(row, mcfg.get("select")): continue ov = _apply_overrides(row, mx) for march in marches: @@ -248,9 +255,10 @@ def status_counts() -> dict: def model_row(model_id: str) -> dict: + from .adapters import model_spec for r in models(): if r["model_id"] == model_id: - return r + return model_spec(r) raise KeyError(model_id) @@ -298,7 +306,8 @@ def data_snapshot(spec: JobSpec) -> dict: t1_spec = ds.get("t1", {}) or {} seed = t1_spec.get("seed", 0) t1_tar = paths.DATA / "t1" / ds_name / f"{seed}.tar" - calib_dir = paths.DATA / "calib" / spec.task / row["input_spec"] + from .adapters import calibration_dir + calib_dir = calibration_dir(row) def sha_of(p: Path, key: str = "sha256") -> str | None: m = p / "manifest.json" diff --git a/harness/rollout.py b/harness/rollout.py index 88c4f80..8363158 100644 --- a/harness/rollout.py +++ b/harness/rollout.py @@ -16,8 +16,8 @@ from pathlib import Path from . import paths, db, watchdog from .registry import gates as gates_cfg -from .stages.s02_baseline import baseline_dir -from .stages.s03_export import onnx_path +from .stages.s02_baseline import baseline_dir, Baseline +from .stages.s03_export import onnx_path, Export from .verify import verify_rollout @@ -32,6 +32,24 @@ def _render(tmpl: str, **kw) -> str: return out +def inputs_ready(run_id: str, job_id: str) -> bool: + """E2E depends on versioned inputs, never on an Oracle INT8 result.""" + from .stages import StageContext + jd = paths.RUNS / run_id / job_id + try: + job = json.loads((jd / "job.json").read_text()) + ctx = StageContext(run_id, job, jd, jd / "oracle/stages", jd / "oracle/artifacts") + data = job["data"] + return bool(Export().cached(ctx) + # Existing subset continuations build their new FP32 reference + # in prepare(), before starting the Agent; no INT8 rerun. + and (Baseline().cached(ctx) or job.get("continuation")) + and Path(data["t1"]["path"]).is_file() + and (Path(data["calib"]["path"]) / "manifest.json").is_file()) + except (OSError, ValueError): + return False # Inputs can still be absent or being written by preparation. + + def setup_models_json(force: bool = False) -> Path: """渲染 ~/.pi/agent/models.json。key 只从 .env.local 来。""" dst = Path(os.path.expanduser("~/.pi/agent/models.json")) @@ -54,8 +72,12 @@ def setup_models_json(force: bool = False) -> Path: def prepare(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_run: bool = False) -> tuple[Path, dict, str]: """dry_run=True 只在内存里渲染,不建目录、不写 rollout.json、不动 DB(踩过:dry-run 把 3 个 kk 占成了空壳 RUNNING)。""" + if mode == "repair": + raise ValueError("Oracle repair collection is retired; existing rows and artifacts remain historical") jd = paths.RUNS / run_id / job_id job = json.loads((jd / "job.json").read_text()) + with db.connect() as con: + db.assert_run_open(con, run_id) rd = rollout_dir(run_id, job_id, kk) if (rd / "rollout.json").exists(): raise SystemExit(f"rollout 已存在(append-only):{rd}。想再答一次就用下一个 kk。") @@ -63,15 +85,9 @@ def prepare(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_run: for d in ("workspace", "agent", "stages", "handles", "workspace/.pi"): (rd / d).mkdir(parents=True, exist_ok=True) - # 前置:model-level 产物必须已在 cache 里(由 oracle 的 01–03 产出) + # 输入缓存可复用;缺少导出或基线时由 Agent 在自己的轨迹中提交。 onnx = onnx_path(job) bdir = baseline_dir(job) - missing = [str(p) for p in (onnx, bdir / "metrics.json") if not p or not p.exists()] - oracle = jd / "oracle" / "result.json" - if mode == "repair" and not oracle.exists(): - missing.append(f"{oracle}(repair 需要 oracle 失败现场)") - if missing: - raise SystemExit("rollout 前置缺失,先跑 oracle(至少 01–03):\n " + "\n ".join(missing)) cfg = paths.config() ep = cfg["endpoints"][policy] @@ -88,19 +104,14 @@ def prepare(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_run: tname = job.get("prompt") or (f"{mode}.md") tmpl = (paths.AGENT / "prompts" / tname).read_text() g = job["gates"] - ores = json.loads(oracle.read_text()) if oracle.exists() else {} - o_stage = ores.get("stage_failed") or "" prompt = _render( tmpl, job_id=job_id, run_id=run_id, model=job["model"], task=job["task"], march=job["march"], backend=job["backend"], qcfg=job["qcfg"], job_json=str(jd / "job.json"), data_dir=str(jd / "data"), - input_spec=job["model_spec"]["input_spec"], onnx=str(onnx), baseline_dir=str(bdir), + input_spec=job["model_spec"]["input_spec"], onnx=str(onnx) if onnx else "尚未生成;由 fetch/export 阶段返回路径", baseline_dir=str(bdir), calib_dir=job["data"]["calib"]["path"], calib_n=job["data"]["calib"]["n"], t1_tar=job["data"]["t1"]["path"], t1_n=job["data"]["t1"]["n"], g2_threshold=g["G2"].get("threshold"), g3_metric=g["G3"].get("metric"), g3_tol=g["G3"].get("tol"), g3_kind=g["G3"].get("kind"), max_turns=cfg["rollouts"]["max_turns"], max_tokens=cfg["rollouts"]["max_tokens"], - oracle_stage_failed=o_stage, oracle_error_class=ores.get("error_class"), - oracle_stage_dir=str(jd / "oracle" / "stages" / o_stage) if o_stage else "", - oracle_compile_yaml=str(jd / "oracle" / "stages" / "04-calibrate" / "compile.yaml"), ) if dry_run: return rd, job, prompt @@ -145,14 +156,10 @@ def agent_env(rd: Path, mode: str = "e2e") -> dict: path_parts.append(os.environ.get("PATH", "/usr/bin:/bin")) # path-guard 的白名单按"项目树内路径"判;state/ 是指向 /home 的符号链接、work_root 是本地盘, # 用真实路径就绕开了 → 把它们的 realpath 作为额外拒绝项传给 guard;本 rollout 自己的编译工作目录放行。 - work_root = cfg["paths"].get("work_root") - deny = [str(Path(paths.STATE).resolve())] - allow = [] - if work_root: - wr = paths.expand(work_root) - deny.append(str(wr)) - rel = rd.relative_to(paths.RUNS) # //rollouts/ - allow.append(str(wr / rel.parts[0] / rel.parts[1] / f"r{int(rel.parts[3]):02d}")) + wr = paths.expand(cfg["paths"]["work_root"]) + rel = rd.relative_to(paths.RUNS) # //rollouts/ + deny = [str(Path(paths.STATE).resolve()), str(wr)] + allow = [str(wr / rel.parts[0] / rel.parts[1] / f"r{int(rel.parts[3]):02d}")] return { "BENCH_GUARD_DENY": ":".join(deny), "BENCH_GUARD_ALLOW": ":".join(allow), @@ -197,7 +204,39 @@ def pi_command(rd: Path, prompt: str, mode: str = "e2e") -> list[str]: prompt] -def run_rollout(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_run: bool = False) -> dict: +def run_rollout(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_run: bool = False, + *, _claimed: bool = False) -> dict: + if mode == "repair": + raise ValueError("Oracle repair collection is retired; existing rows and artifacts remain historical") + with db.connect() as con: + db.assert_run_open(con, run_id) + run = db.get_run(con, run_id) + row = con.execute("SELECT * FROM rollouts WHERE run_id=? AND job_id=? AND kk=?", (run_id, job_id, kk)).fetchone() + if run and json.loads(run["manifest"]).get("continuation"): + if not row or (row["mode"], row["policy"]) != (mode, policy): + raise ValueError("rollout is not in the continuation queue") + if row and (row["mode"], row["policy"]) != (mode, policy): + raise ValueError("rollout mode/policy differs from the queued task") + if not dry_run: + if _claimed: + if not row or row["state"] != "RUNNING": + raise ValueError("collector has not claimed this rollout") + else: + if not row: + db.rollout_upsert(con, run_id, job_id, kk, mode, policy) + con.commit() + if not db.claim_rollout(con, run_id, job_id, kk): + raise ValueError("rollout already claimed or finished") + try: + return _execute_rollout(run_id, job_id, kk, mode, policy, dry_run) + except (SystemExit, Exception): + if not dry_run and not _claimed: + with db.connect() as con: + db.rollout_set(con, run_id, job_id, kk, state="UNVERIFIED", finished_at=time.time()) + raise + + +def _execute_rollout(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_run: bool) -> dict: setup_models_json() rd, job, prompt = prepare(run_id, job_id, kk, mode, policy, dry_run=dry_run) cmd = pi_command(rd, prompt, mode) @@ -209,9 +248,7 @@ def run_rollout(run_id: str, job_id: str, kk: int, mode: str, policy: str, dry_r cfg = paths.config()["rollouts"] from contextlib import nullcontext from .stageworker import Worker, config_bytes - work = paths.expand(paths.config()['paths'].get('work_root')) - if work: - (work / run_id / job_id / f'r{kk:02d}').mkdir(parents=True, exist_ok=True) + (paths.expand(paths.config()['paths']['work_root']) / run_id / job_id / f'r{kk:02d}').mkdir(parents=True, exist_ok=True) jailed = bool(jail_prefix(rd, mode)) with Worker(rd) if jailed else nullcontext(): r = watchdog.run(cmd, cwd=rd / "workspace", stdout=rd / "agent" / "events.jsonl", stderr=rd / "agent" / "pi.stderr.log", diff --git a/harness/scheduler.py b/harness/scheduler.py index 05e37c2..2dd4c19 100644 --- a/harness/scheduler.py +++ b/harness/scheduler.py @@ -43,9 +43,12 @@ def pool_reserve(pool: str) -> int: def _try_slots(pool: str, order: list[int]): d = _lock_dir() for i in order: - f = open(d / f"{pool}-{i}.lock", "w") + # "a" 而非 "w":抢锁失败时也不能动文件。"w" 在 flock 之前就截断,于是每次探测 + # 繁忙槽位都会把持锁者刚写的 "{pid} {ts}" 抹掉。拿到锁之后再 truncate 才安全。 + f = open(d / f"{pool}-{i}.lock", "a") try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + f.seek(0); f.truncate() f.write(f"{os.getpid()} {time.time()}\n"); f.flush() return f, i except BlockingIOError: @@ -124,7 +127,7 @@ def acquire_stage(ctx, pool: str, lane: str = "normal"): @contextmanager def model_lock(model: str): """model-level 阶段(01–03)按模型互斥:同一模型的三个 march job 并行时只有一个真跑,其余看到缓存 SKIPPED。""" - f = open(_lock_dir() / f"model-{model}.lock", "w") + f = open(_lock_dir() / f"model-{model}.lock", "a") # 同 _try_slots:抢锁前不碰文件 fcntl.flock(f, fcntl.LOCK_EX) try: yield diff --git a/harness/stagerun.py b/harness/stagerun.py index 9fe3222..ee3167c 100644 --- a/harness/stagerun.py +++ b/harness/stagerun.py @@ -10,6 +10,7 @@ rollout 上下文由 env BENCH_ROLLOUT_DIR 给定(rollout 驱动器设置) """ from __future__ import annotations +from contextlib import nullcontext import json import os import subprocess @@ -132,7 +133,9 @@ def _exec(hd: Path, *, rd=None, publish_status=None, protected=False) -> int: put_status({"state": "QUEUED", "pool": stage.pool, "attempt": ctx.attempt, "queued_at": time.time()}) with db.connect() as con: db.stage_start(con, ctx.run_id, ctx.job["job_id"], stage.name, ctx.attempt, ctx.rollout, lane="queued") - with scheduler.acquire_stage(ctx, stage.pool, lane="agent") as slot: # agent 优先拿保留槽,不被 oracle 饿死 + # 01–03 retain shared-cache exclusion even when submitted by different Agents. + with (scheduler.model_lock(ctx.job["model"]) if stage.level == "model" else nullcontext()), \ + scheduler.acquire_stage(ctx, stage.pool, lane="agent") as slot: put_status({"state": "RUNNING", "pool": stage.pool, "slot": slot, "attempt": ctx.attempt, "board_name": (ctx.extra.get("board") or {}).get("name"), "started_at": time.time()}) with db.connect() as con: # 拿到槽位才算真的开始跑:看板据此区分"排队"与"在算" diff --git a/harness/stages/__init__.py b/harness/stages/__init__.py index 0227a4c..1bca9a0 100644 --- a/harness/stages/__init__.py +++ b/harness/stages/__init__.py @@ -12,7 +12,7 @@ from .s08_eval import Eval ORDER: list[type[Stage]] = [Fetch, Baseline, Export, Calibrate, Compile, Hbm, Board, Eval] BY_NAME: dict[str, type[Stage]] = {s.name: s for s in ORDER} # agent 通过 `bench stage run` 可提交的阶段(重阶段由 harness 执行) -AGENT_RUNNABLE = {"04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval"} +AGENT_RUNNABLE = set(BY_NAME) def short(name: str) -> str: diff --git a/harness/stages/base.py b/harness/stages/base.py index 83b672b..57ccb67 100644 --- a/harness/stages/base.py +++ b/harness/stages/base.py @@ -42,6 +42,22 @@ class StageContext: return wd +def work_dir(ctx: StageContext) -> Path: + """本地盘上的工作目录:stage 子进程的 cwd,也是 hb_compile 的 working_dir。 + + 必须放本地盘(config.yaml paths.work_root)。实测 2026-09-15:JuiceFS 写 1 MB 要 360 ms、 + 本地盘 3 ms;且 hbdk-lld/ld.lld 用相对路径写 acu_*.elf(~300 MB/次),cwd 落在 JuiceFS 上 + 会打满整条 FUSE 连接,链接从分钟级变小时级。最终产物由 06-hbm 收进 runs/ 的 artifacts/。 + 返回 resolve() 后的真路径:cwd 必须与 compile.yaml 的 working_dir 逐字符相等,hb_compile 靠字符串 + 相等跳过自拷日志(horizon_tc_ui/hb_compile.py:50),不等会在编译成功后删掉日志再抛错。 + 叶子名暂留 05-compile-work:在飞 job 已按此路径写完,上移一层会让 s06_hbm 的 hb_compile.log 落空。 + """ + rel = f"{ctx.run_id}/{ctx.job['job_id']}" + (f"/r{ctx.rollout:02d}" if ctx.rollout >= 0 else "/oracle") + d = paths.expand(paths.config()["paths"]["work_root"]) / rel / "05-compile-work" + d.mkdir(parents=True, exist_ok=True) + return d.resolve() + + @dataclass class StageOutcome: state: str # OK | FAIL | STALLED | SKIPPED @@ -117,9 +133,11 @@ class Stage: out = self.native(ctx, sd) else: wd = ctx.watchdog_params() - r = watchdog.run(cmd, cwd=sd, stdout=sd / "stdout.log", stderr=sd / "stderr.log", + r = watchdog.run(cmd, cwd=work_dir(ctx), stdout=sd / "stdout.log", stderr=sd / "stderr.log", env=self.env(ctx), stall_s=wd["stall_s"], cpu_idle_pct=wd["cpu_idle_pct"], - poll_s=wd["poll_s"], hard_ceiling_s=wd["hard_ceiling_s"]) + poll_s=wd["poll_s"], hard_ceiling_s=wd["hard_ceiling_s"], + io_stall_s=wd["io_stall_s"], + no_progress_ceiling_s=wd["no_progress_ceiling_s"]) ec = None if r.state != "OK": ec = classify(_tail(sd / "stderr.log"), _tail(sd / "stdout.log"), r.exit_code) diff --git a/harness/stages/s01_fetch.py b/harness/stages/s01_fetch.py index 164ea92..1381c02 100644 --- a/harness/stages/s01_fetch.py +++ b/harness/stages/s01_fetch.py @@ -2,6 +2,7 @@ from __future__ import annotations import hashlib +import fcntl import json import shutil from pathlib import Path @@ -53,13 +54,18 @@ class Fetch(Stage): f = files[0] sha = hashlib.sha256(f.read_bytes()).hexdigest() dst = paths.CACHE / "weights" / sha - if not dst.exists(): - dst.mkdir(parents=True) - shutil.move(str(f), dst / f.name) - (dst / "meta.json").write_text(json.dumps({"model": ctx.job["model"], "uri": ctx.job["model_spec"]["weights"], - "sha256": sha, "size": (dst / f.name).stat().st_size}, indent=2)) - shutil.rmtree(tmp, ignore_errors=True) - idx = _index() - idx[ctx.job["model"]] = {"sha256": sha, "file": f.name} - INDEX.write_text(json.dumps(idx, indent=2, ensure_ascii=False)) + # Publish shared content and update the index under one short lock. + with INDEX.with_suffix('.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if not dst.exists(): + dst.mkdir(parents=True) + shutil.move(str(f), dst / f.name) + (dst / "meta.json").write_text(json.dumps({"model": ctx.job["model"], "uri": ctx.job["model_spec"]["weights"], + "sha256": sha, "size": (dst / f.name).stat().st_size}, indent=2)) + shutil.rmtree(tmp, ignore_errors=True) + idx = _index() + idx[ctx.job["model"]] = {"sha256": sha, "file": f.name} + temporary = INDEX.with_suffix('.tmp') + temporary.write_text(json.dumps(idx, indent=2, ensure_ascii=False)) + temporary.replace(INDEX) return {"weights_sha256": sha} diff --git a/harness/stages/s02_baseline.py b/harness/stages/s02_baseline.py index 41ac524..4bc8998 100644 --- a/harness/stages/s02_baseline.py +++ b/harness/stages/s02_baseline.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from pathlib import Path from .. import paths @@ -12,7 +13,17 @@ from .s01_fetch import weights_file def baseline_dir(job: dict) -> Path: - return paths.CACHE / "baseline" / f"{job['model']}--{job['data']['dataset']}" + directory = paths.CACHE / "baseline" / f"{job['model']}--{job['data']['dataset']}" + t1 = job["data"]["t1"] + if not t1.get("eval_set_id"): + return directory # 已有任务继续读取原缓存;显式新子集才启用独立目录。 + content_sha = t1.get("content_sha") + if not isinstance(content_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", content_sha): + raise ValueError("an explicit eval set requires a full SHA256 content_sha") + version = job["evaluator_version"] + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", version): + raise ValueError("invalid evaluator_version for baseline cache") + return directory / content_sha / version class Baseline(Stage): @@ -36,7 +47,7 @@ class Baseline(Stage): job = ctx.job out = baseline_dir(job) out.mkdir(parents=True, exist_ok=True) - return [str(paths.venv_python("bench")), "-m", "harness.eval.run", + return [str(paths.model_python(job)), "-m", "harness.eval.run", "--mode", "fp32", "--task", job["task"], "--model", job["model"], "--source", job["model_spec"]["source"], "--weights", str(weights_file(job["model"])), "--input-spec", job["model_spec"]["input_spec"], "--resize", str(job["model_spec"].get("resize") or 0), diff --git a/harness/stages/s03_export.py b/harness/stages/s03_export.py index 8059a3b..714e3de 100644 --- a/harness/stages/s03_export.py +++ b/harness/stages/s03_export.py @@ -26,10 +26,10 @@ class Export(Stage): if not (p and p.exists()): return None # 导出口径版本要一致,否则重导(踩过:ultralytics 旧导出留下的 ONNX 被新 run 直接复用) - from ..tools.export_onnx import EXPORTER_VERSION + from ..tools.export_onnx import exporter_version meta = p.parent / "meta.json" try: - if json.loads(meta.read_text()).get("exporter_version") != EXPORTER_VERSION: + if json.loads(meta.read_text()).get("exporter_version") != exporter_version(ctx.job["model"]): return None except Exception: return None @@ -45,8 +45,8 @@ class Export(Stage): def measure(self, ctx, stage_dir, outcome) -> dict: p = onnx_path(ctx.job) sha = hashlib.sha256(p.read_bytes()).hexdigest() - from ..tools.export_onnx import EXPORTER_VERSION + from ..tools.export_onnx import exporter_version (p.parent / "meta.json").write_text(json.dumps({"model": ctx.job["model"], "onnx_sha256": sha, "input_spec": ctx.job["model_spec"]["input_spec"], - "exporter_version": EXPORTER_VERSION}, indent=2)) + "exporter_version": exporter_version(ctx.job["model"])}, indent=2)) return {"onnx_sha256": sha} diff --git a/harness/stages/s04_calibrate.py b/harness/stages/s04_calibrate.py index 8d7d8a1..0219c8b 100644 --- a/harness/stages/s04_calibrate.py +++ b/harness/stages/s04_calibrate.py @@ -9,9 +9,8 @@ from pathlib import Path import yaml -from .. import paths from ..backends import ptq -from .base import Stage, StageContext, StageOutcome +from .base import Stage, StageContext, StageOutcome, work_dir from .s03_export import onnx_path @@ -28,21 +27,6 @@ def compile_yaml_path(ctx: StageContext) -> Path: return ctx.stage_root / "04-calibrate" / "compile.yaml" -def work_dir(ctx: StageContext) -> Path: - """hb_compile 的 working_dir:中间产物(几个 ONNX + bc + hbm,几百 MB)都写这里。 - - 默认放本地盘(config.yaml paths.work_root),不放 runs/。实测 2026-09-15: - JuiceFS 写 1 MB 要 360 ms,本地盘 3 ms(慢 118 倍),大模型编译会长时间卡在 D 状态, - 看起来就像挂死——EfficientNet-B4/B5 因此被看门狗误杀。最终产物由 06-hbm 收进 runs/ 的 artifacts/。 - work_root 为空则退回 runs/ 内(单机小规模时够用)。 - """ - root = paths.config()["paths"].get("work_root") - if not root: - return ctx.stage_root / "05-compile" / "work" - rel = f"{ctx.run_id}/{ctx.job['job_id']}" + (f"/r{ctx.rollout:02d}" if ctx.rollout >= 0 else "/oracle") - return paths.expand(root) / rel / "05-compile-work" - - class Calibrate(Stage): name = "04-calibrate" level = "target" diff --git a/harness/stages/s05_compile.py b/harness/stages/s05_compile.py index 03fac9d..1e28713 100644 --- a/harness/stages/s05_compile.py +++ b/harness/stages/s05_compile.py @@ -1,12 +1,10 @@ """05-compile:hb_compile(PTQ)/ plugin calib + hbdk4 compile(QAT)。CPU 密集,只有看门狗,没有超时。""" from __future__ import annotations -from pathlib import Path - from ..backends import ptq, qat -from .base import Stage, StageContext +from .base import Stage, StageContext, work_dir from .s01_fetch import weights_file -from .s04_calibrate import compile_yaml_path, work_dir, calib_dir +from .s04_calibrate import compile_yaml_path, calib_dir class Compile(Stage): @@ -16,7 +14,6 @@ class Compile(Stage): def command(self, ctx: StageContext): wd = work_dir(ctx) - wd.mkdir(parents=True, exist_ok=True) if ctx.job["backend"] == "ptq": return ptq.compile_cmd(compile_yaml_path(ctx)) return qat.calib_compile_cmd(ctx.job, weights_file(ctx.job["model"]), calib_dir(ctx.job), wd, ctx.agent_config) diff --git a/harness/stages/s06_hbm.py b/harness/stages/s06_hbm.py index c4ebc6b..7a66655 100644 --- a/harness/stages/s06_hbm.py +++ b/harness/stages/s06_hbm.py @@ -14,8 +14,8 @@ from pathlib import Path from .. import watchdog from ..backends import get as backend from ..registry import boards -from .base import Stage, StageContext, StageOutcome -from .s04_calibrate import work_dir, calib_npy_dir +from .base import Stage, StageContext, StageOutcome, work_dir +from .s04_calibrate import calib_npy_dir ANSI = re.compile(r"\x1b\[[0-9;]*m") @@ -40,7 +40,7 @@ class Hbm(Stage): def native(self, ctx: StageContext, stage_dir: Path) -> StageOutcome: job = ctx.job wd = work_dir(ctx) - cands = sorted(wd.rglob("*.hbm"), key=lambda p: p.stat().st_mtime, reverse=True) if wd.exists() else [] + cands = sorted(wd.rglob("*.hbm"), key=lambda p: p.stat().st_mtime, reverse=True) m: dict = {"hbm_exists": bool(cands)} if not cands: (stage_dir / "stderr.log").write_text(f"no .hbm under {wd}\n") @@ -56,7 +56,7 @@ class Hbm(Stage): prefix = job["job_id"] # ---- hb_model_info:stdout=静态 perf,stderr=I/O 表 + march - r = watchdog.run(be.model_info_cmd(dst), cwd=stage_dir, stdout=stage_dir / "model_info.log", + r = watchdog.run(be.model_info_cmd(dst), cwd=wd, stdout=stage_dir / "model_info.log", stderr=stage_dir / "model_info.err.log", env=self.env(ctx), stall_s=wdp["stall_s"], poll_s=5) info = _read(stage_dir / "model_info.log") + "\n" + _read(stage_dir / "model_info.err.log") m["model_info_ok"] = r.state == "OK" @@ -72,7 +72,7 @@ class Hbm(Stage): ptq_onnx = next(iter(sorted(wd.rglob("*_ptq_model.onnx"))), None) sample = next(iter(sorted(calib_npy_dir(Path(job["data"]["calib"]["path"])).glob("*.npy"))), None) if ptq_onnx and sample: - rv = watchdog.run(be.verifier_cmd(ptq_onnx, bc, sample), cwd=stage_dir, + rv = watchdog.run(be.verifier_cmd(ptq_onnx, bc, sample), cwd=wd, stdout=stage_dir / "verifier.log", stderr=stage_dir / "verifier.err.log", env=self.env(ctx), stall_s=wdp["stall_s"], poll_s=10) vtext = _read(stage_dir / "verifier.log") + "\n" + _read(stage_dir / "verifier.err.log") @@ -108,7 +108,11 @@ class Hbm(Stage): (stage_dir / "stdout.log").write_text(json.dumps(m, indent=2, ensure_ascii=False) + "\n") (stage_dir / "stderr.log").touch() - # work 目录可能在本地盘(config paths.work_root),把它记下来,事后还能找到中间产物 + # 产物已进 artifacts,就地回收中间 ONNX:约 770 MB/job,而本地盘没有任何清理机制(实测 + # bench-work 121 GB 里 95 GB 是它们)。只删 *_model.onnx——另外三个全仓零引用, + # _ptq_model.onnx 的唯一读者是上面的 hb_verifier。.hbm/.bc/.hbo 与日志留着,重跑 05-compile 会重新生成。 + for f in wd.rglob("*_model.onnx"): + f.unlink(missing_ok=True) return StageOutcome("OK", 0, None, 0.0, {"hbm": str(dst), "size": dst.stat().st_size, "work_dir": str(wd)}, {"G1": m}) @@ -126,7 +130,7 @@ def _hbm_bc_spotcheck(ctx: StageContext, stage_dir: Path, hbm: Path, bc: Path, b for tag, model in (("hbm", hbm), ("bc", bc)): fo = stage_dir / f"spot_{tag}.npz" r = watchdog.run([str(be.python()), "-m", "harness.tools.infer_hbm", "--model", str(model), "--inputs", str(fin), "--out", str(fo)], - cwd=stage_dir, stdout=stage_dir / f"spot_{tag}.log", stderr=stage_dir / f"spot_{tag}.err.log", + cwd=work_dir(ctx), stdout=stage_dir / f"spot_{tag}.log", stderr=stage_dir / f"spot_{tag}.err.log", env=self_env(ctx), stall_s=wdp["stall_s"], poll_s=5) if r.state != "OK": return None diff --git a/harness/stages/s08_eval.py b/harness/stages/s08_eval.py index d35d3b0..e5af06d 100644 --- a/harness/stages/s08_eval.py +++ b/harness/stages/s08_eval.py @@ -26,7 +26,7 @@ class Eval(Stage): if not board: return None out = self.stage_dir(ctx) / "eval.json" - return [str(paths.venv_python("bench")), "-m", "harness.eval.run", + return [str(paths.model_python(job)), "-m", "harness.eval.run", "--runner", "board", "--runtime-dir", board.get("runtime_dir", "/userdata/bench"), "--mode", "quant", "--task", job["task"], "--model", job["model"], "--hbm", str(hbm_path(ctx)), "--onnx", str(onnx_path(job)), diff --git a/harness/tools/board_bench.py b/harness/tools/board_bench.py index 91800ba..61a120f 100644 --- a/harness/tools/board_bench.py +++ b/harness/tools/board_bench.py @@ -21,6 +21,10 @@ import threading import time +def progress(phase: str, **values): + print(json.dumps({'board_progress': phase, **values}), file=sys.stderr, flush=True) + + def read_ratio() -> float | None: for p in ("/sys/class/bpu/bpu_core0/ratio", "/sys/class/bpu/bpu/ratio"): try: @@ -57,7 +61,9 @@ def infer_inputs(m, name: str, inputs: str, output: str) -> int: raise ValueError(f"{key}: input shape {x.shape} != model shape {shape}") if x.dtype != np.float32: raise ValueError(f"{key}: input dtype {x.dtype} != float32") + progress('sample_start', sample=key) res = m.run({name: {input_name: np.ascontiguousarray(x)}})[name] + progress('sample_done', sample=key) for i, output_name in enumerate(out_names): results[f"{key}/out{i}"] = np.array(res[output_name], copy=True) np.savez(output, **results) @@ -76,6 +82,7 @@ def model_metadata(m, name: str, hbm: str) -> dict: "inputs": {n: list(map(int, m.input_shapes[name][n])) for n in names}, "outputs": list(m.output_names[name]), "input_types": {n: str(types.get(n, "?")) for n in names}, "ucp": _ucp_version(), "board_type": _board_type(), "hostname": os.uname().nodename, + "board_guard_version": os.environ.get("BOARD_GUARD_VERSION"), } @@ -92,16 +99,22 @@ def main() -> int: ap.error("--inputs and --out must be supplied together") if not a.inputs and (a.warmup < 0 or a.runs <= 0): ap.error("--warmup must be >= 0 and --runs must be > 0") + import faulthandler + import signal + faulthandler.enable() + faulthandler.register(signal.SIGUSR1, all_threads=True) import numpy as np import hbm_runtime t_load = time.time() + progress('load_start') m = hbm_runtime.HB_HBMRuntime(a.hbm) name = m.model_names[0] in_names = list(m.input_names[name]) shapes = m.input_shapes[name] types = getattr(m, "input_types", {}).get(name, {}) if hasattr(m, "input_types") else {} load_s = time.time() - t_load + progress('load_done', seconds=load_s) if a.inputs: samples = infer_inputs(m, name, a.inputs, a.out) @@ -132,15 +145,24 @@ def main() -> int: ratios.append(r) time.sleep(0.05) - for _ in range(a.warmup): + last_progress = 0.0 + progress('warmup_start', count=a.warmup) + for i in range(a.warmup): m.run(feed) + if time.monotonic() - last_progress >= 10: + progress('warmup_done', completed=i+1, total=a.warmup) + last_progress = time.monotonic() th = threading.Thread(target=sampler, daemon=True); th.start() lat = [] t0 = time.time() - for _ in range(a.runs): + progress('measurement_start', count=a.runs) + for i in range(a.runs): t = time.perf_counter() m.run(feed) lat.append((time.perf_counter() - t) * 1000.0) + if time.monotonic() - last_progress >= 10: + progress('measurement_done', completed=i+1, total=a.runs) + last_progress = time.monotonic() wall = time.time() - t0 stop.set(); th.join(timeout=1) lat_np = np.asarray(lat) @@ -151,7 +173,8 @@ def main() -> int: "fps_single_thread": a.runs / wall, "warmup": a.warmup, "runs": a.runs, "load_s": load_s, "bpu_util": (float(np.mean(ratios)) / 100.0) if ratios else None, "bpu_ratio_samples": len(ratios), } - print(json.dumps(out)) + print(json.dumps(out), flush=True) + progress('result_written') return 0 diff --git a/harness/tools/board_guard.py b/harness/tools/board_guard.py new file mode 100644 index 0000000..282b89c --- /dev/null +++ b/harness/tools/board_guard.py @@ -0,0 +1,155 @@ +"""Board-side lifetime owner. Stdlib only; runs outside the native runtime. + +stdin is a host lease, stdout is the child's result, stderr is live evidence. +EOF, lease expiry, signals and the existing RPC deadline reap the owned group. +The board lock stays held by the child too, even if this owner is killed. +""" +from __future__ import annotations + +import argparse +import ctypes +import fcntl +import json +import os +from pathlib import Path +import select +import signal +import subprocess +import sys +import time + +VERSION = 1 + + +def event(kind: str, **values): + print(json.dumps({'board_guard': VERSION, 'event': kind, **values}), file=sys.stderr, flush=True) + + +def competitors() -> list[int]: + result = [] + for p in Path('/proc').iterdir(): + if not p.name.isdigit() or int(p.name) == os.getpid(): + continue + try: + args = (p / 'cmdline').read_bytes().decode(errors='replace').split('\0') + if any(Path(a).name in ('board_bench.py', 'hrt_model_exec') for a in args): + result.append(int(p.name)) + except OSError: + continue + return result + + +def reap(child: subprocess.Popen, grace_s: float): + # Do not stop escalation just because the group leader already exited. + try: + os.killpg(child.pid, signal.SIGTERM) + except ProcessLookupError: + child.wait() + return + deadline = time.monotonic() + grace_s + while time.monotonic() < deadline: + child.poll() + try: + os.killpg(child.pid, 0) + except ProcessLookupError: + return + time.sleep(.05) + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.wait(timeout=5) + + +def supervise(command: list[str], directory: Path, timeout_s: float, + lease_s: float = 30, grace_s: float = 5, report_s: float = 30) -> int: + directory = directory.resolve() + owner_pid = os.getpid() + lock = open(directory.parent / '.board-inference.lock', 'a') + child = None + previous_handlers = {} + reason = 'exception' + rc = 1 + try: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + event('board_busy', reason='board lock held') + return 75 + others = competitors() + if others: + event('board_busy', reason='unmanaged inference present', pids=others) + return 75 + + def stop(sig, _frame): + nonlocal reason + reason = f'owner_signal_{sig}' + raise SystemExit(128 + sig) + + for sig in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + previous_handlers[sig] = signal.signal(sig, stop) + + def parent_death(): + # Linux arm64 board: kill even inside a native call if the guard dies. + libc = ctypes.CDLL(None, use_errno=True) + if libc.prctl(1, signal.SIGKILL, 0, 0, 0) != 0: # PR_SET_PDEATHSIG + os._exit(126) + if os.getppid() != owner_pid: + os._exit(126) + + t0 = last_lease = last_report = time.monotonic() + child = subprocess.Popen(command, stdin=subprocess.DEVNULL, start_new_session=True, + pass_fds=(lock.fileno(),), preexec_fn=parent_death, + env={**os.environ, 'BOARD_GUARD_VERSION': str(VERSION)}) + event('started', pid=child.pid, directory=str(directory), timeout_s=timeout_s) + while True: + rc = child.poll() + if rc is not None: + reason = 'child_exit' + return rc if rc >= 0 else 128 - rc + now = time.monotonic() + if now - t0 >= timeout_s: + reason = 'deadline' + print('__BENCH_WATCHDOG_STALLED__ board RPC deadline', file=sys.stderr, flush=True) + return 124 + if now - last_lease >= lease_s: + reason = 'lease_expired' + return 125 + ready, _, _ = select.select([sys.stdin.fileno()], [], [], min(.25, lease_s)) + if ready: + if not os.read(sys.stdin.fileno(), 4096): + reason = 'host_disconnected' + return 125 + last_lease = time.monotonic() + if now - last_report >= report_s: + # Transport liveness is not model progress. Keep the deadline independent. + event('alive', pid=child.pid, elapsed_s=round(now-t0, 1)) + last_report = now + finally: + if child is not None: + reap(child, grace_s) + receipt = {'guard_version': VERSION, 'pid': child.pid, 'reason': reason, + 'child_exit_code': child.returncode, 'finished_at': time.time()} + (directory / 'guard-result.json').write_text(json.dumps(receipt)) + event('reaped', **receipt) + lock.close() + for sig, handler in previous_handlers.items(): + signal.signal(sig, handler) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument('--directory', required=True) + ap.add_argument('--timeout', type=float, default=3600) + ap.add_argument('--lease', type=float, default=30) + ap.add_argument('--grace', type=float, default=5) + ap.add_argument('command', nargs=argparse.REMAINDER) + a = ap.parse_args() + command = a.command[1:] if a.command[:1] == ['--'] else a.command + if not command or min(a.timeout, a.lease, a.grace) <= 0: + ap.error('command and positive timeout/lease/grace required') + return supervise(command, Path(a.directory), a.timeout, a.lease, a.grace) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/harness/tools/board_perf.py b/harness/tools/board_perf.py index eb82882..e0eb8ad 100644 --- a/harness/tools/board_perf.py +++ b/harness/tools/board_perf.py @@ -2,13 +2,15 @@ 凭证只从本进程 env 取(BOARD_HOST/PORT/USER/SSH_KEY),密钥登录。 实测 RDK S100P 沙箱:无 hrt_model_exec,用 hbm_runtime python 包计时(见 board_bench.py)。 -板是共享的:开测前探测板上有没有别人的推理进程 / BPU ratio,最多等 --busy-wait-s,仍忙照测但标 contended。 +板是共享的:开测前探测推理进程 / BPU ratio,最多等 --busy-wait-s,仍忙拒绝测量。 +07/08 都通过 board_guard 管理远端进程:断连回收、板端互斥、独立 RPC 上限。 """ from __future__ import annotations import argparse import json import os +import selectors import shlex import subprocess import sys @@ -49,6 +51,56 @@ def result_json(text: str) -> dict: raise RuntimeError("board_bench produced no JSON:\n" + text[-2000:]) from e +def guarded_run(cmd: list[str], timeout: float) -> str: + """Stream board evidence to the host watchdog and renew a separate stdin lease. + + A killed host/SSH cannot leave the native process running indefinitely: the + board-side owner expires the lease and reaps its group independently. + """ + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = bytearray() + deadline = time.monotonic() + timeout + 15 # board deadline includes its cleanup acknowledgement + next_lease = 0.0 + try: + with selectors.DefaultSelector() as selector: + selector.register(proc.stdout, selectors.EVENT_READ, 'stdout') + selector.register(proc.stderr, selectors.EVENT_READ, 'stderr') + while selector.get_map(): + now = time.monotonic() + if now > deadline: + raise subprocess.TimeoutExpired(cmd, timeout) + if now >= next_lease and proc.poll() is None: + try: + proc.stdin.write(b'lease\n'); proc.stdin.flush() + except BrokenPipeError: + pass + next_lease = now + 5 + for key, _ in selector.select(timeout=.25): + data = os.read(key.fileobj.fileno(), 65536) + if not data: + selector.unregister(key.fileobj) + elif key.data == 'stderr': + sys.stderr.write(data.decode(errors='replace')); sys.stderr.flush() + else: + output.extend(data) + # Only the final JSON and a bounded diagnostic tail are needed. + if len(output) > 1024 * 1024: + del output[:-1024 * 1024] + rc = proc.wait(timeout=5) + if rc: + raise SystemExit(rc) + return output.decode(errors='replace') + finally: + proc.stdin.close() + if proc.poll() is None: + # EOF lets the board owner clean up before we tear down transport. + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill(); proc.wait() + proc.stdout.close(); proc.stderr.close() + + class RemoteSession: """一次评测拥有一个临时目录。HBM/脚本只传一次,退出时只清理自己的文件。""" @@ -58,6 +110,7 @@ class RemoteSession: self.scp_opts = ["-P" if o == "-p" else o for o in self.opts] self.directory = runtime_dir.rstrip("/") + "/eval-" + uuid.uuid4().hex self.active = False + self.failed_bench = False def __enter__(self): if self.active: @@ -67,6 +120,7 @@ class RemoteSession: self.execute(["mkdir", "-p", "--", self.directory]) self.upload(self.hbm, "model.hbm") self.upload(HERE / "board_bench.py", "board_bench.py") + self.upload(HERE / "board_guard.py", "board_guard.py") except BaseException: self.close() raise @@ -87,8 +141,15 @@ class RemoteSession: run(["scp", *self.scp_opts, "-q", f"{self.target}:{shlex.quote(self.path(name))}", str(local)]) def bench(self, args: list[str], timeout: int = 3600) -> dict: - return result_json(self.execute(["python3", self.path("board_bench.py"), "--hbm", - self.path("model.hbm"), *args], timeout=timeout)) + command = ["python3", "-u", self.path("board_guard.py"), "--directory", self.directory, + "--timeout", str(timeout), "--", "python3", "-u", self.path("board_bench.py"), + "--hbm", self.path("model.hbm"), *args] + print(f"board session: {self.directory}", file=sys.stderr, flush=True) + try: + return result_json(guarded_run(["ssh", *self.opts, self.target, shlex.join(command)], timeout)) + except BaseException: + self.failed_bench = True + raise def remove(self, *names: str): self.execute(["rm", "-f", "--", *(self.path(n) for n in names)]) @@ -96,6 +157,9 @@ class RemoteSession: def close(self): if self.active: self.active = False + if self.failed_bench: + print(f"board failed session retained for diagnosis: {self.directory}", file=sys.stderr, flush=True) + return try: self.execute(["rm", "-rf", "--", self.directory], timeout=60) except (Exception, SystemExit) as e: @@ -131,7 +195,8 @@ def main() -> int: print(f"board busy: waiting ({waited}/{a.busy_wait_s}s, procs={others}, ratio={ratio})", flush=True) time.sleep(20); waited += 20 if contended: - print(f"WARN board busy after {waited}s (procs={others}, ratio={ratio}); measuring anyway, flag contended", file=sys.stderr) + # A host flock cannot prove that a previous SSH child exited on the board. + raise RuntimeError(f"board busy after {waited}s (procs={others}, ratio={ratio}); refusing contended measurement") with RemoteSession(a.hbm, a.runtime_dir) as session: d = session.bench(["--warmup", str(a.warmup), "--runs", str(a.runs)]) diff --git a/harness/tools/export_onnx.py b/harness/tools/export_onnx.py index 7c77c09..5a73fdb 100644 --- a/harness/tools/export_onnx.py +++ b/harness/tools/export_onnx.py @@ -14,12 +14,36 @@ OPSET = 11 # 实测 OE 3.7 接受 opset 11 EXPORTER_VERSION = "2" # 导出口径版本:变了缓存作废。v2 = ultralytics 走官方 exporter;torchvision 走 TorchScript exporter + arch 名映射 +def exporter_version(model: str) -> str: + from ..adapters import get + if version := getattr(get(model), "EXPORTER_VERSION", None): + return version + # These models previously failed before producing an ONNX. Do not invalidate + # successful exports for the running classification/detection jobs. + return "3-attention" if model.startswith("vit.") or model == "maxvit.t" else EXPORTER_VERSION + + +def prepare_attention_export(model): + """Use explicit attention math instead of unexportable fused inference ops.""" + import torch + torch.backends.mha.set_fastpath_enabled(False) + def with_attention_weights(forward): + def explicit(*args, **kwargs): + kwargs["need_weights"] = True + return forward(*args, **kwargs) + return explicit + for module in model.modules(): + if isinstance(module, torch.nn.MultiheadAttention): + module.forward = with_attention_weights(module.forward) + + def load_model(model: str, source: str, weights: Path, input_spec: str): import torch if source == "torchvision": import torchvision.models as tvm arch = model.replace(".", "_") # model_id 里 '.' 是 builder 名的 '_'(job_id 段不允许下划线以外的分隔) - m = tvm.get_model(arch, weights=None) + kwargs = {"image_size": int(input_spec.split("x")[-1])} if arch.startswith("vit_") else {} + m = tvm.get_model(arch, weights=None, **kwargs) m.load_state_dict(torch.load(weights, map_location="cpu")) return m.eval() if source in ("ultralytics", "github") and weights.suffix == ".pt": @@ -63,6 +87,15 @@ def main() -> int: print(out) return 0 m = load_model(a.model, a.source, Path(a.weights), a.input_spec) + from ..adapters import get + if a.opset == OPSET: + a.opset = getattr(get(a.model), "OPSET", OPSET) + if a.model.startswith("vit."): + prepare_attention_export(m) + if a.opset == OPSET: + a.opset = 13 # Explicit MHA includes aten::unflatten (introduced in 13). + if a.model == "maxvit.t" and a.opset == OPSET: + a.opset = 12 # aten::einsum requires >=12; all other exports keep opset 11. x = torch.randn(*shape) kw = dict(opset_version=a.opset, input_names=["input"], do_constant_folding=True, dynamic_axes=None) with torch.no_grad(): diff --git a/harness/verify.py b/harness/verify.py index ff90556..5e9c4bd 100644 --- a/harness/verify.py +++ b/harness/verify.py @@ -1,7 +1,7 @@ """L3:对一次 rollout 独立核验 → verdict.json。reward 的唯一来源。 输入:rollout/stages/*/meta.json(harness 自己跑的,agent 改不了)、claim.json(agent 自报,不可信)、 - oracle/result.json(参照系)、agent/events.jsonl(成本)。 + job.json(冻结的任务门槛)、agent/events.jsonl(成本)。 不读 workspace 里 agent 写的任何"结果"。 """ from __future__ import annotations @@ -12,7 +12,19 @@ from collections import Counter from pathlib import Path from . import db, gates -from .errors import same_class +VERDICT_SCHEMA = "bench.verdict/3" +VERIFICATION_RULE = "independent-rollout-gates-v1" + + +def eval_set(job: dict) -> dict | None: + """评测口径来自本次任务快照。""" + data = job.get("data") or {} + t1 = data.get("t1") or {} + if not t1: + return None + return {"dataset": data.get("dataset"), "n": t1.get("n"), + "content_sha": t1.get("content_sha"), "manifest_sha": t1.get("manifest_sha"), + "evaluator_version": job.get("evaluator_version"), "eval_set_id": t1.get("eval_set_id")} def _latest_meta(stage_root: Path, name: str) -> dict | None: @@ -25,7 +37,7 @@ def collect_measurements(rd: Path, stage_root: Path | None = None) -> tuple[dict rollout 传 rollout 目录;oracle 传 stage_root=…/oracle/stages。06-hbm 的 G1 若 meta 里没有,回退读它的 stdout.log(就是那份 JSON)。""" root = stage_root or (rd / "stages") meas, summ = {}, {} - for name in ("04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval"): + for name in ("01-fetch", "02-baseline", "03-export", "04-calibrate", "05-compile", "06-hbm", "07-board", "08-eval"): m = _latest_meta(root, name) if not m: continue @@ -119,7 +131,6 @@ def verify_rollout(rd: Path, write: bool = True) -> dict: jd = rd.parent.parent job = json.loads((jd / "job.json").read_text()) kk = int(rd.name) - oracle = json.loads((jd / "oracle" / "result.json").read_text()) if (jd / "oracle" / "result.json").exists() else None claim = json.loads((rd / "claim.json").read_text()) if (rd / "claim.json").exists() else None rmeta = json.loads((rd / "rollout.json").read_text()) if (rd / "rollout.json").exists() else {} @@ -135,13 +146,9 @@ def verify_rollout(rd: Path, write: bool = True) -> dict: terminal = "FAIL" # TIMEOUT 是执行终止原因,不能抹掉独立测量的 PASS/FAIL/UNVERIFIED。 - blocked_correct = None + measured_terminal = terminal if terminal != "PASS" and claim and claim.get("terminal") == "BLOCKED": terminal = "BLOCKED" - o_ec = (oracle or {}).get("error_class") - o_stage = (oracle or {}).get("stage_failed") - blocked_correct = bool(oracle) and oracle.get("terminal") == "FAIL" and \ - same_class(claim.get("error_class"), o_ec) and (claim.get("stage") in (None, o_stage)) elif terminal == "UNVERIFIED" and (claim or {}).get("terminal") == "PASS": terminal = "FAIL" # 说通过但没有可核验的 hbm/eval → 按 FAIL 记,claim≠verdict 另扣 @@ -149,33 +156,28 @@ def verify_rollout(rd: Path, write: bool = True) -> dict: if claim: claim_consistent = (claim.get("terminal") == terminal) if terminal in ("PASS", "FAIL", "BLOCKED") else None - o_g3 = ((oracle or {}).get("gates") or {}).get("G3") or {} - o_g4 = ((oracle or {}).get("gates") or {}).get("G4") or {} - review = bool(oracle) and oracle.get("terminal") == "FAIL" and terminal == "PASS" - v_ec = gates.classify_gate_failure(ev) if terminal == "FAIL" else None for st in summ.values(): if st.get("state") not in ("OK", "SKIPPED") and st.get("error_class"): v_ec = st["error_class"]; break verdict = { - "schema": "bench.verdict/2", + "schema": VERDICT_SCHEMA, "verification_rule": VERIFICATION_RULE, "error_class": v_ec, "job_id": job["job_id"], "run_id": job["run_id"], "rollout": kk, + "eval_set": eval_set(job), + "continuation": job.get("continuation"), "mode": rmeta.get("mode"), "policy": rmeta.get("policy"), "gates": ev["gates"], "required": ev["required"], - "terminal": terminal, + "terminal": terminal, "measured_terminal": measured_terminal, "execution_terminal": rmeta.get("state"), "timeout_reason": rmeta.get("timeout_reason"), "completion": completion_summary(ev["gates"], summ), "failed_gate": ev["failed_gate"], "stages": summ, "claim": claim, "claim_consistent": claim_consistent, - "blocked_correct": blocked_correct, - "oracle_ref": {"terminal": (oracle or {}).get("terminal"), "error_class": (oracle or {}).get("error_class"), - "stage_failed": (oracle or {}).get("stage_failed"), - "G3_delta": o_g3.get("delta"), "G3_kind": o_g3.get("kind"), - "latency_ms_mean": o_g4.get("latency_ms_mean")} if oracle else None, - "review": review, + # BLOCKED is a self-report, not proof that the task is impossible. + "blocked_correct": None, + "review": False, "cost": _cost(rd), "verified_at": time.time(), } @@ -186,7 +188,7 @@ def verify_rollout(rd: Path, write: bool = True) -> dict: db.rollout_set(con, job["run_id"], job["job_id"], kk, state=terminal, claim_terminal=(claim or {}).get("terminal"), verdict_terminal=terminal, claim_consistent=None if claim_consistent is None else int(claim_consistent), - review=int(review), turns=verdict["cost"]["turns"], tokens=verdict["cost"]["tokens"], + review=0, turns=verdict["cost"]["turns"], tokens=verdict["cost"]["tokens"], wall_s=verdict["cost"]["wall_s"], verdict=json.dumps(verdict, default=str), finished_at=rmeta.get("finished_at") or time.time()) return verdict diff --git a/harness/watchdog.py b/harness/watchdog.py index 62cc50d..1e3ce89 100644 --- a/harness/watchdog.py +++ b/harness/watchdog.py @@ -1,7 +1,8 @@ """无固定超时的子进程执行器。 -STALLED 的唯一定义:stdout+stderr 文件 stall_s 秒不增长 且 进程树 CPU 使用率 < cpu_idle_pct%。 -真正在算的编译,跑 6 小时也不会被杀。hard_ceiling_s 是纯保险。 +STALLED 的定义:stdout+stderr 文件 stall_s 秒不增长 且 进程树 CPU < cpu_idle_pct% 且 树内无 D 状态线程。 +树内有 D 线程时算"卡在 I/O",走更宽松的 io_stall_s。两者会互相清零,所以另有 no_progress_ceiling_s +这条不受抖动影响的兜底。真正在算(或在写盘)的编译不会被杀;hard_ceiling_s 是最后保险。 """ from __future__ import annotations @@ -25,11 +26,38 @@ class RunResult: stdout: Path stderr: Path stalled_reason: str | None = None - io_blocked_s: float = 0.0 # 期间最长的连续 I/O 阻塞(D 状态)时长,进 meta.json 供诊断 + io_blocked_s: float = 0.0 # 期间最长的连续 I/O 阻塞(树内 D 线程)时长,进 meta.json 供诊断 + + +def _d_threads(pdir: Path, leader_state: str | None) -> int: + """该进程处于 D 状态的**线程**数。 + + 只读 /proc//stat 拿到的是线程组长的状态。实测(2026-09-17,1001 次 ld.lld 观测)组长是 + S(futex_wait),但 17 个线程里有 6~16 个 D 在 wait_on_page_bit_common / balance_dirty_pages / + fuse_wait_on_page_writeback(都是 FUSE 回写族)。59.7% 的观测是"组长口径 0、线程口径 >0", + 按组长判定会把正在写盘的链接期当成挂死。task/ 含组长自身,是原进程级判定的严格超集。 + + 只数 D,不要把 futex 等待也算豁免:每个 hb_compile 常驻 40+ 个 futex_wait 线程(空闲线程池), + 真挂死时形态完全一样,算进来等于让 stall 判定永久失效。 + """ + try: + n = 0 + for t in (pdir / "task").iterdir(): + try: + # 线程比进程短命得多:读到一半消失会给出空串/截断串,IndexError 必须接住, + # 否则异常会冲出轮询循环,_kill_tree 不执行、子进程变孤儿。 + ts = (t / "stat").read_text() + if ts[ts.rfind(")") + 2:].split()[0] == "D": + n += 1 + except (OSError, IndexError): + continue + return n + except OSError: + return 1 if leader_state == "D" else 0 def _proc_tree_stat(root: int) -> tuple[int, int] | None: - """Linux: (root 及其全部后代的 utime+stime 总和 ticks, 处于 D 状态的进程数)。非 Linux 返回 None。 + """Linux: (root 及其全部后代的 utime+stime 总和 ticks, 树内处于 D 状态的线程数)。非 Linux 返回 None。 D = 不可中断睡眠 = 卡在 I/O。实测(2026-09-15)JuiceFS 写 1 MB 要 360 ms(本地盘 3 ms), hbdk 编译大模型时会长时间待在 D 里:CPU ticks 不涨、日志不涨 —— 与"真挂死"在表象上完全一样。 @@ -45,22 +73,24 @@ def _proc_tree_stat(root: int) -> tuple[int, int] | None: if not p.name.isdigit(): continue try: + # 解析也放进 try:进程随时会消失,read_text 可能返回空串/截断串, + # 那时切片与下标会抛 IndexError/ValueError。异常冲出去会跳过 _kill_tree + # (start_new_session=True),把子进程留成孤儿。 stat = (p / "stat").read_text() - except OSError: + # comm 可能含空格/括号:从最后一个 ')' 之后切 + rest = stat[stat.rfind(")") + 2:].split() + pid = int(p.name) + state[pid] = rest[0] + ppid = int(rest[1]) + ticks[pid] = int(rest[11]) + int(rest[12]) + except (OSError, IndexError, ValueError): continue - # comm 可能含空格/括号:从最后一个 ')' 之后切 - rest = stat[stat.rfind(")") + 2:].split() - pid = int(p.name) - state[pid] = rest[0] - ppid = int(rest[1]) - ticks[pid] = int(rest[11]) + int(rest[12]) children.setdefault(ppid, []).append(pid) total, d_count, stack = 0, 0, [root] while stack: pid = stack.pop() total += ticks.get(pid, 0) - if state.get(pid) == "D": - d_count += 1 + d_count += _d_threads(proc / str(pid), state.get(pid)) stack.extend(children.get(pid, [])) return total, d_count @@ -82,7 +112,11 @@ def _kill_tree(proc: subprocess.Popen) -> None: def run(cmd: list[str] | str, *, cwd: Path, stdout: Path, stderr: Path, env: dict | None = None, stall_s: int = 900, cpu_idle_pct: float = 5.0, poll_s: int = 30, - hard_ceiling_s: int = 86400, shell: bool = False, io_stall_s: int = 21600) -> RunResult: + hard_ceiling_s: int = 86400, shell: bool = False, io_stall_s: int = 21600, + no_progress_ceiling_s: int | None = None) -> RunResult: + """no_progress_ceiling_s 默认关闭:run() 还有 rollout.py / s06_hbm.py 四个调用点, + 其中 rollout.py 刻意把 stall_s 设成 hard_ceiling_s(不 stall-kill pi agent)。 + 改默认值会连带改掉它们的杀进程行为,所以新兜底只由显式传参的调用点开启。""" stdout.parent.mkdir(parents=True, exist_ok=True) t0 = time.time() with open(stdout, "ab") as fo, open(stderr, "ab") as fe: @@ -93,6 +127,7 @@ def run(cmd: list[str] | str, *, cwd: Path, stdout: Path, stderr: Path, env: dic last_ticks = st[0] if st else None stalled_for = io_blocked_for = max_io_blocked = 0.0 stalled_reason = None + last_progress = t0 while True: rc = proc.poll() if rc is not None: @@ -121,6 +156,7 @@ def run(cmd: list[str] | str, *, cwd: Path, stdout: Path, stderr: Path, env: dic io_blocked_for = 0.0 else: stalled_for = io_blocked_for = 0.0 + last_progress = now last_size = size if stalled_for >= stall_s: stalled_reason = f"log unchanged and cpu idle for {int(stalled_for)}s" @@ -128,6 +164,13 @@ def run(cmd: list[str] | str, *, cwd: Path, stdout: Path, stderr: Path, env: dic if io_blocked_for >= io_stall_s: stalled_reason = f"blocked on I/O (D state) for {int(io_blocked_for)}s" break + # stalled_for 与 io_blocked_for 会互相清零:d_count 在两次轮询间抖动时两个阈值都到不了 + # (仿真:P(d>0) 落在 0.02~0.99 就会塌到 24h hard_ceiling)。这条只看"距上次日志增长或 + # CPU 活动",免疫 D 抖动。注意它不免疫"挂死但有线程空转 >cpu_idle_pct"——那种只剩 + # hard_ceiling 兜底。数值必须 > io_stall_s,否则它会抢先命中、盖掉更准确的原因。 + if no_progress_ceiling_s and now - last_progress >= no_progress_ceiling_s: + stalled_reason = f"no log/cpu progress for {int(now - last_progress)}s" + break if stalled_reason: _kill_tree(proc) fe.write(f"\n{STALLED_MARK} {stalled_reason}\n".encode()) diff --git a/scripts/_probe.py b/scripts/_probe.py deleted file mode 100644 index daa6da1..0000000 --- a/scripts/_probe.py +++ /dev/null @@ -1,24 +0,0 @@ -"""一次性诊断:轨迹到底有没有、agent 侧依赖齐不齐。scp 到机器上跑,不进 git 主流程。""" -import glob -import os -import shutil -import sqlite3 - -DB = "/home/ming01.wang/bench-state/jobs.sqlite" -R = "/mnt/data/ming01.wang/model-compile-hbdk/bench" - -c = sqlite3.connect(DB) -c.execute("PRAGMA busy_timeout=30000") -print("rollouts by state:", dict(c.execute("select state,count(*) from rollouts group by 1").fetchall())) -print("rollouts executed (turns not null):", c.execute("select count(*) from rollouts where turns is not null").fetchone()[0]) -print("verdict rows:", c.execute("select count(*) from rollouts where verdict is not null").fetchone()[0]) -print("jobs by state:", dict(c.execute("select state,count(*) from jobs group by 1").fetchall())) -print("datasets/:", os.listdir(R + "/datasets") if os.path.isdir(R + "/datasets") else "-") -print("rollout dirs on disk:", len(glob.glob(R + "/runs/*/*/rollouts/*"))) -print("events.jsonl on disk:", len(glob.glob(R + "/runs/*/*/rollouts/*/agent/events.jsonl"))) -for b in ("pi", "node", "npm", "npx"): - print(f" {b} -> {shutil.which(b) or 'NOT FOUND'}") -od = os.path.expanduser("~/.opencode/bin") -print("~/.opencode/bin:", os.listdir(od) if os.path.isdir(od) else "-") -pi = os.path.expanduser("~/.pi") -print("~/.pi:", os.listdir(pi) if os.path.isdir(pi) else "NOT FOUND") diff --git a/scripts/audit_rollouts.py b/scripts/audit_rollouts.py index 67e2b05..f807d81 100644 --- a/scripts/audit_rollouts.py +++ b/scripts/audit_rollouts.py @@ -15,8 +15,8 @@ "成功读到"= 该次工具调用的结果里没有 path-guard 拒绝语;被 guard 拦下的尝试单独计入 blocked_attempts(不算污染)。 结论列: - RL ✓ = 可进 GRPO 组(干净 + 版本合格 + 不是被误杀) - SFT ✓ = RL ✓ 且 verdict=PASS 且 claim 一致 + RL ✓ = 初步候选(独立执行 + 结果可核验 + 版本合格 + 不是被误杀) + SFT ✓ = RL ✓ 且 verdict=PASS 且 claim 一致;完整性以导出器为准 """ from __future__ import annotations @@ -174,6 +174,8 @@ def audit_one(rd: Path, kk: int, min_version: int, mode: str) -> dict: reasons.append("stalled_killed") if ver < min_version: reasons.append(f"old_version(v{ver}<{min_version})") + if mode == "repair": + reasons.append("historical-oracle-repair-analysis-only") terminal = vj.get("terminal") or rj.get("state") claim_t = (cj or {}).get("terminal") or (vj.get("claim") or {}).get("terminal") cc = vj.get("claim_consistent") @@ -183,7 +185,8 @@ def audit_one(rd: Path, kk: int, min_version: int, mode: str) -> dict: reasons.append("claim_mismatch") contaminated = bool((mode != "repair" and s["oracle_read"]) or s["sibling_read"] or s["state_access"] or s["other_job_read"] or (mode != "repair" and s["oracle_content_match"])) running = rj.get("state") in ("RUNNING", "VERIFYING") or not s["has_agent_end"] and rj.get("state") == "RUNNING" - rl_ok = (not contaminated) and (not stalled) and ver >= min_version and not running + rl_ok = (not contaminated) and (not stalled) and ver >= min_version and not running \ + and mode != "repair" and terminal in ("PASS", "FAIL") sft_ok = rl_ok and terminal == "PASS" and cc is True return {"kk": kk, "version": ver, "state": rj.get("state"), "terminal": terminal, "claim": claim_t, "claim_consistent": cc, "turns": s["turns"], "tool_calls": s["tool_calls"], "blocked": s["blocked_attempts"], diff --git a/scripts/gate_summary.py b/scripts/gate_summary.py deleted file mode 100644 index d4690ab..0000000 --- a/scripts/gate_summary.py +++ /dev/null @@ -1,49 +0,0 @@ -"""每个已结束 job 的四道门一行摘要(看 FAIL 到底败在哪一门、差多少)。 - - python3 scripts/gate_summary.py [--run r…] [--state FAIL] -""" -from __future__ import annotations - -import argparse -import json -import sqlite3 -import sys -from collections import Counter -from pathlib import Path - -BENCH = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(BENCH)) -from harness import paths # noqa: E402 - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--run", action="append") - ap.add_argument("--state", default=None) - a = ap.parse_args() - con = sqlite3.connect(paths.DB_PATH) - runs = a.run or [con.execute("select run_id from runs order by created_at desc limit 1").fetchone()[0]] - q = "select run_id, job_id, state, error_class, stage_failed, result from jobs where mode='oracle' and run_id in (%s)" % ",".join("?" * len(runs)) - args = list(runs) - if a.state: - q += " and state=?"; args.append(a.state) - q += " and state in ('PASS','FAIL','UNVERIFIED') order by run_id, job_id" - fg = Counter() - for rid, jid, st, ec, sf, res in con.execute(q, args): - r = json.loads(res) if res else {} - g = r.get("gates", {}) - g1, g2, g3, g4 = (g.get(k, {}) for k in ("G1", "G2", "G3", "G4")) - fg[(st, r.get("failed_gate") or (sf and f"stage:{sf}") or "-")] += 1 - f1 = f"/{','.join(g1.get('failed', []))}" if g1.get("failed") else (f"/miss:{','.join(g1.get('missing', []))}" if g1.get("missing") else "") - cos = g2.get("cos_min") - print(f"{rid} {jid:<46} {st:<10} fg={r.get('failed_gate') or '-':<3} ec={ec or '-':<14} " - f"G1={g1.get('verdict', '-')}{f1} G2={g2.get('verdict', '-')}{'' if cos is None else f'({cos:.4f})'} " - f"G3={g3.get('verdict', '-')}" - + ("" if g3.get("fp32") is None else f"(fp32={g3['fp32']:.2f} q={g3['quant']:.2f} d={g3['delta']:+.2f} tol={g3.get('effective_tolerance', 0):.2f})") - + (f" lat_est={g1.get('static_perf', {}).get('latency_ms_est')}ms" if g1.get("static_perf", {}).get("latency_ms_est") else "")) - print("\nsummary:", dict(fg)) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/jail_setup.py b/scripts/jail_setup.py index c2892fc..4569902 100644 --- a/scripts/jail_setup.py +++ b/scripts/jail_setup.py @@ -1,5 +1,6 @@ """Mount construction runs before untrusted code, then drops every capability.""" import os +import json import sys import subprocess import tempfile @@ -47,7 +48,9 @@ def main(): save(jd / 'job.json') save(jd / 'data') if mode == 'repair': - save(jd / 'oracle') + ref = json.loads((jd / 'job.json').read_text()).get('oracle_ref') + source = bench / 'runs' / ref['run_id'] / ref['job_id'] / 'oracle' if ref else jd / 'oracle' + save(source, jd / 'oracle') # historical evidence, same read-only sandbox path save(rd) save(rd / 'workspace', writable=True) save(rd / 'handles', writable=True) diff --git a/scripts/make_calib.py b/scripts/make_calib.py index 2223e5b..cdec236 100644 --- a/scripts/make_calib.py +++ b/scripts/make_calib.py @@ -79,6 +79,7 @@ def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--task", required=True) ap.add_argument("--input-spec", required=True) + ap.add_argument("--model", help="Use this model's official input recipe in a separate calibration directory") ap.add_argument("--n", type=int, default=128) ap.add_argument("--from", dest="src", choices=["auto", "train", "val-holdout"], default="auto") ap.add_argument("--seed", type=int, default=20260914) @@ -87,15 +88,47 @@ def main() -> int: from PIL import Image from harness.eval.common import preprocess_cls, letterbox - ds = TASK_DATASET[a.task] + from harness.adapters import calibration_dir + from harness.registry import model_row + row = model_row(a.model) if a.model else {"task": a.task, "input_spec": a.input_spec, "model_id": ""} + if a.model and (row["task"] != a.task or row["input_spec"] != a.input_spec): + raise SystemExit("--task/--input-spec must match the model recipe") + ds = row.get("dataset") or TASK_DATASET[a.task] d = REG["datasets"][ds] ccfg = d.get("calib") or {} pool = POOL / ds / d.get("version", "v1") shape = tuple(int(x) for x in a.input_spec.split("x")) rng = random.Random(a.seed) + out = calibration_dir(row) + if a.model and out.exists(): + manifest = json.loads((out / "manifest.json").read_text()) + files = sorted((out / manifest["npy_dir"]).glob("*.npy")) + digest = hashlib.sha256() + for path in files: + digest.update(path.read_bytes()) + expected_recipe = row.get("preprocess") or "harness.eval.common (same as evaluator)" + if (manifest.get("count") != a.n or len(files) != a.n or manifest.get("seed") != a.seed + or json.dumps(manifest.get("preprocess"), sort_keys=True) != json.dumps(expected_recipe, sort_keys=True) + or manifest.get("sha256") != digest.hexdigest() + or (a.src == "train" and not manifest.get("source", "").startswith("train"))): + raise ValueError(f"existing model calibration differs; refusing to overwrite {out}") + print(f"{out} n={a.n} sha={digest.hexdigest()[:12]} (reused)") + return 0 imgs, desc, source = [], [], None if a.src in ("auto", "train"): + if ds == "cityscapes": + import zipfile + archive = LOCAL / d["sources"][0]["path"] + with zipfile.ZipFile(archive) as source_zip: + names = sorted(name for name in source_zip.namelist() + if name.startswith("leftImg8bit/train/") and name.endswith("_leftImg8bit.png")) + if len(names) < a.n: + raise ValueError(f"only {len(names)} Cityscapes training images") + rng.shuffle(names) + desc = [f"{archive}#{name}" for name in sorted(names[:a.n])] + imgs = [Image.open(io.BytesIO(source_zip.read(name))).convert("RGB") for name in sorted(names[:a.n])] + source = "train-zip" if ccfg.get("train_parquet") and (LOCAL / ccfg["train_parquet"]).exists(): try: imgs, desc = from_train_parquet(ccfg, a.n, a.seed); source = "train-parquet" @@ -122,7 +155,6 @@ def main() -> int: imgs = [Image.open(p).convert("RGB") for p in files]; desc = [str(p.relative_to(pool)) for p in files]; source = "val-holdout" print(f"WARN: {ds} 没有 train 来源,用 val-holdout(排除 T1 的 {len(excl)} 张)。出 T2 交付数前须换 train。", file=sys.stderr) - out = BENCH / "data" / "calib" / a.task / a.input_spec npy_dir = out / "npy_f32" # hb_compile 读 cal_data_dir 下的全部文件 → manifest.json 不能放同一目录;_f32 后缀是 OE 约定 if out.exists(): import shutil; shutil.rmtree(out) @@ -130,7 +162,13 @@ def main() -> int: h = hashlib.sha256() for i, img in enumerate(imgs[:a.n]): if a.task == "cls": - x = preprocess_cls(img, size=shape[-1], resize=round(shape[-1] * 256 / 224)) + x = preprocess_cls(img, size=shape[-1], resize=round(shape[-1] * 256 / 224), model=a.model or "") + elif a.task in ("seg", "det", "mde", "pose") and a.model: + from harness.adapters import get + transform = getattr(get(a.model), "preprocess", None) + if a.task in ("seg", "mde") and transform is None: + raise ValueError(f"no {a.task} preprocessing recipe for {a.model}") + x = transform(img, a.input_spec) if transform else letterbox(img, shape[-1])[0] else: x = letterbox(img, shape[-1])[0] assert x.shape == shape, (x.shape, shape) @@ -140,7 +178,8 @@ def main() -> int: (out / "manifest.json").write_text(json.dumps({ "task": a.task, "input_spec": a.input_spec, "count": len(imgs[:a.n]), "seed": a.seed, "source": source, "dataset": ds, "sha256": h.hexdigest(), "files": desc[:a.n], - "preprocess": "harness.eval.common (same as evaluator)", "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}, indent=2)) + "preprocess": row.get("preprocess") or "harness.eval.common (same as evaluator)", + "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}, indent=2)) print(f"{out} n={len(imgs[:a.n])} source={source} sha={h.hexdigest()[:12]}") return 0 diff --git a/scripts/make_t1.py b/scripts/make_t1.py index c4be7b0..3af1060 100644 --- a/scripts/make_t1.py +++ b/scripts/make_t1.py @@ -1,6 +1,10 @@ """T1 smoke 子集:从 pool 固定 seed 抽样 → data/t1//.tar(单包,避开 JuiceFS 小文件)。 python scripts/make_t1.py --dataset imagenet1k [--dataset coco2017] + python scripts/make_t1.py --dataset imagenet1k --n 100 --seed 20260914 + +显式指定 n/seed 时默认写入独立的 n-s/ 目录;--out-dir 可指定该目录。 +不传覆盖参数时仍使用 registry 的规格及旧路径。 tar 内:manifest.json(items 清单 + 内容 sha)、images/…、(det/iseg/pose)annotations.json(COCO 子集)。 tar 确定性:条目按名排序、mtime=0、uid/gid=0 → 同一 pool 同一 seed 产出同一 sha。 @@ -34,26 +38,45 @@ def _add(tar: tarfile.TarFile, name: str, data: bytes) -> None: def items_imagenet(pool: Path, n: int, seed: int, stratify: str) -> list[dict]: labels = json.loads((pool / "labels.json").read_text()) + _check_count(n, len(labels)) by = defaultdict(list) for fn, lab in labels.items(): by[lab].append(fn) rng = random.Random(seed) items = [] if stratify == "label": - per = max(1, n // len(by)) + per = n // len(by) + remaining = {} for lab in sorted(by): fs = sorted(by[lab]); rng.shuffle(fs) items += [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab} for f in fs[:per]] + remaining[lab] = fs[per:] + # 不足一轮时随机选择类别;余数及样本不足的类别也按随机轮次补齐。 + # n 可整除类别数且各类充足时,保持旧 2000 张的选样和顺序。 + while len(items) < n: + labs = [lab for lab, fs in remaining.items() if fs] + rng.shuffle(labs) + for lab in labs[:n - len(items)]: + f = remaining[lab].pop() + items.append({"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": lab}) else: fs = sorted(labels); rng.shuffle(fs) items = [{"id": f, "file": f"images/{f}", "src": f"images/{f}", "label": labels[f]} for f in fs[:n]] - return items[:n] + return items + + +def _check_count(n: int, available: int) -> None: + if not isinstance(n, int) or isinstance(n, bool) or n < 1: + raise ValueError("n must be a positive integer") + if n > available: + raise ValueError(f"requested {n} samples, only {available} available") def items_coco(pool: Path, n: int, seed: int) -> tuple[list[dict], dict]: ann_p = next(pool.rglob("instances_val2017.json")) ann = json.loads(ann_p.read_text()) imgs = sorted(ann["images"], key=lambda x: x["id"]) + _check_count(n, len(imgs)) rng = random.Random(seed); rng.shuffle(imgs) sel = sorted(imgs[:n], key=lambda x: x["id"]) ids = {im["id"] for im in sel} @@ -70,28 +93,82 @@ def items_coco(pool: Path, n: int, seed: int) -> tuple[list[dict], dict]: return items, {"annotations": sub, "cat_map": cat_map} -def build(ds: str) -> Path: +def items_segmentation(pool: Path, ds: str, n: int, seed: int) -> list[dict]: + """Keep image/label pairs together; never sample PNG labels as input images.""" + if ds == "cityscapes": + files = sorted(pool.glob("leftImg8bit/val/*/*_leftImg8bit.png")) + pairs = [(p, pool / "gtFine/val" / p.parent.name / p.name.replace("_leftImg8bit.png", "_gtFine_labelIds.png")) + for p in files] + else: + pairs = [(p, pool / "annotations/validation" / (p.stem + ".png")) + for p in sorted((pool / "images/validation").glob("*.jpg"))] + _check_count(n, len(pairs)) + random.Random(seed).shuffle(pairs) + items = [] + for image, label in sorted(pairs[:n]): + if not label.is_file(): + raise FileNotFoundError(f"missing segmentation label: {label}") + src, label_src = str(image.relative_to(pool)), str(label.relative_to(pool)) + items.append({"id": image.stem, "file": "images/" + image.name, "src": src, + "label_file": "labels/" + label.name, "label_src": label_src, + "label_format": ds + "_labelIds"}) + return items + + +def build(ds: str, *, n: int | None = None, seed: int | None = None, + out_dir: Path | None = None) -> Path: d = REG["datasets"][ds] ver = d.get("version", "v1") pool = POOL / ds / ver pm = json.loads((pool / "manifest.json").read_text()) t1 = d.get("t1") or {} - n, seed = t1.get("n"), t1.get("seed", 20260914) + custom = n is not None or seed is not None or out_dir is not None + if n is not None: + _check_count(n, n) + n = t1.get("n") if n is None else n + seed = t1.get("seed", 20260914) if seed is None else seed + legacy_dir = T1 / ds + destination = Path(out_dir) if out_dir is not None else ( + legacy_dir / f"n{n}-s{seed}" if custom else legacy_dir) + if custom and destination.resolve() == legacy_dir.resolve(): + raise ValueError("custom subsets require a separate output directory; preserve the legacy T1") extra = {} if ds == "imagenet1k": items = items_imagenet(pool, n, seed, t1.get("stratify", "label")) elif ds == "coco2017": items, extra = items_coco(pool, n, seed) + elif ds in ("cityscapes", "ade20k"): + items = items_segmentation(pool, ds, n, seed) else: # 通用:pool 下全部文件(n=all)或随机 n 个;标注交给各任务评测器按 src 定位 files = sorted(str(p.relative_to(pool)) for p in pool.rglob("*") if p.is_file() and p.name != "manifest.json") if n not in (None, "all", 0): + _check_count(n, len(files)) random.Random(seed).shuffle(files); files = sorted(files[:n]) items = [{"id": f, "file": f"files/{f}", "src": f} for f in files] content_sha = hashlib.sha256(json.dumps([i["src"] for i in items]).encode() + pm["sha256"].encode()).hexdigest() man = {"dataset": ds, "version": ver, "seed": seed, "n": len(items), "pool_sha256": pm["sha256"], "sha256": content_sha, "items": items, **({"cat_map": extra["cat_map"]} if "cat_map" in extra else {})} - out = T1 / ds / f"{seed}.tar" + identity = {"eval_set_id": f"{ds}-n{len(items)}-{content_sha[:12]}"} if custom else {} + man.update(identity) + out = destination / f"{seed}.tar" + manifest_path = destination / "manifest.json" + if custom and manifest_path.exists(): + previous = json.loads(manifest_path.read_text()) + if previous.get("content_sha256") != content_sha or previous.get("seed") != seed: + raise FileExistsError(f"{destination} already contains a different subset; choose a new directory") + if (previous.get("tar") != out.name or previous.get("dataset") != ds + or previous.get("n") != len(items)): + raise ValueError(f"existing subset manifest is inconsistent: {manifest_path}") + if not out.is_file(): + raise ValueError(f"existing subset package is missing: {out}") + fsha = hashlib.sha256(out.read_bytes()).hexdigest() + if fsha != previous.get("sha256"): + raise ValueError(f"existing subset package hash mismatch: {out}") + print(f"{ds}: {out} n={len(items)} sha={fsha[:12]} (reused)") + return out + if custom and out.exists(): + raise FileExistsError(f"subset package exists without its manifest: {out}") out.parent.mkdir(parents=True, exist_ok=True) with tarfile.open(out, "w") as tar: _add(tar, "manifest.json", json.dumps(man, ensure_ascii=False).encode()) @@ -99,10 +176,12 @@ def build(ds: str) -> Path: _add(tar, "annotations.json", json.dumps(extra["annotations"]).encode()) for it in items: _add(tar, it["file"], (pool / it["src"]).read_bytes()) + if "label_file" in it: + _add(tar, it["label_file"], (pool / it["label_src"]).read_bytes()) fsha = hashlib.sha256(out.read_bytes()).hexdigest() - (out.parent / "manifest.json").write_text(json.dumps({"dataset": ds, "seed": seed, "n": len(items), "tar": out.name, + manifest_path.write_text(json.dumps({"dataset": ds, "seed": seed, "n": len(items), "tar": out.name, "sha256": fsha, "content_sha256": content_sha, - "pool_sha256": pm["sha256"]}, indent=2)) + "pool_sha256": pm["sha256"], **identity}, indent=2)) print(f"{ds}: {out} n={len(items)} sha={fsha[:12]}") return out @@ -110,9 +189,14 @@ def build(ds: str) -> Path: def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--dataset", action="append", required=True) + ap.add_argument("--n", type=int, help="exact sample count; registry default when omitted") + ap.add_argument("--seed", type=int, help="fixed sampling seed; registry default when omitted") + ap.add_argument("--out-dir", type=Path, help="independent directory for tar and manifest (one dataset)") a = ap.parse_args() + if a.out_dir is not None and len(a.dataset) != 1: + ap.error("--out-dir requires exactly one --dataset") for ds in a.dataset: - build(ds) + build(ds, n=a.n, seed=a.seed, out_dir=a.out_dir) return 0 diff --git a/scripts/probe_classification_adapters.py b/scripts/probe_classification_adapters.py new file mode 100644 index 0000000..bcf624c --- /dev/null +++ b/scripts/probe_classification_adapters.py @@ -0,0 +1,126 @@ +"""Validate real classification inputs once; reuse the normal preparation cache. + +No quantization, board runs, task insertion, or training traces are synthesized. +The subsequent Agent job reuses these weights, ONNX and measured FP32 baseline. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness import paths, registry, scheduler +from harness.continuation import load_eval_set +from harness.eval import EVALUATOR_VERSION + + +def probe(mid: str, manifest: str, weights_directory: str | None = None) -> dict: + import numpy as np + import onnxruntime as ort + from harness.eval import cls + from harness.eval.common import T1, TorchRunner, cosine + from harness.eval.run import g2_inputs + from harness.stages.s01_fetch import Fetch, weights_file + from harness.stages.s02_baseline import baseline_dir + from harness.stages.s03_export import Export, onnx_path + + row = registry.model_row(mid) + if row["task"] != "cls" or not row.get("checkpoint") or not row.get("preprocess"): + raise ValueError(f"{mid}: explicit checkpoint and image recipe required") + dataset, evaluation = load_eval_set(manifest) + if dataset != row["dataset"]: + raise ValueError("model and evaluation dataset differ") + spec = registry.expand_matrix(["oracle"], [mid])[0] + job = registry.build_job_json(spec, "adapter-probe", EVALUATOR_VERSION) + job["data"]["t1"] = evaluation + ctx = SimpleNamespace(job=job) + directory = paths.CACHE / "adapter-probes" / mid + directory.mkdir(parents=True, exist_ok=True) + result = {"model": mid, "task": "cls", "status": "running", "started_at": time.time(), + "checkpoint": row["checkpoint"], "preprocess": row["preprocess"], "eval_set": evaluation} + proof = directory / "probe.json" + if proof.exists(): + (directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) + proof.write_text(json.dumps(result, indent=2)) + try: + with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log: + fetch = Fetch() + if not fetch.cached(ctx): + cmd = fetch.command(ctx) + if weights_directory: + source = Path(weights_directory) / row["checkpoint"]["filename"] + cmd[cmd.index("--uri") + 1] = source.resolve().as_uri() + subprocess.run(cmd, check=True, stdout=log, stderr=log) + fetch.measure(ctx, directory, None) + weights = weights_file(mid) + actual_sha = hashlib.sha256(weights.read_bytes()).hexdigest() + if actual_sha != row["checkpoint"]["sha256"]: + raise ValueError("checkpoint digest differs from the pinned official recipe") + runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) + export = Export() + if not export.cached(ctx): + subprocess.run(export.command(ctx), check=True, stdout=log, stderr=log) + export.measure(ctx, directory, None) + onnx = onnx_path(job) + options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx), sess_options=options, providers=["CPUExecutionProvider"]) + def exported(x): + return session.run(None, {session.get_inputs()[0].name: x}) + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + xs = g2_inputs("cls", t1, row["input_spec"], job["gates"]["G2"]["n_samples"], model=mid) + outputs, comparisons = {}, [] + for key, x in xs.items(): + a, b = runner(x)[0], exported(x)[0] + if a.shape != (1, 1000) or b.shape != a.shape or not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError(f"invalid classifier output: {a.shape}, {b.shape}") + score = cosine(a, b) + comparisons.append({"sample": key, "cosine": score, "max_abs": float(np.abs(a - b).max())}) + if score < .9999: + raise ValueError(f"ONNX/FP32 output mismatch: {score}") + outputs[f"{key}/out0"] = a + baseline = {"task": "cls", "model": mid, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)} + converted = {**baseline} + cls.run_fp32(runner, t1, row["input_spec"], baseline) + cls.run_fp32(exported, t1, row["input_spec"], converted) + if baseline["metrics"]["top1"] <= 0 or abs(baseline["metrics"]["top1"] - converted["metrics"]["top1"]) > .1: + raise ValueError(f"FP32/ONNX accuracy mismatch: {baseline['metrics']} / {converted['metrics']}") + # Populate precisely the cache consumed by stage 02, after verification. + cache = baseline_dir(job); cache.mkdir(parents=True, exist_ok=True) + np.savez(cache / "g2_inputs.npz", **xs) + np.savez(cache / "g2_fp32.npz", **outputs) + (cache / "metrics.json").write_text(json.dumps(baseline, indent=2)) + result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], + onnx=converted["metrics"], comparisons=comparisons, + weights_path=str(weights), weights_sha256=actual_sha, + onnx_path=str(onnx), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest(), + baseline_path=str(cache), sample_ids=[it["id"] for it in t1.items]) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + result["finished_at"] = time.time() + proof.write_text(json.dumps(result, indent=2)) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", required=True) + parser.add_argument("--t1", required=True) + parser.add_argument("--weights-dir") + args = parser.parse_args() + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(2) + failures = sum(probe(mid, args.t1, args.weights_dir)["status"] != "validated_fp32_export_only" for mid in args.model) + raise SystemExit(bool(failures)) diff --git a/scripts/probe_coco_adapters.py b/scripts/probe_coco_adapters.py new file mode 100644 index 0000000..83a7b47 --- /dev/null +++ b/scripts/probe_coco_adapters.py @@ -0,0 +1,119 @@ +"""Real checkpoint -> ONNX -> paired FP32 inference/COCO score, before queue admission. + +Uses the existing fetch/export/evaluation code and resource lock. Does not label a +model ready, insert tasks, change packages, or claim quantization/board completion. +""" +from __future__ import annotations +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness import paths, registry, scheduler + + +def probe(row, n): + import numpy as np + import onnxruntime as ort + from harness.eval.common import T1, TorchRunner, cosine + from harness.eval.run import g2_inputs + from harness.eval.coco_extra import predict, evaluate, run_fp32, run_quant + mid, task = row["model_id"], row["task"] + directory = paths.CACHE / "adapter-probes" / mid + directory.mkdir(parents=True, exist_ok=True) + onnx = directory / (mid + ".onnx") + result = {"model": mid, "task": task, "started_at": time.time(), "status": "running"} + try: + with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log: + subprocess.run([sys.executable, "-m", "harness.tools.fetch_weights", "--model", mid, + "--uri", row["weights"], "--out-dir", str(directory)], check=True, stdout=log, stderr=log) + weights = directory / row["weights"].split("/")[-1] + subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid, + "--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"], + "--out", str(onnx)], check=True, stdout=log, stderr=log) + torch_runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) + opts = ort.SessionOptions() + opts.intra_op_num_threads = 2 + opts.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx), sess_options=opts, providers=["CPUExecutionProvider"]) + def exported(x): + return session.run(None, {session.get_inputs()[0].name: x}) + manifest_path = paths.DATA / "t1" / row["dataset"] / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"]) + t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} + try: + comparisons = [] + for sample, x in g2_inputs(task, t1, row["input_spec"], n).items(): + raw, converted = torch_runner(x), exported(x) + # TorchRunner can expose extra training/debug tensors. Match the + # official export outputs by shape; require an unambiguous match. + for y in converted: + matches = [v for v in raw if v.shape == y.shape] + if len(matches) != 1: + raise ValueError(f"ambiguous output mapping: {y.shape}, {[v.shape for v in raw]}") + cos = cosine(matches[0], y) + comparisons.append({"sample": sample, "shape": list(y.shape), "cosine": cos, + "max_abs": float(np.abs(matches[0] - y).max())}) + if cos < .9999: + raise ValueError(f"export numeric mismatch: {cos}") + if task == "pose": + baseline = {"task": task, "model": mid} + converted = {"task": task, "model": mid} + run_fp32(torch_runner, t1, row["input_spec"], baseline) + run_quant(exported, t1, row["input_spec"], baseline, converted) + a, b = baseline["metrics"], converted["metrics"] + shift = converted["G3"]["extra"]["keypoint_shift"] + # This compares two FP32 implementations; it is never a + # source=board measurement or a quantization gate pass. + if shift["status"] != "measured" or shift["keypoint_mean_shift_px"] > 2: + raise ValueError(f"export pose geometry mismatch: {shift.get('reason', shift['keypoint_mean_shift_px'])}") + evidence_path = directory / f"pose-export-evidence-{time.time_ns()}.json" + evidence_path.write_text(json.dumps(converted["pose_shift_evidence"], indent=2)) + result["pose_export_shift"] = {**shift, "evidence": str(evidence_path)} + result["pose_export_evidence"] = { + "path": str(evidence_path), "sha256": hashlib.sha256(evidence_path.read_bytes()).hexdigest()} + else: + a = evaluate(t1, predict(torch_runner, t1, row["input_spec"], task), task) + b = evaluate(t1, predict(exported, t1, row["input_spec"], task), task) + metric = "mask_ap" if task == "iseg" else "oks_ap" + if a[metric] <= 0 or abs(a[metric] - b[metric]) > .1: + raise ValueError(f"export task metric mismatch or zero baseline: {a} / {b}") + result.update(status="validated_fp32_export_only", comparisons=comparisons, fp32=a, onnx=b, + sample_ids=[it["id"] for it in t1.items], t1_sha256=manifest["sha256"], + weights_sha256=hashlib.sha256(weights.read_bytes()).hexdigest(), + onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest()) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + result["finished_at"] = time.time() + previous = directory / "probe.json" + if previous.exists(): + import shutil + shutil.copy2(previous, directory / f"probe-history-{time.time_ns()}.json") + previous.write_text(json.dumps(result, indent=2)) + print(json.dumps(result), flush=True) + return result + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--model", action="append", required=True) + ap.add_argument("--n", type=int, default=8) + args = ap.parse_args() + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(2) + failures = 0 + for mid in args.model: + row = registry.model_row(mid) + if row["source"] != "ultralytics" or row["task"] not in ("iseg", "pose") or mid.startswith("fastsam."): + raise SystemExit(f"unsupported probe scope: {mid}") + failures += probe(row, args.n)["status"] == "blocked" + sys.exit(bool(failures)) diff --git a/scripts/probe_da_metric_adapters.py b/scripts/probe_da_metric_adapters.py new file mode 100644 index 0000000..685c26e --- /dev/null +++ b/scripts/probe_da_metric_adapters.py @@ -0,0 +1,417 @@ +"""Probe the six official metric Depth Anything V2 checkpoints on the NYUv2 T1. + +Same admission evidence as probe_depth_adapters.py for the relative models: +strict official checkpoint load, real-data FP32 forward, ONNX export plus +checker, onnxruntime parity on real images, per-image alignment audit under the +frozen nyuv2-relative-inverse-ls-v1 protocol (the affine least squares carries +a signed scale, so metric-metre predictions are aligned, never trusted as +metres) and a 128-image train-split calibration package keyed to the metric +recipe. Sources are the same pinned V2 upstream commit via metric_depth/. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import json +import os +from pathlib import Path +import sys +import time +import traceback + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +MODELS = [f"depth.anything.v2.metric.{domain}.{variant}" + for domain in ("hypersim", "vkitti") for variant in ("small", "base", "large")] + + +def digest(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def save(path: Path, value: dict): + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2, allow_nan=False)) + temp.replace(path) + + +def open_eval_set(t1_manifest: Path) -> dict: + from harness.continuation import load_eval_set + dataset, evaluation = load_eval_set(t1_manifest) + if dataset != "nyuv2" or evaluation.get("n") != 100: + raise ValueError(f"expected the frozen nyuv2 n100 T1, got {dataset}/{evaluation.get('n')}") + manifest = json.loads(t1_manifest.read_text()) + evaluation = dict(evaluation) + evaluation["eval_set_id"] = manifest["eval_set_id"] + evaluation["content_sha"] = manifest["content_sha256"] + evaluation["manifest"] = str(t1_manifest) + return evaluation + + +def calibration(mat: Path, splits: Path, evaluation: dict, models: list[str], out: Path) -> None: + """Deterministic 128-image train-split tensors for the metric recipe.""" + import h5py + import numpy as np + import random + from PIL import Image + from scipy.io import loadmat + from harness.adapters import calibration_dir, get, model_spec + from harness.registry import model_row + + ids = loadmat(splits) + train, test = [sorted(int(v) for v in ids[key].ravel()) for key in ("trainNdxs", "testNdxs")] + if len(train) != 795 or len(test) != 654: + raise ValueError("unexpected NYUv2 official split") + manifest = json.loads(Path(evaluation["manifest"]).read_text()) + raw_sha, split_sha = digest(mat), digest(splits) + if (manifest["source"]["raw_sha256"] != raw_sha or manifest["source"]["split_sha256"] != split_sha): + raise ValueError("raw NYUv2 material differs from the frozen T1 source") + calib_ids = sorted(set(manifest["source"]["calibration_train_ids"])) + if len(calib_ids) != 128 or not set(calib_ids) <= set(train): + raise ValueError("calibration ids must be 128 official train images") + if calib_ids != sorted(random.Random(manifest["source"]["seed"]).sample(train, 128)): + raise ValueError("calibration ids are not the recorded seed sample") + test_ids = set(manifest["source"]["test_ids"]) + with h5py.File(mat, "r") as raw: + if raw["images"].shape != (1449, 3, 640, 480): + raise ValueError("unexpected NYU original HDF5 dimensions") + + def read_image(index: int): + rgb = np.asarray(raw["images"][index - 1]).transpose(2, 1, 0) + if rgb.dtype != np.uint8 or rgb.shape != (480, 640, 3): + raise ValueError("invalid NYU RGB") + return Image.fromarray(rgb) + + for model in models: + row = model_spec(model_row(model)) + recipe = get(model).PREPROCESS + target = calibration_dir(row) + if (target / "manifest.json").exists(): + cm = json.loads((target / "manifest.json").read_text()) + files = sorted((target / "npy_f32").glob("*.npy")) + h = hashlib.sha256() + for p in files: + h.update(p.read_bytes()) + if (cm.get("count") != 128 or len(files) != 128 or cm.get("preprocess") != recipe + or cm.get("sha256") != h.hexdigest() or cm.get("t1_sha256") != digest(Path(evaluation["path"]))): + raise ValueError(f"existing calibration differs; refusing to overwrite {target}") + print(f"{model}: calibration verified at {target}", flush=True) + continue + if target.exists(): + raise FileExistsError(f"partial calibration; refusing to overwrite {target}") + temporary = target.with_name(target.name + f".partial-{os.getpid()}") + (temporary / "npy_f32").mkdir(parents=True, exist_ok=False) + items, h = [], hashlib.sha256() + for index in calib_ids: + image = read_image(index) + if index in test_ids: + raise ValueError("calibration/test split overlap") + x = get(model).preprocess(image, row["input_spec"]) + if x.shape != (1, 3, 518, 518) or not np.isfinite(x).all(): + raise ValueError("invalid real calibration tensor") + path = temporary / "npy_f32" / f"{len(items):05d}.npy" + np.save(path, x) + h.update(path.read_bytes()) + items.append({"id": f"nyuv2/train/{index:04d}", "official_index": index, + "rgb_sha256": hashlib.sha256(np.asarray(image).tobytes()).hexdigest(), + "npy": path.name, "npy_sha256": digest(path)}) + save(temporary / "manifest.json", {"task": "mde", "input_spec": row["input_spec"], + "count": 128, "seed": manifest["source"]["seed"], "source": "official-labeled-train", + "dataset": "nyuv2", "items": items, "files": [item["id"] for item in items], + "sha256": h.hexdigest(), "preprocess": recipe, "npy_dir": "npy_f32", + "layout": "NCHW", "dtype": "float32", "test_content_overlap": 0, + "t1_sha256": digest(Path(evaluation["path"])), + "raw_source": {"raw_mat": str(mat), "raw_sha256": raw_sha, "split_sha256": split_sha}}) + temporary.rename(target) + print(f"{model}: calibration written at {target}", flush=True) + + +def verify_contract(evaluation: dict, models: list[str], out: Path) -> None: + """Independent geometry/preprocessing checks on the frozen T1 package.""" + import numpy as np + from harness.eval.common import T1 + from harness.eval import mde + from harness.adapters import get + from harness.adapters import depth_anything_v2_metric_common as metric_common + + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + root = metric_common.source_root() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import cv2 # noqa: F401 (the pinned metric transform imports cv2) + from metric_depth.depth_anything_v2.util.transform import NormalizeImage, PrepareForNet, Resize + import metric_depth.depth_anything_v2.util.transform as pinned_transform + if Path(pinned_transform.__file__).resolve() != root / "metric_depth/depth_anything_v2/util/transform.py": + raise RuntimeError("a different Depth Anything V2 metric source is already imported") + total_valid = 0 + for item in t1.items: + for key, expected in item["file_sha256"].items(): + assert hashlib.sha256(t1.tar.extractfile(item[key]).read()).hexdigest() == expected + depth, mask = mde.ground_truth(t1, item) + total_valid += int(mask.sum()) + image = t1.image(t1.items[0]) + for model in models: + adapter = get(model) + native = {"image": np.asarray(image, dtype=np.float64) / 255.0} + native = Resize(518, 518, resize_target=False, keep_aspect_ratio=False, ensure_multiple_of=14, + resize_method="lower_bound", image_interpolation_method=cv2.INTER_CUBIC)(native) + native = NormalizeImage(mean=[.485, .456, .406], std=[.229, .224, .225])(native) + native = PrepareForNet()(native)["image"][None] + np.testing.assert_array_equal(adapter.preprocess(image, adapter.INPUT_SPEC), native) + # Exact affine inverse depth must recover GT; a depth-direction + # (metric-like) prediction aligns through the signed least squares and + # is never read as metres without the recorded scale/shift. + truth = np.linspace(.5, 9.5, 10000, dtype=np.float32).reshape(100, 100) + relative = (1.0 / truth - .02) / .04 + score = mde.aligned_metrics(relative, truth, np.ones_like(truth, dtype=bool)) + assert score["absrel"] < 1e-6 and score["delta1"] == 100 + signed = mde.aligned_metrics(3.0 - 1.0 / truth, truth, np.ones_like(truth, dtype=bool)) + np.testing.assert_allclose([signed["scale"], signed["shift"]], [-1, 3], atol=1e-6) + assert signed["absrel"] < 1e-6 and signed["delta1"] == 100 + for bad in [np.ones_like(truth), np.full_like(truth, np.nan)]: + try: + mde.aligned_metrics(bad, truth, np.ones_like(truth, dtype=bool)) + except ValueError: + pass + else: + raise AssertionError("degenerate/nonfinite input accepted") + for model in models: + constant = np.full((1, 518, 518), 2, dtype=np.float32) + np.testing.assert_array_equal(mde.restore_prediction([constant], (480, 640), model), + np.full((480, 640), 2)) + save(out / "contract_checks.json", {"status": "passed", "t1_sha256": digest(Path(evaluation["path"])), + "decoded_real_labels": len(t1.items), "valid_pixels": total_valid, "models": models, + "checks": ["all real RGB/depth member hashes", + "official metric_depth static resize/normalization equality", + "original metre GT and fixed crop", "affine inverse-depth exact recovery", + "signed LS recovers (-1,3) for depth-direction predictions", + "reject constant/NaN prediction", + "restore constant depth at 518 without vector scaling"]}) + finally: + t1.tar.close() + + +def probe(model: str, evaluation: dict, weights_dir: Path, out: Path, n: int, confirm: bool, threads: int): + import numpy as np + import torch + import torchvision + import onnx + import onnxruntime as ort + from harness.eval.common import T1, cosine + from harness.eval import EVALUATOR_VERSION, mde + + recipe = importlib.import_module("harness.adapters." + model.replace(".", "_")) + directory = out / model + directory.mkdir(parents=True, exist_ok=True) + weights = weights_dir / recipe.CHECKPOINT["url"].rsplit("/", 1)[1] + result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": recipe.INPUT_SPEC, + "checkpoint": recipe.CHECKPOINT, "preprocess": recipe.PREPROCESS, "opset": recipe.OPSET, + "exporter_version": recipe.EXPORTER_VERSION, + "weights_path": str(weights), "t1_path": evaluation["path"], + "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, + "evaluation_protocol": mde.PROTOCOL, + "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu", + "threads": threads}, + "input_contract": {"status": "unverified"}, "board_tested": False, "quantization_tested": False} + + def phase(name): + result["phase"] = name + save(directory / "probe.json", result) + print(f"{model}: {name}", flush=True) + + try: + source_files = ["harness/adapters/depth_anything_v2_metric_common.py", + "harness/adapters/" + model.replace(".", "_") + ".py", "harness/eval/mde.py"] + root = Path(__file__).resolve().parents[1] + result["source_sha256"] = {name: digest(root / name) for name in source_files} + phase("strict_official_load") + result["weights_sha256"] = digest(weights) + model_object = recipe.build(weights, recipe.INPUT_SPEC) + result.update(strict_load=True, parameters=sum(p.numel() for p in model_object.parameters())) + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} + if len(t1.items) != n: + raise ValueError("requested sample count differs from actual T1 subset") + result["sample_ids"] = [item["id"] for item in t1.items] + references = {} + + def native(x): + key = hashlib.sha256(x.tobytes()).hexdigest() + if key in references: + return [references[key]] + with torch.inference_mode(): + value = model_object(torch.from_numpy(x)).numpy() + if value.shape != (1, 518, 518) or not np.isfinite(value).all(): + raise ValueError("invalid official forward output") + references[key] = value + print(f"{model}: FP32 {len(references)}/{n}", flush=True) + return [value] + + phase("real_nyu_fp32_forward") + baseline = {"task": "mde", "model": model, "n": n, + "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"]} + for item in t1.items: + native(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) + np.savez(directory / "fp32_predictions.npz", **references) + result["fp32_forward_n"] = len(references) + first = mde.inputs(t1, t1.items[0], recipe.INPUT_SPEC, model) + np.save(directory / "input_first.npy", first) + np.save(directory / "fp32_first.npy", references[hashlib.sha256(first.tobytes()).hexdigest()]) + phase("onnx_export_and_checker") + path = directory / f"{model}.onnx" + started = time.monotonic() + with torch.no_grad(): + torch.onnx.export(model_object, torch.from_numpy(first), str(path), dynamo=False, + opset_version=recipe.OPSET, input_names=["input"], + output_names=["metric_depth"], dynamic_axes=None, + do_constant_folding=True) + result["onnx"] = {"path": str(path), "sha256": digest(path), "size": path.stat().st_size, + "export_seconds": time.monotonic() - started} + onnx.checker.check_model(str(path), full_check=True) + phase("real_nyu_onnxruntime_and_paired_metrics") + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) + result["onnx"]["input_shape"] = session.get_inputs()[0].shape + result["onnx"]["output_shape"] = session.get_outputs()[0].shape + if (result["onnx"]["input_shape"] != [1, 3, 518, 518] + or result["onnx"]["output_shape"] != [1, 518, 518]): + raise ValueError("ONNX static input/output shape mismatch") + comparisons = [] + converted_outputs = {} + result["comparisons"] = comparisons + + def converted(x): + key = hashlib.sha256(x.tobytes()).hexdigest() + if key in converted_outputs: + return [converted_outputs[key]] + value = session.run(None, {"input": x})[0] + ref = references[key] + difference = np.abs(ref - value) + comp = {"sample_id": result["sample_ids"][len(comparisons)], "cosine": cosine(ref, value), + "max_abs": float(difference.max()), "mean_abs": float(difference.mean()), + "relative_max_abs": float(difference.max() / max(float(np.abs(ref).max()), 1e-12))} + comparisons.append(comp) + print(json.dumps(comp), flush=True) + if not np.isfinite(value).all() or comp["cosine"] < .99999 or comp["relative_max_abs"] > .001: + raise ValueError(f"ONNX mismatch: {comp}") + converted_outputs[key] = value + return [value] + + for item in t1.items: + converted(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) + np.savez(directory / "onnx_predictions.npz", **converted_outputs) + result["onnxruntime_forward_n"] = len(converted_outputs) + phase("real_nyu_alignment_audit") + audit, failures = [], [] + for item in t1.items: + x = mde.inputs(t1, item, recipe.INPUT_SPEC, model) + depth, mask = mde.ground_truth(t1, item) + sample = {"id": item["id"]} + for name, runner in (("fp32", native), ("onnxruntime", converted)): + prediction = mde.restore_prediction(runner(x), depth.shape, model) + try: + sample[name] = {"status": "measured", **mde.aligned_metrics(prediction, depth, mask)} + except ValueError as exc: + sample[name] = {"status": "blocked", "error": str(exc)} + failures.append({"id": item["id"], "runner": name, "error": str(exc)}) + audit.append(sample) + save(directory / "alignment_audit.json", {"protocol": mde.PROTOCOL, "n": n, + "samples": audit, "failures": failures, "status": "blocked" if failures else "passed"}) + if failures: + result["evaluation_contract"] = {"status": "blocked", "protocol": mde.PROTOCOL, + "failures": failures} + raise ValueError(f"alignment failed for {len({s['id'] for s in failures})}/{n} real samples") + mde.run_fp32(native, t1, recipe.INPUT_SPEC, baseline) + result["fp32"] = baseline["metrics"] + measured = {"task": "mde", "model": model} + mde.run_quant(converted, t1, recipe.INPUT_SPEC, baseline, measured) + paired = measured["G3"]["extra"]["scale_alignment"]["samples"] + if len(paired) != n or len({sample["id"] for sample in paired}) != n: + raise ValueError("incomplete/duplicate per-image alignment proof") + for sample, fp, quant in zip(paired, baseline["_depth_samples"], measured["_depth_samples"]): + if (sample["id"] != fp["id"] or sample["id"] != quant["id"] + or sample["fp32_scale"] != fp["scale"] or sample["fp32_shift"] != fp["shift"] + or sample["quant_scale"] != quant["scale"] or sample["quant_shift"] != quant["shift"] + or not np.isfinite([sample[key] for key in + ("fp32_scale", "fp32_shift", "quant_scale", "quant_shift")]).all()): + raise ValueError("invalid/mispaired real per-image alignment proof") + if (abs(baseline["metrics"]["absrel"] - measured["metrics"]["absrel"]) > .001 + or abs(baseline["metrics"]["delta1"] - measured["metrics"]["delta1"]) > .1): + raise ValueError("FP32/ONNX paired depth metric mismatch") + checks = json.loads((out / "contract_checks.json").read_text()) + if checks.get("status") != "passed" or checks.get("t1_sha256") != result["t1_sha256"]: + raise ValueError("missing matching real-data and alignment contract checks") + result.update(comparisons=comparisons, onnxruntime=measured["metrics"], G3_probe=measured["G3"], + status="validated_fp32_export_only" if confirm else "validated_export_pending_eval_protocol", + phase="complete") + result["input_contract"] = {"status": "validated", "input_spec": recipe.INPUT_SPEC, + "output_semantics": "nonnegative_metric_depth_m_aligned_not_trusted", + "t1_sha256": result["t1_sha256"], "checks_path": str(out / "contract_checks.json"), + "sample_ids": result["sample_ids"], "onnx_sha256": result["onnx"]["sha256"]} + result["evaluation_contract"] = {"status": "validated" if confirm else "pending_mainline_protocol", + "protocol": mde.PROTOCOL} + baseline["note"] = "probe subset only, not the complete 100-image FP32 baseline" + save(directory / "baseline_probe.json", baseline) + save(directory / "onnx_metrics_probe.json", measured) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + (directory / "error.txt").write_text(traceback.format_exc()) + result["finished_at"] = time.time() + save(directory / "probe.json", result) + print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", choices=MODELS, required=True) + parser.add_argument("--t1-manifest", type=Path, required=True, + help="manifest.json of the existing frozen nyuv2 n100 T1 package") + parser.add_argument("--weights-dir", type=Path, required=True) + parser.add_argument("--data-reader", type=Path, + help="private unpacked h5py/scipy wheels used only to read the source MAT") + parser.add_argument("--mat", type=Path, required=True, help="original nyu_depth_v2_labeled.mat") + parser.add_argument("--splits", type=Path, required=True, help="official splits.mat") + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--n", type=int, default=3) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--confirm-protocol", choices=["nyuv2-relative-inverse-ls-v1"]) + args = parser.parse_args() + if not 1 <= args.n <= 100 or args.threads < 1: + parser.error("require 1 <= n <= 100 and positive threads") + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + if args.data_reader: + sys.path.insert(0, str(args.data_reader.resolve())) + models = list(dict.fromkeys(args.model)) + evaluation = open_eval_set(args.t1_manifest) + from harness.adapters import depth_anything_v2_metric_common + depth_anything_v2_metric_common.source_root() + calibration(args.mat, args.splits, evaluation, models, args.out) + verify_contract(evaluation, models, args.out) + save(args.out / "data_preparation.json", {"t1_path": evaluation["path"], + "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "n": args.n}) + results = [probe(model, evaluation, args.weights_dir, args.out, args.n, + bool(args.confirm_protocol), args.threads) for model in models] + save(args.out / "summary.json", {"results": results, "complete": True}) + return int(any(result["status"] == "blocked" for result in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_depth_adapters.py b/scripts/probe_depth_adapters.py new file mode 100644 index 0000000..44ff25f --- /dev/null +++ b/scripts/probe_depth_adapters.py @@ -0,0 +1,432 @@ +"""Prepare original NYUv2 data and probe the three official relative DA-V2 models. + +All work is private except an explicitly supplied existing --calib-root. No +package install, run/queue mutation, quantization, or board pass is performed. +--confirm-protocol is an explicit mainline choice, not an official-model claim. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import io +import json +import os +from pathlib import Path +import random +import sys +import tarfile +import time +import traceback + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +def digest(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def save(path: Path, value: dict): + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2, allow_nan=False)) + temp.replace(path) + + +def data(mat: Path, splits: Path, out: Path, calib_root: Path, seed: int): + import h5py + import numpy as np + from PIL import Image + from scipy.io import loadmat + from harness.adapters.depth_anything_v2_common import INPUT_SPEC, PREPROCESS, preprocess + from harness.eval.mde import PROTOCOL + + ids = loadmat(splits) + train, test = [sorted(int(v) for v in ids[key].ravel()) for key in ("trainNdxs", "testNdxs")] + if len(train) != 795 or len(test) != 654 or set(train) & set(test) or sorted(train + test) != list(range(1, 1450)): + raise ValueError("unexpected NYUv2 official split; require disjoint 795 train / 654 test") + selected = sorted(random.Random(seed).sample(test, 100)) + calib_ids = sorted(random.Random(seed).sample(train, 128)) + raw_sha, split_sha = digest(mat), digest(splits) + source = {"dataset": "nyuv2", "raw_mat": str(mat), "raw_sha256": raw_sha, + "split_file": str(splits), "split_sha256": split_sha, + "test_total": len(test), "train_total": len(train), "unit": "m", "depth_field": "depths", + "matlab_indices_are_one_based": True, "test_ids": selected, "calibration_train_ids": calib_ids, + "split_overlap": 0, "seed": seed} + destination = out / f"t1-n100-s{seed}" + destination.mkdir(parents=True, exist_ok=True) + package = destination / f"{seed}.tar" + recipe = hashlib.sha256(json.dumps(PREPROCESS, sort_keys=True).encode()).hexdigest()[:16] + calibration = calib_root / "mde" / INPUT_SPEC / recipe + if (destination / "manifest.json").exists() and (calibration / "manifest.json").exists(): + old = json.loads((destination / "manifest.json").read_text()) + cm = json.loads((calibration / "manifest.json").read_text()) + files = sorted((calibration / "npy_f32").glob("*.npy")) + h = hashlib.sha256() + for p in files: + h.update(p.read_bytes()) + if (old["source"]["raw_sha256"] != raw_sha or old["source"]["split_sha256"] != split_sha + or old.get("evaluation_protocol") != PROTOCOL + or old["sha256"] != digest(package) or cm["count"] != 128 or len(files) != 128 + or cm["preprocess"] != PREPROCESS or cm["sha256"] != h.hexdigest()): + raise ValueError("existing depth data differs; refusing to overwrite") + return package, calibration + if package.exists() or calibration.exists(): + raise FileExistsError("partial/existing data; choose a new private --out/calib-root") + temporary = calibration.with_name(calibration.name + f".partial-{os.getpid()}") + (temporary / "npy_f32").mkdir(parents=True, exist_ok=False) + items, members, image_hashes = [], {}, set() + calibration_items, chash = [], hashlib.sha256() + with h5py.File(mat, "r") as raw: + if raw["images"].shape != (1449, 3, 640, 480) or raw["depths"].shape != (1449, 640, 480): + raise ValueError("unexpected NYU original HDF5 dimensions") + def read_image(index): + rgb = np.asarray(raw["images"][index - 1]).transpose(2, 1, 0) + if rgb.dtype != np.uint8 or rgb.shape != (480, 640, 3): + raise ValueError("invalid NYU RGB") + return Image.fromarray(rgb), hashlib.sha256(rgb.tobytes()).hexdigest() + for index in selected: + image, image_sha = read_image(index) + if image_sha in image_hashes: + raise ValueError("duplicate test RGB content") + image_hashes.add(image_sha) + depth = np.ascontiguousarray(raw["depths"][index - 1].T, dtype=np.float32) + if not np.isfinite(depth).all() or not np.any((depth > 0) & (depth < 10)): + raise ValueError("invalid original metre depth") + image_buf, depth_buf = io.BytesIO(), io.BytesIO() + image.save(image_buf, format="PNG") + np.save(depth_buf, depth) + paths = {"file": f"images/{index:04d}.png", "depth": f"depth/{index:04d}.npy"} + members[paths["file"]], members[paths["depth"]] = image_buf.getvalue(), depth_buf.getvalue() + items.append({"id": f"nyuv2/test/{index:04d}", **paths, "official_index": index, + "depth_unit": "m", "depth_format": "npy_float32_m", "rgb_sha256": image_sha, + "file_sha256": {key: hashlib.sha256(members[value]).hexdigest() for key, value in paths.items()}}) + used = set() + for index in calib_ids: + image, image_sha = read_image(index) + if image_sha in image_hashes or image_sha in used: + raise ValueError("calibration/test content overlap or duplicate calibration RGB") + used.add(image_sha) + path = temporary / "npy_f32" / f"{len(calibration_items):05d}.npy" + x = preprocess(image) + if x.shape != (1, 3, 518, 518) or not np.isfinite(x).all(): + raise ValueError("invalid real calibration tensor") + np.save(path, x) + chash.update(path.read_bytes()) + calibration_items.append({"id": f"nyuv2/train/{index:04d}", "official_index": index, + "rgb_sha256": image_sha, "npy": path.name, "npy_sha256": digest(path)}) + content_sha = hashlib.sha256(json.dumps({"items": items, "protocol": PROTOCOL, "source": source}, sort_keys=True).encode()).hexdigest() + manifest = {"dataset": "nyuv2", "version": "test-654", "seed": seed, "n": 100, + "sha256": content_sha, "items": items, "source": source, + "eval_set_id": f"nyuv2-relative-n100-{content_sha[:12]}", "evaluation_protocol": PROTOCOL} + members["manifest.json"] = json.dumps(manifest, sort_keys=True).encode() + with tarfile.open(package.with_suffix(".tmp"), "w") as archive: + for name, value in sorted(members.items()): + info = tarfile.TarInfo(name) + info.size = len(value) + archive.addfile(info, io.BytesIO(value)) + package.with_suffix(".tmp").replace(package) + save(destination / "manifest.json", {key: value for key, value in manifest.items() if key != "items"} | + {"sha256": digest(package), "content_sha256": content_sha, "tar": package.name}) + save(temporary / "manifest.json", {"task": "mde", "input_spec": INPUT_SPEC, "count": 128, + "seed": seed, "source": "official-labeled-train", "dataset": "nyuv2", "items": calibration_items, + "files": [item["id"] for item in calibration_items], "sha256": chash.hexdigest(), + "preprocess": PREPROCESS, "npy_dir": "npy_f32", "layout": "NCHW", "dtype": "float32", + "t1_sha256": digest(package), "test_content_overlap": 0, "raw_source": source}) + temporary.rename(calibration) + save(out / "dataset_checks.json", source | {"status": "passed", "t1_images": 100, + "t1_depth_maps": 100, "calibration_images": 128, "test_calibration_content_overlap": 0, + "t1_path": str(package), "t1_sha256": digest(package), "calibration_dir": str(calibration)}) + return package, calibration + + +def verify_contract(package: Path, out: Path): + import numpy as np + from harness.eval.common import T1 + from harness.eval import mde + from harness.adapters import depth_anything_v2_common as recipe + from depth_anything_v2.util.transform import Resize, NormalizeImage, PrepareForNet + import cv2 + + t1 = T1.open(package) + try: + sample = t1.items[0] + image = t1.image(sample) + native = {"image": np.asarray(image, dtype=np.float64) / 255.0} + native = Resize(518, 518, resize_target=False, keep_aspect_ratio=False, ensure_multiple_of=14, + resize_method="lower_bound", image_interpolation_method=cv2.INTER_CUBIC)(native) + native = NormalizeImage(mean=[.485, .456, .406], std=[.229, .224, .225])(native) + native = PrepareForNet()(native)["image"][None] + np.testing.assert_array_equal(recipe.preprocess(image), native) + total_valid = 0 + for item in t1.items: + for key, expected in item["file_sha256"].items(): + assert hashlib.sha256(t1.tar.extractfile(item[key]).read()).hexdigest() == expected + truth, mask = mde.ground_truth(t1, item) + total_valid += int(mask.sum()) + # Exact affine inverse depth must recover GT; raw prediction is not metres. + truth = np.linspace(.5, 9.5, 10000, dtype=np.float32).reshape(100, 100) + relative = (1.0 / truth - .02) / .04 + score = mde.aligned_metrics(relative, truth, np.ones_like(truth, dtype=bool)) + assert score["absrel"] < 1e-6 and score["delta1"] == 100 + signed = mde.aligned_metrics(3.0 - 1.0 / truth, truth, np.ones_like(truth, dtype=bool)) + np.testing.assert_allclose([signed["scale"], signed["shift"]], [-1, 3], atol=1e-6) + assert signed["absrel"] < 1e-6 and signed["delta1"] == 100 + for bad in [np.ones_like(truth), np.full_like(truth, np.nan)]: + try: + mde.aligned_metrics(bad, truth, np.ones_like(truth, dtype=bool)) + except ValueError: + pass + else: + raise AssertionError("degenerate/nonfinite input accepted") + constant = np.full((1, 518, 518), 2, dtype=np.float32) + np.testing.assert_array_equal(mde.restore_prediction([constant], (480, 640)), np.full((480, 640), 2)) + save(out / "contract_checks.json", {"status": "passed", "t1_sha256": digest(package), + "decoded_real_labels": len(t1.items), "valid_pixels": total_valid, + "checks": ["all real RGB/depth member hashes", "official static resize/normalization equality", + "original metre GT and fixed crop", "affine inverse-depth exact recovery", + "signed LS recovers (-1,3)", "reject constant/NaN prediction", + "restore constant depth without vector scaling"]}) + finally: + t1.tar.close() + + +def probe(model: str, package: Path, weights_dir: Path, out: Path, n: int, confirm: bool, threads: int, + reuse_onnx: bool = False): + import numpy as np + import torch + import torchvision + import onnx + import onnxruntime as ort + from harness.eval.common import T1, cosine + from harness.eval import EVALUATOR_VERSION, mde + + recipe = importlib.import_module("harness.adapters." + model.replace(".", "_")) + directory = out / model if n == 3 else out / model / f"n{n}" + directory.mkdir(parents=True, exist_ok=True) + weights = weights_dir / recipe.CHECKPOINT["url"].rsplit("/", 1)[1] + result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": recipe.INPUT_SPEC, + "checkpoint": recipe.CHECKPOINT, "preprocess": recipe.PREPROCESS, "opset": recipe.OPSET, + "weights_path": str(weights), "t1_path": str(package), + "t1_sha256": digest(package), "evaluation_protocol": mde.PROTOCOL, + "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu", "threads": threads}, + "input_contract": {"status": "unverified"}, "board_tested": False, "quantization_tested": False} + def phase(name): + result["phase"] = name + save(directory / "probe.json", result) + print(f"{model}: {name}", flush=True) + try: + source_files = ["harness/adapters/depth_anything_v2_common.py", + "harness/adapters/" + model.replace(".", "_") + ".py", "harness/eval/mde.py"] + root = Path(__file__).resolve().parents[1] + result["source_sha256"] = {name: digest(root / name) for name in source_files} + phase("strict_official_load") + result["weights_sha256"] = digest(weights) + model_object = recipe.build(weights, recipe.INPUT_SPEC) + result.update(strict_load=True, parameters=sum(p.numel() for p in model_object.parameters())) + t1 = T1.open(package, result["t1_sha256"]) + try: + t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} + if len(t1.items) != n: + raise ValueError("requested sample count differs from actual T1 subset") + result["sample_ids"] = [item["id"] for item in t1.items] + references = {} + def native(x): + key = hashlib.sha256(x.tobytes()).hexdigest() + if key in references: + return [references[key]] + with torch.inference_mode(): + value = model_object(torch.from_numpy(x)).numpy() + if value.shape != (1, 518, 518) or not np.isfinite(value).all(): + raise ValueError("invalid official forward output") + references[key] = value + print(f"{model}: FP32 {len(references)}/{n}", flush=True) + return [value] + phase("real_nyu_fp32_forward") + baseline = {"task": "mde", "model": model, "n": n, + "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"]} + for item in t1.items: + native(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) + np.savez(directory / "fp32_predictions.npz", **references) + result["fp32_forward_n"] = len(references) + first = mde.inputs(t1, t1.items[0], recipe.INPUT_SPEC, model) + np.save(directory / "input_first.npy", first) + np.save(directory / "fp32_first.npy", references[hashlib.sha256(first.tobytes()).hexdigest()]) + if n == 100: + xs = {f"s{i:03d}": mde.inputs(t1, item, recipe.INPUT_SPEC, model) + for i, item in enumerate(t1.g2_subset(32))} + np.savez(directory / "g2_inputs.npz", **xs) + np.savez(directory / "g2_fp32.npz", **{ + f"{key}/out0": references[hashlib.sha256(x.tobytes()).hexdigest()] + for key, x in xs.items()}) + phase("onnx_reuse_and_checker" if reuse_onnx else "onnx_export_and_checker") + if reuse_onnx: + previous_path = out / model / "probe.json" + previous = json.loads(previous_path.read_text()) + path = out / model / f"{model}.onnx" + if (previous.get("input_contract", {}).get("status") != "validated" + or not previous.get("strict_load") + or previous["weights_sha256"] != result["weights_sha256"] + or previous["input_spec"] != recipe.INPUT_SPEC or previous["opset"] != recipe.OPSET + or any(previous["source_sha256"][name] != result["source_sha256"][name] + for name in source_files[:2]) + or digest(path) != previous["onnx"]["sha256"]): + raise ValueError("existing ONNX identity/adapter/weights proof mismatch") + result["onnx"] = {**previous["onnx"], "path": str(path), "reused_from": str(previous_path)} + else: + path = directory / f"{model}.onnx" + started = time.monotonic() + with torch.no_grad(): + torch.onnx.export(model_object, torch.from_numpy(first), str(path), dynamo=False, + opset_version=recipe.OPSET, input_names=["input"], output_names=["inverse_depth"], + dynamic_axes=None, do_constant_folding=True) + result["onnx"] = {"path": str(path), "sha256": digest(path), "size": path.stat().st_size, + "export_seconds": time.monotonic() - started} + onnx.checker.check_model(str(path), full_check=True) + phase("real_nyu_onnxruntime_and_paired_metrics") + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) + result["onnx"]["input_shape"] = session.get_inputs()[0].shape + result["onnx"]["output_shape"] = session.get_outputs()[0].shape + if result["onnx"]["input_shape"] != [1, 3, 518, 518] or result["onnx"]["output_shape"] != [1, 518, 518]: + raise ValueError("ONNX static input/output shape mismatch") + comparisons = [] + converted_outputs = {} + result["comparisons"] = comparisons + def converted(x): + key = hashlib.sha256(x.tobytes()).hexdigest() + if key in converted_outputs: + return [converted_outputs[key]] + value = session.run(None, {"input": x})[0] + ref = references[key] + difference = np.abs(ref - value) + comp = {"sample_id": result["sample_ids"][len(comparisons)], "cosine": cosine(ref, value), + "max_abs": float(difference.max()), "mean_abs": float(difference.mean()), + "relative_max_abs": float(difference.max() / max(float(np.abs(ref).max()), 1e-12))} + comparisons.append(comp) + print(json.dumps(comp), flush=True) + if not np.isfinite(value).all() or comp["cosine"] < .99999 or comp["relative_max_abs"] > .001: + raise ValueError(f"ONNX mismatch: {comp}") + converted_outputs[key] = value + return [value] + for item in t1.items: + converted(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) + np.savez(directory / "onnx_predictions.npz", **converted_outputs) + result["onnxruntime_forward_n"] = len(converted_outputs) + # Audit every real image even if one alignment fails; never omit it from a score. + phase("real_nyu_alignment_audit") + audit, failures = [], [] + for item in t1.items: + x = mde.inputs(t1, item, recipe.INPUT_SPEC, model) + depth, mask = mde.ground_truth(t1, item) + sample = {"id": item["id"]} + for name, runner in (("fp32", native), ("onnxruntime", converted)): + prediction = mde.restore_prediction(runner(x), depth.shape) + try: + sample[name] = {"status": "measured", **mde.aligned_metrics(prediction, depth, mask)} + except ValueError as exc: + sample[name] = {"status": "blocked", "error": str(exc)} + failures.append({"id": item["id"], "runner": name, "error": str(exc)}) + audit.append(sample) + save(directory / "alignment_audit.json", {"protocol": mde.PROTOCOL, "n": n, + "samples": audit, "failures": failures, "status": "blocked" if failures else "passed"}) + if failures: + result["evaluation_contract"] = {"status": "blocked", "protocol": mde.PROTOCOL, + "failures": failures} + raise ValueError(f"alignment failed for {len({s['id'] for s in failures})}/{n} real samples") + mde.run_fp32(native, t1, recipe.INPUT_SPEC, baseline) + result["fp32"] = baseline["metrics"] + if n == 100: + save(directory / "metrics.json", baseline) + measured = {"task": "mde", "model": model} + mde.run_quant(converted, t1, recipe.INPUT_SPEC, baseline, measured) + paired = measured["G3"]["extra"]["scale_alignment"]["samples"] + if len(paired) != n or len({sample["id"] for sample in paired}) != n: + raise ValueError("incomplete/duplicate per-image alignment proof") + for sample, fp, quant in zip(paired, baseline["_depth_samples"], measured["_depth_samples"]): + if (sample["id"] != fp["id"] or sample["id"] != quant["id"] + or sample["fp32_scale"] != fp["scale"] or sample["fp32_shift"] != fp["shift"] + or sample["quant_scale"] != quant["scale"] or sample["quant_shift"] != quant["shift"] + or not np.isfinite([sample[key] for key in ("fp32_scale", "fp32_shift", "quant_scale", "quant_shift")]).all()): + raise ValueError("invalid/mispaired real per-image alignment proof") + if abs(baseline["metrics"]["absrel"] - measured["metrics"]["absrel"]) > .001 or abs(baseline["metrics"]["delta1"] - measured["metrics"]["delta1"]) > .1: + raise ValueError("FP32/ONNX paired depth metric mismatch") + checks = json.loads((out / "contract_checks.json").read_text()) + if checks.get("status") != "passed" or checks.get("t1_sha256") != result["t1_sha256"]: + raise ValueError("missing matching real-data and alignment contract checks") + result.update(comparisons=comparisons, onnxruntime=measured["metrics"], G3_probe=measured["G3"], + status="validated_fp32_export_only" if confirm else "validated_export_pending_eval_protocol", phase="complete") + result["input_contract"] = {"status": "validated", "input_spec": recipe.INPUT_SPEC, + "output_semantics": "affine_invariant_inverse_depth_not_metric", "t1_sha256": result["t1_sha256"], + "checks_path": str(out / "contract_checks.json"), "sample_ids": result["sample_ids"], + "onnx_sha256": result["onnx"]["sha256"]} + result["evaluation_contract"] = {"status": "validated" if confirm else "pending_mainline_protocol", + "protocol": mde.PROTOCOL} + baseline["note"] = ("complete fixed 100-image FP32 baseline; bench protocol, not paper scores" + if n == 100 else "probe subset only, not the complete 100-image FP32 baseline") + save(directory / "baseline_probe.json", baseline) + save(directory / "onnx_metrics_probe.json", measured) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + (directory / "error.txt").write_text(traceback.format_exc()) + result["finished_at"] = time.time() + save(directory / "probe.json", result) + print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", choices=[f"depth.anything.v2.{v}" for v in ("small", "base", "large")], required=True) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--data-reader", type=Path, help="private unpacked h5py/scipy wheels, only used to convert source MAT") + parser.add_argument("--mat", type=Path, required=True) + parser.add_argument("--splits", type=Path, required=True) + parser.add_argument("--weights-dir", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--calib-root", type=Path) + parser.add_argument("--n", type=int, default=3) + parser.add_argument("--seed", type=int, default=20260914) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--confirm-protocol", choices=["nyuv2-relative-inverse-ls-v1"]) + parser.add_argument("--reuse-onnx", action="store_true", help="verify and reuse the original three-image ONNX proof") + parser.add_argument("--prepare-only", action="store_true") + args = parser.parse_args() + if not 1 <= args.n <= 100 or args.threads < 1: + parser.error("require 1 <= n <= 100 and positive threads") + if args.reuse_onnx and args.n == 3: + parser.error("reuse requires a separate sample count to preserve the original three-image proof") + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + os.environ["BENCH_DEPTH_ANYTHING_V2_SOURCE"] = str(args.source.resolve()) + if args.data_reader: + sys.path.insert(0, str(args.data_reader.resolve())) + sys.path.insert(0, str(args.source.resolve())) + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + from harness.adapters.depth_anything_v2_common import source_root + source_root() + package, calibration = data(args.mat, args.splits, args.out, args.calib_root or args.out / "data/calib", args.seed) + verify_contract(package, args.out) + save(args.out / "data_preparation.json", {"t1_path": str(package), "t1_sha256": digest(package), + "n": 100, "calibration_dir": str(calibration), "calibration_n": 128}) + if args.prepare_only: + return 0 + results = [probe(model, package, args.weights_dir, args.out, args.n, bool(args.confirm_protocol), args.threads, args.reuse_onnx) + for model in dict.fromkeys(args.model)] + save(args.out / ("summary.json" if args.n == 3 else f"summary-n{args.n}.json"), {"results": results, "complete": True}) + return int(any(result["status"] == "blocked" for result in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_feature_adapters.py b/scripts/probe_feature_adapters.py new file mode 100644 index 0000000..4bbbf98 --- /dev/null +++ b/scripts/probe_feature_adapters.py @@ -0,0 +1,161 @@ +"""Real official/adapter/ORT + HPatches geometry probe, outside all existing runs.""" +from __future__ import annotations +import argparse +import ast +import hashlib +import importlib +import json +import os +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +def sha(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + for b in iter(lambda: f.read(1 << 20), b''): h.update(b) + return h.hexdigest() + + +def write(path, obj): + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix(path.suffix + '.tmp') + temp.write_text(json.dumps(obj, indent=2, allow_nan=False) + '\n'); temp.replace(path) + + +def prepare_calibration(adapter, selection, destination): + import numpy as np + from PIL import Image + recipe_sha = hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()[:16] + out = destination / adapter.INPUT_SPEC / recipe_sha + s = json.loads(selection.read_text()); source = Path(s['source_pool']) + if (out / 'manifest.json').exists(): + man = json.loads((out / 'manifest.json').read_text()) + assert man['count'] == 128 and man['preprocess'] == adapter.PREPROCESS and man['t1_sha256'] == s['t1_sha256'] + for item in man['items']: assert sha(out / 'npy_f32' / item['npy']) == item['npy_sha256'] + metadata = {'task': 'feat', 'dataset': 'hpatches', 'source': 'val-holdout', + 'calib_source': 'val-holdout', 'files': [it['file'] for it in man['items']]} + if any(man.get(k) != v for k, v in metadata.items()): + man.update(metadata); write(out / 'manifest.json', man) + return out + if out.exists(): raise FileExistsError(f'refuse partial calibration overwrite: {out}') + stage = out.with_name(out.name + '.partial'); (stage / 'npy_f32').mkdir(parents=True, exist_ok=False) + items = []; digest = hashlib.sha256() + for i, record in enumerate(s['items']): + path = source / record['file']; assert sha(path) == record['sha256'] + with Image.open(path) as im: x = adapter.preprocess(im, adapter.INPUT_SPEC) + p = stage / 'npy_f32' / f'{i:05d}.npy'; np.save(p, x); digest.update(p.read_bytes()) + items.append({**record, 'npy': p.name, 'npy_sha256': sha(p)}) + write(stage / 'manifest.json', {**{k: v for k, v in s.items() if k != 'items'}, 'items': items, + 'preprocess': adapter.PREPROCESS, 'count': 128, 'sha256': digest.hexdigest(), 'input_spec': adapter.INPUT_SPEC, + 'npy_dir': 'npy_f32', 'layout': 'NCHW', 'dtype': 'float32', 'task': 'feat', 'dataset': 'hpatches', + 'source': 'val-holdout', 'calib_source': 'val-holdout', 'files': [it['file'] for it in items]}) + stage.rename(out); return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--model', choices=['superpoint', 'disk', 'aliked', 'xfeat', 'd2.net', 'r2d2', 'dedode.v2', 'keynetplushardnet'], required=True) + ap.add_argument('--t1', type=Path, required=True); ap.add_argument('--out', type=Path, required=True) + ap.add_argument('--calibration-selection', type=Path, required=True) + ap.add_argument('--threads', type=int, default=2) + ap.add_argument('--stage', choices=['all', 'probe', 'evaluate'], default='all') + a = ap.parse_args(); a.out.mkdir(parents=True, exist_ok=True) + os.environ['CUDA_VISIBLE_DEVICES'] = '' + import cv2 + import numpy as np + import torch + import onnx + import onnxruntime as ort + from harness.adapters import feature_common as common + from harness.eval.common import T1 + from harness.eval import feat + torch.set_num_threads(a.threads); torch.set_num_interop_threads(1); cv2.setNumThreads(1) + torch.manual_seed(20260914) + from harness.adapters import get + adapter = get(a.model) + weights = common.ROOT / adapter.CHECKPOINT['official_relative_path'] + calib = prepare_calibration(adapter, a.calibration_selection, a.out.parent / 'calibration') + print(json.dumps({'model': a.model, 'stage': 'calibration_ready', 'path': str(calib)}), flush=True) + t1 = T1.open(a.t1); assert len(t1.items) == 100 + graph = adapter.build(weights, adapter.INPUT_SPEC).eval() + def torch_run(x): + with torch.no_grad(): output = graph(torch.from_numpy(x)) + return [t.detach().numpy() for t in (output if isinstance(output, tuple) else (output,))] + def compare_features(left, right): + # Coordinates uniquely identify sparse detections; row order is not a semantic contract. + x, y = left['keypoints'], right['keypoints'] + assert len(x) == len(y), (len(x), len(y)) + aidx = np.lexsort((x[:, 0], x[:, 1])); bidx = np.lexsort((y[:, 0], y[:, 1])) + result = {} + for key in ('keypoints', 'scores', 'descriptors'): + p, q = left[key][aidx], right[key][bidx] + result[key + '_max_abs'] = float(np.max(abs(p - q), initial=0)) + np.testing.assert_allclose(p, q, atol=2e-3, rtol=2e-3) + return result + x = feat.inputs(t1, t1.items[0], adapter.INPUT_SPEC, a.model) + onnx_path = a.out / 'model.onnx' + checks = [] + if a.stage != 'evaluate': + official = adapter.official_model(weights) if hasattr(adapter, 'official_model') else common.official_model(a.model, weights) + for item in [t1.items[i] for i in (0, 49, 99)]: + value = adapter.preprocess(t1.image({'file': item['file2']}), adapter.INPUT_SPEC) + with torch.no_grad(): + k, s, d = (adapter.official_features(official, torch.from_numpy(value)) if hasattr(adapter, 'official_features') + else common.official_features(a.model, official, torch.from_numpy(value))) + expected = {'keypoints': k.numpy(), 'scores': s.numpy(), 'descriptors': d.numpy()} + actual = common.decode(a.model, torch_run(value)) + checks.append({'id': item['id'], 'official_vs_adapter': compare_features(expected, actual)}) + del official + print(json.dumps({'model': a.model, 'stage': 'official_numeric_passed', 'checks': checks}), flush=True) + with torch.no_grad(): + torch.onnx.export(graph, torch.from_numpy(x), str(onnx_path), opset_version=adapter.OPSET, + input_names=['input'], dynamic_axes=None, do_constant_folding=True, dynamo=False) + onnx.checker.check_model(str(onnx_path)) + so = ort.SessionOptions(); so.intra_op_num_threads = a.threads; so.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx_path), so, providers=['CPUExecutionProvider']) + def ort_run(value): return session.run(None, {'input': value}) + if a.stage != 'evaluate': + for idx, item in enumerate([t1.items[i] for i in (0, 49, 99)]): + value = adapter.preprocess(t1.image({'file': item['file2']}), adapter.INPUT_SPEC) + eager, ort_out = torch_run(value), ort_run(value) + assert all(p.shape == q.shape for p, q in zip(eager, ort_out)) + checks[idx]['dense_or_sparse_output_max_abs'] = [float(np.max(abs(p-q))) for p, q in zip(eager, ort_out)] + checks[idx]['adapter_vs_ort'] = compare_features(common.decode(a.model, eager), common.decode(a.model, ort_out)) + static = onnx.load(str(onnx_path)) + ops = sorted({node.op_type for node in static.graph.node}) + output_shapes = [[dim.dim_value for dim in output.type.tensor_type.shape.dim] for output in static.graph.output] + assert all(all(n > 0 for n in shape) for shape in output_shapes), output_shapes + write(a.out / 'probe.json', {'status': 'official_adapter_ort_verified', 'model': a.model, + 'checkpoint': adapter.CHECKPOINT, 'checkpoint_sha256': sha(weights), 'input_spec': adapter.INPUT_SPEC, + 'output_shapes': output_shapes, 'onnx_sha256': sha(onnx_path), 'onnx_ops': ops, 'checks': checks, + 'calibration': str(calib), 't1': str(a.t1), 't1_sha256': sha(a.t1), 'feature_protocol': feat.PROTOCOL, + 'feature_protocol_sha256': feat.PROTOCOL_SHA, 'torch_version': torch.__version__, + 'ort_version': ort.__version__, 'cv2_version': cv2.__version__, 'board_tested': False, + 'quant_compile_tested': False, 'admitted': False}) + print(json.dumps({'model': a.model, 'stage': 'onnx_probe_passed', 'shapes': output_shapes}), flush=True) + if a.stage == 'probe': return + for label, runner in [('fp32', torch_run), ('ort', ort_run)]: + started = time.time() + out = {'model': a.model, 'task': 'feat', 't1_manifest_sha': t1.manifest['sha256'], 'evaluator_version': 'feature-static-1'} + feat.run_fp32(runner, t1, adapter.INPUT_SPEC, out) + out['wall_seconds'] = time.time() - started + write(a.out / f'{label}-metrics.json', out) + print(json.dumps({'model': a.model, 'stage': label + '_100pair_done', 'metrics': out['metrics'], 'seconds': out['wall_seconds']}), flush=True) + baseline = json.loads((a.out / 'fp32-metrics.json').read_text()); measured = json.loads((a.out / 'ort-metrics.json').read_text()) + b = baseline['metrics'][feat.METRIC]; m = measured['metrics'][feat.METRIC] + loss_pct = 100 * (b - m) / b if b > 0 else None + write(a.out / 'receipt.json', {'model': a.model, 'status': 'fp32_ort_100pair_complete', 'n': 100, + 'fp32': baseline['metrics'], 'ort': measured['metrics'], 'ort_relative_auc_loss_pct': loss_pct, + 'within_existing_5pct_tolerance': loss_pct is not None and loss_pct <= 5, + 'protocol_sha256': feat.PROTOCOL_SHA, 'calibration_count': 128, 'calibration': str(calib), + 't1_sha256': sha(a.t1), 'onnx_sha256': sha(onnx_path), 'probe': str(a.out/'probe.json'), + 'admitted': False, 'quant_compile_tested': False, 'board_tested': False, 'agent_trajectory_collected': False}) + t1.tar.close() + if loss_pct is None or loss_pct > 5: raise ValueError(f'ORT geometric AUC exceeds 5% degradation: {loss_pct}') + + +if __name__ == '__main__': main() diff --git a/scripts/probe_midas_adapters.py b/scripts/probe_midas_adapters.py new file mode 100644 index 0000000..d677e37 --- /dev/null +++ b/scripts/probe_midas_adapters.py @@ -0,0 +1,420 @@ +"""Probe the three pinned official MiDaS models on the existing NYUv2 T1. + +Same admission evidence as probe_depth_adapters.py: strict official checkpoint +load, real-data FP32 forward, ONNX export plus checker, onnxruntime parity on +real images, per-image alignment audit against the frozen nyuv2 protocol and a +128-image train-split calibration package. No installs, no run/queue mutation, +no quantization and no board pass. Static deployment uses each variant's own +midas.model_loader normalization (v3 DPT mean=std=0.5, v2.1 small ImageNet). +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import json +import os +from pathlib import Path +import sys +import time +import traceback + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +VARIANTS = {"midas.small": "small", "dpt.hybrid": "hybrid", "dpt.large": "large"} + + +def digest(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def save(path: Path, value: dict): + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2, allow_nan=False)) + temp.replace(path) + + +def open_eval_set(t1_manifest: Path) -> dict: + from harness.continuation import load_eval_set + dataset, evaluation = load_eval_set(t1_manifest) + if dataset != "nyuv2" or evaluation.get("n") != 100: + raise ValueError(f"expected the frozen nyuv2 n100 T1, got {dataset}/{evaluation.get('n')}") + manifest = json.loads(t1_manifest.read_text()) + evaluation = dict(evaluation) + evaluation["eval_set_id"] = manifest["eval_set_id"] + evaluation["content_sha"] = manifest["content_sha256"] + evaluation["manifest"] = str(t1_manifest) + return evaluation + + +def calibration(mat: Path, splits: Path, evaluation: dict, models: list[str], out: Path) -> None: + """Deterministic 128-image train-split tensors for each new recipe.""" + import h5py + import numpy as np + from PIL import Image + from scipy.io import loadmat + from harness import paths + from harness.adapters import calibration_dir, get, model_spec + from harness.registry import model_row + + ids = loadmat(splits) + train, test = [sorted(int(v) for v in ids[key].ravel()) for key in ("trainNdxs", "testNdxs")] + if len(train) != 795 or len(test) != 654: + raise ValueError("unexpected NYUv2 official split") + manifest = json.loads(Path(evaluation["manifest"]).read_text()) + raw_sha, split_sha = digest(mat), digest(splits) + if (manifest["source"]["raw_sha256"] != raw_sha or manifest["source"]["split_sha256"] != split_sha): + raise ValueError("raw NYUv2 material differs from the frozen T1 source") + calib_ids = sorted(set(manifest["source"]["calibration_train_ids"])) + if len(calib_ids) != 128 or not set(calib_ids) <= set(train): + raise ValueError("calibration ids must be 128 official train images") + import random + if calib_ids != sorted(random.Random(manifest["source"]["seed"]).sample(train, 128)): + raise ValueError("calibration ids are not the recorded seed sample") + test_ids = set(manifest["source"]["test_ids"]) + with h5py.File(mat, "r") as raw: + if raw["images"].shape != (1449, 3, 640, 480): + raise ValueError("unexpected NYU original HDF5 dimensions") + + def read_image(index: int): + rgb = np.asarray(raw["images"][index - 1]).transpose(2, 1, 0) + if rgb.dtype != np.uint8 or rgb.shape != (480, 640, 3): + raise ValueError("invalid NYU RGB") + return Image.fromarray(rgb) + + for model in models: + row = model_spec(model_row(model)) + recipe = get(model).PREPROCESS + target = calibration_dir(row) + if (target / "manifest.json").exists(): + cm = json.loads((target / "manifest.json").read_text()) + files = sorted((target / "npy_f32").glob("*.npy")) + h = hashlib.sha256() + for p in files: + h.update(p.read_bytes()) + if (cm.get("count") != 128 or len(files) != 128 or cm.get("preprocess") != recipe + or cm.get("sha256") != h.hexdigest() or cm.get("t1_sha256") != digest(Path(evaluation["path"]))): + raise ValueError(f"existing calibration differs; refusing to overwrite {target}") + print(f"{model}: calibration verified at {target}", flush=True) + continue + if target.exists(): + raise FileExistsError(f"partial calibration; refusing to overwrite {target}") + temporary = target.with_name(target.name + f".partial-{os.getpid()}") + (temporary / "npy_f32").mkdir(parents=True, exist_ok=False) + items, h = [], hashlib.sha256() + for index in calib_ids: + image = read_image(index) + if index in test_ids: + raise ValueError("calibration/test split overlap") + x = get(model).preprocess(image, row["input_spec"]) + if x.shape != tuple(int(v) for v in row["input_spec"].split("x")) or not np.isfinite(x).all(): + raise ValueError("invalid real calibration tensor") + path = temporary / "npy_f32" / f"{len(items):05d}.npy" + np.save(path, x) + h.update(path.read_bytes()) + items.append({"id": f"nyuv2/train/{index:04d}", "official_index": index, + "rgb_sha256": hashlib.sha256(np.asarray(image).tobytes()).hexdigest(), + "npy": path.name, "npy_sha256": digest(path)}) + save(temporary / "manifest.json", {"task": "mde", "input_spec": row["input_spec"], + "count": 128, "seed": manifest["source"]["seed"], "source": "official-labeled-train", + "dataset": "nyuv2", "items": items, "files": [item["id"] for item in items], + "sha256": h.hexdigest(), "preprocess": recipe, "npy_dir": "npy_f32", + "layout": "NCHW", "dtype": "float32", "test_content_overlap": 0, + "t1_sha256": digest(Path(evaluation["path"])), + "raw_source": {"raw_mat": str(mat), "raw_sha256": raw_sha, "split_sha256": split_sha}}) + temporary.rename(target) + print(f"{model}: calibration written at {target}", flush=True) + + +def verify_contract(evaluation: dict, models: list[str], out: Path) -> None: + """Independent geometry/preprocessing checks on the frozen T1 package.""" + import numpy as np + from harness.eval.common import T1 + from harness.eval import mde + from harness.adapters import get + + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + from harness.adapters import midas_common + root = midas_common.source_root() + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + import cv2 # noqa: F401 (the pinned midas transform imports cv2) + from midas.transforms import NormalizeImage, PrepareForNet, Resize + import midas.transforms as pinned_transforms + if Path(pinned_transforms.__file__).resolve() != root / "midas/transforms.py": + raise RuntimeError("a different MiDaS source is already imported") + total_valid = 0 + for item in t1.items: + for key, expected in item["file_sha256"].items(): + assert hashlib.sha256(t1.tar.extractfile(item[key]).read()).hexdigest() == expected + depth, mask = mde.ground_truth(t1, item) + total_valid += int(mask.sum()) + image = t1.image(t1.items[0]) + for model in models: + adapter = get(model) + variant = VARIANTS[model] + size = int(adapter.INPUT_SPEC.split("x")[-1]) + mean, std = (([.5, .5, .5], [.5, .5, .5]) if variant != "small" + else ([.485, .456, .406], [.229, .224, .225])) + native = {"image": np.asarray(image, dtype=np.float64) / 255.0} + native = Resize(size, size, resize_target=None, keep_aspect_ratio=False, ensure_multiple_of=32, + resize_method="upper_bound" if variant == "small" else "minimal", + image_interpolation_method=cv2.INTER_CUBIC)(native) + native = NormalizeImage(mean=mean, std=std)(native) + native = PrepareForNet()(native)["image"][None] + np.testing.assert_array_equal(adapter.preprocess(image, adapter.INPUT_SPEC), native) + # Exact affine inverse depth must recover GT; raw prediction is not metres. + truth = np.linspace(.5, 9.5, 10000, dtype=np.float32).reshape(100, 100) + relative = (1.0 / truth - .02) / .04 + score = mde.aligned_metrics(relative, truth, np.ones_like(truth, dtype=bool)) + assert score["absrel"] < 1e-6 and score["delta1"] == 100 + signed = mde.aligned_metrics(3.0 - 1.0 / truth, truth, np.ones_like(truth, dtype=bool)) + np.testing.assert_allclose([signed["scale"], signed["shift"]], [-1, 3], atol=1e-6) + for bad in [np.ones_like(truth), np.full_like(truth, np.nan)]: + try: + mde.aligned_metrics(bad, truth, np.ones_like(truth, dtype=bool)) + except ValueError: + pass + else: + raise AssertionError("degenerate/nonfinite input accepted") + for model in models: + size = int(get(model).INPUT_SPEC.split("x")[-1]) + constant = np.full((1, size, size), 2, dtype=np.float32) + restored = mde.restore_prediction([constant], (480, 640), model) + np.testing.assert_allclose(restored, np.full((480, 640), 2), atol=1e-5) + save(out / "contract_checks.json", {"status": "passed", "t1_sha256": digest(Path(evaluation["path"])), + "decoded_real_labels": len(t1.items), "valid_pixels": total_valid, + "models": models, + "checks": ["all real RGB/depth member hashes", + "official per-variant static resize/normalization equality (model_loader statistics)", + "original metre GT and fixed crop", "affine inverse-depth exact recovery", + "signed LS recovers (-1,3)", "reject constant/NaN prediction", + "restore constant depth per family shape without vector scaling"]}) + finally: + t1.tar.close() + + +def probe(model: str, evaluation: dict, weights_dir: Path, out: Path, n: int, confirm: bool, threads: int): + import numpy as np + import torch + import torchvision + import onnx + import onnxruntime as ort + from harness.eval.common import T1, cosine + from harness.eval import EVALUATOR_VERSION, mde + + recipe = importlib.import_module("harness.adapters." + model.replace(".", "_")) + directory = out / model + directory.mkdir(parents=True, exist_ok=True) + weights = weights_dir / recipe.CHECKPOINT["url"].rsplit("/", 1)[1] + size = int(recipe.INPUT_SPEC.split("x")[-1]) + result = {"model": model, "status": "running", "started_at": time.time(), "input_spec": recipe.INPUT_SPEC, + "checkpoint": recipe.CHECKPOINT, "preprocess": recipe.PREPROCESS, "opset": recipe.OPSET, + "exporter_version": recipe.EXPORTER_VERSION, + "weights_path": str(weights), "t1_path": evaluation["path"], + "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, + "evaluation_protocol": mde.PROTOCOL, + "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__, "device": "cpu", + "threads": threads}, + "input_contract": {"status": "unverified"}, "board_tested": False, "quantization_tested": False} + + def phase(name): + result["phase"] = name + save(directory / "probe.json", result) + print(f"{model}: {name}", flush=True) + + try: + source_files = ["harness/adapters/midas_common.py", + "harness/adapters/" + model.replace(".", "_") + ".py", "harness/eval/mde.py"] + root = Path(__file__).resolve().parents[1] + result["source_sha256"] = {name: digest(root / name) for name in source_files} + phase("strict_official_load") + result["weights_sha256"] = digest(weights) + model_object = recipe.build(weights, recipe.INPUT_SPEC) + result.update(strict_load=True, parameters=sum(p.numel() for p in model_object.parameters())) + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} + if len(t1.items) != n: + raise ValueError("requested sample count differs from actual T1 subset") + result["sample_ids"] = [item["id"] for item in t1.items] + references = {} + + def native(x): + key = hashlib.sha256(x.tobytes()).hexdigest() + if key in references: + return [references[key]] + with torch.inference_mode(): + value = model_object(torch.from_numpy(x)).numpy() + if value.shape != (1, size, size) or not np.isfinite(value).all(): + raise ValueError("invalid official forward output") + references[key] = value + print(f"{model}: FP32 {len(references)}/{n}", flush=True) + return [value] + + phase("real_nyu_fp32_forward") + baseline = {"task": "mde", "model": model, "n": n, + "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"]} + for item in t1.items: + native(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) + np.savez(directory / "fp32_predictions.npz", **references) + result["fp32_forward_n"] = len(references) + first = mde.inputs(t1, t1.items[0], recipe.INPUT_SPEC, model) + np.save(directory / "input_first.npy", first) + np.save(directory / "fp32_first.npy", references[hashlib.sha256(first.tobytes()).hexdigest()]) + phase("onnx_export_and_checker") + path = directory / f"{model}.onnx" + started = time.monotonic() + with torch.no_grad(): + torch.onnx.export(model_object, torch.from_numpy(first), str(path), dynamo=False, + opset_version=recipe.OPSET, input_names=["input"], + output_names=["inverse_depth"], dynamic_axes=None, + do_constant_folding=True) + result["onnx"] = {"path": str(path), "sha256": digest(path), "size": path.stat().st_size, + "export_seconds": time.monotonic() - started} + onnx.checker.check_model(str(path), full_check=True) + phase("real_nyu_onnxruntime_and_paired_metrics") + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) + result["onnx"]["input_shape"] = session.get_inputs()[0].shape + result["onnx"]["output_shape"] = session.get_outputs()[0].shape + if (result["onnx"]["input_shape"] != [1, 3, size, size] + or result["onnx"]["output_shape"] != [1, size, size]): + raise ValueError("ONNX static input/output shape mismatch") + comparisons = [] + converted_outputs = {} + result["comparisons"] = comparisons + + def converted(x): + key = hashlib.sha256(x.tobytes()).hexdigest() + if key in converted_outputs: + return [converted_outputs[key]] + value = session.run(None, {"input": x})[0] + ref = references[key] + difference = np.abs(ref - value) + comp = {"sample_id": result["sample_ids"][len(comparisons)], "cosine": cosine(ref, value), + "max_abs": float(difference.max()), "mean_abs": float(difference.mean()), + "relative_max_abs": float(difference.max() / max(float(np.abs(ref).max()), 1e-12))} + comparisons.append(comp) + print(json.dumps(comp), flush=True) + if not np.isfinite(value).all() or comp["cosine"] < .99999 or comp["relative_max_abs"] > .001: + raise ValueError(f"ONNX mismatch: {comp}") + converted_outputs[key] = value + return [value] + + for item in t1.items: + converted(mde.inputs(t1, item, recipe.INPUT_SPEC, model)) + np.savez(directory / "onnx_predictions.npz", **converted_outputs) + result["onnxruntime_forward_n"] = len(converted_outputs) + phase("real_nyu_alignment_audit") + audit, failures = [], [] + for item in t1.items: + x = mde.inputs(t1, item, recipe.INPUT_SPEC, model) + depth, mask = mde.ground_truth(t1, item) + sample = {"id": item["id"]} + for name, runner in (("fp32", native), ("onnxruntime", converted)): + prediction = mde.restore_prediction(runner(x), depth.shape, model) + try: + sample[name] = {"status": "measured", **mde.aligned_metrics(prediction, depth, mask)} + except ValueError as exc: + sample[name] = {"status": "blocked", "error": str(exc)} + failures.append({"id": item["id"], "runner": name, "error": str(exc)}) + audit.append(sample) + save(directory / "alignment_audit.json", {"protocol": mde.PROTOCOL, "n": n, + "samples": audit, "failures": failures, "status": "blocked" if failures else "passed"}) + if failures: + result["evaluation_contract"] = {"status": "blocked", "protocol": mde.PROTOCOL, + "failures": failures} + raise ValueError(f"alignment failed for {len({s['id'] for s in failures})}/{n} real samples") + mde.run_fp32(native, t1, recipe.INPUT_SPEC, baseline) + result["fp32"] = baseline["metrics"] + measured = {"task": "mde", "model": model} + mde.run_quant(converted, t1, recipe.INPUT_SPEC, baseline, measured) + paired = measured["G3"]["extra"]["scale_alignment"]["samples"] + if len(paired) != n or len({sample["id"] for sample in paired}) != n: + raise ValueError("incomplete/duplicate per-image alignment proof") + for sample, fp, quant in zip(paired, baseline["_depth_samples"], measured["_depth_samples"]): + if (sample["id"] != fp["id"] or sample["id"] != quant["id"] + or sample["fp32_scale"] != fp["scale"] or sample["fp32_shift"] != fp["shift"] + or sample["quant_scale"] != quant["scale"] or sample["quant_shift"] != quant["shift"] + or not np.isfinite([sample[key] for key in + ("fp32_scale", "fp32_shift", "quant_scale", "quant_shift")]).all()): + raise ValueError("invalid/mispaired real per-image alignment proof") + if (abs(baseline["metrics"]["absrel"] - measured["metrics"]["absrel"]) > .001 + or abs(baseline["metrics"]["delta1"] - measured["metrics"]["delta1"]) > .1): + raise ValueError("FP32/ONNX paired depth metric mismatch") + checks = json.loads((out / "contract_checks.json").read_text()) + if checks.get("status") != "passed" or checks.get("t1_sha256") != result["t1_sha256"]: + raise ValueError("missing matching real-data and alignment contract checks") + result.update(comparisons=comparisons, onnxruntime=measured["metrics"], G3_probe=measured["G3"], + status="validated_fp32_export_only" if confirm else "validated_export_pending_eval_protocol", + phase="complete") + result["input_contract"] = {"status": "validated", "input_spec": recipe.INPUT_SPEC, + "output_semantics": "nonnegative_relative_inverse_depth_not_metric", + "t1_sha256": result["t1_sha256"], "checks_path": str(out / "contract_checks.json"), + "sample_ids": result["sample_ids"], "onnx_sha256": result["onnx"]["sha256"]} + result["evaluation_contract"] = {"status": "validated" if confirm else "pending_mainline_protocol", + "protocol": mde.PROTOCOL} + baseline["note"] = "probe subset only, not the complete 100-image FP32 baseline" + save(directory / "baseline_probe.json", baseline) + save(directory / "onnx_metrics_probe.json", measured) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + (directory / "error.txt").write_text(traceback.format_exc()) + result["finished_at"] = time.time() + save(directory / "probe.json", result) + print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", choices=sorted(VARIANTS), required=True) + parser.add_argument("--t1-manifest", type=Path, required=True, + help="manifest.json of the existing frozen nyuv2 n100 T1 package") + parser.add_argument("--weights-dir", type=Path, required=True) + parser.add_argument("--data-reader", type=Path, + help="private unpacked h5py/scipy wheels used only to read the source MAT") + parser.add_argument("--mat", type=Path, required=True, help="original nyu_depth_v2_labeled.mat") + parser.add_argument("--splits", type=Path, required=True, help="official splits.mat") + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--n", type=int, default=3) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--confirm-protocol", choices=["nyuv2-relative-inverse-ls-v1"]) + args = parser.parse_args() + if not 1 <= args.n <= 100 or args.threads < 1: + parser.error("require 1 <= n <= 100 and positive threads") + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + if args.data_reader: + sys.path.insert(0, str(args.data_reader.resolve())) + models = list(dict.fromkeys(args.model)) + evaluation = open_eval_set(args.t1_manifest) + from harness.adapters import midas_common + midas_common.source_root() + calibration(args.mat, args.splits, evaluation, models, args.out) + verify_contract(evaluation, models, args.out) + save(args.out / "data_preparation.json", {"t1_path": evaluation["path"], + "t1_sha256": evaluation["manifest_sha"], "eval_set": evaluation, "n": args.n}) + results = [probe(model, evaluation, args.weights_dir, args.out, args.n, + bool(args.confirm_protocol), args.threads) for model in models] + save(args.out / "summary.json", {"results": results, "complete": True}) + return int(any(result["status"] == "blocked" for result in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_mmdet_dense_adapters.py b/scripts/probe_mmdet_dense_adapters.py new file mode 100644 index 0000000..a3189bd --- /dev/null +++ b/scripts/probe_mmdet_dense_adapters.py @@ -0,0 +1,375 @@ +"""Validate exact MMDetection dense detectors in private output directories. + +Run each variant with the same COCO n100 manifest and train128 recipe. +All generated files stay under --out; existing official weights may be reused. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import sys +import time +import traceback +import urllib.request +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness.adapters import get +from harness.adapters.openmmlab import get_recipe +from harness.adapters.mmdet_dense_common import RUNTIME_PYTHON, geometry, output_names, MODELS + + +def sha256(path): + h = hashlib.sha256() + with Path(path).open("rb") as f: + for part in iter(lambda: f.read(1 << 20), b""): + h.update(part) + return h.hexdigest() + + +def save(path, value): + Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") + + +def weights_for(model, args): + info = get(model).CHECKPOINT + path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"] + if not path.is_file(): + if args.weights_dir: + raise FileNotFoundError(path) + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(".partial") + with urllib.request.urlopen(info["url"], timeout=120) as response, partial.open("wb") as f: + headers = dict(response.headers) + final_url = response.url + shutil.copyfileobj(response, f) + partial.replace(path) + save(path.with_suffix(".download.json"), { + "model": model, "url": info["url"], "final_url": final_url, + "headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path), + }) + digest = sha256(path) + if info.get("sha256") and digest != info["sha256"]: + raise ValueError(f"{model}: checkpoint hash mismatch") + return path, digest + + +def official_input(network, pipeline, image, image_id): + import numpy as np + data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id)) + batch = network.data_preprocessor({ + "inputs": [data["inputs"]], "data_samples": [data["data_samples"]], + }, training=False) + return batch["inputs"].numpy(), batch["data_samples"][0].metainfo + + +def check_input(adapter, network, pipeline, image, image_id): + import numpy as np + x = adapter.preprocess(image, adapter.INPUT_SPEC) + expected, meta = official_input(network, pipeline, image, image_id) + _, scales, img_shape = geometry(image.size) + error = float(np.max(np.abs(x - expected))) + if (x.shape != (1, 3, 640, 640) or error != 0 or tuple(meta["img_shape"]) != img_shape + or tuple(meta["pad_shape"]) != (640, 640) + or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8)): + raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}") + return x, {"input_max_abs": error, "original_size": list(image.size), + "scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"])} + + +def calibration(args, adapter, network, pipeline, t1): + import numpy as np + from PIL import Image + files = sorted(Path(args.train_dir).glob("*.jpg")) + if len(files) < 128: + raise ValueError(f"Need 128 actual training images, found {len(files)}") + random.Random(20260914).shuffle(files) + files = sorted(files[:128]) + with zipfile.ZipFile(args.annotations_zip) as archive: + with archive.open("annotations/captions_train2017.json") as f: + annotation = json.load(f) + official_train = {im["file_name"]: im for im in annotation["images"]} + train_ids = {int(p.stem) for p in files} + if train_ids & {it["id"] for it in t1.items}: + raise ValueError("Train calibration overlaps evaluation IDs") + directory = Path(args.out) / adapter.MODEL_ID / "calibration" + npy_dir = directory / "npy_f32" + npy_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + records = [] + for i, path in enumerate(files): + item = official_train.get(path.name) + if not item or item["id"] != int(path.stem): + raise ValueError(f"Not in official COCO train2017 annotations: {path}") + with Image.open(path) as opened: + image = opened.convert("RGB") + if image.size != (item["width"], item["height"]): + raise ValueError(f"Training image geometry differs from annotation: {path}") + x, comparison = check_input(adapter, network, pipeline, image, item["id"]) + target = npy_dir / f"{i:05d}.npy" + if target.exists(): + if not np.array_equal(np.load(target), x): + raise ValueError(f"Existing calibration changed: {target}") + else: + np.save(target, x) + digest.update(target.read_bytes()) + records.append({"source": str(path), "image_id": item["id"], + "image_sha256": sha256(path), "file": target.name, + "npy_sha256": sha256(target), **comparison}) + if len(list(npy_dir.glob("*.npy"))) != 128: + raise ValueError("Calibration directory contains unexpected NPY files") + manifest = {"task": "det", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC, + "count": 128, "seed": 20260914, "source": "train-dir", + "sha256": digest.hexdigest(), "files": [str(p) for p in files], + "preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32", + "npy_dir": "npy_f32", "train_dir": args.train_dir, + "annotations_zip": args.annotations_zip, + "annotation_member": "annotations/captions_train2017.json", + "official_input_exact_matches": 128, "eval_overlap": [], "items": records} + save(directory / "manifest.json", manifest) + return {"path": str(directory), "sha256": manifest["sha256"], "count": 128, + "official_input_exact_matches": 128, "eval_overlap": [], + "recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()} + + +def probe(model_id, args): + import numpy as np + import onnx + import onnxruntime as ort + import torch + from mmcv.transforms import Compose + from mmdet.apis import inference_detector + from mmdet.datasets import CocoDataset + from mmdet.utils import get_test_pipeline_cfg + from harness.eval import EVALUATOR_VERSION + from harness.eval.common import T1, cosine + from harness.eval.det import coco_eval + + adapter = get(model_id) + directory = Path(args.out) / model_id + directory.mkdir(parents=True, exist_ok=True) + proof = directory / "probe.json" + if proof.exists(): + proof.rename(directory / f"probe-history-{time.time_ns()}.json") + bench = Path(__file__).resolve().parents[1] + sources = ["harness/adapters/mmdet_dense_common.py", "harness/adapters/openmmlab.py", + "harness/adapters/__init__.py", f"harness/adapters/{model_id.replace('.', '_')}.py", + "harness/eval/det.py", "harness/eval/common.py", "harness/eval/run.py", + "scripts/make_calib.py", "scripts/probe_mmdet_dense_adapters.py"] + if model_id.startswith(("vfnet.", "reppoints.")): + sources.append("harness/adapters/vfnet_deformconv.py") + result = {"model": model_id, "task": "det", "status": "running", "started_at": time.time(), + "runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT, + "preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"], + "source_sha256": {p: sha256(bench / p) for p in sources}, + "recipe_snapshot": get_recipe(model_id), "sample_count": args.n, + "runtime": {"torch": torch.__version__, "onnx": onnx.__version__, + "onnxruntime": ort.__version__, "cpu_threads": args.threads}, + "scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"} + save(proof, result) + t1 = None + try: + weights, digest = weights_for(model_id, args) + network = adapter.build(weights, adapter.INPUT_SPEC).eval() + pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg)) + pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray" + resizes = [step for step in pipeline_cfg if step["type"] == "Resize"] + if len(resizes) != 1 or tuple(resizes[0]["scale"]) != (1333, 800) or not resizes[0]["keep_ratio"]: + raise ValueError("Original MMDetection test geometry changed") + # Explicit CSV deployment-size override, not paper/native-scale accuracy. + resizes[0]["scale"] = (640, 640) + network.upstream.data_preprocessor.pad_size_divisor = 640 + result["deployment_geometry"] = { + "upstream_resize": [1333, 800], "upstream_pad_divisor": 32, + "resize": [640, 640], "keep_ratio": True, "normalized_pad_value": 0, + "pad_divisor": 640, "pad_placement": "bottom_right", + "reference": "official pipeline and model under identical CSV static640 contract", + "metric_scope": "fixed COCO100 deployment score; not paper/full-COCO accuracy"} + pipeline = Compose(pipeline_cfg) + manifest_path = Path(args.t1) + manifest = json.loads(manifest_path.read_text()) + if manifest["dataset"] != "coco2017" or manifest["n"] != 100: + raise ValueError("This probe requires the frozen real COCO n100 manifest") + t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"]) + if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]: + raise ValueError("COCO T1 content identity/count mismatch") + categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]} + cat_map = t1.manifest["cat_map"] + if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]): + raise ValueError("Official COCO class order differs from T1 category mapping") + evaluation = {"path": str(manifest_path.parent / manifest["tar"]), + "manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"], + "eval_set_id": manifest["eval_set_id"], "n": 100} + result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True, + eval_set=evaluation, sample_ids=[it["id"] for it in t1.items[:args.n]]) + save(proof, result) + export_path = directory / "model.onnx" + first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC) + names = output_names(model_id) + with torch.inference_mode(): + torch.onnx.export(network, torch.from_numpy(first), str(export_path), opset_version=11, + input_names=["images"], output_names=names) + onnx.checker.check_model(str(export_path)) + graph = onnx.load(str(export_path)) + forbidden = {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"} + ops = sorted({node.op_type for node in graph.graph.node}) + if forbidden.intersection(ops): + raise ValueError(f"Dynamic postprocessing leaked into raw graph: {ops}") + result["graph"] = {"input": [d.dim_value for d in graph.graph.input[0].type.tensor_type.shape.dim], + "output_names": names, "opset": 11, "ops": ops, + "postprocessing_in_graph": False} + if result["graph"]["input"] != [1, 3, 640, 640]: + raise ValueError("ONNX input is not static CSV geometry") + del graph + options = ort.SessionOptions() + options.intra_op_num_threads = args.threads + options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"]) + predictions = {key: [] for key in ("official", "fp32", "onnx")} + comparisons = [] + g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))} + g2_x, g2_y = {}, {} + + def add_predictions(key, image_id, values): + boxes, scores, labels = values + for box, score, label in zip(boxes, scores, labels): + predictions[key].append({"image_id": image_id, "category_id": cat_map[str(int(label))], + "score": float(score), "bbox": [float(box[0]), float(box[1]), + float(box[2] - box[0]), float(box[3] - box[1])]}) + + for index, item in enumerate(t1.items[:args.n]): + image = t1.image(item) + x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"]) + with torch.inference_mode(): + raw = [value.numpy() for value in network(torch.from_numpy(x))] + official = inference_detector(network.upstream, np.asarray(image)[:, :, ::-1].copy(), + test_pipeline=pipeline).pred_instances + reference = (official.bboxes.numpy(), official.scores.numpy(), official.labels.numpy()) + decoded = adapter.decode(raw, image.size) + if not np.array_equal(reference[2], decoded[2]): + raise ValueError(f"Official/adapter selected classes differ at image {item['id']}") + box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0)) + score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0)) + if box_error > 1e-4 or score_error > 1e-6: + raise ValueError(f"Official/adapter detections differ: image={item['id']} boxes={box_error} scores={score_error}") + exported = session.run(None, {session.get_inputs()[0].name: x}) + head_comparisons = [] + for name, a, b in zip(names, raw, exported): + if a.shape != b.shape or not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError(f"ONNX head shape/nonfinite output: {name}") + similarity = cosine(a, b) + head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity, + "max_abs": float(np.max(np.abs(a - b)))}) + if similarity < .9999: + raise ValueError(f"FP32/ONNX {name} cosine below .9999: {similarity}") + add_predictions("official", item["id"], reference) + add_predictions("fp32", item["id"], decoded) + add_predictions("onnx", item["id"], adapter.decode(exported, image.size)) + comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error, + "official_score_max_abs": score_error, "detections": len(decoded[1]), + "onnx_heads": head_comparisons}) + if item["id"] in g2_keys: + key = g2_keys[item["id"]] + g2_x[key] = x + g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)}) + if (index + 1) % 10 == 0: + print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True) + save(directory / "comparisons.json", comparisons) + metrics = {} + save(directory / "comparisons.json", comparisons) + eval_t1 = T1({**t1.manifest, "items": t1.items[:args.n]}, t1.tar) + for key, values in predictions.items(): + save(directory / f"{key}-predictions.json", values) + # pycocotools.loadRes adds area/id/iscrowd/segmentation in place. + # Keep the saved prediction contract intact for the shared hook check. + metrics[key] = coco_eval(eval_t1, deepcopy(values)) + save(directory / "metrics-all.json", metrics) + if metrics["fp32"]["map_50_95"] <= 0 or metrics["official"] != metrics["fp32"]: + raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}") + if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("map_50_95", "ap50")): + raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}") + baseline_dir = directory / "baseline" + baseline_dir.mkdir(exist_ok=True) + baseline = {"task": "det", "model": model_id, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": args.n, "metrics": metrics["fp32"]} + save(baseline_dir / "metrics.json", baseline) + np.savez(baseline_dir / "g2_inputs.npz", **g2_x) + np.savez(baseline_dir / "g2_fp32.npz", **g2_y) + if args.n == 100: + from harness.eval.run import g2_inputs + shared_g2 = g2_inputs("det", t1, adapter.INPUT_SPEC, 32, model=model_id) + if len(g2_x) != 32 or len(g2_y) != 32 * len(names) or g2_x.keys() != shared_g2.keys(): + raise ValueError("Missing real G2 inputs/outputs") + for key in g2_x: + np.testing.assert_array_equal(g2_x[key], shared_g2[key]) + result["g2"] = {"count": 32, "outputs_per_sample": len(names), + "shared_input_hook_exact": True, + "inputs_sha256": sha256(baseline_dir / "g2_inputs.npz"), + "outputs_sha256": sha256(baseline_dir / "g2_fp32.npz")} + result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"], + baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path), + comparisons_path=str(directory / "comparisons.json"), + official_exact_input_count=args.n, official_predict_matches=args.n, + onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_heads"]), + official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons), + official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons)) + save(proof, result) + if args.n == 100: + result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1) + # Exercise the unchanged shared det prediction route on real inputs. + # Run the last real image again through ONNX and the shared decoder hook. + from harness.eval.det import predict + one = T1({**t1.manifest, "items": [t1.items[-1]]}, t1.tar) + actual = predict(lambda value: session.run(None, {session.get_inputs()[0].name: value}), + one, adapter.INPUT_SPEC, model=model_id) + expected = [p for p in predictions["onnx"] if p["image_id"] == t1.items[-1]["id"]] + if actual != expected: + raise ValueError("Shared det.predict differs from validated ONNX route") + result["shared_det_hook_verified"] = True + result["status"] = "validated_fp32_export_only" if args.n == 100 else "smoke_passed" + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) + finally: + if t1: + t1.tar.close() + result["finished_at"] = time.time() + save(proof, result) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=MODELS, action="append", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--weights-dir") + parser.add_argument("--download-only", action="store_true") + parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json") + parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017") + parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip") + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--n", type=int, choices=[3, 100], default=100) + args = parser.parse_args() + if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON): + os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]]) + Path(args.out).mkdir(parents=True, exist_ok=True) + if args.download_only: + for model in args.model: + path, digest = weights_for(model, args) + print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True) + return 0 + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + return int(any([probe(model, args)["status"] not in ("validated_fp32_export_only", "smoke_passed") for model in args.model])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_mmdet_detr2_adapters.py b/scripts/probe_mmdet_detr2_adapters.py new file mode 100644 index 0000000..5889d7e --- /dev/null +++ b/scripts/probe_mmdet_detr2_adapters.py @@ -0,0 +1,410 @@ +"""Validate the batch-6 exact MMDetection DETR extension in private output dirs. + +Same COCO n100 manifest and train128 recipe as the batch-1 DETR probe; the +shared export geometry (static 640, in-graph padding mask, raw query outputs) +is imported from mmdet_detr_common without modifying it. All generated files +stay under --out; existing official weights may be reused. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import sys +import time +import traceback +import urllib.request +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness.adapters import get +from harness.adapters.openmmlab import get_recipe +from harness.adapters.mmdet_detr_common import RUNTIME_PYTHON, geometry, padding_mask + +MODELS = ("conditional.detr.r50",) + + +def output_names(model): + return get(model).output_names() + + +def sha256(path): + h = hashlib.sha256() + with Path(path).open("rb") as f: + for part in iter(lambda: f.read(1 << 20), b""): + h.update(part) + return h.hexdigest() + + +def save(path, value): + Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") + + +def weights_for(model, args): + info = get(model).CHECKPOINT + path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"] + if not path.is_file(): + if args.weights_dir: + raise FileNotFoundError(path) + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(".partial") + with urllib.request.urlopen(info["url"], timeout=120) as response, partial.open("wb") as f: + headers = dict(response.headers) + final_url = response.url + shutil.copyfileobj(response, f) + partial.replace(path) + save(path.with_suffix(".download.json"), { + "model": model, "url": info["url"], "final_url": final_url, + "headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path), + }) + digest = sha256(path) + if info.get("sha256") and digest != info["sha256"]: + raise ValueError(f"{model}: checkpoint hash mismatch") + return path, digest + + +def official_input(network, pipeline, image, image_id): + import numpy as np + data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id)) + batch = network.data_preprocessor({ + "inputs": [data["inputs"]], "data_samples": [data["data_samples"]], + }, training=False) + return batch["inputs"].numpy(), batch["data_samples"][0].metainfo + + +def check_input(adapter, network, pipeline, image, image_id): + import numpy as np + import torch + x = adapter.preprocess(image, adapter.INPUT_SPEC) + expected, meta = official_input(network, pipeline, image, image_id) + _, scales, img_shape = geometry(image.size) + error = float(np.max(np.abs(x - expected))) + if (x.shape != (1, 3, 640, 640) or error != 0 or tuple(meta["img_shape"]) != img_shape + or tuple(meta["pad_shape"]) != (640, 640) + or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8)): + raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}") + expected_mask = np.ones((1, 640, 640), dtype=bool) + expected_mask[:, :img_shape[0], :img_shape[1]] = False + np.testing.assert_array_equal(padding_mask(torch.from_numpy(x)).numpy(), expected_mask) + return x, {"input_max_abs": error, "original_size": list(image.size), + "scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"]), + "in_graph_padding_mask_exact": True} + + +def calibration(args, adapter, network, pipeline, t1): + import numpy as np + from PIL import Image + files = sorted(Path(args.train_dir).glob("*.jpg")) + if len(files) < 128: + raise ValueError(f"Need 128 actual training images, found {len(files)}") + random.Random(20260914).shuffle(files) + files = sorted(files[:128]) + with zipfile.ZipFile(args.annotations_zip) as archive: + with archive.open("annotations/captions_train2017.json") as f: + annotation = json.load(f) + official_train = {im["file_name"]: im for im in annotation["images"]} + train_ids = {int(p.stem) for p in files} + if train_ids & {it["id"] for it in t1.items}: + raise ValueError("Train calibration overlaps evaluation IDs") + directory = Path(args.out) / adapter.MODEL_ID / "calibration" + npy_dir = directory / "npy_f32" + npy_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + records = [] + for i, path in enumerate(files): + item = official_train.get(path.name) + if not item or item["id"] != int(path.stem): + raise ValueError(f"Not in official COCO train2017 annotations: {path}") + with Image.open(path) as opened: + image = opened.convert("RGB") + if image.size != (item["width"], item["height"]): + raise ValueError(f"Training image geometry differs from annotation: {path}") + x, comparison = check_input(adapter, network, pipeline, image, item["id"]) + target = npy_dir / f"{i:05d}.npy" + if target.exists(): + if not np.array_equal(np.load(target), x): + raise ValueError(f"Existing calibration changed: {target}") + else: + np.save(target, x) + digest.update(target.read_bytes()) + records.append({"source": str(path), "image_id": item["id"], + "image_sha256": sha256(path), "file": target.name, + "npy_sha256": sha256(target), **comparison}) + if len(list(npy_dir.glob("*.npy"))) != 128: + raise ValueError("Calibration directory contains unexpected NPY files") + manifest = {"task": "det", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC, + "count": 128, "seed": 20260914, "source": "train-dir", + "sha256": digest.hexdigest(), "files": [str(p) for p in files], + "preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32", + "npy_dir": "npy_f32", "train_dir": args.train_dir, + "annotations_zip": args.annotations_zip, + "annotation_member": "annotations/captions_train2017.json", + "official_input_exact_matches": 128, "eval_overlap": [], "items": records} + save(directory / "manifest.json", manifest) + return {"path": str(directory), "sha256": manifest["sha256"], "count": 128, + "official_input_exact_matches": 128, "eval_overlap": [], + "recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()} + + +def probe(model_id, args): + import numpy as np + import onnx + import onnxruntime as ort + import torch + from mmcv.transforms import Compose + from mmdet.apis import inference_detector + from mmdet.datasets import CocoDataset + from mmdet.utils import get_test_pipeline_cfg + from harness.eval import EVALUATOR_VERSION + from harness.eval.common import T1, cosine + from harness.eval.det import coco_eval + + adapter = get(model_id) + directory = Path(args.out) / model_id + directory.mkdir(parents=True, exist_ok=True) + proof = directory / "probe.json" + if proof.exists(): + proof.rename(directory / f"probe-history-{time.time_ns()}.json") + bench = Path(__file__).resolve().parents[1] + sources = ["harness/adapters/conditional_detr_r50.py", "harness/adapters/mmdet_detr_common.py", + "harness/adapters/openmmlab.py", "harness/adapters/__init__.py", + "harness/eval/det.py", "harness/eval/common.py", "harness/eval/run.py", + "scripts/make_calib.py", "scripts/probe_mmdet_detr2_adapters.py", + "harness/adapters/mmdet_dense_common.py"] + result = {"model": model_id, "task": "det", "status": "running", "started_at": time.time(), + "runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT, + "preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"], + "source_sha256": {p: sha256(bench / p) for p in sources}, + "recipe_snapshot": get_recipe(model_id), "sample_count": args.n, + "runtime": {"torch": torch.__version__, "onnx": onnx.__version__, + "onnxruntime": ort.__version__, "cpu_threads": args.threads}, + "scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"} + save(proof, result) + t1 = None + try: + weights, digest = weights_for(model_id, args) + network = adapter.build(weights, adapter.INPUT_SPEC).eval() + pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg)) + pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray" + resizes = [step for step in pipeline_cfg if step["type"] == "Resize"] + if len(resizes) != 1 or tuple(resizes[0]["scale"]) != (1333, 800) or not resizes[0]["keep_ratio"]: + raise ValueError("Original MMDetection test geometry changed") + # Explicit CSV deployment-size override, not paper/native-scale accuracy. + resizes[0]["scale"] = (640, 640) + network.upstream.data_preprocessor.pad_size_divisor = 640 + result["deployment_geometry"] = { + "upstream_resize": [1333, 800], "upstream_pad_divisor": 1, + "resize": [640, 640], "keep_ratio": True, "normalized_pad_value": 0, + "pad_divisor": 640, "pad_placement": "bottom_right", + "reference": "official pipeline and model under identical CSV static640 contract", + "metric_scope": "fixed COCO100 deployment score; not paper/full-COCO accuracy"} + pipeline = Compose(pipeline_cfg) + manifest_path = Path(args.t1) + manifest = json.loads(manifest_path.read_text()) + if manifest["dataset"] != "coco2017" or manifest["n"] != 100: + raise ValueError("This probe requires the frozen real COCO n100 manifest") + t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"]) + if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]: + raise ValueError("COCO T1 content identity/count mismatch") + categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]} + cat_map = t1.manifest["cat_map"] + if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]): + raise ValueError("Official COCO class order differs from T1 category mapping") + evaluation = {"path": str(manifest_path.parent / manifest["tar"]), + "manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"], + "eval_set_id": manifest["eval_set_id"], "n": 100} + result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True, + eval_set=evaluation, sample_ids=[it["id"] for it in t1.items[:args.n]]) + save(proof, result) + export_path = directory / "model.onnx" + first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC) + names = output_names(model_id) + with torch.inference_mode(): + torch.onnx.export(network, torch.from_numpy(first), str(export_path), opset_version=adapter.OPSET, do_constant_folding=False, + input_names=["images"], output_names=names) + onnx.checker.check_model(str(export_path)) + graph = onnx.load(str(export_path)) + forbidden = {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"} + ops = sorted({node.op_type for node in graph.graph.node}) + if forbidden.intersection(ops): + raise ValueError(f"Dynamic postprocessing leaked into raw graph: {ops}") + result["graph"] = {"input": [d.dim_value for d in graph.graph.input[0].type.tensor_type.shape.dim], + "output_names": names, "opset": adapter.OPSET, "ops": ops, + "postprocessing_in_graph": False} + if result["graph"]["input"] != [1, 3, 640, 640]: + raise ValueError("ONNX input is not static CSV geometry") + # Check graph reachability, including exporter Identity aliases for + # equal-valued parameters. Every learned inference tensor must feed an + # output; declaring it on the wrapper alone does not count as export. + producers = {name: node for node in graph.graph.node for name in node.output} + reachable = set() + pending = [value.name for value in graph.graph.output] + while pending: + name = pending.pop() + if name in reachable: + continue + reachable.add(name) + if name in producers: + pending.extend(producers[name].input) + parameters = dict(network.named_parameters()) + missing = sorted(set(parameters) - reachable) + result["learned_parameter_graph_coverage"] = { + "parameter_tensors": len(parameters), "parameter_elements": sum(p.numel() for p in parameters.values()), + "reachable_parameter_tensors": len(parameters) - len(missing), "missing": missing, + "constant_folding": False, "host_learned_network": False} + if missing: + raise ValueError(f"Learned parameters absent from output graph: {missing}") + if any(node.domain not in ("", "ai.onnx") for node in graph.graph.node): + raise ValueError("Custom ONNX domain remains") + del graph + options = ort.SessionOptions() + options.intra_op_num_threads = args.threads + options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"]) + predictions = {key: [] for key in ("official", "fp32", "onnx")} + comparisons = [] + g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))} + g2_x, g2_y = {}, {} + + def add_predictions(key, image_id, values): + boxes, scores, labels = values + for box, score, label in zip(boxes, scores, labels): + predictions[key].append({"image_id": image_id, "category_id": cat_map[str(int(label))], + "score": float(score), "bbox": [float(box[0]), float(box[1]), + float(box[2] - box[0]), float(box[3] - box[1])]}) + + for index, item in enumerate(t1.items[:args.n]): + image = t1.image(item) + x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"]) + with torch.inference_mode(): + raw = [value.numpy() for value in network(torch.from_numpy(x))] + official = inference_detector(network.upstream, np.asarray(image)[:, :, ::-1].copy(), + test_pipeline=pipeline).pred_instances + reference = (official.bboxes.numpy(), official.scores.numpy(), official.labels.numpy()) + decoded = adapter.decode(raw, image.size) + if not np.array_equal(reference[2], decoded[2]): + raise ValueError(f"Official/adapter selected classes differ at image {item['id']}") + box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0)) + score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0)) + if box_error > 1e-4 or score_error > 1e-6: + raise ValueError(f"Official/adapter detections differ: image={item['id']} boxes={box_error} scores={score_error}") + exported = session.run(None, {session.get_inputs()[0].name: x}) + head_comparisons = [] + for name, a, b in zip(names, raw, exported): + if a.shape != b.shape or not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError(f"ONNX head shape/nonfinite output: {name}") + similarity = cosine(a, b) + head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity, + "max_abs": float(np.max(np.abs(a - b)))}) + if similarity < .9999: + raise ValueError(f"FP32/ONNX {name} cosine below .9999: {similarity}") + add_predictions("official", item["id"], reference) + add_predictions("fp32", item["id"], decoded) + add_predictions("onnx", item["id"], adapter.decode(exported, image.size)) + comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error, + "official_score_max_abs": score_error, "detections": len(decoded[1]), + "onnx_heads": head_comparisons}) + if item["id"] in g2_keys: + key = g2_keys[item["id"]] + g2_x[key] = x + g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)}) + if (index + 1) % 10 == 0: + print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True) + save(directory / "comparisons.json", comparisons) + metrics = {} + save(directory / "comparisons.json", comparisons) + eval_t1 = T1({**t1.manifest, "items": t1.items[:args.n]}, t1.tar) + for key, values in predictions.items(): + save(directory / f"{key}-predictions.json", values) + # pycocotools.loadRes adds area/id/iscrowd/segmentation in place. + # Keep the saved prediction contract intact for the shared hook check. + metrics[key] = coco_eval(eval_t1, deepcopy(values)) + save(directory / "metrics-all.json", metrics) + if metrics["fp32"]["map_50_95"] <= 0 or metrics["official"] != metrics["fp32"]: + raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}") + if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("map_50_95", "ap50")): + raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}") + baseline_dir = directory / "baseline" + baseline_dir.mkdir(exist_ok=True) + baseline = {"task": "det", "model": model_id, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": args.n, "metrics": metrics["fp32"]} + save(baseline_dir / "metrics.json", baseline) + np.savez(baseline_dir / "g2_inputs.npz", **g2_x) + np.savez(baseline_dir / "g2_fp32.npz", **g2_y) + if args.n == 100: + from harness.eval.run import g2_inputs + shared_g2 = g2_inputs("det", t1, adapter.INPUT_SPEC, 32, model=model_id) + if len(g2_x) != 32 or len(g2_y) != 32 * len(names) or g2_x.keys() != shared_g2.keys(): + raise ValueError("Missing real G2 inputs/outputs") + for key in g2_x: + np.testing.assert_array_equal(g2_x[key], shared_g2[key]) + result["g2"] = {"count": 32, "outputs_per_sample": len(names), + "shared_input_hook_exact": True, + "inputs_sha256": sha256(baseline_dir / "g2_inputs.npz"), + "outputs_sha256": sha256(baseline_dir / "g2_fp32.npz")} + result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"], + baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path), + comparisons_path=str(directory / "comparisons.json"), + official_exact_input_count=args.n, official_predict_matches=args.n, + onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_heads"]), + official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons), + official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons)) + save(proof, result) + if args.n == 100: + result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1) + # Exercise the unchanged shared det prediction route on real inputs. + # Run the last real image again through ONNX and the shared decoder hook. + from harness.eval.det import predict + one = T1({**t1.manifest, "items": [t1.items[-1]]}, t1.tar) + actual = predict(lambda value: session.run(None, {session.get_inputs()[0].name: value}), + one, adapter.INPUT_SPEC, model=model_id) + expected = [p for p in predictions["onnx"] if p["image_id"] == t1.items[-1]["id"]] + if actual != expected: + raise ValueError("Shared det.predict differs from validated ONNX route") + result["shared_det_hook_verified"] = True + result["status"] = "validated_fp32_export_only" if args.n == 100 else "smoke_passed" + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) + finally: + if t1: + t1.tar.close() + result["finished_at"] = time.time() + save(proof, result) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=MODELS, action="append", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--weights-dir") + parser.add_argument("--download-only", action="store_true") + parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json") + parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017") + parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip") + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--n", type=int, choices=[3, 100], default=100) + args = parser.parse_args() + if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON): + os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]]) + Path(args.out).mkdir(parents=True, exist_ok=True) + if args.download_only: + for model in args.model: + path, digest = weights_for(model, args) + print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True) + return 0 + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + return int(any([probe(model, args)["status"] not in ("validated_fp32_export_only", "smoke_passed") for model in args.model])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_mmdet_detr_adapters.py b/scripts/probe_mmdet_detr_adapters.py new file mode 100644 index 0000000..4aad605 --- /dev/null +++ b/scripts/probe_mmdet_detr_adapters.py @@ -0,0 +1,403 @@ +"""Validate exact MMDetection DETR models in private output directories. + +Run each variant with the same COCO n100 manifest and train128 recipe. +All generated files stay under --out; existing official weights may be reused. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import sys +import time +import traceback +import urllib.request +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness.adapters import get +from harness.adapters.openmmlab import get_recipe +from harness.adapters.mmdet_detr_common import RUNTIME_PYTHON, geometry, output_names, MODELS + + +def sha256(path): + h = hashlib.sha256() + with Path(path).open("rb") as f: + for part in iter(lambda: f.read(1 << 20), b""): + h.update(part) + return h.hexdigest() + + +def save(path, value): + Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") + + +def weights_for(model, args): + info = get(model).CHECKPOINT + path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"] + if not path.is_file(): + if args.weights_dir: + raise FileNotFoundError(path) + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(".partial") + with urllib.request.urlopen(info["url"], timeout=120) as response, partial.open("wb") as f: + headers = dict(response.headers) + final_url = response.url + shutil.copyfileobj(response, f) + partial.replace(path) + save(path.with_suffix(".download.json"), { + "model": model, "url": info["url"], "final_url": final_url, + "headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path), + }) + digest = sha256(path) + if info.get("sha256") and digest != info["sha256"]: + raise ValueError(f"{model}: checkpoint hash mismatch") + return path, digest + + +def official_input(network, pipeline, image, image_id): + import numpy as np + data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id)) + batch = network.data_preprocessor({ + "inputs": [data["inputs"]], "data_samples": [data["data_samples"]], + }, training=False) + return batch["inputs"].numpy(), batch["data_samples"][0].metainfo + + +def check_input(adapter, network, pipeline, image, image_id): + import numpy as np + import torch + from harness.adapters.mmdet_detr_common import padding_mask + x = adapter.preprocess(image, adapter.INPUT_SPEC) + expected, meta = official_input(network, pipeline, image, image_id) + _, scales, img_shape = geometry(image.size) + error = float(np.max(np.abs(x - expected))) + if (x.shape != (1, 3, 640, 640) or error != 0 or tuple(meta["img_shape"]) != img_shape + or tuple(meta["pad_shape"]) != (640, 640) + or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8)): + raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}") + expected_mask = np.ones((1, 640, 640), dtype=bool) + expected_mask[:, :img_shape[0], :img_shape[1]] = False + np.testing.assert_array_equal(padding_mask(torch.from_numpy(x)).numpy(), expected_mask) + return x, {"input_max_abs": error, "original_size": list(image.size), + "scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"]), + "in_graph_padding_mask_exact": True} + + +def calibration(args, adapter, network, pipeline, t1): + import numpy as np + from PIL import Image + files = sorted(Path(args.train_dir).glob("*.jpg")) + if len(files) < 128: + raise ValueError(f"Need 128 actual training images, found {len(files)}") + random.Random(20260914).shuffle(files) + files = sorted(files[:128]) + with zipfile.ZipFile(args.annotations_zip) as archive: + with archive.open("annotations/captions_train2017.json") as f: + annotation = json.load(f) + official_train = {im["file_name"]: im for im in annotation["images"]} + train_ids = {int(p.stem) for p in files} + if train_ids & {it["id"] for it in t1.items}: + raise ValueError("Train calibration overlaps evaluation IDs") + directory = Path(args.out) / adapter.MODEL_ID / "calibration" + npy_dir = directory / "npy_f32" + npy_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + records = [] + for i, path in enumerate(files): + item = official_train.get(path.name) + if not item or item["id"] != int(path.stem): + raise ValueError(f"Not in official COCO train2017 annotations: {path}") + with Image.open(path) as opened: + image = opened.convert("RGB") + if image.size != (item["width"], item["height"]): + raise ValueError(f"Training image geometry differs from annotation: {path}") + x, comparison = check_input(adapter, network, pipeline, image, item["id"]) + target = npy_dir / f"{i:05d}.npy" + if target.exists(): + if not np.array_equal(np.load(target), x): + raise ValueError(f"Existing calibration changed: {target}") + else: + np.save(target, x) + digest.update(target.read_bytes()) + records.append({"source": str(path), "image_id": item["id"], + "image_sha256": sha256(path), "file": target.name, + "npy_sha256": sha256(target), **comparison}) + if len(list(npy_dir.glob("*.npy"))) != 128: + raise ValueError("Calibration directory contains unexpected NPY files") + manifest = {"task": "det", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC, + "count": 128, "seed": 20260914, "source": "train-dir", + "sha256": digest.hexdigest(), "files": [str(p) for p in files], + "preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32", + "npy_dir": "npy_f32", "train_dir": args.train_dir, + "annotations_zip": args.annotations_zip, + "annotation_member": "annotations/captions_train2017.json", + "official_input_exact_matches": 128, "eval_overlap": [], "items": records} + save(directory / "manifest.json", manifest) + return {"path": str(directory), "sha256": manifest["sha256"], "count": 128, + "official_input_exact_matches": 128, "eval_overlap": [], + "recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()} + + +def probe(model_id, args): + import numpy as np + import onnx + import onnxruntime as ort + import torch + from mmcv.transforms import Compose + from mmdet.apis import inference_detector + from mmdet.datasets import CocoDataset + from mmdet.utils import get_test_pipeline_cfg + from harness.eval import EVALUATOR_VERSION + from harness.eval.common import T1, cosine + from harness.eval.det import coco_eval + + adapter = get(model_id) + directory = Path(args.out) / model_id + directory.mkdir(parents=True, exist_ok=True) + proof = directory / "probe.json" + if proof.exists(): + proof.rename(directory / f"probe-history-{time.time_ns()}.json") + bench = Path(__file__).resolve().parents[1] + sources = ["harness/adapters/mmdet_detr_common.py", "harness/adapters/openmmlab.py", + "harness/adapters/__init__.py", f"harness/adapters/{model_id.replace('.', '_')}.py", + "harness/eval/det.py", "harness/eval/common.py", "harness/eval/run.py", + "scripts/make_calib.py", "scripts/probe_mmdet_detr_adapters.py"] + sources.append("harness/adapters/mmdet_dense_common.py") + result = {"model": model_id, "task": "det", "status": "running", "started_at": time.time(), + "runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT, + "preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"], + "source_sha256": {p: sha256(bench / p) for p in sources}, + "recipe_snapshot": get_recipe(model_id), "sample_count": args.n, + "runtime": {"torch": torch.__version__, "onnx": onnx.__version__, + "onnxruntime": ort.__version__, "cpu_threads": args.threads}, + "scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"} + save(proof, result) + t1 = None + try: + weights, digest = weights_for(model_id, args) + network = adapter.build(weights, adapter.INPUT_SPEC).eval() + pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg)) + pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray" + resizes = [step for step in pipeline_cfg if step["type"] == "Resize"] + if len(resizes) != 1 or tuple(resizes[0]["scale"]) != (1333, 800) or not resizes[0]["keep_ratio"]: + raise ValueError("Original MMDetection test geometry changed") + # Explicit CSV deployment-size override, not paper/native-scale accuracy. + resizes[0]["scale"] = (640, 640) + network.upstream.data_preprocessor.pad_size_divisor = 640 + result["deployment_geometry"] = { + "upstream_resize": [1333, 800], "upstream_pad_divisor": 1, + "resize": [640, 640], "keep_ratio": True, "normalized_pad_value": 0, + "pad_divisor": 640, "pad_placement": "bottom_right", + "reference": "official pipeline and model under identical CSV static640 contract", + "metric_scope": "fixed COCO100 deployment score; not paper/full-COCO accuracy"} + pipeline = Compose(pipeline_cfg) + manifest_path = Path(args.t1) + manifest = json.loads(manifest_path.read_text()) + if manifest["dataset"] != "coco2017" or manifest["n"] != 100: + raise ValueError("This probe requires the frozen real COCO n100 manifest") + t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"]) + if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]: + raise ValueError("COCO T1 content identity/count mismatch") + categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]} + cat_map = t1.manifest["cat_map"] + if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]): + raise ValueError("Official COCO class order differs from T1 category mapping") + evaluation = {"path": str(manifest_path.parent / manifest["tar"]), + "manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"], + "eval_set_id": manifest["eval_set_id"], "n": 100} + result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True, + eval_set=evaluation, sample_ids=[it["id"] for it in t1.items[:args.n]]) + save(proof, result) + export_path = directory / "model.onnx" + first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC) + names = output_names(model_id) + with torch.inference_mode(): + torch.onnx.export(network, torch.from_numpy(first), str(export_path), opset_version=adapter.OPSET, do_constant_folding=False, + input_names=["images"], output_names=names) + onnx.checker.check_model(str(export_path)) + graph = onnx.load(str(export_path)) + forbidden = {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"} + ops = sorted({node.op_type for node in graph.graph.node}) + if forbidden.intersection(ops): + raise ValueError(f"Dynamic postprocessing leaked into raw graph: {ops}") + result["graph"] = {"input": [d.dim_value for d in graph.graph.input[0].type.tensor_type.shape.dim], + "output_names": names, "opset": adapter.OPSET, "ops": ops, + "postprocessing_in_graph": False} + if result["graph"]["input"] != [1, 3, 640, 640]: + raise ValueError("ONNX input is not static CSV geometry") + # Check graph reachability, including exporter Identity aliases for + # equal-valued parameters. Every learned inference tensor must feed an + # output; declaring it on the wrapper alone does not count as export. + producers = {name: node for node in graph.graph.node for name in node.output} + reachable = set() + pending = [value.name for value in graph.graph.output] + while pending: + name = pending.pop() + if name in reachable: + continue + reachable.add(name) + if name in producers: + pending.extend(producers[name].input) + parameters = dict(network.named_parameters()) + missing = sorted(set(parameters) - reachable) + result["learned_parameter_graph_coverage"] = { + "parameter_tensors": len(parameters), "parameter_elements": sum(p.numel() for p in parameters.values()), + "reachable_parameter_tensors": len(parameters) - len(missing), "missing": missing, + "constant_folding": False, "host_learned_network": False} + if missing: + raise ValueError(f"Learned parameters absent from output graph: {missing}") + if any(node.domain not in ("", "ai.onnx") for node in graph.graph.node): + raise ValueError("Custom ONNX domain remains") + del graph + options = ort.SessionOptions() + options.intra_op_num_threads = args.threads + options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"]) + predictions = {key: [] for key in ("official", "fp32", "onnx")} + comparisons = [] + g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))} + g2_x, g2_y = {}, {} + + def add_predictions(key, image_id, values): + boxes, scores, labels = values + for box, score, label in zip(boxes, scores, labels): + predictions[key].append({"image_id": image_id, "category_id": cat_map[str(int(label))], + "score": float(score), "bbox": [float(box[0]), float(box[1]), + float(box[2] - box[0]), float(box[3] - box[1])]}) + + for index, item in enumerate(t1.items[:args.n]): + image = t1.image(item) + x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"]) + with torch.inference_mode(): + raw = [value.numpy() for value in network(torch.from_numpy(x))] + official = inference_detector(network.upstream, np.asarray(image)[:, :, ::-1].copy(), + test_pipeline=pipeline).pred_instances + reference = (official.bboxes.numpy(), official.scores.numpy(), official.labels.numpy()) + decoded = adapter.decode(raw, image.size) + if not np.array_equal(reference[2], decoded[2]): + raise ValueError(f"Official/adapter selected classes differ at image {item['id']}") + box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0)) + score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0)) + if box_error > 1e-4 or score_error > 1e-6: + raise ValueError(f"Official/adapter detections differ: image={item['id']} boxes={box_error} scores={score_error}") + exported = session.run(None, {session.get_inputs()[0].name: x}) + head_comparisons = [] + for name, a, b in zip(names, raw, exported): + if a.shape != b.shape or not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError(f"ONNX head shape/nonfinite output: {name}") + similarity = cosine(a, b) + head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity, + "max_abs": float(np.max(np.abs(a - b)))}) + if similarity < .9999: + raise ValueError(f"FP32/ONNX {name} cosine below .9999: {similarity}") + add_predictions("official", item["id"], reference) + add_predictions("fp32", item["id"], decoded) + add_predictions("onnx", item["id"], adapter.decode(exported, image.size)) + comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error, + "official_score_max_abs": score_error, "detections": len(decoded[1]), + "onnx_heads": head_comparisons}) + if item["id"] in g2_keys: + key = g2_keys[item["id"]] + g2_x[key] = x + g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)}) + if (index + 1) % 10 == 0: + print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True) + save(directory / "comparisons.json", comparisons) + metrics = {} + save(directory / "comparisons.json", comparisons) + eval_t1 = T1({**t1.manifest, "items": t1.items[:args.n]}, t1.tar) + for key, values in predictions.items(): + save(directory / f"{key}-predictions.json", values) + # pycocotools.loadRes adds area/id/iscrowd/segmentation in place. + # Keep the saved prediction contract intact for the shared hook check. + metrics[key] = coco_eval(eval_t1, deepcopy(values)) + save(directory / "metrics-all.json", metrics) + if metrics["fp32"]["map_50_95"] <= 0 or metrics["official"] != metrics["fp32"]: + raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}") + if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("map_50_95", "ap50")): + raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}") + baseline_dir = directory / "baseline" + baseline_dir.mkdir(exist_ok=True) + baseline = {"task": "det", "model": model_id, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": args.n, "metrics": metrics["fp32"]} + save(baseline_dir / "metrics.json", baseline) + np.savez(baseline_dir / "g2_inputs.npz", **g2_x) + np.savez(baseline_dir / "g2_fp32.npz", **g2_y) + if args.n == 100: + from harness.eval.run import g2_inputs + shared_g2 = g2_inputs("det", t1, adapter.INPUT_SPEC, 32, model=model_id) + if len(g2_x) != 32 or len(g2_y) != 32 * len(names) or g2_x.keys() != shared_g2.keys(): + raise ValueError("Missing real G2 inputs/outputs") + for key in g2_x: + np.testing.assert_array_equal(g2_x[key], shared_g2[key]) + result["g2"] = {"count": 32, "outputs_per_sample": len(names), + "shared_input_hook_exact": True, + "inputs_sha256": sha256(baseline_dir / "g2_inputs.npz"), + "outputs_sha256": sha256(baseline_dir / "g2_fp32.npz")} + result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"], + baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path), + comparisons_path=str(directory / "comparisons.json"), + official_exact_input_count=args.n, official_predict_matches=args.n, + onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_heads"]), + official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons), + official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons)) + save(proof, result) + if args.n == 100: + result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1) + # Exercise the unchanged shared det prediction route on real inputs. + # Run the last real image again through ONNX and the shared decoder hook. + from harness.eval.det import predict + one = T1({**t1.manifest, "items": [t1.items[-1]]}, t1.tar) + actual = predict(lambda value: session.run(None, {session.get_inputs()[0].name: value}), + one, adapter.INPUT_SPEC, model=model_id) + expected = [p for p in predictions["onnx"] if p["image_id"] == t1.items[-1]["id"]] + if actual != expected: + raise ValueError("Shared det.predict differs from validated ONNX route") + result["shared_det_hook_verified"] = True + result["status"] = "validated_fp32_export_only" if args.n == 100 else "smoke_passed" + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) + finally: + if t1: + t1.tar.close() + result["finished_at"] = time.time() + save(proof, result) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=MODELS, action="append", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--weights-dir") + parser.add_argument("--download-only", action="store_true") + parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json") + parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017") + parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip") + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--n", type=int, choices=[3, 100], default=100) + args = parser.parse_args() + if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON): + os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]]) + Path(args.out).mkdir(parents=True, exist_ok=True) + if args.download_only: + for model in args.model: + path, digest = weights_for(model, args) + print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True) + return 0 + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + return int(any([probe(model, args)["status"] not in ("validated_fp32_export_only", "smoke_passed") for model in args.model])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_mmdet_iseg_adapters.py b/scripts/probe_mmdet_iseg_adapters.py new file mode 100644 index 0000000..3210568 --- /dev/null +++ b/scripts/probe_mmdet_iseg_adapters.py @@ -0,0 +1,479 @@ +"""Validate exact MMDetection query-based instance segmentation in private dirs. + +Run each variant with the frozen COCO n100 manifest and the train128 recipe. +The probe compares the official Mask2Former predict route against the adapter +decode of the raw FP32/ONNX query tensors, then exercises the shared +harness/eval/coco_extra.py predict hook. All generated files stay under --out; +the final probe.json is mirrored to the shared adapter-probes cache. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import sys +import time +import traceback +import urllib.request +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness.adapters import get +from harness.adapters.openmmlab import get_recipe +from harness.adapters.mmdet_iseg_common import (RUNTIME_PYTHON, MODELS as MASK2FORMER_MODELS, + output_names as mask2former_output_names) +from harness.adapters.mmdet_solo_common import MODELS as SOLO_MODELS, output_names as solo_output_names + +MODELS = MASK2FORMER_MODELS + SOLO_MODELS + + +def output_names(model): + return mask2former_output_names(model) if model in MASK2FORMER_MODELS else solo_output_names(model) + + +def sha256(path): + h = hashlib.sha256() + with Path(path).open("rb") as f: + for part in iter(lambda: f.read(1 << 20), b""): + h.update(part) + return h.hexdigest() + + +def save(path, value): + Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") + + +def weights_for(model, args): + info = get(model).CHECKPOINT + path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"] + if not path.is_file(): + if args.weights_dir: + raise FileNotFoundError(path) + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(".partial") + with urllib.request.urlopen(info["url"], timeout=300) as response, partial.open("wb") as f: + headers = dict(response.headers) + final_url = response.url + shutil.copyfileobj(response, f) + partial.replace(path) + save(path.with_suffix(".download.json"), { + "model": model, "url": info["url"], "final_url": final_url, + "headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path), + }) + digest = sha256(path) + if info.get("sha256") and digest != info["sha256"]: + raise ValueError(f"{model}: checkpoint hash mismatch") + return path, digest + + +def official_input(network, pipeline, image, image_id): + import numpy as np + data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id)) + batch = network.data_preprocessor({ + "inputs": [data["inputs"]], "data_samples": [data["data_samples"]], + }, training=False) + return batch["inputs"].numpy(), batch["data_samples"][0].metainfo + + +def check_input(adapter, network, pipeline, image, image_id): + import numpy as np + from harness.adapters.mmdet_iseg_common import geometry + x = adapter.preprocess(image, adapter.INPUT_SPEC) + expected, meta = official_input(network, pipeline, image, image_id) + _, scales, img_shape = geometry(image.size) + error = float(np.max(np.abs(x - expected))) + if (x.shape != (1, 3, 640, 640) or error != 0 or tuple(meta["img_shape"]) != img_shape + or tuple(meta["pad_shape"]) != (640, 640) + or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8)): + raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}") + return x, {"input_max_abs": error, "original_size": list(image.size), + "scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"])} + + +def calibration(args, adapter, network, pipeline, t1): + """Materialize the shared data/calib iseg directory for this recipe.""" + import numpy as np + from PIL import Image + from harness import paths, registry + from harness.adapters import calibration_dir + files = sorted(Path(args.train_dir).glob("*.jpg")) + if len(files) < 128: + raise ValueError(f"Need 128 actual training images, found {len(files)}") + random.Random(20260914).shuffle(files) + files = sorted(files[:128]) + with zipfile.ZipFile(args.annotations_zip) as archive: + with archive.open("annotations/captions_train2017.json") as f: + annotation = json.load(f) + official_train = {im["file_name"]: im for im in annotation["images"]} + train_ids = {int(p.stem) for p in files} + if train_ids & {it["id"] for it in t1.items}: + raise ValueError("Train calibration overlaps evaluation IDs") + directory = calibration_dir(registry.model_row(adapter.MODEL_ID)) + npy_dir = directory / "npy_f32" + npy_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + records = [] + for i, path in enumerate(files): + item = official_train.get(path.name) + if not item or item["id"] != int(path.stem): + raise ValueError(f"Not in official COCO train2017 annotations: {path}") + with Image.open(path) as opened: + image = opened.convert("RGB") + if image.size != (item["width"], item["height"]): + raise ValueError(f"Training image geometry differs from annotation: {path}") + x, comparison = check_input(adapter, network, pipeline, image, item["id"]) + target = npy_dir / f"{i:05d}.npy" + if target.exists(): + if not np.array_equal(np.load(target), x): + raise ValueError(f"Existing calibration changed: {target}") + else: + np.save(target, x) + digest.update(target.read_bytes()) + records.append({"source": str(path), "image_id": item["id"], + "image_sha256": sha256(path), "file": target.name, + "npy_sha256": sha256(target), **comparison}) + if len(list(npy_dir.glob("*.npy"))) != 128: + raise ValueError("Calibration directory contains unexpected NPY files") + manifest = {"task": "iseg", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC, + "count": 128, "seed": 20260914, "source": "train-dir", + "sha256": digest.hexdigest(), "files": [str(p) for p in files], + "preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32", + "npy_dir": "npy_f32", "train_dir": args.train_dir, + "annotations_zip": args.annotations_zip, + "annotation_member": "annotations/captions_train2017.json", + "official_input_exact_matches": 128, "eval_overlap": [], "items": records} + save(directory / "manifest.json", manifest) + return {"path": str(directory), "sha256": manifest["sha256"], "count": 128, + "official_input_exact_matches": 128, "eval_overlap": [], + "recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()} + + +def segm_predictions(values, cat_map, image_id): + import numpy as np + from pycocotools import mask as mask_utils + boxes, scores, labels, masks = values + predictions = [] + for mask, score, label in zip(masks, scores, labels): + rle = mask_utils.encode(np.asfortranarray(mask.astype(np.uint8))) + rle["counts"] = rle["counts"].decode("ascii") + predictions.append({"image_id": image_id, "category_id": cat_map[str(int(label))], + "score": float(score), "segmentation": rle}) + return predictions + + +def probe(model_id, args): + import numpy as np + import onnx + import onnxruntime as ort + import torch + from mmcv.transforms import Compose + from mmdet.datasets import CocoDataset + from mmdet.utils import get_test_pipeline_cfg + from harness import paths + from harness.eval import EVALUATOR_VERSION + from harness.eval.common import T1, cosine + from harness.eval.coco_extra import evaluate + + adapter = get(model_id) + directory = Path(args.out) / model_id + directory.mkdir(parents=True, exist_ok=True) + proof = directory / "probe.json" + if proof.exists(): + proof.rename(directory / f"probe-history-{time.time_ns()}.json") + bench = Path(__file__).resolve().parents[1] + sources = ["harness/adapters/mmdet_iseg_common.py", "harness/adapters/openmmlab.py", + "harness/adapters/__init__.py", f"harness/adapters/{model_id.replace('.', '_')}.py", + "harness/eval/coco_extra.py", "harness/eval/common.py", "harness/eval/run.py", + "scripts/probe_mmdet_iseg_adapters.py"] + result = {"model": model_id, "task": "iseg", "status": "running", "started_at": time.time(), + "runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT, + "preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"], + "source_sha256": {p: sha256(bench / p) for p in sources}, + "recipe_snapshot": get_recipe(model_id), "sample_count": args.n, + "runtime": {"torch": torch.__version__, "onnx": onnx.__version__, + "onnxruntime": ort.__version__, "cpu_threads": args.threads}, + "scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"} + save(proof, result) + t1 = None + try: + weights, digest = weights_for(model_id, args) + network = adapter.build(weights, adapter.INPUT_SPEC).eval() + pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg)) + pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray" + for step in pipeline_cfg: + # Ground-truth loaders are no-ops for bare inference dicts; the + # panoptic variant additionally demands an unused panopticapi + # install. Plain LoadAnnotations keeps the exact same no-op. + if step.get("type") == "LoadPanopticAnnotations": + step["type"] = "LoadAnnotations" + resizes = [step for step in pipeline_cfg if step["type"] == "Resize"] + if len(resizes) != 1 or tuple(resizes[0]["scale"]) != (1333, 800) or not resizes[0]["keep_ratio"]: + raise ValueError("Original MMDetection test geometry changed") + # Explicit CSV deployment-size override, not paper/native-scale accuracy. + resizes[0]["scale"] = (640, 640) + network.upstream.data_preprocessor.pad_size_divisor = 640 + result["deployment_geometry"] = { + "upstream_resize": [1333, 800], "upstream_pad_divisor": 32, + "resize": [640, 640], "keep_ratio": True, "normalized_pad_value": 0, + "pad_divisor": 640, "pad_placement": "bottom_right", + "reference": "official pipeline and model under identical CSV static640 contract", + "metric_scope": "fixed COCO100 deployment score; not paper/full-COCO accuracy"} + pipeline = Compose(pipeline_cfg) + manifest_path = Path(args.t1) + manifest = json.loads(manifest_path.read_text()) + if manifest["dataset"] != "coco2017" or manifest["n"] != 100: + raise ValueError("This probe requires the frozen real COCO n100 manifest") + t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"]) + if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]: + raise ValueError("COCO T1 content identity/count mismatch") + categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]} + cat_map = t1.manifest["cat_map"] + if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]): + raise ValueError("Official COCO class order differs from T1 category mapping") + evaluation = {"path": str(manifest_path.parent / manifest["tar"]), + "manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"], + "eval_set_id": manifest["eval_set_id"], "n": 100} + result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True, + eval_set=evaluation, sample_ids=[it["id"] for it in t1.items[:args.n]]) + save(proof, result) + export_path = directory / "model.onnx" + first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC) + names = output_names(model_id) + opset = getattr(adapter, "OPSET", 11) + with torch.inference_mode(): + torch.onnx.export(network, torch.from_numpy(first), str(export_path), + opset_version=opset, input_names=["images"], output_names=names) + onnx.checker.check_model(str(export_path)) + graph = onnx.load(str(export_path)) + forbidden = {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"} + ops = sorted({node.op_type for node in graph.graph.node}) + if forbidden.intersection(ops): + raise ValueError(f"Dynamic postprocessing leaked into raw graph: {ops}") + result["graph"] = {"input": [d.dim_value for d in graph.graph.input[0].type.tensor_type.shape.dim], + "output_names": names, "opset": opset, "ops": ops, + "postprocessing_in_graph": False} + if result["graph"]["input"] != [1, 3, 640, 640]: + raise ValueError("ONNX input is not static CSV geometry") + del graph + # Debug graph with continuous-segment taps: the backbone/FPN or + # backbone/pixel-decoder part must match FP32 at .9999 cosine. The + # heads themselves amplify fp reassociation across frameworks through + # discontinuities (Mask2Former: boolean attention masks derived from + # near-threshold sigmoid logits; SOLO: GroupNorm+ReLU kinks over + # stacked convs), so raw head cosine is recorded, not gated; head + # equivalence is judged on decoded predictions and the COCO metric. + masked_family = model_id in MASK2FORMER_MODELS + if masked_family: + from harness.adapters.mmdet_iseg_common import DEBUG_OUTPUT_NAMES, build_debug_variant + else: + from harness.adapters.mmdet_solo_common import DEBUG_OUTPUT_NAMES, build_debug_variant + debug_names = DEBUG_OUTPUT_NAMES + debug_path = directory / "debug-taps.onnx" + debug_module = build_debug_variant(model_id, weights, adapter.INPUT_SPEC) + with torch.inference_mode(): + torch.onnx.export(debug_module, torch.from_numpy(first), str(debug_path), + opset_version=opset, input_names=["images"], + output_names=debug_names) + onnx.checker.check_model(str(debug_path)) + options = ort.SessionOptions() + options.intra_op_num_threads = args.threads + options.inter_op_num_threads = 1 + debug_session = ort.InferenceSession(str(debug_path), sess_options=options, providers=["CPUExecutionProvider"]) + session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"]) + predictions = {key: [] for key in ("official", "fp32", "onnx")} + comparisons = [] + g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))} + g2_x, g2_y = {}, {} + for index, item in enumerate(t1.items[:args.n]): + image = t1.image(item) + x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"]) + data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=item["id"])) + batch = network.upstream.data_preprocessor( + {"inputs": [data["inputs"]], "data_samples": [data["data_samples"]]}, training=False) + with torch.inference_mode(): + raw = [value.numpy() for value in network(torch.from_numpy(x))] + results = network.upstream(batch["inputs"], batch["data_samples"], + mode="predict") + official = results[0].pred_instances + reference = (official.bboxes.numpy(), official.scores.numpy(), + official.labels.numpy(), official.masks.numpy()) + decoded = adapter.decode(raw, image.size) + if not np.array_equal(reference[2], decoded[2]): + raise ValueError(f"Official/adapter selected classes differ at image {item['id']}") + box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0)) + score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0)) + masks_exact = (len(reference[3]) == len(decoded[3]) + and all(np.array_equal(a, b) for a, b in zip(reference[3], decoded[3]))) + if box_error > 1e-4 or score_error > 1e-6 or not masks_exact: + raise ValueError(f"Official/adapter instance results differ: image={item['id']} " + f"boxes={box_error} scores={score_error} masks_exact={masks_exact}") + exported = session.run(None, {session.get_inputs()[0].name: x}) + with torch.inference_mode(): + debug_raw = [value.numpy() for value in debug_module(torch.from_numpy(x))] + if masked_family: + # The debug module must reproduce the production forward exactly. + for a, b in zip((debug_raw[-2], debug_raw[-1]), raw): + if not np.array_equal(a, b): + raise ValueError("Debug-tap module diverges from the production graph path") + debug_exported = debug_session.run(None, {debug_session.get_inputs()[0].name: x}) + pre_decoder = [] + # Only the continuous backbone/FPN or pre-decoder taps are gated; + # the Mask2Former debug graph's final two outputs repeat the + # flip-sensitive decoder tensors and stay recorded-only. + gated = slice(5) if masked_family else slice(None) + for name, a, b in zip(debug_names[gated], debug_raw[gated], debug_exported[gated]): + if a.shape != b.shape or not np.isfinite(b).all(): + raise ValueError(f"ONNX debug output shape/nonfinite: {name}") + similarity = cosine(a, b) + pre_decoder.append({"output": name, "shape": list(a.shape), "cosine": similarity, + "max_abs": float(np.max(np.abs(a - b)))}) + if similarity < .9999: + raise ValueError(f"FP32/ONNX continuous-segment {name} cosine below .9999: {similarity}") + head_comparisons = [] + for name, a, b in zip(names, raw, exported): + if a.shape != b.shape or not np.isfinite(b).all(): + raise ValueError(f"ONNX output shape/nonfinite: {name}") + similarity = cosine(a, b) + head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity, + "max_abs": float(np.max(np.abs(a - b)))}) + predictions["official"].extend(segm_predictions(reference, cat_map, item["id"])) + predictions["fp32"].extend(segm_predictions(decoded, cat_map, item["id"])) + onnx_decoded = adapter.decode(exported, image.size) + predictions["onnx"].extend(segm_predictions(onnx_decoded, cat_map, item["id"])) + onnx_stats = {"fp32_instances": len(decoded[1]), "onnx_instances": len(onnx_decoded[1]), + "label_order_equal": bool(np.array_equal(decoded[2], onnx_decoded[2]))} + if onnx_stats["label_order_equal"]: + onnx_stats["score_max_abs"] = float(np.max(np.abs(decoded[1] - onnx_decoded[1]), initial=0)) + ious = [float((a & b).sum()) / float((a | b).sum()) if (a | b).sum() else 1.0 + for a, b in zip(decoded[3], onnx_decoded[3])] + onnx_stats["mask_iou_min"] = min(ious) + onnx_stats["mask_iou_mean"] = float(np.mean(ious)) + comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error, + "official_score_max_abs": score_error, "official_masks_exact": bool(masks_exact), + "instances": len(decoded[1]), "onnx_decoded": onnx_stats, + "onnx_outputs": head_comparisons, "pre_decoder": pre_decoder}) + if item["id"] in g2_keys: + key = g2_keys[item["id"]] + g2_x[key] = x + g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)}) + if (index + 1) % 10 == 0: + print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True) + save(directory / "comparisons.json", comparisons) + metrics = {} + save(directory / "comparisons.json", comparisons) + eval_t1 = T1({**t1.manifest, "items": t1.items[:args.n]}, t1.tar) + for key, values in predictions.items(): + save(directory / f"{key}-predictions.json", values) + # pycocotools.loadRes adds area/id/iscrowd in place. Keep the saved + # prediction contract intact for the shared hook check. + metrics[key] = evaluate(eval_t1, deepcopy(values), "iseg") + save(directory / "metrics-all.json", metrics) + if metrics["fp32"]["mask_ap"] <= 0 or metrics["official"] != metrics["fp32"]: + raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}") + if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("mask_ap", "ap50")): + raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}") + baseline_dir = directory / "baseline" + baseline_dir.mkdir(exist_ok=True) + baseline = {"task": "iseg", "model": model_id, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": args.n, "metrics": metrics["fp32"]} + save(baseline_dir / "metrics.json", baseline) + np.savez(baseline_dir / "g2_inputs.npz", **g2_x) + np.savez(baseline_dir / "g2_fp32.npz", **g2_y) + if args.n == 100: + from harness.eval.run import g2_inputs + shared_g2 = g2_inputs("iseg", t1, adapter.INPUT_SPEC, 32, model=model_id) + if len(g2_x) != 32 or len(g2_y) != 32 * len(names) or g2_x.keys() != shared_g2.keys(): + raise ValueError("Missing real G2 inputs/outputs") + for key in g2_x: + np.testing.assert_array_equal(g2_x[key], shared_g2[key]) + result["g2"] = {"count": 32, "outputs_per_sample": len(names), + "shared_input_hook_exact": True, + "inputs_sha256": sha256(baseline_dir / "g2_inputs.npz"), + "outputs_sha256": sha256(baseline_dir / "g2_fp32.npz")} + result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"], + baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path), + comparisons_path=str(directory / "comparisons.json"), + official_exact_input_count=args.n, official_predict_matches=args.n, + onnx_raw_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_outputs"]), + official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons), + official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons), + official_masks_exact=all(c["official_masks_exact"] for c in comparisons)) + if masked_family: + result.update(pre_decoder_onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["pre_decoder"]), + onnx_raw_cosine_note=("decoder attention masks come from sigmoid(mask_logits)<0.5; " + "fp reassociation flips borderline bits and shifts outputs " + "discretely, so raw decoder cosine is recorded, not gated; " + "decoder equivalence is the decoded-prediction and COCO-AP " + "comparison above")) + else: + result.update(fpn_onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["pre_decoder"]), + onnx_raw_cosine_note=("seven stacked GroupNorm+ReLU blocks over generated " + "coordinate grids flip near-zero pre-activations under fp " + "reassociation (identical at opset 11 and 16), so raw head " + "cosine is recorded, not gated; head equivalence is the " + "decoded-prediction and COCO-AP comparison above")) + save(proof, result) + if args.n == 100: + result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1) + # Exercise the unchanged shared iseg prediction route on real inputs. + from harness.eval.coco_extra import predict + one = T1({**t1.manifest, "items": [t1.items[-1]]}, t1.tar) + actual = predict(lambda value: session.run(None, {session.get_inputs()[0].name: value}), + one, adapter.INPUT_SPEC, "iseg", model=model_id) + expected = [p for p in predictions["onnx"] if p["image_id"] == t1.items[-1]["id"]] + if actual != expected: + raise ValueError("Shared iseg predict hook differs from validated ONNX route") + result["shared_iseg_hook_verified"] = True + result["status"] = "validated_fp32_export_only" if args.n == 100 else "smoke_passed" + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) + finally: + if t1: + t1.tar.close() + result["finished_at"] = time.time() + save(proof, result) + # Mirror the finished proof into the shared adapter-probes cache location + # that harness/coverage.py reads; keep one history copy per overwrite. + shared = paths.CACHE / "adapter-probes" / model_id / "probe.json" + shared.parent.mkdir(parents=True, exist_ok=True) + if shared.exists(): + shutil.copy2(shared, shared.parent / f"probe-history-{time.time_ns()}.json") + shutil.copy2(proof, shared) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=MODELS, action="append", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--weights-dir") + parser.add_argument("--download-only", action="store_true") + parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json") + parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017") + parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip") + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--n", type=int, choices=[3, 100], default=100) + args = parser.parse_args() + if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON): + os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]]) + Path(args.out).mkdir(parents=True, exist_ok=True) + if args.download_only: + for model in args.model: + path, digest = weights_for(model, args) + print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True) + return 0 + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + return int(any([probe(model, args)["status"] not in ("validated_fp32_export_only", "smoke_passed") for model in args.model])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_raft_adapter.py b/scripts/probe_raft_adapter.py new file mode 100644 index 0000000..033a6ce --- /dev/null +++ b/scripts/probe_raft_adapter.py @@ -0,0 +1,400 @@ +"""Private RAFT checkpoint/real KITTI pair/ONNX probe; never touches run state. + +Example (CPU only, no installs): + python scripts/probe_raft_adapter.py --model raft.small --model raft.large \ + --kitti-root /data/kitti_2015 --out /private/raft-probe --t1-n 100 --n 3 + +The resulting T1 contains 100 *pairs*, but the probe scores only --n pairs. +It is not a completed 100-pair FP32 baseline or board/quantization measurement. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import io +import json +import os +from pathlib import Path +import random +import sys +import tarfile +import time +import traceback +import urllib.request + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def write_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2, allow_nan=False)) + temp.replace(path) + + +def prepare_t1(root: Path, out: Path, n: int, seed: int) -> Path: + """Package complete temporal pairs and labels, never arbitrary PNG files.""" + training = root / "training" + audit = {"root": str(root.resolve()), "split": "training", "dataset": "kitti2015"} + complete, incomplete = [], [] + for first in sorted((training / "image_2").glob("*_10.png")): + sid = first.stem.removesuffix("_10") + rel = {"file": f"image_2/{sid}_10.png", "file2": f"image_2/{sid}_11.png", + "flow": f"flow_occ/{sid}_10.png"} + missing = [v for v in rel.values() if not (training / v).is_file()] + if missing: + incomplete.append({"id": sid, "missing": missing}) + else: + complete.append({"id": sid, **rel, "flow_format": "kitti_png16"}) + audit.update(complete_pairs=len(complete), incomplete_pairs=incomplete, + image10_count=len(list((training / "image_2").glob("*_10.png"))), + image11_count=len(list((training / "image_2").glob("*_11.png"))), + flow_occ_count=len(list((training / "flow_occ").glob("*_10.png"))), + flow_noc_count=len(list((training / "flow_noc").glob("*_10.png")))) + write_json(out / "dataset_audit.json", audit) + if not 0 < n <= len(complete) or incomplete: + raise ValueError(f"cannot build {n} complete KITTI pairs: {audit}") + items = sorted(random.Random(seed).sample(complete, n), key=lambda it: it["id"]) + destination = out / f"t1-n{n}-s{seed}" + destination.mkdir(parents=True, exist_ok=True) + tar_path = destination / f"{seed}.tar" + manifest_path = destination / "manifest.json" + if manifest_path.exists(): + previous = json.loads(manifest_path.read_text()) + if previous.get("source_root") != str(root.resolve()) or previous.get("n") != n or sha256(tar_path) != previous["sha256"]: + raise ValueError("existing private T1 differs; choose another --out") + return tar_path + blobs = {} + for item in items: + item["file_sha256"] = {} + for field in ("file", "file2", "flow"): + rel = item[field] + blobs[rel] = (training / rel).read_bytes() + item["file_sha256"][field] = hashlib.sha256(blobs[rel]).hexdigest() + content_sha = hashlib.sha256(json.dumps(items, sort_keys=True).encode()).hexdigest() + man = {"dataset": "kitti2015", "version": "data_scene_flow", "task": "flow", "split": "training", + "seed": seed, "n": n, "sha256": content_sha, "items": items, + "eval_set_id": f"kitti2015-flow-n{n}-{content_sha[:12]}", "source_root": str(root.resolve())} + blobs["manifest.json"] = json.dumps(man, sort_keys=True).encode() + with tarfile.open(tar_path.with_suffix(".tar.tmp"), "w") as tar: + for name, data in sorted(blobs.items()): + info = tarfile.TarInfo(name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + tar_path.with_suffix(".tar.tmp").replace(tar_path) + write_json(manifest_path, {k: v for k, v in man.items() if k != "items"} | { + "content_sha256": content_sha, "sha256": sha256(tar_path), "tar": tar_path.name}) + return tar_path + + +def verify_contract(t1_path: Path, root: Path, out: Path) -> None: + """Check geometry independently of model accuracy; compare real PNG decoding.""" + import numpy as np + import torch + from PIL import Image + from torchvision.datasets._optical_flow import _read_16bits_png_with_flow_and_valid_mask + from torchvision.models.optical_flow import Raft_Small_Weights + from torchvision.transforms import functional as F + from harness.adapters.raft_common import INPUT_SPEC, preprocess, input_shape + from harness.eval.common import T1 + from harness.eval import flow + + t1 = T1.open(t1_path) + try: + item = t1.items[0] + truth, valid = flow.ground_truth(t1, item) + official, official_valid = _read_16bits_png_with_flow_and_valid_mask(str(root / "training" / item["flow"])) + np.testing.assert_array_equal(truth, official) + np.testing.assert_array_equal(valid, official_valid) + first, second = t1.image(item), t1.image({"file": item["file2"]}) + x1, x2 = [F.resize(F.convert_image_dtype(F.pil_to_tensor(im), torch.float32), [384, 1024], antialias=False) + for im in (first, second)] + x1, x2 = Raft_Small_Weights.C_T_V2.transforms()(x1, x2) + packed = flow.inputs(t1, item, INPUT_SPEC, "raft.small") + np.testing.assert_array_equal(packed, torch.cat([x1, x2])[None].numpy()) + # Anisotropic scaling must change u and v independently, not only resize. + const = np.empty((1, 2, 384, 1024), dtype=np.float32) + const[:, 0], const[:, 1] = 8.0, -4.0 + restored = flow.restore_flow([const], INPUT_SPEC, (192, 2048)) + np.testing.assert_array_equal(restored[0], np.full((192, 2048), 16.0)) + np.testing.assert_array_equal(restored[1], np.full((192, 2048), -2.0)) + pixels = np.array([[[10, 30, 50]]], dtype=np.uint8) + x = preprocess(Image.fromarray(pixels), Image.fromarray(255 - pixels), "1x6x128x128") + np.testing.assert_allclose(x[0, :, 0, 0], np.r_[pixels.flatten(), (255 - pixels).flatten()] / 127.5 - 1, atol=1e-7) + pred = np.zeros((2, 2, 2), dtype=np.float32) + pred[0] = [[3, 4], [100, 100]] + mask = np.array([[True, True], [False, False]]) + metrics = flow.sample_metrics(pred, np.zeros_like(pred), mask) + assert metrics["epe2d"] == 3.5 and metrics["fl_all"] == 50 and metrics["valid_pixels"] == 2 + truth_fixture = np.full((2, 1, 1), 0, dtype=np.float32) + truth_fixture[0] = 100 + pred_fixture = truth_fixture.copy() + pred_fixture[0] += 4 + assert flow.sample_metrics(pred_fixture, truth_fixture, np.ones((1, 1), dtype=bool))["fl_all"] == 0 + try: + input_shape("1x3x384x1024") + except ValueError: + pass + else: + raise AssertionError("legacy single-frame specification was accepted") + pred[0, 0, 0] = np.nan + try: + flow.sample_metrics(pred, np.zeros_like(pred), mask) + except ValueError: + pass + else: + raise AssertionError("non-finite prediction was accepted") + write_json(out / "contract_checks.json", {"status": "passed", "t1_sha256": sha256(t1_path), + "real_sample_id": item["id"], + "real_image_shape": list(truth.shape[-2:]), "real_valid_pixels": int(valid.sum()), + "checks": ["real_uint16_decode_matches_torchvision", "real_preprocess_matches_official_transform", + "anisotropic_flow_resize_and_vector_scale", "RGB_temporal_channel_order", + "valid_mask_excludes_invalid_pixels", "strict_outlier_threshold_3px_and_5pct", + "reject_legacy_single_image_input", "reject_nonfinite_prediction"]}) + finally: + t1.tar.close() + + +def prepare_calibration(root: Path, t1_path: Path, calib_root: Path, seed: int, n: int = 128) -> Path: + """Use distinct testing pairs; no labels are needed for calibration.""" + import numpy as np + from PIL import Image + from harness.adapters.raft_common import INPUT_SPEC, PREPROCESS, preprocess + + recipe = hashlib.sha256(json.dumps(PREPROCESS, sort_keys=True).encode()).hexdigest()[:16] + destination = calib_root / "flow" / INPUT_SPEC / recipe + manifest_path = destination / "manifest.json" + if manifest_path.exists(): + manifest = json.loads(manifest_path.read_text()) + digest = hashlib.sha256() + files = sorted((destination / "npy_f32").glob("*.npy")) + for path in files: + digest.update(path.read_bytes()) + if (manifest.get("count") != n or len(files) != n or manifest.get("seed") != seed + or manifest.get("t1_sha256") != sha256(t1_path) or manifest.get("preprocess") != PREPROCESS + or manifest.get("sha256") != digest.hexdigest()): + raise ValueError(f"existing calibration differs; refusing to overwrite {destination}") + return destination + if destination.exists(): + raise FileExistsError(f"partial/existing calibration; refusing to overwrite {destination}") + with tarfile.open(t1_path) as tar: + t1 = json.load(tar.extractfile("manifest.json")) + excluded = {it["file_sha256"][key] for it in t1["items"] for key in ("file", "file2")} + image_dir = root / "testing" / "image_2" + candidates = [p for p in sorted(image_dir.glob("*_10.png")) + if p.with_name(p.name.replace("_10.png", "_11.png")).is_file()] + random.Random(seed).shuffle(candidates) + if len(candidates) < n: + raise ValueError(f"only {len(candidates)} unique testing pairs available; need {n}") + # Build alongside the final path and publish only after all real pairs pass. + temporary = destination.with_name(destination.name + f".partial-{os.getpid()}") + npy_dir = temporary / "npy_f32" + npy_dir.mkdir(parents=True, exist_ok=False) + digest, items, seen = hashlib.sha256(), [], set() + for first in candidates: + second = first.with_name(first.name.replace("_10.png", "_11.png")) + raw = [first.read_bytes(), second.read_bytes()] + hashes = tuple(hashlib.sha256(data).hexdigest() for data in raw) + if any(h in excluded for h in hashes) or hashes in seen: + continue + seen.add(hashes) + images = [Image.open(io.BytesIO(data)).convert("RGB") for data in raw] + packed = preprocess(*images, INPUT_SPEC) + if packed.shape != (1, 6, 384, 1024) or not np.isfinite(packed).all(): + raise ValueError("invalid real calibration pair") + path = npy_dir / f"{len(items):05d}.npy" + np.save(path, packed) + digest.update(path.read_bytes()) + items.append({"id": f"kitti2015/testing/{first.stem.removesuffix('_10')}", + "file": str(first), "file2": str(second), "file_sha256": list(hashes), + "npy": path.name, "npy_sha256": sha256(path)}) + if len(items) == n: + break + if len(items) != n: + raise ValueError(f"only {len(items)} unique, nonoverlapping calibration pairs; need {n}") + write_json(temporary / "manifest.json", {"task": "flow", "input_spec": INPUT_SPEC, + "count": n, "seed": seed, "source": "testing-unlabeled", "dataset": "kitti2015", + "sha256": digest.hexdigest(), "files": [it["file"] + "|" + it["file2"] for it in items], + "items": items, "t1_sha256": sha256(t1_path), "preprocess": PREPROCESS, + "t1_frame_content_overlap": 0, "unique_pair_content_count": len(seen), + "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32"}) + temporary.rename(destination) + return destination + + +def probe(model: str, t1_path: Path, out: Path, n: int, threads: int) -> dict: + import numpy as np + import onnx + import onnxruntime as ort + import torch + import torchvision + from harness.eval.common import T1, cosine + from harness.eval import flow + + adapter = importlib.import_module("harness.adapters." + model.replace(".", "_")) + directory = out / model + directory.mkdir(parents=True, exist_ok=True) + result = {"model": model, "status": "running", "started_at": time.time(), + "input_spec": adapter.INPUT_SPEC, "preprocess": adapter.PREPROCESS, + "opset": adapter.OPSET, "num_flow_updates": 12, "checkpoint": adapter.CHECKPOINT, + "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__, + "device": "cpu", "threads": threads}, + "t1_path": str(t1_path), "t1_sha256": sha256(t1_path), + "board_tested": False, "quantization_tested": False, + "input_contract": {"status": "unverified", "input_spec": adapter.INPUT_SPEC}} + def phase(name): + result["phase"] = name + write_json(directory / "probe.json", result) + print(f"{model}: {name}", flush=True) + + try: + phase("download_official_checkpoint") + weights = directory / adapter.CHECKPOINT["url"].rsplit("/", 1)[1] + if not weights.exists(): + with urllib.request.urlopen(adapter.CHECKPOINT["url"], timeout=90) as remote, weights.with_suffix(".part").open("wb") as target: + while block := remote.read(1 << 20): + target.write(block) + weights.with_suffix(".part").replace(weights) + result["weights_path"] = str(weights) + result["weights_sha256"] = sha256(weights) + phase("strict_load") + m = adapter.build(weights, adapter.INPUT_SPEC) + result["strict_load"] = True + result["parameters"] = sum(p.numel() for p in m.parameters()) + t1 = T1.open(t1_path, result["t1_sha256"]) + try: + if not 0 < n <= len(t1.items): + raise ValueError("--n must be positive and <= --t1-n") + result["prepared_t1_n"] = len(t1.items) + t1.manifest = {**t1.manifest, "items": t1.g2_subset(n)} + result["sample_ids"] = [it["id"] for it in t1.items] + phase("real_pair_forward") + first = flow.inputs(t1, t1.items[0], adapter.INPUT_SPEC, model) + np.save(directory / "input_first_pair.npy", first) + start = time.monotonic() + with torch.inference_mode(): + reference = m(torch.from_numpy(first)).numpy() + native = m.core(torch.from_numpy(first[:, :3]), torch.from_numpy(first[:, 3:6]), num_flow_updates=12)[-1].numpy() + if reference.shape != (1, 2, 384, 1024) or not np.isfinite(reference).all(): + raise ValueError("invalid RAFT forward output") + np.testing.assert_array_equal(reference, native) + result["native_pair_wrapper_max_abs"] = float(np.abs(reference - native).max()) + result["first_and_native_forward_seconds"] = time.monotonic() - start + np.save(directory / "fp32_first_flow.npy", reference) + phase("onnx_export_and_check") + onnx_path = directory / f"{model}.onnx" + start = time.monotonic() + with torch.no_grad(): + torch.onnx.export(m, torch.from_numpy(first), str(onnx_path), dynamo=False, + opset_version=adapter.OPSET, input_names=["input"], output_names=["flow"], + dynamic_axes=None, do_constant_folding=True) + onnx.checker.check_model(str(onnx_path), full_check=True) + graph = onnx.load(onnx_path) + from collections import Counter + result["onnx"] = {"path": str(onnx_path), "sha256": sha256(onnx_path), + "size_bytes": onnx_path.stat().st_size, + "operators": dict(Counter(node.op_type for node in graph.graph.node)), + "export_and_check_seconds": time.monotonic() - start, + "input_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim] for t in graph.graph.input], + "output_shapes": [[v.dim_value for v in t.type.tensor_type.shape.dim] for t in graph.graph.output]} + del graph + phase("onnxruntime_real_pairs_and_ground_truth") + opts = ort.SessionOptions() + opts.intra_op_num_threads = threads + opts.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx_path), sess_options=opts, providers=["CPUExecutionProvider"]) + comparisons, fp32_samples, ort_samples = [], [], [] + for item in t1.items: + x = flow.inputs(t1, item, adapter.INPUT_SPEC, model) + with torch.inference_mode(): + native = m(torch.from_numpy(x)).numpy() + exported = session.run(None, {"input": x})[0] + truth, valid = flow.ground_truth(t1, item) + a = flow.sample_metrics(flow.restore_flow([native], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid) + b = flow.sample_metrics(flow.restore_flow([exported], adapter.INPUT_SPEC, truth.shape[-2:]), truth, valid) + diff = np.abs(native - exported) + comparison = {"id": item["id"], "cosine": cosine(native, exported), + "max_abs": float(diff.max()), "mean_abs": float(diff.mean()), + "epe2d_difference": abs(a["epe2d"] - b["epe2d"]), + "fl_all_difference_pp": abs(a["fl_all"] - b["fl_all"])} + comparisons.append(comparison) + fp32_samples.append({"id": item["id"], **a}) + ort_samples.append({"id": item["id"], **b}) + print(json.dumps(comparison), flush=True) + if (comparison["cosine"] < .9999 or comparison["mean_abs"] > .005 + or comparison["max_abs"] > .5 or comparison["epe2d_difference"] > .01 + or comparison["fl_all_difference_pp"] > .1): + raise ValueError(f"FP32 export numerical mismatch: {comparison}") + result.update(status="validated_fp32_export_only", phase="complete", comparisons=comparisons, + fp32=flow.aggregate(fp32_samples), onnxruntime=flow.aggregate(ort_samples), + fp32_samples=fp32_samples, onnxruntime_samples=ort_samples) + checks = json.loads((out / "contract_checks.json").read_text()) + if checks["status"] != "passed" or checks["t1_sha256"] != result["t1_sha256"]: + raise ValueError("missing matching real data/geometry contract checks") + result["input_contract"] = {"status": "validated", "input_spec": adapter.INPUT_SPEC, + "packing": "RGB(t) channels 0:3; RGB(t+1) channels 3:6; NCHW float32 [-1,1]", + "output": "one N,2,H,W tensor (u,v), 12th update; input-image pixel displacements", + "ground_truth": "KITTI2015 flow_occ uint16 RGB, (RG-32768)/64, B=valid", + "evaluation_coordinates": "original image; bilinear resize plus independent u/v scaling", + "sample_ids": result["sample_ids"], "t1_sha256": result["t1_sha256"], + "checks_path": str(out / "contract_checks.json"), + "onnx_sha256": result["onnx"]["sha256"], "opset": adapter.OPSET} + write_json(directory / "baseline_probe.json", {"task": "flow", "model": model, + "t1_manifest_sha": t1.manifest["sha256"], "probe_subset_ids": result["sample_ids"], + "metrics": result["fp32"], "_flow_samples": fp32_samples, + "note": "probe subset only; not a completed T1 baseline"}) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + (directory / "error.txt").write_text(traceback.format_exc()) + result["finished_at"] = time.time() + write_json(directory / "probe.json", result) + print(json.dumps({"model": model, "status": result["status"], "error": result.get("error")}), flush=True) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", choices=["raft.small", "raft.large"], required=True) + parser.add_argument("--kitti-root", type=Path, required=True, help="directory containing training/image_2 and training/flow_occ") + parser.add_argument("--out", type=Path, required=True, help="new private probe directory, outside existing run/cache state") + parser.add_argument("--n", type=int, default=3) + parser.add_argument("--t1-n", type=int, default=100) + parser.add_argument("--seed", type=int, default=20260914) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--calib-root", type=Path, help="existing data/calib root; defaults to private --out/data/calib") + parser.add_argument("--prepare-only", action="store_true", help="prepare real T1/calibration without model inference") + args = parser.parse_args() + if args.threads < 1: + parser.error("--threads must be positive") + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + args.out.mkdir(parents=True, exist_ok=True) + t1_path = prepare_t1(args.kitti_root, args.out, args.t1_n, args.seed) + verify_contract(t1_path, args.kitti_root, args.out) + calib = prepare_calibration(args.kitti_root, t1_path, args.calib_root or args.out / "data" / "calib", args.seed) + write_json(args.out / "data_preparation.json", {"t1": str(t1_path), "t1_n": args.t1_n, + "t1_sha256": sha256(t1_path), "calibration_dir": str(calib), + "calibration_manifest": str(calib / "manifest.json"), "calibration_count": 128}) + if args.prepare_only: + return 0 + results = [probe(model, t1_path, args.out, args.n, args.threads) for model in dict.fromkeys(args.model)] + write_json(args.out / "summary.json", {"results": results, "complete": True}) + return int(any(r["status"] != "validated_fp32_export_only" for r in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_rtmdet_adapters.py b/scripts/probe_rtmdet_adapters.py new file mode 100644 index 0000000..4a0310b --- /dev/null +++ b/scripts/probe_rtmdet_adapters.py @@ -0,0 +1,312 @@ +"""Validate RTMDet inputs in an isolated runtime, without admission or cache writes. + +Run each variant with the same COCO n100 manifest and train128 recipe. +All generated files stay under --out; existing official weights may be reused. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import sys +import time +import traceback +import urllib.request +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness.adapters import get +from harness.adapters.openmmlab import get_recipe +from harness.adapters.rtmdet_common import RUNTIME_PYTHON, geometry + + +def sha256(path): + h = hashlib.sha256() + with Path(path).open("rb") as f: + for part in iter(lambda: f.read(1 << 20), b""): + h.update(part) + return h.hexdigest() + + +def save(path, value): + Path(path).write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n") + + +def weights_for(model, args): + info = get(model).CHECKPOINT + path = Path(args.weights_dir or Path(args.out) / "checkpoints") / info["filename"] + if not path.is_file(): + if args.weights_dir: + raise FileNotFoundError(path) + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(".partial") + with urllib.request.urlopen(info["url"], timeout=120) as response, partial.open("wb") as f: + headers = dict(response.headers) + final_url = response.url + shutil.copyfileobj(response, f) + partial.replace(path) + save(path.with_suffix(".download.json"), { + "model": model, "url": info["url"], "final_url": final_url, + "headers": headers, "bytes": path.stat().st_size, "sha256": sha256(path), + }) + digest = sha256(path) + if info.get("sha256") and digest != info["sha256"]: + raise ValueError(f"{model}: checkpoint hash mismatch") + return path, digest + + +def official_input(network, pipeline, image, image_id): + import numpy as np + data = pipeline(dict(img=np.asarray(image)[:, :, ::-1].copy(), img_id=image_id)) + batch = network.data_preprocessor({ + "inputs": [data["inputs"]], "data_samples": [data["data_samples"]], + }, training=False) + return batch["inputs"].numpy(), batch["data_samples"][0].metainfo + + +def check_input(adapter, network, pipeline, image, image_id): + import numpy as np + x = adapter.preprocess(image, adapter.INPUT_SPEC) + expected, meta = official_input(network, pipeline, image, image_id) + _, scales, padded_shape = geometry(image.size) + error = float(np.max(np.abs(x - expected))) + if error != 0 or tuple(meta["img_shape"]) != padded_shape or not np.allclose(meta["scale_factor"], scales, rtol=0, atol=1e-8): + raise ValueError(f"Preprocessing differs from official pipeline: id={image_id}, error={error}, meta={meta}") + return x, {"input_max_abs": error, "original_size": list(image.size), + "scale_factor": list(meta["scale_factor"]), "img_shape": list(meta["img_shape"])} + + +def calibration(args, adapter, network, pipeline, t1): + import numpy as np + from PIL import Image + files = sorted(Path(args.train_dir).glob("*.jpg")) + if len(files) < 128: + raise ValueError(f"Need 128 actual training images, found {len(files)}") + random.Random(20260914).shuffle(files) + files = sorted(files[:128]) + with zipfile.ZipFile(args.annotations_zip) as archive: + with archive.open("annotations/captions_train2017.json") as f: + annotation = json.load(f) + official_train = {im["file_name"]: im for im in annotation["images"]} + train_ids = {int(p.stem) for p in files} + if train_ids & {it["id"] for it in t1.items}: + raise ValueError("Train calibration overlaps evaluation IDs") + directory = Path(args.out) / "calibration" + npy_dir = directory / "npy_f32" + npy_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + records = [] + for i, path in enumerate(files): + item = official_train.get(path.name) + if not item or item["id"] != int(path.stem): + raise ValueError(f"Not in official COCO train2017 annotations: {path}") + with Image.open(path) as opened: + image = opened.convert("RGB") + if image.size != (item["width"], item["height"]): + raise ValueError(f"Training image geometry differs from annotation: {path}") + x, comparison = check_input(adapter, network, pipeline, image, item["id"]) + target = npy_dir / f"{i:05d}.npy" + if target.exists(): + if not np.array_equal(np.load(target), x): + raise ValueError(f"Existing calibration changed: {target}") + else: + np.save(target, x) + digest.update(target.read_bytes()) + records.append({"source": str(path), "image_id": item["id"], + "image_sha256": sha256(path), "file": target.name, + "npy_sha256": sha256(target), **comparison}) + if len(list(npy_dir.glob("*.npy"))) != 128: + raise ValueError("Calibration directory contains unexpected NPY files") + manifest = {"task": "det", "dataset": "coco2017", "input_spec": adapter.INPUT_SPEC, + "count": 128, "seed": 20260914, "source": "train-dir", + "sha256": digest.hexdigest(), "files": [str(p) for p in files], + "preprocess": adapter.PREPROCESS, "layout": "NCHW", "dtype": "float32", + "npy_dir": "npy_f32", "train_dir": args.train_dir, + "annotations_zip": args.annotations_zip, + "annotation_member": "annotations/captions_train2017.json", + "official_input_exact_matches": 128, "eval_overlap": [], "items": records} + save(directory / "manifest.json", manifest) + return {"path": str(directory), "sha256": manifest["sha256"], "count": 128, + "official_input_exact_matches": 128, "eval_overlap": [], + "recipe_hash": hashlib.sha256(json.dumps(adapter.PREPROCESS, sort_keys=True).encode()).hexdigest()} + + +def probe(model_id, args): + import numpy as np + import onnx + import onnxruntime as ort + import torch + from mmcv.transforms import Compose + from mmdet.apis import inference_detector + from mmdet.datasets import CocoDataset + from mmdet.utils import get_test_pipeline_cfg + from harness.eval import EVALUATOR_VERSION + from harness.eval.common import T1, cosine + from harness.eval.det import coco_eval + + adapter = get(model_id) + directory = Path(args.out) / model_id + directory.mkdir(parents=True, exist_ok=True) + proof = directory / "probe.json" + if proof.exists(): + proof.rename(directory / f"probe-history-{time.time_ns()}.json") + result = {"model": model_id, "task": "det", "status": "running", "started_at": time.time(), + "runtime_python": sys.executable, "checkpoint": adapter.CHECKPOINT, + "preprocess": adapter.PREPROCESS, "source": get_recipe(model_id)["source"], + "scope": "FP32 input/export validation only; no quantization, board, admission or Agent run"} + save(proof, result) + t1 = None + try: + weights, digest = weights_for(model_id, args) + network = adapter.build(weights, adapter.INPUT_SPEC).eval() + pipeline_cfg = deepcopy(get_test_pipeline_cfg(network.cfg)) + pipeline_cfg[0]["type"] = "mmdet.LoadImageFromNDArray" + pipeline = Compose(pipeline_cfg) + manifest_path = Path(args.t1) + manifest = json.loads(manifest_path.read_text()) + if manifest["dataset"] != "coco2017" or manifest["n"] != 100: + raise ValueError("This probe requires the frozen real COCO n100 manifest") + t1 = T1.open(manifest_path.parent / manifest["tar"], manifest["sha256"]) + if len(t1.items) != 100 or t1.manifest["sha256"] != manifest["content_sha256"]: + raise ValueError("COCO T1 content identity/count mismatch") + categories = {c["id"]: c["name"] for c in json.loads(t1.tar.extractfile("annotations.json").read())["categories"]} + cat_map = t1.manifest["cat_map"] + if [categories[cat_map[str(i)]] for i in range(80)] != list(CocoDataset.METAINFO["classes"]): + raise ValueError("Official COCO class order differs from T1 category mapping") + evaluation = {"path": str(manifest_path.parent / manifest["tar"]), + "manifest_sha": manifest["sha256"], "content_sha": manifest["content_sha256"], + "eval_set_id": manifest["eval_set_id"], "n": 100} + result.update(weights_path=str(weights), weights_sha256=digest, strict_load=True, + eval_set=evaluation, sample_ids=[it["id"] for it in t1.items]) + save(proof, result) + export_path = directory / "model.onnx" + first = adapter.preprocess(t1.image(t1.items[0]), adapter.INPUT_SPEC) + output_names = [f"cls_s{s}" for s in (8, 16, 32)] + [f"dist_s{s}" for s in (8, 16, 32)] + with torch.inference_mode(): + torch.onnx.export(network, torch.from_numpy(first), str(export_path), opset_version=11, + input_names=["images"], output_names=output_names) + onnx.checker.check_model(str(export_path)) + options = ort.SessionOptions() + options.intra_op_num_threads = args.threads + options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(export_path), sess_options=options, providers=["CPUExecutionProvider"]) + predictions = {key: [] for key in ("official", "fp32", "onnx")} + comparisons = [] + g2_keys = {it["id"]: f"s{i:03d}" for i, it in enumerate(t1.g2_subset(32))} + g2_x, g2_y = {}, {} + + def add_predictions(key, image_id, values): + boxes, scores, labels = values + for box, score, label in zip(boxes, scores, labels): + predictions[key].append({"image_id": image_id, "category_id": cat_map[str(int(label))], + "score": float(score), "bbox": [float(box[0]), float(box[1]), + float(box[2] - box[0]), float(box[3] - box[1])]}) + + for index, item in enumerate(t1.items): + image = t1.image(item) + x, comparison = check_input(adapter, network.upstream, pipeline, image, item["id"]) + with torch.inference_mode(): + raw = [value.numpy() for value in network(torch.from_numpy(x))] + official = inference_detector(network.upstream, np.asarray(image)[:, :, ::-1].copy(), + test_pipeline=pipeline).pred_instances + reference = (official.bboxes.numpy(), official.scores.numpy(), official.labels.numpy()) + decoded = adapter.decode(raw, image.size) + if not np.array_equal(reference[2], decoded[2]): + raise ValueError(f"Official/adapter selected classes differ at image {item['id']}") + box_error = float(np.max(np.abs(reference[0] - decoded[0]), initial=0)) + score_error = float(np.max(np.abs(reference[1] - decoded[1]), initial=0)) + if box_error > 1e-4 or score_error > 1e-6: + raise ValueError(f"Official/adapter detections differ: image={item['id']} boxes={box_error} scores={score_error}") + exported = session.run(None, {session.get_inputs()[0].name: x}) + head_comparisons = [] + for name, a, b in zip(output_names, raw, exported): + if a.shape != b.shape or not np.isfinite(a).all() or not np.isfinite(b).all(): + raise ValueError(f"ONNX head shape/nonfinite output: {name}") + similarity = cosine(a, b) + head_comparisons.append({"output": name, "shape": list(a.shape), "cosine": similarity, + "max_abs": float(np.max(np.abs(a - b)))}) + if similarity < .9999: + raise ValueError(f"FP32/ONNX {name} cosine below .9999: {similarity}") + add_predictions("official", item["id"], reference) + add_predictions("fp32", item["id"], decoded) + add_predictions("onnx", item["id"], adapter.decode(exported, image.size)) + comparisons.append({"image_id": item["id"], **comparison, "official_box_max_abs": box_error, + "official_score_max_abs": score_error, "detections": len(decoded[1]), + "onnx_heads": head_comparisons}) + if item["id"] in g2_keys: + key = g2_keys[item["id"]] + g2_x[key] = x + g2_y.update({f"{key}/out{i}": value for i, value in enumerate(raw)}) + if (index + 1) % 10 == 0: + print(json.dumps({"model": model_id, "n100_progress": index + 1}), flush=True) + save(directory / "comparisons.json", comparisons) + metrics = {} + for key, values in predictions.items(): + save(directory / f"{key}-predictions.json", values) + metrics[key] = coco_eval(t1, values) + save(directory / "metrics-all.json", metrics) + if metrics["fp32"]["map_50_95"] <= 0 or metrics["official"] != metrics["fp32"]: + raise ValueError(f"Official/adapter COCO evaluation differs: {metrics}") + if any(abs(metrics["fp32"][key] - metrics["onnx"][key]) > .1 for key in ("map_50_95", "ap50")): + raise ValueError(f"FP32/ONNX COCO metrics differ by over .1 percentage points: {metrics}") + baseline_dir = directory / "baseline" + baseline_dir.mkdir(exist_ok=True) + baseline = {"task": "det", "model": model_id, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": 100, "metrics": metrics["fp32"]} + save(baseline_dir / "metrics.json", baseline) + np.savez(baseline_dir / "g2_inputs.npz", **g2_x) + np.savez(baseline_dir / "g2_fp32.npz", **g2_y) + result.update(official=metrics["official"], fp32=metrics["fp32"], onnx=metrics["onnx"], + baseline_path=str(baseline_dir), onnx_path=str(export_path), onnx_sha256=sha256(export_path), + comparisons_path=str(directory / "comparisons.json"), + official_exact_input_count=100, official_predict_matches=100, + onnx_min_cosine=min(h["cosine"] for c in comparisons for h in c["onnx_heads"]), + official_box_max_abs=max(c["official_box_max_abs"] for c in comparisons), + official_score_max_abs=max(c["official_score_max_abs"] for c in comparisons)) + save(proof, result) + result["calibration"] = calibration(args, adapter, network.upstream, pipeline, t1) + result["status"] = "validated_fp32_export_only" + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) + finally: + if t1: + t1.tar.close() + result["finished_at"] = time.time() + save(proof, result) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", choices=["rtmdet.tiny", "rtmdet.s", "rtmdet.m", "rtmdet.l", "rtmdet.x"], action="append", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--weights-dir") + parser.add_argument("--download-only", action="store_true") + parser.add_argument("--t1", default="/home/ming01.wang/bench-data/t1/coco2017/n100-s20260914/manifest.json") + parser.add_argument("--train-dir", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/train2017") + parser.add_argument("--annotations-zip", default="/mnt/data/chao01.wu/Data/Datasets/COCO2017/annotations_trainval2017.zip") + parser.add_argument("--threads", type=int, default=2) + args = parser.parse_args() + if Path(sys.executable).absolute() != Path(RUNTIME_PYTHON): + os.execv(RUNTIME_PYTHON, [RUNTIME_PYTHON, str(Path(__file__).resolve()), *sys.argv[1:]]) + Path(args.out).mkdir(parents=True, exist_ok=True) + if args.download_only: + for model in args.model: + path, digest = weights_for(model, args) + print(json.dumps({"model": model, "path": str(path), "sha256": digest}), flush=True) + return 0 + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(args.threads) + torch.set_num_interop_threads(1) + return int(any([probe(model, args)["status"] != "validated_fp32_export_only" for model in args.model])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_rtmpose_adapters.py b/scripts/probe_rtmpose_adapters.py new file mode 100644 index 0000000..54ce4d7 --- /dev/null +++ b/scripts/probe_rtmpose_adapters.py @@ -0,0 +1,299 @@ +"""Real RTMPose preparation: official crops, COCO100, 32 G2, train128. + +Run in the pinned OpenMMLab runtime. Outputs stay in the requested directory; +no queue, catalog, shared environment, or board state is changed. Full results +include standard baseline NPZ/metrics files for the main task to install. +""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import random +import subprocess +import sys +import time +import traceback +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + + +def sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def save(path, value): + Path(path).write_text(json.dumps(value, indent=2) + "\n") + + +def check_shared_g2(mid, t1, inputs): + import numpy as np + from harness.eval import run + from harness.registry import model_row + + shared = run.g2_inputs("pose", t1, model_row(mid)["input_spec"], 32, model=mid) + if list(shared) != list(inputs) or len(shared) != 32: + raise ValueError("Shared G2 keys/count differ from saved probe inputs") + for key in inputs: + if not np.array_equal(shared[key], inputs[key]): + raise ValueError(f"Shared G2 input differs from probe input: {key}") + return {"count": 32, "elementwise_identical": True, + "shared_run_sha256": sha(run.__file__)} + + +def calibration(root: Path, output: Path, model: str): + import numpy as np + from PIL import Image + from harness.adapters import get + from harness.adapters.rtmpose_common import crop + + recipe = get(model).PREPROCESS + input_spec = recipe["input_spec"] + shape = tuple(int(v) for v in input_spec.split("x")) + recipe_sha = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest() + directory = output / "calib" / recipe_sha[:16] + manifest_path = directory / "manifest.json" + if manifest_path.exists(): + manifest = json.loads(manifest_path.read_text()) + files = sorted((directory / "npy_f32").glob("*.npy")) + digest = hashlib.sha256() + for path in files: + digest.update(path.read_bytes()) + if (len(files) != 128 or manifest["preprocess"] != recipe + or manifest["sha256"] != digest.hexdigest() + or len({x["image_id"] for x in manifest["crops"]}) != 128): + raise ValueError(f"Existing calibration differs: {directory}") + return {"path": str(directory), "sha256": manifest["sha256"], "count": 128} + if directory.exists(): + raise ValueError(f"Incomplete calibration requires inspection: {directory}") + member = "annotations/person_keypoints_train2017.json" + with zipfile.ZipFile(root / "annotations_trainval2017.zip") as archive: + annotation_bytes = archive.read(member) + annotation_sha = hashlib.sha256(annotation_bytes).hexdigest() + annotations = json.loads(annotation_bytes) + del annotation_bytes + by_image = {} + for ann in annotations["annotations"]: + box = ann["bbox"] + if (ann["category_id"] == 1 and not ann.get("iscrowd", 0) + and ann.get("num_keypoints", 0) > 0 and min(box[2:]) > 0): + by_image.setdefault(ann["image_id"], []).append(ann) + images = {im["id"]: im for im in annotations["images"]} + digest, records, sources = hashlib.sha256(), [], [] + with zipfile.ZipFile(root / "train2017.zip") as archive: + available = set(archive.namelist()) + eligible = sorted(i for i in by_image if f"train2017/{images[i]['file_name']}" in available) + selected = sorted(random.Random(20260914).sample(eligible, 128)) + (directory / "npy_f32").mkdir(parents=True) + for index, image_id in enumerate(selected): + # One deterministic, annotated person per distinct training image. + ann = max(by_image[image_id], key=lambda a: (a["area"], -a["id"])) + name = f"train2017/{images[image_id]['file_name']}" + content = archive.read(name) + image = Image.open(io.BytesIO(content)).convert("RGB") + bbox = np.asarray(ann["bbox"], np.float32) + bbox[2:] += bbox[:2] + x, matrix, _, _ = crop(image, input_spec, bbox) + if x.shape != shape or x.dtype != np.float32 or not np.isfinite(x).all(): + raise ValueError("Invalid calibration crop") + path = directory / "npy_f32" / f"{index:05d}.npy" + np.save(path, x) + digest.update(path.read_bytes()) + sources.append(f"{root / 'train2017.zip'}#{name}") + records.append({"image_id": image_id, "annotation_id": ann["id"], + "bbox_xyxy": bbox.tolist(), "affine": matrix.tolist(), + "image_sha256": hashlib.sha256(content).hexdigest(), "npy_sha256": sha(path)}) + manifest = {"task": "pose", "dataset": "coco2017", "input_spec": input_spec, + "count": 128, "seed": 20260914, "source": "train-zip-person-annotations", + "sha256": digest.hexdigest(), "files": sources, "preprocess": recipe, + "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32", + "annotations": f"{root / 'annotations_trainval2017.zip'}#{member}", + "annotations_sha256": annotation_sha, "selection": "128 distinct images; largest noncrowd labeled person, area then annotation ID", + "crops": records} + save(manifest_path, manifest) + return {"path": str(directory), "sha256": digest.hexdigest(), "count": 128} + + +def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict): + import numpy as np + import onnx + import onnxruntime as ort + import torch + from mmengine.dataset import Compose, pseudo_collate + from mmpose.apis.inference import inference_topdown + from harness import registry, scheduler + from harness.adapters import get + from harness.adapters.openmmlab import get_recipe + from harness.adapters.rtmpose_common import (crop, decode, components, diagnostic_box, + boxes_for, BBOX_FILE, BBOX_SHA256) + from harness.continuation import load_eval_set + from harness.eval import EVALUATOR_VERSION, coco_extra + from harness.eval.common import T1, TorchRunner, cosine + from harness.tools.export_onnx import exporter_version + + directory = output / mid + directory.mkdir(parents=True, exist_ok=True) + proof = directory / "probe.json" + if proof.exists(): + (directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) + result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()} + save(proof, result) + try: + adapter, row = get(mid), registry.model_row(mid) + dataset, evaluation = load_eval_set(manifest) + if dataset != "coco2017" or row["dataset"] != dataset: + raise ValueError("RTMPose requires the fixed COCO2017 T1") + weights = weights_dir / row["checkpoint"]["filename"] + if sha(weights) != row["checkpoint"]["sha256"]: + raise ValueError("Checkpoint bytes are not pinned in the shared catalog") + result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation, + weights_path=str(weights), weights_sha256=sha(weights), calibration=calib, + bbox_file=str(BBOX_FILE), bbox_sha256=BBOX_SHA256, + official_recipe=get_recipe(mid), + implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p) + for p in (Path(__file__).resolve(), Path(adapter.__file__), + Path(__file__).resolve().parents[1] / "harness/adapters/rtmpose_common.py", + Path(coco_extra.__file__))}) + with scheduler.acquire("cpu"): + runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) + result["strict_load"] = True + onnx_path = directory / "model.onnx" + meta_path = directory / "meta.json" + version = exporter_version(mid) + previous = json.loads(meta_path.read_text()) if meta_path.exists() else {} + if not (onnx_path.exists() and previous.get("exporter_version") == version + and previous.get("weights_sha256") == sha(weights) + and previous.get("onnx_sha256") == sha(onnx_path)): + with (directory / "export.log").open("a") as log: + subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid, + "--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"], + "--out", str(onnx_path)], check=True, stdout=log, stderr=log) + save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"], + "weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)}) + onnx.checker.check_model(str(onnx_path)) + options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"]) + + def exported(x): + return session.run(None, {session.get_inputs()[0].name: x}) + + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + if len(t1.items) != 100: + raise ValueError("This admission probe requires exactly the fixed 100 COCO images") + eligible = [i for i, item in enumerate(t1.items) if diagnostic_box(t1, item) is not None] + if len(eligible) < 32: + raise ValueError("Fewer than 32 images have genuine detector boxes") + indices = sorted(np.random.default_rng(20260914).permutation(eligible)[:32].tolist()) + native = runner.m.upstream + native.cfg = runner.m.cfg + native.dataset_meta = components(row["input_spec"])[3] + pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline) + inputs, outputs, comparisons, references = {}, {}, [], [] + for i, index in enumerate(indices): + item = t1.items[index] + image = t1.image(item) + box = diagnostic_box(t1, item) + x, matrix, center, scale = crop(image, row["input_spec"], box["bbox_xyxy"]) + key = f"s{i:03d}" + inputs[key] = x + a, b = runner(x), exported(x) + if len(a) != 2 or len(b) != 2: + raise ValueError("Expected both SimCC X/Y tensors") + for j, (aa, bb) in enumerate(zip(a, b)): + if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all(): + raise ValueError("Invalid FP32/ONNX output") + score = cosine(aa, bb) + comparisons.append({"sample": key, "image_id": item["id"], "output": j, + "cosine": score, "max_abs": float(np.abs(aa-bb).max())}) + if score < .9999: + raise ValueError(f"SimCC parity failed: {score}") + outputs[f"{key}/out{j}"] = aa + bgr = np.asarray(image)[:, :, ::-1].copy() + original = {"img": bgr, "bbox": box["bbox_xyxy"][None].copy(), + "bbox_score": np.ones(1, np.float32), **native.dataset_meta} + packed = pipeline(original) + with torch.no_grad(): + official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy() + input_delta = float(np.abs(x - official_input).max()) + ref = inference_topdown(native, bgr, bboxes=box["bbox_xyxy"][None].copy(), bbox_format="xyxy")[0].pred_instances + points = decode(a, runner(np.ascontiguousarray(x[..., ::-1])), center, scale, row["input_spec"]) + coordinate_delta = float(np.abs(points[:, :2] - ref.keypoints[0]).max()) + score_delta = float(np.abs(points[:, 2] - ref.keypoint_scores[0]).max()) + references.append({"sample": key, "image_id": item["id"], "source_index": box["source_index"], + "affine": matrix.tolist(), "input_max_abs": input_delta, + "coordinate_max_abs_px": coordinate_delta, "score_max_abs": score_delta}) + if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4: + raise ValueError(f"Official pipeline mismatch: {references[-1]}") + result.update(comparisons=comparisons, official_reference=references, + no_bbox_image_ids=[it["id"] for it in t1.items if not boxes_for(t1, it)], + g2_selection="seed20260914 sample32 eligible detector images; sorted T1 order") + save(proof, result) + print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True) + baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)} + converted = dict(baseline) + coco_extra.run_fp32(runner, t1, row["input_spec"], baseline) + save(directory / "metrics.json", baseline) + print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True) + coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted) + converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"} + save(directory / "eval.json", converted) + shift = converted["pose_shift_evidence"] + delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"]) + if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured" + or shift["keypoint_mean_shift_px"] > 2): + raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}") + np.savez(directory / "g2_inputs.npz", **inputs) + np.savez(directory / "g2_fp32.npz", **outputs) + result["shared_g2"] = check_shared_g2(mid, t1, inputs) + evidence_path = directory / "pose-export-evidence.json" + save(evidence_path, shift) + result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"], + pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)}, + pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)}, + onnx_path=str(onnx_path), + onnx_sha256=sha(onnx_path), baseline_path=str(directory), + sample_ids=[it["id"] for it in t1.items]) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) + result["finished_at"] = time.time() + save(proof, result) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", required=True, choices=["rtmpose.tiny", "rtmpose.s", "rtmpose.m", "rtmpose.l", "rtmpose.x"]) + parser.add_argument("--t1", required=True) + parser.add_argument("--weights-dir", type=Path, required=True) + parser.add_argument("--train-root", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if len(args.model) > 3 or len(set(args.model)) != len(args.model): + parser.error("at most three distinct models per batch") + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(2) + args.out.mkdir(parents=True, exist_ok=True) + results = [] + for mid in args.model: + calib = calibration(args.train_root, args.out, mid) + print(json.dumps({"model": mid, "calibration": calib}), flush=True) + results.append(probe(mid, args.t1, args.weights_dir, args.out, calib)) + return int(any(r["status"] != "validated_fp32_export_only" for r in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/probe_seg_adapters.py b/scripts/probe_seg_adapters.py new file mode 100644 index 0000000..f2d879b --- /dev/null +++ b/scripts/probe_seg_adapters.py @@ -0,0 +1,147 @@ +"""Validate real segmentation preparation through the existing stage caches. + +Requires an explicit labeled T1 with at least 100 images. No quantization, +board run, task insertion, or training trace is synthesized. One dataset per +invocation; at most three models. Run B0/Cityscapes before B1+B2/ADE20K. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness import paths, registry, scheduler +from harness.continuation import load_eval_set +from harness.eval import EVALUATOR_VERSION + + +def probe(mid: str, manifest: str, weights_directory: str | None = None) -> dict: + import numpy as np + import onnxruntime as ort + from harness.eval import seg + from harness.eval.common import T1, TorchRunner, cosine + from harness.stages.s01_fetch import Fetch, weights_file + from harness.stages.s02_baseline import baseline_dir + from harness.stages.s03_export import Export, onnx_path + + directory = paths.CACHE / "adapter-probes" / mid + directory.mkdir(parents=True, exist_ok=True) + result = {"model": mid, "task": "seg", "status": "running", "started_at": time.time()} + proof = directory / "probe.json" + if proof.exists(): + (directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) + proof.write_text(json.dumps(result, indent=2)) + try: + row = registry.model_row(mid) + if row["task"] != "seg" or not row.get("checkpoint") or not row.get("preprocess"): + raise ValueError(f"{mid}: explicit segmentation checkpoint and recipe required") + dataset, evaluation = load_eval_set(manifest) + if dataset != row["dataset"]: + raise ValueError(f"model/T1 dataset mismatch: {row['dataset']} / {dataset}") + spec = registry.expand_matrix(["oracle"], [mid])[0] + job = registry.build_job_json(spec, "adapter-probe", EVALUATOR_VERSION) + job["data"]["t1"] = evaluation + ctx = SimpleNamespace(job=job) + result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], + model_spec=row, eval_set=evaluation) + with scheduler.acquire("cpu"), (directory / "probe.log").open("a") as log: + fetch = Fetch() + if not fetch.cached(ctx): + cmd = fetch.command(ctx) + if weights_directory: + source = Path(weights_directory) / row["checkpoint"]["filename"] + cmd[cmd.index("--uri") + 1] = source.resolve().as_uri() + subprocess.run(cmd, check=True, stdout=log, stderr=log) + fetch.measure(ctx, directory, None) + weights = weights_file(mid) + actual_sha = hashlib.sha256(weights.read_bytes()).hexdigest() + if actual_sha != row["checkpoint"]["sha256"]: + raise ValueError("checkpoint digest differs from the pinned official segmentation recipe") + runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) + export = Export() + if not export.cached(ctx): + subprocess.run(export.command(ctx), check=True, stdout=log, stderr=log) + export.measure(ctx, directory, None) + onnx = onnx_path(job) + options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx), sess_options=options, providers=["CPUExecutionProvider"]) + diagnostic_outputs = {} + + def native(x): + cached = diagnostic_outputs.get(hashlib.sha256(x.tobytes()).digest()) + return [cached[0]] if cached is not None else runner(x) + + def exported(x): + cached = diagnostic_outputs.get(hashlib.sha256(x.tobytes()).digest()) + return [cached[1]] if cached is not None else session.run(None, {session.get_inputs()[0].name: x}) + + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + if len(t1.items) < 100: + raise ValueError("segmentation admission requires at least 100 real labeled T1 images") + xs = {f"s{i:03d}": seg.inputs(t1, it, row["input_spec"], mid) + for i, it in enumerate(t1.g2_subset(job["gates"]["G2"]["n_samples"]))} + outputs, comparisons = {}, [] + for key, x in xs.items(): + a, b = runner(x)[0], exported(x)[0] + if (a.ndim != 4 or a.shape[:2] != (1, row["checkpoint"]["classes"]) + or b.shape != a.shape or not np.isfinite(a).all() or not np.isfinite(b).all()): + raise ValueError(f"invalid segmentation outputs: {a.shape}, {b.shape}") + score = cosine(a, b) + agreement = float((a.argmax(1) == b.argmax(1)).mean()) + comparisons.append({"sample": key, "cosine": score, + "max_abs": float(np.abs(a-b).max()), "argmax_agreement": agreement}) + if score < .9999 or agreement < .999: + raise ValueError(f"ONNX/FP32 segmentation output mismatch: cosine={score}, agreement={agreement}") + outputs[f"{key}/out0"] = a + # The same 32 images also belong to T1: reuse their measured + # outputs instead of running both models on them twice. + diagnostic_outputs[hashlib.sha256(x.tobytes()).digest()] = (a, b) + baseline = {"task": "seg", "model": mid, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)} + converted = dict(baseline) + seg.run_fp32(native, t1, row["input_spec"], baseline) + seg.run_fp32(exported, t1, row["input_spec"], converted) + delta = abs(baseline["metrics"]["miou"] - converted["metrics"]["miou"]) + if baseline["metrics"]["miou"] <= 0 or delta > .01: + raise ValueError(f"FP32/ONNX mIoU mismatch: delta={delta} pp") + cache = baseline_dir(job); cache.mkdir(parents=True, exist_ok=True) + np.savez(cache / "g2_inputs.npz", **xs) + np.savez(cache / "g2_fp32.npz", **outputs) + (cache / "metrics.json").write_text(json.dumps(baseline, indent=2)) + result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], + onnx=converted["metrics"], comparisons=comparisons, + segmentation_protocol=baseline["seg_protocol"], + weights_path=str(weights), weights_sha256=actual_sha, + onnx_path=str(onnx), onnx_sha256=hashlib.sha256(onnx.read_bytes()).hexdigest(), + baseline_path=str(cache), sample_ids=[it["id"] for it in t1.items]) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + result["finished_at"] = time.time() + proof.write_text(json.dumps(result, indent=2)) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", required=True) + parser.add_argument("--t1", required=True) + parser.add_argument("--weights-dir") + args = parser.parse_args() + if len(args.model) > 3: + parser.error("at most three segmentation models per batch") + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "2"))) + failures = sum(probe(mid, args.t1, args.weights_dir)["status"] != "validated_fp32_export_only" for mid in args.model) + raise SystemExit(bool(failures)) diff --git a/scripts/probe_ssd_yolox_adapters.py b/scripts/probe_ssd_yolox_adapters.py new file mode 100644 index 0000000..e8f5378 --- /dev/null +++ b/scripts/probe_ssd_yolox_adapters.py @@ -0,0 +1,281 @@ +"""Validate SSD300/YOLOX adapters on the existing fixed COCO n100 package. + +This writes probe artifacts only: no jobs, collectors, registry edits, or installs. +Use --n 3 for a smoke check before the full --n 100 validation. A successful +adapter probe is not admission proof until --verify-shared-hooks also succeeds. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from dataclasses import replace +import hashlib +import json +import os +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import importlib + + + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def detections(image_id, prediction, cat_map=None): + boxes, scores, labels = prediction + return [{"image_id": image_id, "category_id": cat_map[str(int(label))] if cat_map else int(label), "score": float(score), + "bbox": [float(b[0]), float(b[1]), float(b[2] - b[0]), float(b[3] - b[1])]} + for b, score, label in zip(boxes, scores, labels)] + + +def calibration(source_manifest, output, eval_ids): + """Validate the 128 real train inputs, reusing an identical shared recipe.""" + import numpy as np + from PIL import Image + + source = json.loads(Path(source_manifest).read_text()) + files = [Path(p) for p in source["files"]] + if source.get("dataset") != "coco2017" or not source.get("source", "").startswith("train"): + raise ValueError("calibration must come from the real COCO train split") + if len(files) != 128 or len(set(files)) != 128 or any(not p.is_file() for p in files): + raise ValueError("128 distinct existing calibration source images are required") + if any(int(p.stem) in eval_ids for p in files): + raise ValueError("calibration and evaluation image IDs overlap") + from harness.adapters import calibration_dir + from harness.registry import model_row + shared = calibration_dir(model_row(MODEL)) + existing = shared / "manifest.json" + reused = existing.is_file() + if reused: + saved = json.loads(existing.read_text()) + if (saved.get("preprocess") != adapter.PREPROCESS or saved.get("input_spec") != INPUT_SPEC + or saved.get("count") != 128 or saved.get("files") != source["files"] + or saved.get("seed") != source["seed"] or saved.get("source") != source["source"] + or saved.get("npy_dir") != "npy_f32"): + raise ValueError("existing shared calibration differs from the requested official recipe") + output = shared + output.mkdir(parents=True, exist_ok=True) + tensors = output / "npy_f32"; tensors.mkdir(exist_ok=True) + digest = hashlib.sha256() + for i, path in enumerate(files): + with Image.open(path) as image: + x = adapter.preprocess(image, INPUT_SPEC) + if x.shape != tuple(map(int, INPUT_SPEC.split("x"))) or x.dtype != np.float32 or not np.isfinite(x).all(): + raise ValueError(f"invalid calibration tensor: {path}") + destination = tensors / f"{i:05d}.npy" + if reused and not destination.is_file(): + raise ValueError(f"shared calibration tensor missing: {destination}") + if destination.exists(): + np.testing.assert_array_equal(np.load(destination), x) + else: + np.save(destination, x) + digest.update(destination.read_bytes()) + result = {"task": "det", "input_spec": INPUT_SPEC, "count": 128, "seed": source["seed"], + "source": source["source"], "dataset": "coco2017", "sha256": digest.hexdigest(), + "files": [str(p) for p in files], "preprocess": adapter.PREPROCESS, + "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32", + "source_manifest": str(source_manifest), "source_manifest_sha256": sha(source_manifest)} + if reused: + if saved["sha256"] != result["sha256"] or len(list(tensors.glob("*.npy"))) != 128: + raise ValueError("shared calibration content digest/count mismatch") + else: + (output / "manifest.json").write_text(json.dumps(result, indent=2)) + return {"path": str(output), "count": 128, "sha256": result["sha256"], "overlap_with_t1": 0, "reused": reused} + + +def probe(args): + import numpy as np + import onnx + import onnxruntime as ort + import torch + import torchvision + from harness.continuation import load_eval_set + from harness.eval import EVALUATOR_VERSION + from harness.eval.common import T1, cosine + from harness.eval.det import coco_eval + + output = Path(args.out); output.mkdir(parents=True, exist_ok=True) + proof = output / "probe.json" + if proof.exists(): + (output / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) + result = {"model": MODEL, "task": "det", "status": "running", "started_at": time.time(), + "checkpoint": adapter.CHECKPOINT, "preprocess": adapter.PREPROCESS, + "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__, + "cpu_threads": torch.get_num_threads()}, + "shared_hooks_verified": False} + proof.write_text(json.dumps(result, indent=2)) + t1 = None + try: + dataset, evaluation = load_eval_set(args.t1) + if dataset != "coco2017" or evaluation["n"] != 100: + raise ValueError("use the existing fixed COCO n100 evaluation package") + result["eval_set"] = evaluation + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + cat_map = t1.manifest["cat_map"] + subset = replace(t1, manifest={**t1.manifest, "items": t1.items[:args.n], "n": args.n}) + model = adapter.build(Path(args.weights), INPUT_SPEC) + reference = adapter.reference_model(Path(args.weights)) + + def forward(x): + with torch.inference_mode(): + return [model(torch.from_numpy(x)).numpy()] + + onnx_path = output / f"{MODEL}.onnx" + with torch.no_grad(): + torch.onnx.export(model, torch.zeros(*map(int, INPUT_SPEC.split("x"))), str(onnx_path), dynamo=False, + opset_version=adapter.OPSET, input_names=["input"], output_names=["detections"], + do_constant_folding=True, dynamic_axes=None) + graph = onnx.load(str(onnx_path)); onnx.checker.check_model(graph, full_check=True) + shapes = {v.name: [d.dim_value for d in v.type.tensor_type.shape.dim] + for v in list(graph.graph.input) + list(graph.graph.output)} + if shapes != {"input": list(map(int, INPUT_SPEC.split("x"))), "detections": list(adapter.OUTPUT_SHAPE)}: + raise ValueError(f"unexpected ONNX shapes: {shapes}") + ops = dict(Counter(n.op_type for n in graph.graph.node)) + if set(ops) & {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"}: + raise ValueError(f"dynamic postprocessing left in ONNX: {ops}") + options = ort.SessionOptions(); options.intra_op_num_threads = torch.get_num_threads(); options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"]) + + def exported(x): + return session.run(None, {"input": x}) + + fp_dets, onnx_dets, official_dets, comparisons = [], [], [], [] + for i, item in enumerate(subset.items): + image = subset.image(item) + x = adapter.preprocess(image, INPUT_SPEC) + expected_x, official = adapter.reference_predict(reference, image) + np.testing.assert_array_equal(x, expected_x) + raw, converted = forward(x)[0], exported(x)[0] + if raw.shape != adapter.OUTPUT_SHAPE or not np.isfinite(raw).all() or not np.isfinite(converted).all(): + raise ValueError("invalid raw output") + # Background YOLOX regressions are unconstrained by detection loss. + # Check every raw coordinate by cosine, and retain strict pointwise + # checks on the union of candidates either runtime could keep. + box_cosine = cosine(raw[:, :, :4], converted[:, :, :4]) + if box_cosine < .99999: + raise ValueError(f"raw box tensor diverged: cosine={box_cosine}") + candidates = np.ones(raw.shape[:2], dtype=bool) + if MODEL.startswith("yolox."): + candidates = ((raw[:, :, 4] * raw[:, :, 5:].max(2) >= .01) + | (converted[:, :, 4] * converted[:, :, 5:].max(2) >= .01)) + np.testing.assert_allclose(raw[:, :, :4][candidates], converted[:, :, :4][candidates], rtol=2e-4, atol=2e-3) + np.testing.assert_allclose(raw[:, :, 4:], converted[:, :, 4:], rtol=3e-4, atol=3e-5) + predicted = adapter.decode([raw], image.size) + np.testing.assert_array_equal(predicted[2], official[2]) + np.testing.assert_allclose(predicted[0], official[0], rtol=1e-6, atol=1e-4) + np.testing.assert_allclose(predicted[1], official[1], rtol=1e-6, atol=1e-7) + transformed = adapter.decode([converted], image.size) + fp_dets.extend(detections(item["id"], predicted, cat_map)) + official_dets.extend(detections(item["id"], official, cat_map)) + onnx_dets.extend(detections(item["id"], transformed, cat_map)) + comparisons.append({"image_id": item["id"], "original_size": list(image.size), + "boxes_max_abs": float(np.max(np.abs(raw[:, :, :4] - converted[:, :, :4]))), + "candidate_boxes_max_abs": float(np.max(np.abs(raw[:, :, :4][candidates] - converted[:, :, :4][candidates]), initial=0)), + "candidate_count": int(candidates.sum()), + "scores_max_abs": float(np.max(np.abs(raw[:, :, 4:] - converted[:, :, 4:]))), + "boxes_cosine": cosine(raw[:, :, :4], converted[:, :, :4]), + "scores_cosine": cosine(raw[:, :, 4:], converted[:, :, 4:]), + "official_predictions": len(official[1]), "official_match": True}) + if (i + 1) % 25 == 0 or i == 0: + print(f"validated {i + 1}/{args.n}", flush=True) + metrics = {} + for key, ds in (("fp32", fp_dets), ("onnx", onnx_dets), ("official", official_dets)): + (output / f"{key}-predictions.json").write_text(json.dumps(ds)) + metrics[key] = coco_eval(subset, ds) + if metrics["fp32"]["map_50_95"] <= 0 or abs(metrics["fp32"]["map_50_95"] - metrics["onnx"]["map_50_95"]) > .02: + raise ValueError(f"FP32/ONNX AP mismatch: {metrics}") + result.update(**metrics, comparisons=comparisons, sample_ids=[it["id"] for it in subset.items], + weights_path=str(Path(args.weights).resolve()), weights_sha256=sha(args.weights), + onnx_path=str(onnx_path), onnx_sha256=sha(onnx_path), shapes=shapes, onnx_ops=ops) + if args.n == 100: + if not args.calib_manifest: + raise ValueError("full validation requires --calib-manifest for 128 real COCO train images") + result["calibration"] = calibration(args.calib_manifest, output / "calibration", {it["id"] for it in t1.items}) + baseline = {"task": "det", "model": MODEL, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": 100, "metrics": metrics["fp32"]} + baseline_dir = output / "baseline"; baseline_dir.mkdir(exist_ok=True) + xs = {f"s{i:03d}": adapter.preprocess(t1.image(it), INPUT_SPEC) for i, it in enumerate(t1.g2_subset(32))} + np.savez(baseline_dir / "g2_inputs.npz", **xs) + np.savez(baseline_dir / "g2_fp32.npz", **{f"{key}/out0": forward(x)[0] for key, x in xs.items()}) + (baseline_dir / "metrics.json").write_text(json.dumps(baseline, indent=2)) + result["baseline_path"] = str(baseline_dir) + if args.verify_shared_hooks: + from harness.adapters import calibration_dir + from harness.eval import det + from harness.eval.run import g2_inputs + from harness.registry import model_row + row = model_row(MODEL) + if (row["source"] != "custom" or row["input_spec"] != INPUT_SPEC + or row.get("checkpoint") != adapter.CHECKPOINT or row.get("preprocess") != adapter.PREPROCESS): + raise ValueError("shared model recipe differs from the validated adapter") + check = {**baseline}; det.run_fp32(forward, t1, INPUT_SPEC, check) + if check["metrics"] != baseline["metrics"]: + raise ValueError("shared detection FP32 hook differs from adapter predictions") + quant = {"model": MODEL}; det.run_quant(exported, t1, INPUT_SPEC, baseline, quant) + if quant["metrics"] != metrics["onnx"]: + raise ValueError("shared detection quant hook differs from adapter predictions") + shared_xs = g2_inputs("det", t1, INPUT_SPEC, 32, model=MODEL) + for key in xs: + np.testing.assert_array_equal(xs[key], shared_xs[key]) + shared_calib = calibration_dir(row) + shared_manifest = json.loads((shared_calib / "manifest.json").read_text()) + if (shared_manifest.get("preprocess") != adapter.PREPROCESS or shared_manifest.get("count") != 128 + or shared_manifest.get("sha256") != result["calibration"]["sha256"]): + raise ValueError("shared calibration recipe or tensor identity differs from this probe") + digest = hashlib.sha256() + tensors = sorted((shared_calib / shared_manifest["npy_dir"]).glob("*.npy")) + if len(tensors) != 128: + raise ValueError("shared calibration requires exactly 128 actual tensors") + for path in tensors: + digest.update(path.read_bytes()) + if digest.hexdigest() != shared_manifest["sha256"]: + raise ValueError("shared calibration files do not match their recorded digest") + result["calibration"]["shared_path"] = str(shared_calib) + result["shared_source_sha256"] = { + str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p) + for p in (Path(det.__file__), Path(sys.modules[g2_inputs.__module__].__file__), + Path(__file__).with_name("make_calib.py"))} + result["shared_hooks_verified"] = True + result["status"] = "validated_fp32_export_only" if args.verify_shared_hooks else "validated_adapter_only" + else: + result["status"] = "validated_smoke_only" + except Exception as exc: + import traceback + traceback.print_exc() + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + finally: + if t1 is not None: + t1.tar.close() + result["finished_at"] = time.time() + common = "yolox_common.py" if MODEL.startswith("yolox.") else "ssdlite_common.py" + result["source_sha256"] = {str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p) + for p in (Path(__file__), Path(adapter.__file__), Path(adapter.__file__).with_name(common))} + proof.write_text(json.dumps(result, indent=2)) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + print("ALL DONE", flush=True) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True, choices=("ssd300.vgg16", "yolox.tiny", "yolox.s", "yolox.m", "yolox.l", "yolox.x")) + parser.add_argument("--weights", required=True) + parser.add_argument("--t1", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--n", type=int, choices=(3, 100), default=100) + parser.add_argument("--calib-manifest") + parser.add_argument("--verify-shared-hooks", action="store_true") + args = parser.parse_args() + MODEL = args.model + adapter = importlib.import_module("harness.adapters." + MODEL.replace(".", "_")) + INPUT_SPEC = adapter.MODEL_SPEC["input_spec"] + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", "2"))) + raise SystemExit(probe(args)["status"] == "blocked") diff --git a/scripts/probe_ssdlite_adapter.py b/scripts/probe_ssdlite_adapter.py new file mode 100644 index 0000000..3468cc5 --- /dev/null +++ b/scripts/probe_ssdlite_adapter.py @@ -0,0 +1,252 @@ +"""Validate the exact SSDLite adapter on the existing fixed COCO n100 package. + +This writes probe artifacts only: no jobs, collectors, registry edits, or installs. +Use --n 3 for a smoke check before the full --n 100 validation. A successful +adapter probe is not admission proof until --verify-shared-hooks also succeeds. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from dataclasses import replace +import hashlib +import json +import os +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness.adapters import ssdlite320_mobilenetv3_large as adapter + +MODEL = "ssdlite320.mobilenetv3.large" +INPUT_SPEC = adapter.MODEL_SPEC["input_spec"] + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def detections(image_id, prediction, cat_map=None): + boxes, scores, labels = prediction + return [{"image_id": image_id, "category_id": cat_map[str(int(label))] if cat_map else int(label), "score": float(score), + "bbox": [float(b[0]), float(b[1]), float(b[2] - b[0]), float(b[3] - b[1])]} + for b, score, label in zip(boxes, scores, labels)] + + +def calibration(source_manifest, output, eval_ids): + """Reuse the existing 128 COCO train selections, not the old YOLO tensors.""" + import numpy as np + from PIL import Image + + source = json.loads(Path(source_manifest).read_text()) + files = [Path(p) for p in source["files"]] + if source.get("dataset") != "coco2017" or not source.get("source", "").startswith("train"): + raise ValueError("calibration must come from the real COCO train split") + if len(files) != 128 or len(set(files)) != 128 or any(not p.is_file() for p in files): + raise ValueError("128 distinct existing calibration source images are required") + if any(int(p.stem) in eval_ids for p in files): + raise ValueError("calibration and evaluation image IDs overlap") + output.mkdir(parents=True, exist_ok=True) + tensors = output / "npy_f32"; tensors.mkdir(exist_ok=True) + digest = hashlib.sha256() + for i, path in enumerate(files): + with Image.open(path) as image: + x = adapter.preprocess(image, INPUT_SPEC) + if x.shape != (1, 3, 320, 320) or x.dtype != np.float32 or not np.isfinite(x).all(): + raise ValueError(f"invalid calibration tensor: {path}") + destination = tensors / f"{i:05d}.npy" + if destination.exists(): + np.testing.assert_array_equal(np.load(destination), x) + else: + np.save(destination, x) + digest.update(destination.read_bytes()) + result = {"task": "det", "input_spec": INPUT_SPEC, "count": 128, "seed": source["seed"], + "source": source["source"], "dataset": "coco2017", "sha256": digest.hexdigest(), + "files": [str(p) for p in files], "preprocess": adapter.PREPROCESS, + "layout": "NCHW", "dtype": "float32", "npy_dir": "npy_f32", + "source_manifest": str(source_manifest), "source_manifest_sha256": sha(source_manifest)} + (output / "manifest.json").write_text(json.dumps(result, indent=2)) + return {"path": str(output), "count": 128, "sha256": result["sha256"], "overlap_with_t1": 0} + + +def probe(args): + import numpy as np + import onnx + import onnxruntime as ort + import torch + import torchvision + from torchvision.transforms.functional import pil_to_tensor + from harness.continuation import load_eval_set + from harness.eval import EVALUATOR_VERSION + from harness.eval.common import T1, cosine + from harness.eval.det import coco_eval + + output = Path(args.out); output.mkdir(parents=True, exist_ok=True) + proof = output / "probe.json" + if proof.exists(): + (output / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) + result = {"model": MODEL, "task": "det", "status": "running", "started_at": time.time(), + "checkpoint": adapter.CHECKPOINT, "preprocess": adapter.PREPROCESS, + "runtime": {"torch": torch.__version__, "torchvision": torchvision.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__}, + "shared_hooks_verified": False} + proof.write_text(json.dumps(result, indent=2)) + t1 = None + try: + dataset, evaluation = load_eval_set(args.t1) + if dataset != "coco2017" or evaluation["n"] != 100: + raise ValueError("use the existing fixed COCO n100 evaluation package") + result["eval_set"] = evaluation + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + cat_map = t1.manifest["cat_map"] + subset = replace(t1, manifest={**t1.manifest, "items": t1.items[:args.n], "n": args.n}) + model = adapter.build(Path(args.weights), INPUT_SPEC) + reference = adapter.official_model(Path(args.weights), adapter.CHECKPOINT["sha256"]) + + def forward(x): + with torch.inference_mode(): + return [model(torch.from_numpy(x)).numpy()] + + onnx_path = output / f"{MODEL}.onnx" + with torch.no_grad(): + torch.onnx.export(model, torch.zeros(1, 3, 320, 320), str(onnx_path), dynamo=False, + opset_version=adapter.OPSET, input_names=["input"], output_names=["detections"], + do_constant_folding=True, dynamic_axes=None) + graph = onnx.load(str(onnx_path)); onnx.checker.check_model(graph, full_check=True) + shapes = {v.name: [d.dim_value for d in v.type.tensor_type.shape.dim] + for v in list(graph.graph.input) + list(graph.graph.output)} + if shapes != {"input": [1, 3, 320, 320], "detections": [1, 3234, 95]}: + raise ValueError(f"unexpected ONNX shapes: {shapes}") + ops = dict(Counter(n.op_type for n in graph.graph.node)) + if set(ops) & {"NonMaxSuppression", "TopK", "NonZero", "Loop", "If"}: + raise ValueError(f"dynamic postprocessing left in ONNX: {ops}") + options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"]) + + def exported(x): + return session.run(None, {"input": x}) + + fp_dets, onnx_dets, official_dets, comparisons = [], [], [], [] + for i, item in enumerate(subset.items): + image = subset.image(item) + x = adapter.preprocess(image, INPUT_SPEC) + image_tensor = pil_to_tensor(image).float() / 255 + with torch.inference_mode(): + expected_x = reference.transform([image_tensor])[0].tensors.numpy() + expected = reference([image_tensor])[0] + np.testing.assert_array_equal(x, expected_x) + raw, converted = forward(x)[0], exported(x)[0] + if raw.shape != (1, 3234, 95) or not np.isfinite(raw).all() or not np.isfinite(converted).all(): + raise ValueError("invalid raw output") + # Validate boxes and probabilities separately; boxes must not hide a bad classifier. + np.testing.assert_allclose(raw[:, :, :4], converted[:, :, :4], rtol=2e-4, atol=2e-3) + np.testing.assert_allclose(raw[:, :, 4:], converted[:, :, 4:], rtol=3e-4, atol=3e-5) + predicted = adapter.decode([raw], image.size) + official = tuple(expected[key].numpy() for key in ("boxes", "scores", "labels")) + mapped_labels = np.asarray([cat_map[str(int(label))] for label in predicted[2]]) + np.testing.assert_array_equal(mapped_labels, official[2]) + np.testing.assert_allclose(predicted[0], official[0], rtol=1e-6, atol=1e-4) + np.testing.assert_allclose(predicted[1], official[1], rtol=1e-6, atol=1e-7) + transformed = adapter.decode([converted], image.size) + fp_dets.extend(detections(item["id"], predicted, cat_map)) + official_dets.extend(detections(item["id"], official)) + onnx_dets.extend(detections(item["id"], transformed, cat_map)) + comparisons.append({"image_id": item["id"], "original_size": list(image.size), + "boxes_max_abs": float(np.max(np.abs(raw[:, :, :4] - converted[:, :, :4]))), + "scores_max_abs": float(np.max(np.abs(raw[:, :, 4:] - converted[:, :, 4:]))), + "boxes_cosine": cosine(raw[:, :, :4], converted[:, :, :4]), + "scores_cosine": cosine(raw[:, :, 4:], converted[:, :, 4:]), + "official_predictions": len(official[1]), "official_match": True}) + if (i + 1) % 25 == 0 or i == 0: + print(f"validated {i + 1}/{args.n}", flush=True) + metrics = {} + for key, ds in (("fp32", fp_dets), ("onnx", onnx_dets), ("official", official_dets)): + (output / f"{key}-predictions.json").write_text(json.dumps(ds)) + metrics[key] = coco_eval(subset, ds) + if metrics["fp32"]["map_50_95"] <= 0 or abs(metrics["fp32"]["map_50_95"] - metrics["onnx"]["map_50_95"]) > .02: + raise ValueError(f"FP32/ONNX AP mismatch: {metrics}") + result.update(**metrics, comparisons=comparisons, sample_ids=[it["id"] for it in subset.items], + weights_path=str(Path(args.weights).resolve()), weights_sha256=sha(args.weights), + onnx_path=str(onnx_path), onnx_sha256=sha(onnx_path), shapes=shapes, onnx_ops=ops) + if args.n == 100: + if not args.calib_manifest: + raise ValueError("full validation requires --calib-manifest for 128 real COCO train images") + result["calibration"] = calibration(args.calib_manifest, output / "calibration", {it["id"] for it in t1.items}) + baseline = {"task": "det", "model": MODEL, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": 100, "metrics": metrics["fp32"]} + baseline_dir = output / "baseline"; baseline_dir.mkdir(exist_ok=True) + xs = {f"s{i:03d}": adapter.preprocess(t1.image(it), INPUT_SPEC) for i, it in enumerate(t1.g2_subset(32))} + np.savez(baseline_dir / "g2_inputs.npz", **xs) + np.savez(baseline_dir / "g2_fp32.npz", **{f"{key}/out0": forward(x)[0] for key, x in xs.items()}) + (baseline_dir / "metrics.json").write_text(json.dumps(baseline, indent=2)) + result["baseline_path"] = str(baseline_dir) + if args.verify_shared_hooks: + from harness.adapters import calibration_dir + from harness.eval import det + from harness.eval.run import g2_inputs + from harness.registry import model_row + row = model_row(MODEL) + if (row["source"] != "custom" or row["input_spec"] != INPUT_SPEC + or row.get("checkpoint") != adapter.CHECKPOINT or row.get("preprocess") != adapter.PREPROCESS): + raise ValueError("shared model recipe differs from the validated adapter") + check = {**baseline}; det.run_fp32(forward, t1, INPUT_SPEC, check) + if check["metrics"] != baseline["metrics"]: + raise ValueError("shared detection FP32 hook differs from adapter predictions") + quant = {"model": MODEL}; det.run_quant(exported, t1, INPUT_SPEC, baseline, quant) + if quant["metrics"] != metrics["onnx"]: + raise ValueError("shared detection quant hook differs from adapter predictions") + shared_xs = g2_inputs("det", t1, INPUT_SPEC, 32, model=MODEL) + for key in xs: + np.testing.assert_array_equal(xs[key], shared_xs[key]) + shared_calib = calibration_dir(row) + shared_manifest = json.loads((shared_calib / "manifest.json").read_text()) + if (shared_manifest.get("preprocess") != adapter.PREPROCESS or shared_manifest.get("count") != 128 + or shared_manifest.get("sha256") != result["calibration"]["sha256"]): + raise ValueError("shared calibration recipe or tensor identity differs from this probe") + digest = hashlib.sha256() + tensors = sorted((shared_calib / shared_manifest["npy_dir"]).glob("*.npy")) + if len(tensors) != 128: + raise ValueError("shared calibration requires exactly 128 actual tensors") + for path in tensors: + digest.update(path.read_bytes()) + if digest.hexdigest() != shared_manifest["sha256"]: + raise ValueError("shared calibration files do not match their recorded digest") + result["calibration"]["shared_path"] = str(shared_calib) + result["shared_source_sha256"] = { + str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p) + for p in (Path(det.__file__), Path(sys.modules[g2_inputs.__module__].__file__), + Path(__file__).with_name("make_calib.py"))} + result["shared_hooks_verified"] = True + result["status"] = "validated_fp32_export_only" if args.verify_shared_hooks else "validated_adapter_only" + else: + result["status"] = "validated_smoke_only" + except Exception as exc: + import traceback + traceback.print_exc() + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}") + finally: + if t1 is not None: + t1.tar.close() + result["finished_at"] = time.time() + result["source_sha256"] = {p.name: sha(p) for p in (Path(__file__), Path(adapter.__file__), + Path(adapter.__file__).with_name("ssdlite_common.py"))} + proof.write_text(json.dumps(result, indent=2)) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + print("ALL DONE", flush=True) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--weights", required=True) + parser.add_argument("--t1", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--n", type=int, choices=(3, 100), default=100) + parser.add_argument("--calib-manifest") + parser.add_argument("--verify-shared-hooks", action="store_true") + args = parser.parse_args() + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(2) + raise SystemExit(probe(args)["status"] == "blocked") diff --git a/scripts/probe_topdown_adapters.py b/scripts/probe_topdown_adapters.py new file mode 100644 index 0000000..3c91b1b --- /dev/null +++ b/scripts/probe_topdown_adapters.py @@ -0,0 +1,237 @@ +"""Real official HRNet/SimCC validation; private outputs, fixed COCO100.""" +from __future__ import annotations +import argparse, json, os, subprocess, sys, time, traceback +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from probe_rtmpose_adapters import check_shared_g2, save, sha + + +def calibration(root, output, model): + """Render the same recorded train128 people with canonical official normalization.""" + import hashlib, io, zipfile + import numpy as np + from PIL import Image + from harness.adapters import get + from harness.adapters.topdown_pose_common import crop + + recipe = get(model).PREPROCESS + recipe_sha = hashlib.sha256(json.dumps(recipe, sort_keys=True).encode()).hexdigest() + directory = output / "calib" / recipe_sha[:16] + manifest_path = directory / "manifest.json" + if not manifest_path.exists(): + if directory.exists(): + raise ValueError(f"Incomplete calibration requires inspection: {directory}") + prior_path = Path("/home/ming01.wang/outputs/pose/verified/calib/e28f899732c4ebf2/manifest.json") + prior = json.loads(prior_path.read_text()) + if len(prior["crops"]) != 128 or len({r["image_id"] for r in prior["crops"]}) != 128: + raise ValueError("Expected verified train128 person identities") + with zipfile.ZipFile(root / "annotations_trainval2017.zip") as archive: + annotations = archive.read("annotations/person_keypoints_train2017.json") + assert hashlib.sha256(annotations).hexdigest() == prior["annotations_sha256"] + (directory / "npy_f32").mkdir(parents=True) + records, digest = [], hashlib.sha256() + with zipfile.ZipFile(root / "train2017.zip") as archive: + for i, (source, record) in enumerate(zip(prior["files"], prior["crops"])): + content = archive.read(source.split("#", 1)[1]) + assert hashlib.sha256(content).hexdigest() == record["image_sha256"] + x, affine, _, _ = crop(Image.open(io.BytesIO(content)).convert("RGB"), recipe["input_spec"], record["bbox_xyxy"]) + assert x.shape == (1, 3, 256, 192) and x.dtype == np.float32 and np.isfinite(x).all() + path = directory / "npy_f32" / f"{i:05d}.npy" + np.save(path, x); digest.update(path.read_bytes()) + records.append({**record, "affine": affine.tolist(), "npy_sha256": sha(path)}) + save(manifest_path, {**prior, "preprocess": recipe, "crops": records, "sha256": digest.hexdigest(), + "selection_manifest": str(prior_path), "selection_manifest_sha256": sha(prior_path)}) + manifest = json.loads(manifest_path.read_text()) + files = sorted((directory / "npy_f32").glob("*.npy")) + digest = hashlib.sha256() + for path in files: + digest.update(path.read_bytes()) + if len(files) != 128 or manifest["preprocess"] != recipe or digest.hexdigest() != manifest["sha256"]: + raise ValueError("Calibration bytes do not match recipe") + return {"path": str(directory), "sha256": digest.hexdigest(), "count": 128} + + +def probe(mid: str, manifest: str, weights_dir: Path, output: Path, calib: dict): + import numpy as np + import onnx + import onnxruntime as ort + import torch + from mmengine.dataset import Compose, pseudo_collate + from mmpose.apis.inference import inference_topdown + from harness import registry, scheduler + from harness.adapters import get + from harness.adapters.openmmlab import get_recipe + from harness.adapters.topdown_pose_common import (crop, components, diagnostic_box, + boxes_for, BBOX_FILE, BBOX_SHA256) + from harness.continuation import load_eval_set + from harness.eval import EVALUATOR_VERSION, coco_extra + from harness.eval.common import T1, TorchRunner, cosine + from harness.tools.export_onnx import exporter_version + + directory = output / mid + directory.mkdir(parents=True, exist_ok=True) + proof = directory / "probe.json" + if proof.exists(): + (directory / f"probe-history-{time.time_ns()}.json").write_bytes(proof.read_bytes()) + result = {"model": mid, "task": "pose", "status": "running", "started_at": time.time()} + save(proof, result) + try: + adapter, row = get(mid), registry.model_row(mid) + dataset, evaluation = load_eval_set(manifest) + if dataset != "coco2017" or row["dataset"] != dataset: + raise ValueError("RTMPose requires the fixed COCO2017 T1") + weights = weights_dir / row["checkpoint"]["filename"] + if sha(weights) != row["checkpoint"]["sha256"]: + raise ValueError("Checkpoint bytes are not pinned in the shared catalog") + audit_path = output.parent / "canonical-audit.json" + audit = json.loads(audit_path.read_text()) + normal = next(r for r in audit["models"] if r["model"] == mid) + assert normal["checkpoint_sha256"] == sha(weights) + assert normal["crop_vs_stock_official_preprocessor_bitwise_equal"] + result.update(checkpoint=row["checkpoint"], preprocess=row["preprocess"], eval_set=evaluation, + weights_path=str(weights), weights_sha256=sha(weights), calibration=calib, + bbox_file=str(BBOX_FILE), bbox_sha256=BBOX_SHA256, + official_recipe=get_recipe(mid), + canonical_audit={"path": str(audit_path), "sha256": sha(audit_path)}, + official_reference_scope=audit["reference_scope"], + implementation_sha256={str(p.relative_to(Path(__file__).resolve().parents[1])): sha(p) + for p in (Path(__file__).resolve(), Path(adapter.__file__), + Path(__file__).resolve().parents[1] / "harness/adapters/topdown_pose_common.py", + Path(__file__).resolve().parents[1] / "harness/adapters/openmmlab.py", + Path(__file__).resolve().parents[1] / "harness/adapters/rtmpose_common.py", + Path(coco_extra.__file__))}) + with scheduler.acquire("cpu"): + runner = TorchRunner(mid, row["source"], weights, row["input_spec"]) + result["strict_load"] = True + onnx_path = directory / "model.onnx" + meta_path = directory / "meta.json" + version = exporter_version(mid) + previous = json.loads(meta_path.read_text()) if meta_path.exists() else {} + if not (onnx_path.exists() and previous.get("exporter_version") == version + and previous.get("weights_sha256") == sha(weights) + and previous.get("onnx_sha256") == sha(onnx_path)): + with (directory / "export.log").open("a") as log: + subprocess.run([sys.executable, "-m", "harness.tools.export_onnx", "--model", mid, + "--source", row["source"], "--weights", str(weights), "--input-spec", row["input_spec"], + "--out", str(onnx_path)], check=True, stdout=log, stderr=log) + save(meta_path, {"model": mid, "exporter_version": version, "input_spec": row["input_spec"], + "weights_sha256": sha(weights), "onnx_sha256": sha(onnx_path)}) + onnx.checker.check_model(str(onnx_path)) + options = ort.SessionOptions(); options.intra_op_num_threads = 2; options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(onnx_path), sess_options=options, providers=["CPUExecutionProvider"]) + + def exported(x): + return session.run(None, {session.get_inputs()[0].name: x}) + + t1 = T1.open(evaluation["path"], evaluation["manifest_sha"]) + try: + if len(t1.items) != 100: + raise ValueError("This admission probe requires exactly the fixed 100 COCO images") + eligible = [i for i, item in enumerate(t1.items) if diagnostic_box(t1, item) is not None] + if len(eligible) < 32: + raise ValueError("Fewer than 32 images have genuine detector boxes") + indices = sorted(np.random.default_rng(20260914).permutation(eligible)[:32].tolist()) + native = runner.m.upstream + native.cfg = runner.m.cfg + native.dataset_meta = components(mid)[2] + pipeline = Compose(native.cfg.test_dataloader.dataset.pipeline) + inputs, outputs, comparisons, references = {}, {}, [], [] + for i, index in enumerate(indices): + item = t1.items[index] + image = t1.image(item) + box = diagnostic_box(t1, item) + x, matrix, center, scale = crop(image, row["input_spec"], box["bbox_xyxy"]) + key = f"s{i:03d}" + inputs[key] = x + a, b = runner(x), exported(x) + if len(a) != len(b) or len(a) != (1 if mid.startswith("hrnet.") else 2): + raise ValueError("Unexpected official head tensor count") + for j, (aa, bb) in enumerate(zip(a, b)): + if aa.shape != bb.shape or not np.isfinite(aa).all() or not np.isfinite(bb).all(): + raise ValueError("Invalid FP32/ONNX output") + score = cosine(aa, bb) + comparisons.append({"sample": key, "image_id": item["id"], "output": j, + "cosine": score, "max_abs": float(np.abs(aa-bb).max())}) + if score < .9999: + raise ValueError(f"SimCC parity failed: {score}") + outputs[f"{key}/out{j}"] = aa + bgr = np.asarray(image)[:, :, ::-1].copy() + original = {"img": bgr, "bbox": box["bbox_xyxy"][None].copy(), + "bbox_score": np.ones(1, np.float32), **native.dataset_meta} + packed = pipeline(original) + with torch.no_grad(): + official_input = native.data_preprocessor(pseudo_collate([packed]), training=False)["inputs"].cpu().numpy() + input_delta = float(np.abs(x - official_input).max()) + ref = inference_topdown(native, bgr, bboxes=box["bbox_xyxy"][None].copy(), bbox_format="xyxy")[0].pred_instances + points = adapter.decode(a, runner(np.ascontiguousarray(x[..., ::-1])), center, scale, row["input_spec"]) + coordinate_delta = float(np.abs(points[:, :2] - ref.keypoints[0]).max()) + score_delta = float(np.abs(points[:, 2] - ref.keypoint_scores[0]).max()) + references.append({"sample": key, "image_id": item["id"], "source_index": box["source_index"], + "affine": matrix.tolist(), "input_max_abs": input_delta, + "coordinate_max_abs_px": coordinate_delta, "score_max_abs": score_delta}) + if input_delta > 1e-6 or coordinate_delta > .01 or score_delta > 1e-4: + raise ValueError(f"Official pipeline mismatch: {references[-1]}") + result.update(comparisons=comparisons, official_reference=references, + no_bbox_image_ids=[it["id"] for it in t1.items if not boxes_for(t1, it)], + g2_selection="seed20260914 sample32 eligible detector images; sorted T1 order") + save(proof, result) + print(f"{mid}: strict load/export and 32 official-reference/G2 comparisons passed", flush=True) + baseline = {"task": "pose", "model": mid, "evaluator_version": EVALUATOR_VERSION, + "t1_manifest_sha": t1.manifest["sha256"], "n": len(t1.items)} + converted = dict(baseline) + coco_extra.run_fp32(runner, t1, row["input_spec"], baseline) + save(directory / "metrics.json", baseline) + print(f"{mid}: FP32 COCO100 {baseline['metrics']}", flush=True) + coco_extra.run_quant(exported, t1, row["input_spec"], baseline, converted) + converted["execution"] = {"source": "onnxruntime_cpu", "purpose": "export_parity_only"} + save(directory / "eval.json", converted) + shift = converted["pose_shift_evidence"] + delta = abs(baseline["metrics"]["oks_ap"] - converted["metrics"]["oks_ap"]) + if (baseline["metrics"]["oks_ap"] <= 0 or delta > .1 or shift["status"] != "measured" + or shift["keypoint_mean_shift_px"] > 2): + raise ValueError(f"COCO100/shift parity failed: delta={delta}, shift={coco_extra.pose_shift_summary(shift)}") + np.savez(directory / "g2_inputs.npz", **inputs) + np.savez(directory / "g2_fp32.npz", **outputs) + result["shared_g2"] = check_shared_g2(mid, t1, inputs) + evidence_path = directory / "pose-export-evidence.json" + save(evidence_path, shift) + result.update(status="validated_fp32_export_only", fp32=baseline["metrics"], onnx=converted["metrics"], + pose_export_shift={**coco_extra.pose_shift_summary(shift), "evidence": str(evidence_path)}, + pose_export_evidence={"path": str(evidence_path), "sha256": sha(evidence_path)}, + onnx_path=str(onnx_path), + onnx_sha256=sha(onnx_path), baseline_path=str(directory), + sample_ids=[it["id"] for it in t1.items]) + finally: + t1.tar.close() + except Exception as exc: + result.update(status="blocked", error=f"{type(exc).__name__}: {exc}", traceback=traceback.format_exc()) + result["finished_at"] = time.time() + save(proof, result) + print(json.dumps({k: result[k] for k in ("model", "status", "fp32", "onnx", "error") if k in result}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", action="append", required=True, choices=["hrnet.w32", "hrnet.w48.2", "simcc.resnet50"]) + parser.add_argument("--t1", required=True) + parser.add_argument("--weights-dir", type=Path, required=True) + parser.add_argument("--train-root", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if len(args.model) > 3 or len(set(args.model)) != len(args.model): + parser.error("at most three distinct models per batch") + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + import torch + torch.set_num_threads(2) + args.out.mkdir(parents=True, exist_ok=True) + results = [] + for mid in args.model: + calib = calibration(args.train_root, args.out, mid) + print(json.dumps({"model": mid, "calibration": calib}), flush=True) + results.append(probe(mid, args.t1, args.weights_dir, args.out, calib)) + return int(any(r["status"] != "validated_fp32_export_only" for r in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start_run.sh b/scripts/start_run.sh index 28f0304..eb41318 100755 --- a/scripts/start_run.sh +++ b/scripts/start_run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# 一键启动 S100P(nash-m)oracle / 续跑已有 oracle: +# 一键启动 S100P(nash-m)输入准备 + Agent 首轮采集: # scripts/start_run.sh --note "P4 batch 1" [--parallel 8] 新 run:preflight → new-run → expand → 后台 run # scripts/start_run.sh --resume [--parallel 8] 续跑最新 run 的 PENDING(调度进程崩了先 reset-stale) # 之后:bench watch(实时看板)· bench status · bench status --fail · bench show · bench report @@ -15,13 +15,15 @@ with db.connect() as c: print(db.latest_run_id(c) or '')") [ -n "$RID" ] || { echo "没有 run 可续跑"; exit 1; } bench reset-stale --run "$RID" || true else - bench preflight --modes oracle --oracle-only || { echo "preflight 有 ERROR,先修"; exit 1; } + bench preflight --modes oracle,e2e || { echo "preflight 有 ERROR,先修"; exit 1; } RID=$(bench new-run --note "${NOTE:-start_run $(date +%F)}") - bench expand --run "$RID" --modes oracle --marches nash-m + bench expand --run "$RID" --modes oracle,e2e --marches nash-m fi LOG="state/run-$RID.log" setsid nohup bench run --run "$RID" $PAR >> "$LOG" 2>&1 < /dev/null & disown echo "run $RID 已在后台启动(pid $!)" +setsid nohup bench collect --run "$RID" --modes e2e --coverage all --first-pass --watch >> "state/collect-$RID.log" 2>&1 < /dev/null & disown +echo "Agent 首轮采集已启动(pid $!):state/collect-$RID.log" echo " 实时看板: bench watch --run $RID" echo " 调度日志: tail -f $LOG" echo " 一眼汇总: bench status --run $RID ;失败清单 bench status --fail" diff --git a/scripts/sync.sh b/scripts/sync.sh index 781f913..0be6478 100755 --- a/scripts/sync.sh +++ b/scripts/sync.sh @@ -9,6 +9,12 @@ HERE="$(cd "$(dirname "$0")/.." && pwd)"; cd "$HERE" REMOTE="${BENCH_REMOTE:-ming01.wang@120.48.151.164}" RDIR="${BENCH_REMOTE_DIR:-/mnt/data/ming01.wang/model-compile-hbdk/bench}" STAMP=".sync/last_pushed.md5" +# Scoped fixes must not overwrite unrelated remote additions. Same entry point, +# per-file preimage checks and backups; the ordinary whole-tree guard is unchanged. +if [ "${1:-}" = "--files" ]; then + shift + exec python3 scripts/sync_patch.py "$@" +fi # 模式带单引号:本地用 eval 展开,远端在 ssh 的双引号里由远端 shell 展开——两边都不会被本地 glob 吃掉(踩过:*.md 被展开成 README.md) # *.js 与 VERSION 是 2026-09-15 补上的:guard-core.js / guard-core.test.js / dump_tools.js / agent/VERSION # 原来不在 TRACK 里 —— 判定核心本身游离在同步与"远端被改过"守卫之外,等于守卫有个洞。 @@ -43,13 +49,13 @@ if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then git add -A && git commit -qm "sync $(date +%F_%T) $(whoami)@$(hostname -s)" && echo "committed $(git rev-parse --short HEAD)" fi fi -find harness agent export scripts -name __pycache__ -type d -prune -exec rm -rf {} + 2>/dev/null || true +find harness agent export scripts bin -name __pycache__ -type d -prune -exec rm -rf {} + 2>/dev/null || true # 注意:rsync 多个源到一个目录会把 registry/x.yaml 摊平到 $RDIR/x.yaml(踩过),所以整目录同步 + 排除 models.csv # 踩过两次:--delete-excluded 会把"被排除的文件"从远端删掉 → registry/models.csv 两次消失。必须用 --delete(排除项在远端受保护)。 rsync -az --delete --exclude '__pycache__' --exclude '.sync' --exclude '.git' --exclude 'registry/models.csv' --exclude 'registry/models_源清单.xlsx' \ - harness agent export scripts bin registry config.yaml CONTRACT.md HANDBOOK.md REPORT-codex.md README.md requirements.txt "$REMOTE:$RDIR/" 2>&1 | tail -1 + harness agent export scripts bin registry docs config.yaml CONTRACT.md HANDBOOK.md CONTINUE.md README.md requirements.txt "$REMOTE:$RDIR/" 2>&1 | tail -1 # registry/models.csv 是机器上生成的(excel_to_models),只在 --pull 方向走 # 只在代码目录里清 __pycache__:find 整棵树会扫进 runs/(JuiceFS 上 190 个 job 目录,几分钟) -ssh -o BatchMode=yes "$REMOTE" "cd $RDIR && chmod +x bin/bench scripts/*.sh agent/*.sh 2>/dev/null; find harness agent export scripts -name __pycache__ -type d -prune -exec rm -rf {} + 2>/dev/null; true" +ssh -o BatchMode=yes "$REMOTE" "cd $RDIR && chmod +x bin/bench scripts/*.sh agent/*.sh 2>/dev/null; find harness agent export scripts bin -name __pycache__ -type d -prune -exec rm -rf {} + 2>/dev/null; true" manifest_remote > "$STAMP" echo "synced → $REMOTE:$RDIR ($(wc -l < "$STAMP") tracked files; stamp saved)" diff --git a/scripts/sync_patch.py b/scripts/sync_patch.py new file mode 100644 index 0000000..f3643e0 --- /dev/null +++ b/scripts/sync_patch.py @@ -0,0 +1,81 @@ +"""Scoped sync behind scripts/sync.sh --files; refuse remote preimage drift.""" +import base64 +import datetime +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + +REMOTE_APPLY = r''' +import base64,fcntl,hashlib,json,os,sys +from pathlib import Path +p=json.load(sys.stdin);root=Path(p['root']);sync=root/'.sync';sync.mkdir(exist_ok=True) +with (sync/'patch.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX) + for x in p['files']: + f=root/x['path'];actual=hashlib.md5(f.read_bytes()).hexdigest() if f.exists() else None + if actual!=x['expected']:raise SystemExit('remote drift: '+x['path']) + backup=sync/p['id'];backup.mkdir() + (backup/'request.json').write_text(json.dumps(p)) + result=[] + for x in p['files']: + f=root/x['path'];old=f.read_bytes() if f.exists() else None + actual=hashlib.md5(old).hexdigest() if old is not None else None + if actual!=x['expected']:raise SystemExit('concurrent remote change: '+x['path']) + if old is not None: + b=backup/x['path'];b.parent.mkdir(parents=True,exist_ok=True);b.write_bytes(old) + raw=base64.b64decode(x['data']);f.parent.mkdir(parents=True,exist_ok=True) + tmp=f.with_name(f.name+'.'+p['id']);tmp.write_bytes(raw) + tmp.chmod(f.stat().st_mode & 0o777 if f.exists() else 0o644);os.replace(tmp,f) + result.append({'path':x['path'],'md5':hashlib.md5(f.read_bytes()).hexdigest(),'sha256':hashlib.sha256(f.read_bytes()).hexdigest()}) + (backup/'receipt.json').write_text(json.dumps(result,indent=2)) + print(json.dumps({'backup':str(backup),'files':result},indent=2)) +''' + + +def main(): + root = Path(__file__).resolve().parents[1] + names = list(dict.fromkeys(sys.argv[1:])) + if not names: + raise SystemExit('usage: scripts/sync.sh --files path ... (committed code only)') + stamp = root / '.sync/last_pushed.md5' + known = {s.split(None, 1)[1]: s.split(None, 1)[0] for s in stamp.read_text().splitlines()} + entries = [] + for name in names: + p = Path(name) + if p.is_absolute() or '..' in p.parts or p.as_posix() != name: + raise SystemExit('invalid relative path: ' + name) + if p.parts[0] not in ('harness', 'scripts', 'docs') and name not in ('README.md', 'CONTRACT.md'): + raise SystemExit('not a scoped code/document path: ' + name) + data = (root / name).read_bytes() + committed = subprocess.check_output(['git', 'show', 'HEAD:' + name], cwd=root) + if data != committed: + raise SystemExit('commit selected files before sync: ' + name) + entries.append({'path': name, 'expected': known.get('./' + name), + 'data': base64.b64encode(data).decode()}) + payload = {'root': os.environ.get('BENCH_REMOTE_DIR', '/mnt/data/ming01.wang/model-compile-hbdk/bench'), + 'id': 'patch-' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S-%f'), 'files': entries} + remote = os.environ.get('BENCH_REMOTE', 'ming01.wang@120.48.151.164') + r = subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', remote, + 'python3 -c ' + shlex.quote(REMOTE_APPLY)], input=json.dumps(payload), + capture_output=True, text=True, timeout=120) + if r.returncode: + sys.stderr.write(r.stderr) + raise SystemExit(r.returncode) + receipt = json.loads(r.stdout) + for x in receipt['files']: + expected = hashlib.md5((root / x['path']).read_bytes()).hexdigest() + if x['md5'] != expected: + raise SystemExit('post-sync hash mismatch: ' + x['path']) + known['./' + x['path']] = expected + # Only selected entries advance; unrelated drift remains detectable. + stamp.write_text(''.join(f'{h} {p}\n' for p, h in sorted(known.items()))) + (root / '.sync' / (payload['id'] + '.json')).write_text(r.stdout) + print(r.stdout) + + +if __name__ == '__main__': + main() diff --git a/scripts/sync_status.py b/scripts/sync_status.py new file mode 100644 index 0000000..6f1ef67 --- /dev/null +++ b/scripts/sync_status.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""sync_status —— 把 registry/models.csv 的 status 列刷新成实机事实。 + + PYTHONPATH=. python3 scripts/sync_status.py # dry-run:只打印将发生的变化 + PYTHONPATH=. python3 scripts/sync_status.py --apply # 真正写 models.csv + +models.csv 的 status 是 Excel 导入时(excel_to_models.py)按来源启发式打的: +torchvision/cls→ready、ultralytics/det→ready、其余 needs_adapter/needs_eval/skip。 +跑了一阵子之后它和实机进度脱节:适配器、probe、t1/calibration 都陆续补齐了, +但 status 没人回头改,于是 `bench expand` 默认只展开 status=ready,看不到这些 +实际已具备准入条件的模型。 + +判定不重新发明:直接复用 harness/coverage.py 的 model_blockers()(与 +master_inventory 同一调用方式:models() → model_spec() → model_blockers(row, cache))。 +规则: + blockers 为空 且 status ∈ {needs_adapter, needs_eval} → 改 ready + blockers 非空 → 保持原 status + status = skip → 一律不动(非神经网络,不量化) + status = ready → 不动(只升不降;是否有 blocker + 与 label 无关,不做降级) +写入用 csv 模块 round-trip:列结构、行序、其他列字节不变。写入前先验证未修改 +内容能被同一套 writer 逐字节还原(防格式漂移),再只改 status 单元格, +临时文件 + os.replace 原子落盘。 + +默认 dry-run 的原因:models.csv 参与 registry_fingerprint(),跑着的 collector / +scheduler 对指纹变化敏感——当前 run 有 rollout 在采时改 registry 可能扰动它们。 +何时可以 --apply:当前 run 收尾(rollout 全部终态、调度进程退出)之后再跑; +改完后用 `PYTHONPATH=. python3 -c "from harness.registry import status_counts; print(status_counts())"` +复核,再正常 scripts/start_run.sh --resume。 +""" +from __future__ import annotations + +import argparse +import csv +import io +import os +import sys +import tempfile +import time +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from harness import paths # noqa: E402 +from harness.adapters import model_spec # noqa: E402 +from harness.coverage import model_blockers # noqa: E402 +from harness.registry import models, status_counts # noqa: E402 + +PROMOTE = ("needs_adapter", "needs_eval") + + +def roundtrip(rows: list[list[str]], terminator: str) -> bytes: + """csv 模块序列化;与读入同一 dialect,只有 terminator 按文件实测。""" + buf = io.StringIO(newline="") + csv.writer(buf, lineterminator=terminator).writerows(rows) + return buf.getvalue().encode("utf-8") + + +def main() -> int: + ap = argparse.ArgumentParser(description="按 model_blockers() 实况刷新 models.csv 的 status") + ap.add_argument("--csv", default=str(paths.REGISTRY / "models.csv"), + help="models.csv 路径(默认 registry/models.csv)") + ap.add_argument("--apply", action="store_true", + help="真正写入(默认 dry-run 只打印)") + a = ap.parse_args() + csv_path = Path(a.csv) + + raw = csv_path.read_bytes() + terminator = "\r\n" if b"\r\n" in raw else "\n" + text = raw.decode("utf-8") + rows = list(csv.reader(io.StringIO(text, newline=""))) + pristine = [list(r) for r in rows] # 改动前的逐字节快照 + header = rows[0] + col = {name: i for i, name in enumerate(header)} + for need in ("model_id", "task", "status"): + if need not in col: + raise SystemExit(f"{csv_path} 缺列 {need!r},现有:{header}") + + # ---- 判定:与 master_inventory() 相同的调用链,共享一个 prerequisite cache + cache: dict = {} + t0 = time.time() + admission: dict[str, list[str]] = {} + for row in models(): + spec = model_spec(row) + admission[spec["model_id"]] = model_blockers(spec, cache) + elapsed = time.time() - t0 + + # ---- 规则应用(改的是原始 CSV 行,不是 models() 的 strip 副本) + st_before = Counter((r[col["status"]].strip() or "ready") + for r in pristine[1:] if r[col["model_id"]].strip()) + changes: list[tuple[str, str, str, str]] = [] # (model, task, old, new) + for r in rows[1:]: + mid = r[col["model_id"]].strip() + if not mid or mid not in admission: + continue + old = r[col["status"]].strip() + if old in ("skip", "ready"): + continue # skip 不动;ready 只升不降 + if not admission[mid] and old in PROMOTE: + r[col["status"]] = "ready" + changes.append((mid, r[col["task"]].strip(), old, "ready")) + + # ---- 报告 + print(f"models.csv: {csv_path} · {len(rows) - 1} 行 · model_blockers() 实测 {len(admission)} 个模型,耗时 {elapsed:.1f}s") + print(f"status 现状: {dict(st_before.most_common())}") + print() + if not changes: + print("没有需要变更的模型(blockers 为空的都已 ready,其余按规则保持)") + return 0 + for mid, task, old, new in changes: + print(f" {mid:<42} {task:<6} {old:<14} → {new} 依据: model_blockers()=0(适配器/probe/t1/calibration 就绪)") + st_after = dict(st_before) + for _, _, old, new in changes: + st_after[old] -= 1 + st_after[new] += 1 + print() + print(f"将变更 {len(changes)} 个模型 · 按 task: {dict(Counter(t for _, t, _, _ in changes))}") + print(f"变更后 status: {dict(Counter({k: v for k, v in st_after.items() if v}).most_common())}") + + if not a.apply: + print("\ndry-run,未写盘。确认后加 --apply(等当前 run 收尾、rollout 全部终态之后再动 registry)。") + return 0 + + # ---- 写入:先证明这套 writer 配置能逐字节还原原文件,再落盘 + if roundtrip(pristine, terminator) != raw: + raise SystemExit("round-trip 校验失败:csv 序列化无法逐字节还原原文件,拒绝写入(防格式漂移)") + fd, tmp = tempfile.mkstemp(dir=csv_path.parent, prefix=csv_path.name + ".") + try: + with os.fdopen(fd, "w", newline="", encoding="utf-8") as stream: + csv.writer(stream, lineterminator=terminator).writerows(rows) + os.replace(tmp, csv_path) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + print(f"\nwrote {len(changes)} 处 status 变更 → {csv_path}") + with csv_path.open(encoding="utf-8", newline="") as stream: + written = list(csv.reader(stream)) + if written != rows: + raise SystemExit("写入后回读与预期行不一致,请立即检查文件") + if csv_path == paths.REGISTRY / "models.csv": + models.cache_clear() + print(f"复核 registry.models() 重新加载: {status_counts()}") # 只认 REGISTRY 下的真身 + else: + print(f"复核回读: {dict(Counter((r[col['status']].strip() or 'ready') for r in written[1:] if r[col['model_id']].strip()).most_common())}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_accuracy_policy.py b/scripts/test_accuracy_policy.py index 5a8eebd..bc7fcb2 100644 --- a/scripts/test_accuracy_policy.py +++ b/scripts/test_accuracy_policy.py @@ -1,5 +1,4 @@ """Accuracy acceptance boundaries, frozen policies, and reward direction.""" -import copy import importlib.util from pathlib import Path import unittest @@ -43,7 +42,9 @@ class AccuracyPolicy(unittest.TestCase): self.assertEqual(gates.g3(self.measurement(.57, fp32=.6), cfg)["verdict"], "pass-screen") def test_lower_is_better_metrics(self): - cfg = self.snapshot("mde")["G3"] + # Exercise scalar direction here; test_depth_gate covers the independent + # required alignment and delta1 measurements for real MDE acceptance. + cfg = {**self.snapshot("mde")["G3"], "extra": {}} for quant, verdict in ((.21, "pass-screen"), (.21001, "fail"), (.15, "pass-screen")): with self.subTest(quant=quant): self.assertEqual(gates.g3(self.measurement(quant, fp32=.2), cfg)["verdict"], verdict) @@ -55,6 +56,21 @@ class AccuracyPolicy(unittest.TestCase): with self.subTest(measurement=m): self.assertEqual(gates.g3(m, cfg)["verdict"], "unverified") + def test_flow_requires_both_original_metrics(self): + cfg = self.snapshot("flow")["G3"] + measurement = self.measurement(1, fp32=1, per_metric={ + "epe2d": {"fp32": 1, "quant": 1}, "fl_all": {"fp32": 10, "quant": 10.6}}) + result = gates.g3(measurement, cfg) + self.assertEqual(result["verdict"], "fail") + self.assertEqual(result["metric"], "fl_all") + self.assertEqual(result["per_metric"]["epe2d"]["verdict"], "pass-screen") + measurement["per_metric"]["fl_all"]["quant"] = 10.5 + self.assertEqual(gates.g3(measurement, cfg)["verdict"], "pass-screen") + del measurement["per_metric"]["fl_all"] + self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified") + measurement["source"] = "sim" + self.assertEqual(gates.g3(measurement, cfg)["verdict"], "unverified") + def test_cosine_and_performance_do_not_block_new_jobs(self): cfg = self.snapshot() measurements = { @@ -92,20 +108,17 @@ class AccuracyPolicy(unittest.TestCase): self.assertTrue(jobs) self.assertTrue(all(j.gates_required == ["G1", "G3"] for j in jobs if "gates" not in j.overrides)) - def test_reward_favors_accuracy_improvement_for_both_kinds(self): - for kind, agent, oracle, tol in (("rel_pct", 2, 4, 5), ("abs_pp", -.2, -.6, -1)): + def test_reward_uses_absolute_gate_evidence_for_both_kinds(self): + for kind in ("rel_pct", "abs_pp"): with self.subTest(kind=kind): - verdict = {"gates": {"G3": {"kind": kind, "delta": agent, "tolerance": tol}}, - "oracle_ref": {"G3_delta": oracle, "G3_kind": kind}} - self.assertGreater(reward.compute(verdict)["terms"]["acc_rel"], 0) - worse = copy.deepcopy(verdict) - worse["gates"]["G3"]["delta"] = oracle - worse["oracle_ref"]["G3_delta"] = agent - self.assertLess(reward.compute(worse)["terms"]["acc_rel"], 0) - verdict["oracle_ref"]["G3_kind"] = "different_policy" - self.assertEqual(reward.compute(verdict)["terms"]["acc_rel"], 0) - verdict["oracle_ref"].pop("G3_kind") - self.assertGreater(reward.compute(verdict)["terms"]["acc_rel"], 0) + verdict = {"gates": {"G3": {"verdict": "pass-screen", "kind": kind, "delta": 2, "tolerance": 5}}, + "oracle_ref": {"G3_delta": 99, "G3_kind": kind}} + expected = reward.compute(verdict) + self.assertEqual(expected["terms"]["G3"], 2.0) + verdict["oracle_ref"]["G3_delta"] = -99 + self.assertEqual(reward.compute(verdict), expected) + verdict["gates"]["G3"]["verdict"] = "fail" + self.assertEqual(reward.compute(verdict)["terms"]["G3"], 0.0) if __name__ == "__main__": diff --git a/scripts/test_adapter_recipes.py b/scripts/test_adapter_recipes.py new file mode 100644 index 0000000..1181bf5 --- /dev/null +++ b/scripts/test_adapter_recipes.py @@ -0,0 +1,130 @@ +"""New input recipes must not alter frozen legacy inputs or evaluation sets.""" +from contextlib import ExitStack +import hashlib +import io +import json +from pathlib import Path +import tarfile +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np +from PIL import Image + +from harness import adapters, db, paths, pipeline, registry +from harness.eval.common import preprocess_cls +from harness.eval import cls, det +from harness.eval.run import g2_inputs + + +class Recipes(unittest.TestCase): + def test_diagnostic_sampling_uses_only_real_eligible_crops(self): + t1 = SimpleNamespace(items=[{"id": i} for i in range(9)]) + transform = lambda t1, item, spec: (np.full((1, 3, 2, 2), item["id"], dtype=np.float32) + if item["id"] % 2 else None) + adapter = SimpleNamespace(g2_input=transform) + with patch.object(adapters, "get", return_value=adapter): + actual = g2_inputs("pose", t1, "1x3x2x2", 3, model="crop.model") + chosen = sorted(np.random.default_rng(20260914).permutation([1, 3, 5, 7])[:3]) + self.assertEqual([int(x[0, 0, 0, 0]) for x in actual.values()], chosen) + with self.assertRaisesRegex(ValueError, "only 4 valid"): + g2_inputs("pose", t1, "1x3x2x2", 5, model="crop.model") + + def test_checkpoint_resolution_keeps_csv_and_old_models_identical(self): + raw = next(row for row in registry.models() if row["model_id"] == "efficientvit.b0") + before = dict(raw) + resolved = registry.model_row("efficientvit.b0") + self.assertIn("/resolve/", resolved["weights"]) + self.assertEqual(raw, before) + self.assertEqual(resolved["status"], "needs_adapter") + legacy = registry.model_row("resnet18") + self.assertNotIn("preprocess", legacy) + self.assertEqual(adapters.calibration_dir(legacy), paths.DATA / "calib/cls/1x3x224x224") + self.assertNotEqual(adapters.calibration_dir(raw), adapters.calibration_dir(legacy)) + self.assertEqual(adapters.calibration_dir(raw), adapters.calibration_dir(registry.model_row("efficientvit.b3"))) + + def test_same_recipe_reaches_fp32_quant_and_g2(self): + img = Image.new("RGB", (17, 31), (10, 20, 30)) + tensor = np.arange(12, dtype=np.float32).reshape(1, 3, 2, 2) + adapter = SimpleNamespace(preprocess=lambda image, spec: tensor.copy()) + t1 = SimpleNamespace(items=[{"label": 1}], image=lambda item: img, + g2_subset=lambda n: [{"label": 1}]) + seen = [] + def runner(x): + seen.append(x) + return [np.array([[0., 3., 1., 2., -1.]])] + baseline, quant = {"model": "new.model"}, {"model": "new.model"} + legacy = preprocess_cls(img, size=2, resize=3) + with patch.object(adapters, "get", return_value=adapter): + cls.run_fp32(runner, t1, "1x3x2x2", baseline) + cls.run_quant(runner, t1, "1x3x2x2", baseline, quant) + diagnostic = g2_inputs("cls", t1, "1x3x2x2", 1, model="new.model") + for actual in [*seen, diagnostic["s000"]]: + np.testing.assert_array_equal(actual, tensor) + np.testing.assert_array_equal(legacy, preprocess_cls(img, size=2, resize=3, model="resnet18")) + + def test_detector_adapter_owns_geometry_and_keeps_coco_category_mapping(self): + image = Image.new("RGB", (20, 40)) + tensor = np.ones((1, 3, 2, 2), dtype=np.float32) * 3 + original_box = np.array([[2., 4., 6., 12.]]) + def decode(outputs, original_size): + self.assertEqual(original_size, (20, 40)) + return original_box, np.array([.9]), np.array([0]) + adapter = SimpleNamespace(preprocess=lambda image, spec: tensor.copy(), decode=decode) + t1 = SimpleNamespace(items=[{"id": 17}], image=lambda item: image, + manifest={"cat_map": {"0": 13}}, g2_subset=lambda n: [{"id": 17}]) + seen = [] + def runner(x): + seen.append(x) + return [np.zeros((1, 4, 1))] + with patch.object(adapters, "get", return_value=adapter): + prediction = det.predict(runner, t1, "1x3x2x2", model="custom.det") + diagnostic = g2_inputs("det", t1, "1x3x2x2", 1, model="custom.det") + self.assertEqual(prediction[0]["bbox"], [2., 4., 4., 8.]) + self.assertEqual(prediction[0]["category_id"], 13) + np.testing.assert_array_equal(seen[0], diagnostic["s000"]) + + def test_new_batch_freezes_eval_set_and_refuses_rewrite(self): + with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: + root = Path(tmp) + for name in ("STATE", "RUNS", "DATA"): + stack.enter_context(patch.object(paths, name, root / name.lower())) + database = root / "jobs.sqlite" + connect = db.connect + stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database))) + stack.enter_context(patch.object(pipeline, "build_manifest", return_value={})) + stack.enter_context(patch("harness.coverage.model_blockers", return_value=[])) + rid = pipeline.new_run("adapter recipe test") + package = root / "subset.tar" + items = [{"id": "1", "file": "one.jpg", "label": 1}] + content = hashlib.sha256(json.dumps(items).encode()).hexdigest() + inner = {"dataset": "imagenet1k", "n": 1, "sha256": content, "items": items} + with tarfile.open(package, "w") as archive: + for name, payload in [("manifest.json", json.dumps(inner).encode()), ("one.jpg", b"image")]: + info = tarfile.TarInfo(name); info.size = len(payload) + archive.addfile(info, io.BytesIO(payload)) + manifest = root / "manifest.json" + manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 1, "seed": 1, + "tar": package.name, "sha256": hashlib.sha256(package.read_bytes()).hexdigest(), + "content_sha256": content})) + args = dict(model_ids=["efficientvit.b0"], eval_manifests=[str(manifest)]) + result = pipeline.expand(rid, ["oracle", "e2e"], **args) + self.assertEqual(result["job_ids"], 1) + job_path = next((paths.RUNS / rid).glob("*/job.json")) + job = json.loads(job_path.read_text()) + self.assertEqual(job["data"]["t1"]["n"], 1) + self.assertEqual(job["rollout_plans"], [{"mode": "e2e", "policy": "teacher", "k": 4}]) + self.assertEqual(job["model_spec"]["status"], "needs_adapter") + self.assertEqual(pipeline.expand(rid, ["oracle", "e2e"], **args)["jobs_new"], 0) + original = job_path.read_bytes() + data = json.loads(manifest.read_text()); data["seed"] = 2 + manifest.write_text(json.dumps(data)) + with self.assertRaisesRegex(ValueError, "cannot change"): + pipeline.expand(rid, ["oracle", "e2e"], **args) + self.assertEqual(job_path.read_bytes(), original) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_agent_first.py b/scripts/test_agent_first.py new file mode 100644 index 0000000..cac4f06 --- /dev/null +++ b/scripts/test_agent_first.py @@ -0,0 +1,163 @@ +"""Preparation is reusable input evidence, never a synthetic Oracle verdict.""" +from contextlib import ExitStack, nullcontext +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from harness import cli, db, paths, pipeline, rollout +from harness.stages import Stage, StageOutcome +from harness.stages import s01_fetch + + +class AgentFirst(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.stack = ExitStack() + self.addCleanup(self.stack.close) + for name in ("STATE", "RUNS", "CACHE"): + directory = self.root / name.lower() + directory.mkdir() + self.stack.enter_context(patch.object(paths, name, directory)) + self.stack.enter_context(patch.object(db, "STATE", paths.STATE)) + connect = db.connect + self.stack.enter_context(patch.object(db, "connect", lambda: connect(paths.STATE / "jobs.sqlite"))) + self.stack.enter_context(patch.object(s01_fetch, "INDEX", paths.CACHE / "weights/index.json")) + self.stack.enter_context(patch.object(pipeline.scheduler, "model_lock", lambda _: nullcontext())) + self.stack.enter_context(patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext())) + self.stack.enter_context(patch.object(pipeline.scheduler, "acquire_stage", lambda *_: nullcontext())) + self.jd = paths.RUNS / "run/model" + (self.jd / "oracle").mkdir(parents=True) + self.job = {"job_id": "model", "run_id": "run", "model": "resnet18", "task": "cls", + "evaluator_version": "0.5.0", "data": {"dataset": "imagenet1k", + "t1": {"path": str(self.root / "t1.tar"), "content_sha": "t1-sha"}, + "calib": {"path": str(self.root / "calib")}}, "gates": {}} + self.save_job() + with db.connect() as con: + con.execute("INSERT INTO jobs(run_id,job_id,mode,state) VALUES ('run','model','oracle','PENDING')") + + def save_job(self): + (self.jd / "job.json").write_text(json.dumps(self.job)) + + def execute(self, **kw): + self.called = [] + def run(stage, ctx): + self.called.append(stage.name) + return StageOutcome("OK") + with patch.object(Stage, "run", run): + return pipeline.run_oracle_job("run", "model", log=lambda _: None, **kw) + + def test_prepare_stops_after_export_and_never_creates_oracle_verdict(self): + result = self.execute(prepare_only=True) + self.assertEqual(self.called, ["01-fetch", "02-baseline", "03-export"]) + self.assertEqual(result["terminal"], "PREPARED") + self.assertNotIn("gates", result) + self.assertFalse((self.jd / "oracle/result.json").exists()) + self.assertEqual(json.loads((self.jd / "oracle/preparation.json").read_text()), result) + self.assertFalse(rollout.inputs_ready("run", "model")) # State alone never admits an Agent. + + def test_default_preparation_preserves_all_finished_states_even_when_forced(self): + original = b'{"terminal":"PASS","evidence":"original"}' + (self.jd / "oracle/result.json").write_bytes(original) + for state in ("PASS", "FAIL", "UNVERIFIED", "PREPARED", "ORACLE_RUNNING"): + with self.subTest(state=state): + with db.connect() as con: + db.set_job_state(con, "run", "model", state, result={"saved": True}) + before = db.get_job(con, "run", "model") + self.assertEqual(self.execute(prepare_only=True, force=True)["terminal"], "SKIPPED") + self.assertEqual(self.called, []) + with db.connect() as con: + self.assertEqual(db.get_job(con, "run", "model"), before) + self.assertEqual((self.jd / "oracle/result.json").read_bytes(), original) + + def test_dry_preparation_is_not_prepared(self): + self.assertEqual(self.execute(prepare_only=True, dry_run=True)["terminal"], "UNVERIFIED") + + def test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them(self): + self.job.update(continuation={'source_run_id': 'old'}, march='nash-m', backend='ptq', qcfg='int8', + model_spec={'input_spec': '1x3x224x224'}, + gates={'G2': {'threshold': .99}, 'G3': {'metric': 'top1', 'tol': 5, 'kind': 'rel_pct'}}) + self.job['data']['t1']['n'] = 100 + self.job['data']['calib']['n'] = 128 + self.save_job() + with patch.object(rollout.Baseline, 'run', side_effect=AssertionError('implicit model execution')): + _, _, prompt = rollout.prepare('run', 'model', 0, 'e2e', 'teacher', dry_run=True) + self.assertIn('bench stage run export', prompt) + self.assertIn('bench stage run baseline', prompt) + self.assertFalse((self.jd/'preparation').exists()) + self.assertFalse((self.jd/'rollouts').exists()) + + def test_e2e_prompt_ignores_missing_and_corrupt_oracle_results(self): + self.test_agent_prompt_starts_without_export_or_baseline_and_does_not_run_them() + _, _, expected = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True) + self.job["oracle_ref"] = {"run_id": "legacy", "job_id": "model"} + self.save_job() + for result in (self.jd / "oracle/result.json", paths.RUNS / "legacy/model/oracle/result.json"): + result.parent.mkdir(parents=True, exist_ok=True) + result.write_text("deliberately invalid historical JSON") + _, _, prompt = rollout.prepare("run", "model", 0, "e2e", "teacher", dry_run=True) + self.assertEqual(prompt, expected) + + def test_repair_rejected_before_any_claim_or_file_write(self): + with db.connect() as con: + db.rollout_upsert(con, "run", "model", 7, "repair", "teacher") + before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()) + for function in (rollout.prepare, rollout.run_rollout): + for dry_run in (False, True): + with self.subTest(function=function.__name__, dry_run=dry_run), \ + patch.object(db, "connect", side_effect=AssertionError("unexpected DB access")): + with self.assertRaisesRegex(ValueError, "repair collection is retired"): + function("run", "model", 7, "repair", "teacher", dry_run=dry_run) + with db.connect() as con: + self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before) + self.assertFalse((self.jd / "rollouts").exists()) + + def test_explicit_oracle_can_run_all_stages_after_preparation(self): + self.execute(prepare_only=True) + with patch.object(pipeline.gates, "evaluate", return_value={ + "terminal": "UNVERIFIED", "gates": {}, "required": [], "failed_gate": None}): + self.execute() + self.assertEqual(self.called, [stage.name for stage in pipeline.ORDER]) + self.assertTrue((self.jd / "oracle/result.json").exists()) + + def test_cli_defaults_to_preparation_and_oracle_is_explicit(self): + with patch.object(pipeline, "run_oracle", return_value={}) as run: + cli.main(["run", "--run", "run"]) + self.assertTrue(run.call_args.kwargs["prepare_only"]) + cli.main(["run", "--run", "run", "--oracle"]) + self.assertFalse(run.call_args.kwargs["prepare_only"]) + + def test_readiness_requires_versioned_inputs_not_oracle_result(self): + s01_fetch.INDEX.parent.mkdir(parents=True) + s01_fetch.INDEX.write_text('{"resnet18":{"sha256":"weights-sha"}}') + onnx = rollout.onnx_path(self.job) + onnx.parent.mkdir(parents=True) + onnx.touch() + (onnx.parent / "meta.json").write_text('{"exporter_version":"2"}') + baseline = rollout.baseline_dir(self.job) / "metrics.json" + baseline.parent.mkdir(parents=True) + meta = {"t1_manifest_sha": "t1-sha", "evaluator_version": "0.5.0", "metrics": {"top1": 80}} + baseline.write_text(json.dumps(meta)) + Path(self.job["data"]["t1"]["path"]).touch() + calib = self.root / "calib/manifest.json" + calib.parent.mkdir() + calib.write_text('{}') + self.assertTrue(rollout.inputs_ready("run", "model")) + self.assertFalse((self.jd / "oracle/result.json").exists()) + for file in (onnx, baseline, calib, Path(self.job["data"]["t1"]["path"])): + content = file.read_bytes() + file.unlink() + self.assertFalse(rollout.inputs_ready("run", "model")) + file.write_bytes(content) + baseline.write_text(json.dumps({**meta, "t1_manifest_sha": "wrong-subset"})) + self.assertFalse(rollout.inputs_ready("run", "model")) + baseline.write_text(json.dumps(meta)) + (onnx.parent / "meta.json").write_text('{"exporter_version":"obsolete"}') + self.assertFalse(rollout.inputs_ready("run", "model")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_board_guard.py b/scripts/test_board_guard.py new file mode 100644 index 0000000..c7606c1 --- /dev/null +++ b/scripts/test_board_guard.py @@ -0,0 +1,144 @@ +"""Real process fault injection. No SSH or BPU used by these tests.""" +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest.mock import patch +from io import StringIO + +from harness.tools.board_perf import guarded_run + +GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py' + + +class TransportTests(unittest.TestCase): + def test_lease_and_streamed_evidence_and_json(self): + code = "import sys; assert sys.stdin.readline()=='lease\\n'; print('live-progress',file=sys.stderr,flush=True); print('{\"ok\":true}')" + with patch('sys.stderr', new_callable=StringIO) as err: + self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True}) + self.assertIn('live-progress', err.getvalue()) + + def test_nonzero_not_hidden_by_json(self): + with self.assertRaises(SystemExit) as raised: + guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2) + self.assertEqual(raised.exception.code, 17) + + +@unittest.skipUnless(sys.platform == 'linux', 'Linux process ownership and /proc required') +class GuardTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) + self.directory = self.root / 'eval-test' + self.directory.mkdir() + self.processes = [] + self.children = [] + + def tearDown(self): + for proc in self.processes: + if proc.poll() is None: + proc.kill() + proc.communicate(timeout=3) + for pid in self.children: + try: os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: pass + self.tmp.cleanup() + + def start(self, timeout=3, lease=.6, command=None): + code = ("import os,signal,time; from pathlib import Path; " + "signal.signal(signal.SIGTERM,signal.SIG_IGN); " + f"Path({str(self.directory/'pid')!r}).write_text(str(os.getpid())); " + "time.sleep(30)") + argv = [sys.executable, '-u', str(GUARD), '--directory', str(self.directory), + '--timeout', str(timeout), '--lease', str(lease), '--grace', '.15', '--', + *(command or [sys.executable, '-u', '-c', code])] + proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.processes.append(proc) + return proc + + def child(self): + for _ in range(100): + p = self.directory / 'pid' + if p.exists(): + pid = int(p.read_text()); self.children.append(pid); return pid + time.sleep(.02) + self.fail('child did not start') + + def assert_dead(self, pid): + for _ in range(100): + try: + st = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0] + if st == 'Z': return + except FileNotFoundError: return + time.sleep(.02) + self.fail(f'child {pid} survived') + + def test_eof_reaps_term_ignoring_child(self): + proc = self.start(); pid = self.child() + proc.stdin.close(); proc.stdin = None + proc.communicate(timeout=3) + self.assertEqual(proc.returncode, 125); self.assert_dead(pid) + self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'host_disconnected') + + def test_missing_lease_expires_without_eof(self): + proc = self.start(); pid = self.child() + proc.wait(timeout=3) + self.assertEqual(proc.returncode, 125); self.assert_dead(pid) + self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'lease_expired') + + def test_deadline_reaps_even_with_valid_lease(self): + proc = self.start(timeout=.35, lease=2); pid = self.child() + proc.stdin.write(b'lease\n'); proc.stdin.flush() + proc.wait(timeout=3) + self.assertEqual(proc.returncode, 124); self.assert_dead(pid) + + def test_sigterm_owner_reaps_child(self): + proc = self.start(); pid = self.child() + proc.terminate(); proc.wait(timeout=3); self.assert_dead(pid) + + def test_sigkill_owner_kernel_kills_child(self): + proc = self.start(); pid = self.child() + proc.kill(); proc.wait(timeout=3); self.assert_dead(pid) + + def test_exited_group_leader_does_not_leave_grandchild(self): + leaf = "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(30)" + code = ("import subprocess,sys,time;from pathlib import Path;" + f"p=subprocess.Popen([sys.executable,'-c',{leaf!r}]);" + f"Path({str(self.directory/'pid')!r}).write_text(str(p.pid));time.sleep(.1)") + proc = self.start(command=[sys.executable, '-c', code]); pid = self.child() + proc.wait(timeout=3); self.assert_dead(pid) + + def test_unmanaged_inference_is_rejected(self): + script = self.root / 'board_bench.py' + script.write_text('import time; time.sleep(30)') + unmanaged = subprocess.Popen([sys.executable, str(script)]) + self.processes.append(unmanaged) + proc = self.start() + proc.wait(timeout=3) + self.assertEqual(proc.returncode, 75) + self.assertIsNone(unmanaged.poll()) + + def test_board_mutex_rejects_second_owner(self): + first = self.start(lease=3); pid = self.child() + second = self.start() + second.wait(timeout=2) + self.assertEqual(second.returncode, 75) + self.assertIsNone(first.poll()) + + def test_success_preserves_output_and_unlocks(self): + command = [sys.executable, '-c', "print('{\"ok\":true}')"] + for _ in range(2): + proc = self.start(command=command) + proc.wait(timeout=3) + out, err = proc.communicate() + self.assertEqual(proc.returncode, 0, err) + self.assertEqual(json.loads(out), {'ok': True}) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_board_pipeline.py b/scripts/test_board_pipeline.py index 3bac467..f15d1ff 100644 --- a/scripts/test_board_pipeline.py +++ b/scripts/test_board_pipeline.py @@ -30,7 +30,8 @@ class BoardPipeline(unittest.TestCase): self.config = { "paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"}, "perf": {"warmup": 2, "runs": 3}, - "watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60}, + "watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60, + "io_stall_s": 50, "no_progress_ceiling_s": 55}, } for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"), ("BENCH_ROOT", self.root), ("STATE", self.root / "state")): diff --git a/scripts/test_board_runtime.py b/scripts/test_board_runtime.py index 55ad965..3be16a4 100644 --- a/scripts/test_board_runtime.py +++ b/scripts/test_board_runtime.py @@ -178,18 +178,21 @@ class BoardRuntimeTests(unittest.TestCase): calls.append(command) return 'runtime startup\n{"samples": 1}\n' if "python3 " in command[-1] else "" - with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), patch.object(board_perf, "run", side_effect=fake_run): + with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), \ + patch.object(board_perf, "run", side_effect=fake_run), \ + patch.object(board_perf, "guarded_run", side_effect=fake_run): with board_perf.RemoteSession("/local/model.hbm", "/userdata/bench space") as session: session.__enter__() self.assertEqual(session.bench(["--runs", "1"])["samples"], 1) scps = [c for c in calls if c[0] == "scp"] - self.assertEqual(len(scps), 2) + self.assertEqual(len(scps), 3) self.assertEqual(scps[0][1:3], ["-P", "2222"]) self.assertIn("'/userdata/bench space/", calls[0][-1]) self.assertTrue(calls[-1][-1].startswith("rm -rf -- ")) bench = next(c[-1] for c in calls if c[-1].startswith("python3 ")) self.assertNotIn("|", bench) self.assertNotIn("2>/dev/null", bench) + self.assertIn('board_guard.py', bench) def test_remote_setup_failure_keeps_exit_code_and_cleans_owned_directory(self): calls = [] @@ -230,6 +233,28 @@ class BoardRuntimeTests(unittest.TestCase): self.assertEqual(result["busy_waited_s"], 20) self.assertFalse(result["contended"]) + def test_busy_board_refuses_to_measure_after_wait_budget(self): + with tempfile.TemporaryDirectory() as td: + out = Path(td) / 'perf.json' + argv = ['board_perf', '--hbm', 'm.hbm', '--job-id', 'j', '--out', str(out), '--busy-wait-s', '0'] + with patch.object(board_perf.sys, 'argv', argv), \ + patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \ + patch.object(board_perf, 'run', return_value='1\n0\n'), \ + patch.object(board_perf, 'RemoteSession') as session: + with self.assertRaisesRegex(RuntimeError, 'refusing contended'): + board_perf.main() + session.assert_not_called() + self.assertFalse(out.exists()) + + def test_failed_guard_retains_evidence_without_erasing_live_directory(self): + with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \ + patch.object(board_perf, 'run', return_value='') as run, \ + patch.object(board_perf, 'guarded_run', side_effect=SystemExit(124)): + with self.assertRaises(SystemExit): + with board_perf.RemoteSession('model.hbm') as session: + session.bench(['--runs', '1']) + self.assertFalse(any(c.args[0][-1].startswith('rm ') for c in run.call_args_list)) + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_coco_extra.py b/scripts/test_coco_extra.py new file mode 100644 index 0000000..8fdbe52 --- /dev/null +++ b/scripts/test_coco_extra.py @@ -0,0 +1,58 @@ +"""Metric contract tests: official COCOeval golden predictions, not mock scores.""" +import io +import json +from types import SimpleNamespace +import unittest +import numpy as np +from harness.eval.coco_extra import decode, evaluate, masks_to_original + + +def fixture(): + points = sum(([10 + i % 4, 10 + i // 4, 2] for i in range(17)), []) + category = {"id": 1, "name": "person", "keypoints": [str(i) for i in range(17)], "skeleton": []} + a = {"id": 1, "image_id": 1, "category_id": 1, "bbox": [5, 5, 20, 20], "area": 400, + "segmentation": [[5, 5, 25, 5, 25, 25, 5, 25]], "iscrowd": 0, + "keypoints": points, "num_keypoints": 17} + ann = {"images": [{"id": 1, "width": 32, "height": 32}], "annotations": [a], "categories": [category], + "info": {}, "keypoints": {"annotations": [a], "categories": [category]}} + tar = SimpleNamespace(extractfile=lambda _: io.BytesIO(json.dumps(ann).encode())) + return SimpleNamespace(tar=tar, items=[{"id": 1}]), a + + +class CocoExtra(unittest.TestCase): + def test_official_coco_perfect_masks_and_keypoints(self): + from pycocotools import mask as mu + t1, a = fixture() + rle = mu.merge(mu.frPyObjects(a["segmentation"], 32, 32)) + mask = [{"image_id": 1, "category_id": 1, "score": .99, "segmentation": rle}] + pose = [{"image_id": 1, "category_id": 1, "score": .99, "keypoints": a["keypoints"]}] + self.assertAlmostEqual(evaluate(t1, mask, "iseg")["mask_ap"], 100) + self.assertAlmostEqual(evaluate(t1, pose, "pose")["oks_ap"], 100) + self.assertEqual(evaluate(t1, [], "iseg")["mask_ap"], 0) + shifted = [{**pose[0], "keypoints": [v + 100 if i % 3 != 2 else v for i, v in enumerate(a["keypoints"])]}] + self.assertEqual(evaluate(t1, shifted, "pose")["oks_ap"], 0) + + def test_coefficients_are_not_class_scores_and_nms_preserves_extras(self): + p = np.zeros((1, 116, 2), dtype=np.float32) + p[:, :4, :] = np.array([20, 20, 10, 10])[None, :, None] + p[0, 9, :] = [.9, .8] + p[0, 84:, :] = 100 + boxes, scores, labels, coeff, proto = decode([p, np.ones((1, 32, 4, 4), dtype=np.float32)], "iseg") + self.assertEqual(labels.tolist(), [5]) + self.assertEqual(coeff.shape, (1, 32)) + self.assertTrue(np.all(coeff == 100)) + self.assertEqual(boxes.tolist(), [[15, 15, 25, 25]]) + with self.assertRaises(ValueError): + decode([np.zeros((1, 37, 2))], "iseg") # class-agnostic FastSAM + + def test_masks_undo_integer_letterbox_and_crop_original_coordinates(self): + masks = list(masks_to_original(np.ones((32, 2, 2), dtype=np.float32), + np.ones((1, 32), dtype=np.float32), np.array([[1, 1, 4, 3]]), + 8, (5, 3), 1.6, (0, 1))) + expected = np.zeros((3, 5), dtype=np.uint8) + expected[1:3, 1:4] = 1 + np.testing.assert_array_equal(masks[0], expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_codex.py b/scripts/test_codex.py index 8d7abea..1d70865 100644 --- a/scripts/test_codex.py +++ b/scripts/test_codex.py @@ -17,6 +17,18 @@ exporter = module('exporter', 'export/to_agentrl.py') audit = module('audit', 'scripts/audit_rollouts.py') class Regression(unittest.TestCase): + def test_audit_does_not_offer_historical_repair_or_blocked_as_training(self): + with tempfile.TemporaryDirectory() as d: + rd=Path(d)/'runs/r/job/rollouts/00';(rd/'agent').mkdir(parents=True) + (rd/'rollout.json').write_text('{"agent_config_version":7,"state":"DONE"}') + (rd/'agent/events.jsonl').write_text('{"type":"agent_end","messages":[]}') + for mode, terminal, expected in [('e2e','PASS',True),('repair','PASS',False),('e2e','BLOCKED',False)]: + (rd/'verdict.json').write_text(json.dumps({'terminal':terminal,'claim_consistent':True})) + (rd/'claim.json').write_text(json.dumps({'terminal':terminal})) + result=audit.audit_one(rd,0,7,mode) + self.assertEqual(result['rl_ok'],expected) + self.assertEqual(result['sft_ok'],expected) + def test_annotations_scope(self): rows=[('run','job','rollout#02 contaminated:oracle'),('run','job','rollout#01 stalled_killed')] self.assertEqual(exporter.annotation_reasons(rows,'run','job',2),{'contaminated'}) @@ -82,7 +94,7 @@ class Regression(unittest.TestCase): con.executescript(SCHEMA);con.commit();con.close() rd=root/'runs/r/job/rollouts/01';rd.mkdir(parents=True) (rd/'rollout.json').write_text(json.dumps({'agent_config_version':6,'max_turns':60,'max_tokens':100})) - (rd/'verdict.json').write_text(json.dumps({'terminal':'FAIL','run_id':'r','job_id':'job','rollout':1,'gates':{},'cost':{'turns':30,'tokens':50}})) + (rd/'verdict.json').write_text(json.dumps({'schema':'bench.verdict/3','terminal':'FAIL','run_id':'r','job_id':'job','rollout':1,'gates':{},'cost':{'turns':30,'tokens':50}})) (rd.parents[1]/'job.json').write_text(json.dumps({'task':'cls','march':'e','backend':'ptq'})) with patch.object(exporter,'BENCH',root), patch.object(exporter,'scan',return_value=[rd]), patch.object(exporter,'load_trajectory',return_value=([{}], {'partial':False,'source':'test'})), patch.object(exporter,'clean',return_value=([{'role':'assistant','content':'test'}],{'drop':False,'tool_pairing':{'valid':True}})), patch.object(exporter,'system_prompt',return_value=('', 'none')), patch.object(exporter,'tool_defs',return_value=([], 'none')), patch.object(sys,'argv',['export','--runs','r','--dataset-id','test']): self.assertEqual(exporter.main(),0) diff --git a/scripts/test_collection.py b/scripts/test_collection.py index 06ba81f..ef99ad8 100644 --- a/scripts/test_collection.py +++ b/scripts/test_collection.py @@ -32,6 +32,7 @@ class Collection(unittest.TestCase): patch.object(pipeline.paths, "RUNS", self.root / "runs").start() patch.object(pipeline.paths, "config", lambda: {"pools": {"agent": {"workers": 3}}}).start() patch.object(rollout, "setup_models_json").start() + self.ready = patch.object(rollout, "inputs_ready", return_value=True).start() patch.object(pipeline.scheduler, "acquire", lambda _: nullcontext()).start() self.export = patch.object(pipeline, "refresh_run_dataset").start() with self.connect(): @@ -120,24 +121,22 @@ class Collection(unittest.TestCase): self.assertTrue(db.claim_rollout(con, "run", "b", 0, max_active=1)) self.assertFalse(db.claim_rollout(con, "run", "b", 0, max_active=3)) - def test_rescans_new_oracle_before_repeating_previous_model(self): + def test_missing_inputs_do_not_delay_first_agent_attempt(self): self.job("first", k=4) self.job("late", state="ORACLE_RUNNING", estimate=1, k=4) self.job("covered", estimate=0, k=4) with self.connect() as con: db.rollout_set(con, "run", "covered", 0, state="PASS", finished_at=1) started = [] + self.ready.side_effect = lambda rid, jid: jid != "late" or bool(started) def run(*args, **kw): started.append(args[1]) - if args[1] == "first": - with self.connect() as con: - con.execute("UPDATE jobs SET state='PASS' WHERE job_id='late'") return self.complete(*args, **kw) with patch.object(rollout, "run_rollout", run): self.assertEqual(self.collect(parallel=1, limit=2), {"PASS": 2}) - self.assertEqual(started, ["first", "late"]) + self.assertEqual(started, ["late", "first"]) def test_three_slots_cover_three_distinct_models(self): for jid in ("a", "b", "c"): @@ -154,13 +153,40 @@ class Collection(unittest.TestCase): self.assertEqual(self.collect(parallel=3, limit=3), {"PASS": 3}) self.assertEqual(set(started), {"a", "b", "c"}) - def test_prefers_uncovered_passed_oracle_and_shorter_estimate(self): + def test_prefers_shorter_ready_inputs_independent_of_oracle_verdict(self): self.job("failed", state="FAIL", estimate=0, k=1) self.job("slow", estimate=100, k=1) self.job("fast", estimate=1, k=1) - rows, pending = pipeline._collection_snapshot("run", None, None, set()) + rows, pending = pipeline._collection_snapshot("run", None, None) + self.assertFalse(pending) + self.assertEqual([r["job_id"] for r in rows], ["failed", "fast", "slow"]) + + def test_e2e_prepares_its_own_inputs_and_retired_repair_is_never_ready(self): + self.job("ready", state="ORACLE_RUNNING", k=1) + self.job("missing", state="PASS", k=1) + self.ready.side_effect = lambda rid, jid: jid == "ready" + with self.connect() as con: + db.rollout_upsert(con, "run", "ready", 1, "repair", "teacher") + rows, pending = pipeline._collection_snapshot("run", None, None) + self.assertEqual([(r["job_id"], r["mode"]) for r in rows], [("missing", "e2e"), ("ready", "e2e")]) + self.assertFalse(pending) + with self.connect() as con: + self.assertEqual(con.execute("SELECT COUNT(*) FROM rollouts WHERE state!='QUEUED'").fetchone()[0], 0) + + def test_collect_preserves_and_skips_historical_queued_repairs(self): + self.job("failed", state="FAIL", k=1) + with self.connect() as con: + db.rollout_upsert(con, "run", "failed", 7, "repair", "teacher") + before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()) + with patch.object(rollout, "run_rollout", self.complete), \ + patch.object(pipeline, "enqueue_repairs") as enqueue: + self.assertEqual(self.collect(parallel=1, modes=["e2e", "repair"]), {"PASS": 1}) + enqueue.assert_not_called() + with self.connect() as con: + self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before) + rows, pending = pipeline._collection_snapshot("run", None, ["repair"]) + self.assertEqual(rows, []) self.assertFalse(pending) - self.assertEqual([r["job_id"] for r in rows], ["fast", "slow", "failed"]) def test_exception_releases_claim_and_does_not_retry_same_row(self): self.job("broken", k=1) @@ -195,6 +221,7 @@ class Collection(unittest.TestCase): self.assertEqual(self.collect(parallel=1), {"PASS": 1}) verdict = json.loads((rd / "verdict.json").read_text()) self.assertEqual(verdict["terminal"], "PASS") + self.assertNotIn("oracle_ref", verdict) self.assertEqual(verdict["execution_terminal"], "ERROR") self.assertTrue(verdict["completion"]["workflow_completed"]) self.export.assert_called_with("run") @@ -242,6 +269,36 @@ class Collection(unittest.TestCase): states = [r["state"] for r in db.list_rollouts(con, "run", "model")] self.assertEqual(states, ["PASS", "QUEUED", "QUEUED", "QUEUED"]) + def test_first_pass_preserves_repeats_and_failed_attempts_consume_one_slot(self): + self.job("fresh", k=4) + self.job("covered", k=4) + self.job("failed", k=4) + with self.connect() as con: + db.rollout_set(con, "run", "covered", 0, state="PASS", started_at=1, finished_at=2) + db.rollout_set(con, "run", "failed", 0, state="UNVERIFIED", started_at=1, finished_at=2) + with patch.object(rollout, "run_rollout", self.complete): + self.assertEqual(self.collect(parallel=1, first_pass=True), {"PASS": 1}) + with self.connect() as con: + for model in ("fresh", "covered", "failed"): + rows = db.list_rollouts(con, "run", model) + self.assertEqual([r["state"] for r in rows[1:]], ["QUEUED"] * 3) + + def test_first_pass_ignores_completed_repair_but_keeps_e2e_history(self): + self.job("repair-only", k=2) + self.job("e2e-covered", k=2) + with self.connect() as con: + db.rollout_upsert(con, "run", "repair-only", 7, "repair", "teacher", "PASS") + db.rollout_set(con, "run", "repair-only", 7, started_at=1, finished_at=2) + before = dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()) + db.rollout_set(con, "run", "e2e-covered", 0, state="PASS", started_at=1, finished_at=2) + with patch.object(rollout, "run_rollout", side_effect=self.complete) as run: + self.assertEqual(self.collect(parallel=1, modes=["e2e"], first_pass=True), {"PASS": 1}) + self.assertEqual(run.call_args.args[1:4], ("repair-only", 0, "e2e")) + with self.connect() as con: + self.assertEqual(dict(con.execute("SELECT * FROM rollouts WHERE kk=7").fetchone()), before) + for jid in ("repair-only", "e2e-covered"): + self.assertEqual(db.list_rollouts(con, "run", jid)[1]["state"], "QUEUED") + def test_sigterm_drains_inflight_and_restores_previous_handler(self): self.job("model", k=2) old_handler = signal.getsignal(signal.SIGTERM) @@ -295,22 +352,12 @@ class Collection(unittest.TestCase): self.assertEqual(self.collect(parallel=3), {"PASS": 1}) self.assertEqual(started, ["fresh"]) - def test_watch_waits_for_oracle_and_terminates_after_queue_drained(self): + def test_watch_finishes_independent_of_oracle_running_state(self): self.job("late", state="ORACLE_RUNNING", k=1) - original_snapshot = pipeline._collection_snapshot - calls = [] - - def snapshot(*args): - result = original_snapshot(*args) - calls.append(1) - if len(calls) == 1: - with self.connect() as con: - con.execute("UPDATE jobs SET state='PASS' WHERE job_id='late'") - return result - - with patch.object(rollout, "run_rollout", self.complete), patch.object(pipeline, "_collection_snapshot", snapshot): + with patch.object(rollout, "run_rollout", self.complete): self.assertEqual(self.collect(parallel=1, watch=True), {"PASS": 1}) - self.assertGreaterEqual(len(calls), 2) + with self.connect() as con: + self.assertEqual(db.get_job(con, "run", "late")["state"], "ORACLE_RUNNING") if __name__ == "__main__": diff --git a/scripts/test_completion.py b/scripts/test_completion.py index 289d5b2..06e8abf 100644 --- a/scripts/test_completion.py +++ b/scripts/test_completion.py @@ -101,6 +101,20 @@ class CompletionTests(unittest.TestCase): self.assertEqual(v["terminal"], "PASS") self.assertFalse(v["claim_consistent"]) + def test_blocked_is_preserved_as_unproven_self_report(self): + self.stage("05-compile", state="FAIL", error="unsupported_op") + (self.rd / "claim.json").write_text('{"terminal":"BLOCKED","stage":"05-compile","error_class":"unsupported_op"}') + od = self.jd / "oracle" + od.mkdir() + (od / "result.json").write_text('{"terminal":"FAIL","stage_failed":"05-compile","error_class":"unsupported_op"}') + v = verify.verify_rollout(self.rd, write=False) + self.assertEqual(v["terminal"], "BLOCKED") + self.assertEqual(v["measured_terminal"], "FAIL") + self.assertIsNone(v["blocked_correct"]) + self.assertEqual(v["claim"]["terminal"], "BLOCKED") + self.assertEqual(v["error_class"], "unsupported_op") + self.assertNotIn("oracle_ref", v) + def test_statistics_distinguish_completion_from_acceptance(self): self.complete(68) v = verify.verify_rollout(self.rd, write=False) diff --git a/scripts/test_continuation.py b/scripts/test_continuation.py new file mode 100644 index 0000000..94bb378 --- /dev/null +++ b/scripts/test_continuation.py @@ -0,0 +1,611 @@ +"""Remaining-work handoff against real temporary SQLite and T1 packages.""" +from contextlib import ExitStack, nullcontext +import copy +import fcntl +import hashlib +import io +import json +import os +from pathlib import Path +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +from harness import continuation, coverage, db, paths, pipeline, registry, rollout + + +class ContinuationTests(unittest.TestCase): + source = "old-batch" + target = "screen-100" + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.stack = ExitStack() + self.addCleanup(self.stack.close) + for name in ("RUNS", "STATE", "DATA", "CACHE"): + directory = self.root / name.lower() + directory.mkdir() + self.stack.enter_context(patch.object(paths, name, directory)) + self.stack.enter_context(patch.object(db, "STATE", paths.STATE)) + database = paths.STATE / "jobs.sqlite" + connect, readonly = db.connect, db.readonly + self.stack.enter_context(patch.object(db, "connect", side_effect=lambda: connect(database))) + self.stack.enter_context(patch.object(db, "readonly", side_effect=lambda: readonly(database))) + self.mx = {"marches": ["nash-m"], "backends": ["ptq"], "modes": { + "e2e": {"enabled": True, "k": 4, "policies": ["teacher"]}, + "repair": {"enabled": False}}} + self.stack.enter_context(patch.object(registry, "matrix", side_effect=lambda: copy.deepcopy(self.mx))) + self.stack.enter_context(patch.object(pipeline, "matrix", side_effect=lambda: copy.deepcopy(self.mx))) + self.real_fingerprint = registry.registry_fingerprint + self.stack.enter_context(patch.object(registry, "registry_fingerprint", return_value="frozen-registry")) + self.manifest = {"registry_task_fingerprint": "frozen-registry", "venv": {}, + "evaluator_version": "0.5.0"} + self.stack.enter_context(patch.object(pipeline, "build_manifest", side_effect=lambda: copy.deepcopy(self.manifest))) + with db.connect() as con: + db.create_run(con, self.source, self.manifest) + self.eval_manifest = self.make_t1() + + def make_t1(self): + directory = paths.DATA / "t1/imagenet1k/n100" + directory.mkdir(parents=True) + items = [{"id": str(i), "file": f"images/{i}.jpg", "label": i} for i in range(100)] + content_sha = hashlib.sha256(json.dumps(items).encode()).hexdigest() + inner = {"dataset": "imagenet1k", "n": 100, "seed": 17, "sha256": content_sha, "items": items} + package = directory / "17.tar" + with tarfile.open(package, "w") as archive: + for name, payload in [("manifest.json", json.dumps(inner).encode()), + *((item["file"], item["id"].encode()) for item in items)]: + member = tarfile.TarInfo(name) + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + manifest = directory / "manifest.json" + manifest.write_text(json.dumps({"dataset": "imagenet1k", "n": 100, "seed": 17, "tar": package.name, + "sha256": hashlib.sha256(package.read_bytes()).hexdigest(), + "content_sha256": content_sha, "eval_set_id": "imagenet-fixed-100"})) + return manifest + + def job(self, name="model-a", state="PENDING", error_class=None): + jid = f"cls--{name}--nash-m--ptq--int8" + job = {"job_id": jid, "run_id": self.source, "task": "cls", "model": name, "march": "nash-m", + "backend": "ptq", "qcfg": "int8", "priority": "A", "mode": "oracle", "policy": None, + "evaluator_version": "0.5.0", "model_spec": {"model_id": name, "task": "cls", "priority": "A"}, + "data": {"dataset": "imagenet1k", "t1": {"path": "/old/2000.tar", "n": 2000, + "manifest_sha": "1" * 64, "content_sha": "2" * 64}}, + "rollout_plans": [{"mode": "e2e", "policy": "teacher", "k": 4}]} + directory = paths.RUNS / self.source / jid + directory.mkdir(parents=True) + (directory / "job.json").write_text(json.dumps(job)) + with db.connect() as con: + db.upsert_job(con, self.source, job, 10) + if state != "PENDING": + db.set_job_state(con, self.source, jid, state, error_class=error_class, + result={"terminal": state}) + if state in ("PASS", "FAIL", "UNVERIFIED"): + (directory / "oracle/artifacts").mkdir(parents=True) + (directory / "oracle/artifacts/model.hbm").write_bytes(b"original HBM") + (directory / "oracle/result.json").write_text(json.dumps({"terminal": state, "n": 2000})) + if state == "PREPARED": + (directory / "oracle").mkdir() + (directory / "oracle/preparation.json").write_text(json.dumps({ + "schema": "bench.preparation/1", "terminal": state, "n": 2000})) + return jid + + def rollout(self, jid, kk, state="QUEUED", mode="e2e", policy="teacher"): + with db.connect() as con: + db.rollout_upsert(con, self.source, jid, kk, mode, policy, state) + if state != "QUEUED": + fields = {"started_at": 1.0} + if state not in ("RUNNING", "VERIFYING"): + fields.update(finished_at=2.0, verdict=json.dumps({"terminal": state})) + db.rollout_set(con, self.source, jid, kk, **fields) + + def drain(self): + pipeline.drain_file(self.source).touch() + pipeline.collect_drain_file(self.source).touch() + + def plan(self): + return continuation.plan(self.source, [str(self.eval_manifest)], self.target) + + def file_snapshot(self): + return {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob("*") if p.is_file()} + + def test_already_frozen_subset_continues_without_another_replacement(self): + jid = self.job() + self.rollout(jid, 0) + job_path = paths.RUNS / self.source / jid / "job.json" + job = json.loads(job_path.read_text()) + _, frozen = continuation.load_eval_set(self.eval_manifest) + job["data"]["t1"] = frozen + job_path.write_text(json.dumps(job)) + plan = continuation.plan(self.source, [], self.target) + self.assertEqual(plan["blockers"], []) + self.assertEqual(plan["jobs"][0]["job"]["data"]["t1"], frozen) + self.drain() + with patch.object(pipeline, "scheduler_alive", return_value=[]): + continuation.apply(plan) + check = continuation.check(self.target) + self.assertTrue(check["identities_match"], check) + + def test_implicit_or_modified_subset_still_requires_explicit_identity(self): + jid = self.job() + self.rollout(jid, 0) + with self.assertRaisesRegex(ValueError, "missing replacement T1"): + continuation.plan(self.source, [], self.target) + job_path = paths.RUNS / self.source / jid / "job.json" + job = json.loads(job_path.read_text()) + _, frozen = continuation.load_eval_set(self.eval_manifest) + job["data"]["t1"] = {**frozen, "content_sha": "0" * 64} + job_path.write_text(json.dumps(job)) + with self.assertRaisesRegex(ValueError, "frozen explicit T1 changed"): + continuation.plan(self.source, [], self.target) + + def test_preview_is_readonly_and_preserves_remaining_sampling_identity(self): + jid = self.job(state="PASS") + self.rollout(jid, 0, "FAIL") + for kk in (1, 2, 3): + self.rollout(jid, kk) + before = self.file_snapshot() + plan = self.plan() + self.assertEqual(self.file_snapshot(), before) + self.assertEqual(plan["blockers"], []) + self.assertEqual(plan["counts"], {"oracle": 0, "agent_queued": 3, "agent_due_repairs": 0, + "conditional_repairs": 0}) + self.assertEqual(len(plan["jobs"]), 1) + entry = plan["jobs"][0] + self.assertFalse(entry["oracle"]) + self.assertEqual([(r["kk"], r["mode"], r["policy"]) for r in entry["rollouts"]], + [(1, "e2e", "teacher"), (2, "e2e", "teacher"), (3, "e2e", "teacher")]) + self.assertEqual(entry["job"]["oracle_ref"], {"run_id": self.source, "job_id": jid}) + self.assertEqual(entry["job"]["data"]["t1"]["n"], 100) + + def test_legacy_board_snapshot_allows_only_inventory_changes(self): + self.job() + directory = self.root / "registry" + directory.mkdir() + boards = b"version: 1\nmarch_by_board: {s100p: nash-m}\nboards: {one: {slot: 0}}\n" + (directory / "boards.yaml").write_bytes(boards) + (directory / "matrix.yaml").write_text("modes: {e2e: {k: 4}}\n") + (directory / "models.csv").write_text("model_id,task\nmodel-a,cls\n") + with patch.object(registry, "REGISTRY", directory), \ + patch.object(registry, "registry_fingerprint", side_effect=self.real_fingerprint): + manifest = {"registry_fingerprint": registry.registry_fingerprint()} + with db.connect() as con: + con.execute("UPDATE runs SET manifest=? WHERE run_id=?", (json.dumps(manifest), self.source)) + initial = paths.RUNS / self.source / "board_registry.initial.yaml" + initial.write_bytes(boards) + expanded = boards.replace(b"one: {slot: 0}", b"one: {slot: 0}, two: {slot: 1}") + (directory / "boards.yaml").write_bytes(expanded) + self.assertNotEqual(registry.registry_fingerprint(), manifest["registry_fingerprint"]) + self.assertEqual(self.plan()["blockers"], []) + # A saved board file cannot excuse changed k, model data or march rules. + for name, content in (("matrix.yaml", b"modes: {e2e: {k: 5}}\n"), + ("models.csv", b"model_id,task\nmodel-a,det\n"), + ("boards.yaml", expanded.replace(b"s100p: nash-m", b"s100p: nash-p"))): + with self.subTest(changed=name): + path = directory / name + original = path.read_bytes() + path.write_bytes(content) + with self.assertRaisesRegex(ValueError, "source registry changed"): + self.plan() + path.write_bytes(original) + initial.unlink() + with self.assertRaisesRegex(ValueError, "source registry changed"): + self.plan() + + def test_prepared_source_continues_exact_slots_without_oracle_result_and_exports(self): + from export import to_agentrl + from harness import verify + jid = self.job(state="PREPARED") + self.rollout(jid, 0, "FAIL") + for kk in (1, 2, 3): + self.rollout(jid, kk) + self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2} + jp = paths.RUNS / self.source / jid / "job.json" + job = json.loads(jp.read_text()) + job["gates"] = {"required": [], "G1": {}, "G2": {}, "G3": {}, "G4": {}} + jp.write_text(json.dumps(job)) + handoff = self.plan() + self.assertEqual(handoff["blockers"], []) + self.assertEqual(handoff["counts"], {"oracle": 0, "agent_queued": 3, + "agent_due_repairs": 0, "conditional_repairs": 0}) + self.drain() + self.assertTrue(continuation.apply(handoff)["identities_match"]) + jd = paths.RUNS / self.target / jid + with patch.object(rollout, "inputs_ready", return_value=True): + ready, pending = pipeline._collection_snapshot(self.target, None, None) + self.assertFalse(pending) + self.assertEqual([(r["kk"], r["job_state"], r["completed"], r["started"]) for r in ready], + [(kk, "PREPARED", 1, 1) for kk in (1, 2, 3)]) + self.assertEqual(pipeline.enqueue_repairs(self.target), 0) + rd = jd / "rollouts/01" + (rd / "agent").mkdir(parents=True) + for name, value in { + "rollout.json": {"state": "DONE", "mode": "e2e", "policy": "teacher", "agent_config_version": 1}, + "agent/events.jsonl": {"type": "agent_end", "messages": [ + {"role": "user", "content": "Synthetic test task."}, + {"role": "assistant", "content": "Synthetic test response."}]}, + "agent/tools.json": [{"name": "bash"}], + }.items(): + (rd / name).write_text(json.dumps(value)) + (rd / "agent/system.md").write_text("Synthetic test system.") + verdict = verify.verify_rollout(rd) + self.assertNotIn("oracle_ref", verdict) + with patch.object(to_agentrl, "BENCH", self.root), \ + patch.dict(os.environ, {"BENCH_STATE_DIR": str(paths.STATE)}): + exported = to_agentrl.materialize([self.target], "prepared-test", min_agent_version=1) + self.assertEqual(exported["stats"]["agent_written"], 1) + self.assertEqual(exported["stats"]["pipeline_written"], 0) + record = json.loads((self.root / "datasets/prepared-test/teacher.jsonl").read_text()) + self.assertEqual(record["eval_set"]["n"], 100) + self.assertEqual(record["continuation"], {"source_run_id": self.source, + "source_job_id": jid, "source_rollout": 1}) + self.assertNotIn("oracle_ref", record) + self.assertFalse((jp.parent / "oracle/result.json").exists()) + self.assertFalse((jd / "oracle").exists()) + with db.readonly() as con: + self.assertIsNone(db.get_job(con, self.target, jid)) + + def test_continuation_needs_no_oracle_evidence_and_never_releases_repair(self): + jid = self.job(state="PREPARED") + self.rollout(jid, 1, mode="repair") + evidence = paths.RUNS / self.source / jid / "oracle/preparation.json" + evidence.unlink() + self.assertEqual(self.plan()["blockers"], []) + evidence.write_text(json.dumps({"terminal": "FAIL"})) + self.assertEqual(self.plan()["blockers"], []) + evidence.write_text(json.dumps({"terminal": "PREPARED"})) + self.drain() + continuation.apply(self.plan()) + with patch.object(rollout, "inputs_ready", return_value=True): + ready, pending = pipeline._collection_snapshot(self.target, None, None) + self.assertEqual(ready, []) + self.assertFalse(pending) + + def test_first_pass_carries_source_attempts_for_prepared_and_pending_jobs(self): + jobs = {} + for name, state, history in (("fresh", "PREPARED", None), ("covered", "PREPARED", "PASS"), + ("failed", "PREPARED", "UNVERIFIED"), ("pending", "PENDING", "FAIL")): + jid = jobs[name] = self.job(name, state=state) + if history: + self.rollout(jid, 0, history) + if history == "UNVERIFIED": + with db.connect() as con: + db.rollout_set(con, self.source, jid, 0, verdict=None) + for kk in (1, 2, 3): + self.rollout(jid, kk) + self.drain() + continuation.apply(self.plan()) + started = [] + + def complete(run_id, job_id, kk, *args, **kwargs): + started.append((job_id, kk)) + with db.connect() as con: + db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=3.0) + return {"terminal": "PASS"} + + with patch.object(rollout, "inputs_ready", return_value=True): + ready, _ = pipeline._collection_snapshot(self.target, None, None) + self.assertEqual([r["job_id"] for r in ready[:3]], [jobs["fresh"]] * 3) + self.assertEqual({r["job_id"] for r in ready if not r["started"] and not r["completed"]}, + {jobs["fresh"]}) + # The collector's actual first-pass filter starts only the fresh model. + with patch.object(rollout, "run_rollout", side_effect=complete), \ + patch.object(rollout, "setup_models_json"), \ + patch.object(pipeline, "refresh_run_dataset"), \ + patch.object(pipeline.scheduler, "acquire", return_value=nullcontext()), \ + patch.object(paths, "config", return_value={"pools": {"agent": {"workers": 3}}}): + result = pipeline.collect_rollouts(self.target, parallel=1, first_pass=True, log=lambda _: None) + self.assertEqual(result, {"PASS": 1}) + self.assertEqual(started, [(jobs["fresh"], 1)]) + + def test_first_pass_ignores_source_repair_but_keeps_source_e2e_history(self): + repair_only = self.job("repair-only", state="PREPARED") + e2e_covered = self.job("e2e-covered", state="PREPARED") + self.rollout(repair_only, 7, "PASS", mode="repair") + self.rollout(e2e_covered, 7, "PASS", mode="e2e") + for jid in (repair_only, e2e_covered): + self.rollout(jid, 8) + self.rollout(jid, 9) + with db.readonly() as con: + before = db.list_rollouts(con, self.source, repair_only)[0] + self.drain() + continuation.apply(self.plan()) + started = [] + + def complete(run_id, job_id, kk, *args, **kwargs): + started.append((job_id, kk)) + with db.connect() as con: + db.rollout_set(con, run_id, job_id, kk, state="PASS", finished_at=3) + return {"terminal": "PASS"} + + with patch.object(rollout, "run_rollout", side_effect=complete), \ + patch.object(rollout, "setup_models_json"), \ + patch.object(pipeline, "refresh_run_dataset"), \ + patch.object(pipeline.scheduler, "acquire", return_value=nullcontext()), \ + patch.object(paths, "config", return_value={"pools": {"agent": {"workers": 3}}}): + result = pipeline.collect_rollouts(self.target, parallel=1, modes=["e2e"], first_pass=True, log=lambda _: None) + self.assertEqual(result, {"PASS": 1}) + self.assertEqual(started, [(repair_only, 8)]) + with db.readonly() as con: + self.assertEqual(db.list_rollouts(con, self.source, repair_only)[0], before) + self.assertEqual([r["state"] for r in db.list_rollouts(con, self.target, repair_only)], ["PASS", "QUEUED"]) + self.assertEqual([r["state"] for r in db.list_rollouts(con, self.target, e2e_covered)], ["QUEUED", "QUEUED"]) + + def test_apply_agent_only_preserves_history_and_repeated_apply_is_idempotent(self): + jid = self.job(state="FAIL", error_class="compile_fail") + self.rollout(jid, 0, "FAIL") + for kk in (1, 2, 3): + self.rollout(jid, kk) + old_directory = paths.RUNS / self.source / jid + before = {p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()} + with db.readonly() as con: + old_job = db.get_job(con, self.source, jid) + old_finished = db.list_rollouts(con, self.source, jid)[0] + self.drain() + handoff = self.plan() + report = continuation.apply(handoff) + self.assertTrue(report["identities_match"]) + self.assertEqual(report["agent_states"], {"QUEUED": 3}) + self.assertEqual(continuation.apply(handoff), report) + with db.readonly() as con: + self.assertEqual(db.list_jobs(con, self.target), []) + self.assertEqual(db.get_job(con, self.source, jid), old_job) + original = db.list_rollouts(con, self.source, jid) + self.assertEqual(original[0], old_finished) + self.assertEqual([r["state"] for r in original[1:]], ["TRANSFERRED"] * 3) + moved = db.list_rollouts(con, self.target, jid) + self.assertEqual([r["kk"] for r in moved], [1, 2, 3]) + self.assertEqual({p.relative_to(old_directory): p.read_bytes() for p in old_directory.rglob("*") if p.is_file()}, before) + with patch.object(rollout, "inputs_ready", return_value=True): + ready, pending = pipeline._collection_snapshot(self.target, None, None) + self.assertEqual([r["kk"] for r in ready], [1, 2, 3]) + self.assertFalse(pending) + + def test_apply_requires_both_drains_and_stopped_schedulers(self): + self.job() + handoff = self.plan() + for mark in (None, pipeline.drain_file(self.source)): + if mark: + mark.touch() + with self.assertRaisesRegex(ValueError, "drain both"): + continuation.apply(handoff) + self.assertFalse((paths.RUNS / self.target).exists()) + self.drain() + with patch.object(pipeline, "scheduler_alive", return_value=True): + with self.assertRaisesRegex(ValueError, "scheduler still running"): + continuation.apply(handoff) + lock = paths.STATE / "locks" / f"collect-{self.source}.lock" + lock.parent.mkdir(exist_ok=True) + with lock.open("a") as holder: + fcntl.flock(holder, fcntl.LOCK_EX | fcntl.LOCK_NB) + with self.assertRaisesRegex(ValueError, "collector still running"): + continuation.apply(handoff) + self.assertTrue(continuation.apply(handoff)["identities_match"]) + + def test_active_work_and_requeued_history_block_handoff(self): + jid = self.job(state="PASS") + self.rollout(jid, 0, "RUNNING") + self.rollout(jid, 1) + self.drain() + handoff = self.plan() + self.assertTrue(any("still active" in b for b in handoff["blockers"])) + with self.assertRaisesRegex(ValueError, "blocked"): + continuation.apply(handoff) + with db.connect() as con: + db.rollout_set(con, self.source, jid, 0, state="QUEUED") + self.assertTrue(any("execution history" in b for b in self.plan()["blockers"])) + + def test_stale_or_edited_plan_is_rejected_without_creating_target(self): + jid = self.job(state="PASS") + self.rollout(jid, 0) + self.rollout(jid, 1) + self.drain() + handoff = self.plan() + modified = copy.deepcopy(handoff) + modified["jobs"][0]["rollouts"].pop() + with self.assertRaisesRegex(ValueError, "edited"): + continuation.apply(modified) + with db.connect() as con: + db.rollout_set(con, self.source, jid, 0, state="FAIL", started_at=1.0, finished_at=2.0) + with self.assertRaisesRegex(ValueError, "source changed"): + continuation.apply(handoff) + self.assertFalse((paths.RUNS / self.target).exists()) + + def test_no_due_or_conditional_repairs_even_with_legacy_config(self): + self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2, + "policies": ["teacher", "student"], "error_classes": ["compile_fail"]} + jid = self.job() + self.rollout(jid, 0, "FAIL") + self.rollout(jid, 1) + failed = self.job("failed", state="FAIL", error_class="compile_fail") + self.rollout(failed, 4, "FAIL", mode="repair") + self.rollout(failed, 7, mode="repair") + # Neither missing nor corrupt historical Oracle evidence blocks E2E handoff. + (paths.RUNS / self.source / failed / "oracle/result.json").write_text("corrupt JSON") + self.drain() + handoff = self.plan() + self.assertEqual(handoff["counts"]["conditional_repairs"], 0) + self.assertEqual(handoff["counts"]["agent_due_repairs"], 0) + self.assertEqual(handoff["counts"]["agent_queued"], 2) + self.assertEqual(handoff["conditional_repairs"], []) + continuation.apply(handoff) + self.assertEqual(pipeline.enqueue_repairs(self.target), 0) + self.assertEqual(continuation.check(self.target)["activated_repairs"], 0) + with db.readonly() as con: + moved = db.list_rollouts(con, self.target, failed) + original = db.list_rollouts(con, self.source, failed) + self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in moved], + [(7, "repair", "teacher", "QUEUED")]) + self.assertEqual([(r["kk"], r["state"]) for r in original], [(4, "FAIL"), (7, "TRANSFERRED")]) + ready, _ = pipeline._collection_snapshot(self.target, None, None) + self.assertEqual([(r["job_id"], r["kk"]) for r in ready], [(jid, 1)]) + + def test_legacy_conditional_receipt_is_checked_but_never_activated(self): + jid = self.job() + self.rollout(jid, 1) + self.drain() + handoff = self.plan() + continuation.apply(handoff) + # Model a receipt written by the previous implementation, including a + # repair which may already have been activated before the upgrade. + handoff["conditional_repairs"] = [{"job_id": jid, "error_classes": ["compile_fail"], "rollouts": [ + {"kk": 2, "mode": "repair", "policy": "teacher", "source_exists": False}, + {"kk": 3, "mode": "repair", "policy": "teacher", "source_exists": False}]}] + handoff["counts"]["conditional_repairs"] = 2 + handoff["plan_id"] = continuation._sha({k: v for k, v in handoff.items() if k != "plan_id"}) + (paths.RUNS / self.target / "continuation.json").write_text(json.dumps(handoff)) + with db.connect() as con: + db.set_job_state(con, self.target, jid, "FAIL", error_class="compile_fail") + db.rollout_upsert(con, self.target, jid, 2, "repair", "teacher") + self.assertEqual(continuation.enqueue_repairs(self.target, handoff), 0) + self.assertEqual(pipeline.enqueue_repairs(self.target), 0) + checked = continuation.check(self.target) + self.assertTrue(checked["identities_match"]) + self.assertEqual(checked["activated_repairs"], 1) + ready, _ = pipeline._collection_snapshot(self.target, None, None) + self.assertEqual([(r["kk"], r["mode"]) for r in ready], [(1, "e2e")]) + with db.readonly() as con: + self.assertEqual([(r["kk"], r["mode"]) for r in db.list_rollouts(con, self.target, jid)], + [(1, "e2e"), (2, "repair")]) + + def test_agent_only_coverage_counts_queue_and_exports_without_fake_oracle(self): + jid = self.job(state="PASS") + self.rollout(jid, 0, "FAIL") + for kk in (1, 2, 3): + self.rollout(jid, kk) + self.drain() + continuation.apply(self.plan()) + with db.connect() as con: + db.rollout_set(con, self.target, jid, 1, state="PASS", started_at=3.0, finished_at=4.0) + models = [{"model_id": "model-a", "task": "cls", "priority": "A", "status": "ready"}] + with patch.object(coverage, "models", return_value=models), \ + patch.object(coverage, "exported_keys", return_value={(self.target, jid, 1), (self.source, jid, 0)}): + actual = coverage.collect(self.target)["model-a"] + self.assertEqual(actual["jobs"], []) + self.assertEqual(dict(actual["oracle"]), {}) + self.assertEqual(actual["planned_rollouts"], 3) + self.assertEqual(dict(actual["rollout"]), {"PASS": 1, "QUEUED": 2}) + self.assertEqual(actual["exported"], 1) + self.assertTrue(continuation.check(self.target)["identities_match"]) + + def test_interrupted_filesystem_preparation_retries_without_losing_old_queue(self): + jid = self.job() + self.rollout(jid, 0) + self.drain() + handoff = self.plan() + with patch.object(db, "create_run", side_effect=RuntimeError("injected failure before commit")): + with self.assertRaisesRegex(RuntimeError, "injected failure"): + continuation.apply(handoff) + self.assertEqual(continuation.receipt(self.target), handoff) + self.assertTrue((paths.RUNS / self.target / jid / "job.json").exists()) + with db.readonly() as con: + self.assertIsNone(db.get_run(con, self.target)) + self.assertEqual(db.list_jobs(con, self.target), []) + self.assertEqual(db.list_rollouts(con, self.target, jid), []) + self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING") + self.assertEqual(db.list_rollouts(con, self.source, jid)[0]["state"], "QUEUED") + self.assertNotIn("continued_to", json.loads(db.get_run(con, self.source)["manifest"])) + report = continuation.apply(handoff) + self.assertTrue(report["identities_match"]) + self.assertEqual(report["oracle_states"], {"PENDING": 1}) + self.assertEqual(report["agent_states"], {"QUEUED": 1}) + + def test_interrupted_readonly_job_temp_file_can_be_replaced_on_retry(self): + jid = self.job() + self.drain() + handoff = self.plan() + replace = Path.replace + + def interrupt_job_replace(source, target): + if source.name == ".job.json.tmp": + raise OSError("interrupted after chmod before replace") + return replace(source, target) + + with patch.object(Path, "replace", autospec=True, side_effect=interrupt_job_replace): + with self.assertRaisesRegex(OSError, "after chmod"): + continuation.apply(handoff) + temporary = paths.RUNS / self.target / jid / ".job.json.tmp" + self.assertEqual(temporary.stat().st_mode & 0o777, 0o444) + self.assertFalse(temporary.with_name("job.json").exists()) + with db.readonly() as con: + self.assertIsNone(db.get_run(con, self.target)) + self.assertEqual(db.get_job(con, self.source, jid)["state"], "PENDING") + self.assertTrue(continuation.apply(handoff)["identities_match"]) + self.assertFalse(temporary.exists()) + self.assertEqual(temporary.with_name("job.json").stat().st_mode & 0o777, 0o444) + + def test_orphan_queued_rollout_blocks_plan_instead_of_being_omitted(self): + jid = self.job(state="PASS") + self.rollout(jid, 3) + with db.connect() as con: + con.execute("DELETE FROM jobs WHERE run_id=? AND job_id=?", (self.source, jid)) + handoff = self.plan() + self.assertTrue(any("no source Oracle" in item for item in handoff["blockers"])) + self.drain() + with self.assertRaisesRegex(ValueError, "blocked"): + continuation.apply(handoff) + + def test_direct_legacy_rollout_claims_before_preparation_and_closes_failed_attempt(self): + jid = self.job(state="PASS") + for kk, error in ((4, SystemExit("missing preparation inputs")), (5, RuntimeError("preparation failed"))): + with self.subTest(error=type(error).__name__): + def fail_preparation(*args): + with db.readonly() as con: + current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk) + self.assertEqual(current["state"], "RUNNING") + self.assertIsNotNone(current["started_at"]) + self.assertTrue(any("still active" in item for item in self.plan()["blockers"])) + raise error + + with patch.object(rollout, "_execute_rollout", side_effect=fail_preparation) as execute: + with self.assertRaises(type(error)): + rollout.run_rollout(self.source, jid, kk, "e2e", "teacher") + with self.assertRaisesRegex(ValueError, "claimed or finished"): + rollout.run_rollout(self.source, jid, kk, "e2e", "teacher") + self.assertEqual(execute.call_count, 1) + with db.readonly() as con: + current = next(r for r in db.list_rollouts(con, self.source, jid) if r["kk"] == kk) + self.assertEqual(current["state"], "UNVERIFIED") + self.assertIsNotNone(current["finished_at"]) + + def test_closed_source_and_target_full_expansion_are_rejected(self): + jid = self.job() + self.rollout(jid, 0) + self.drain() + continuation.apply(self.plan()) + source_calls = [lambda: pipeline.expand(self.source), lambda: pipeline.enqueue_repairs(self.source), + lambda: continuation.enqueue_repairs(self.source, {}), + lambda: pipeline.enqueue_e2e_coverage(self.source), + lambda: pipeline.collect_rollouts(self.source, dry_run=True), + lambda: pipeline.run_oracle(self.source, dry_run=True)] + for call in source_calls: + with self.assertRaisesRegex(ValueError, "execution is closed"): + call() + with db.connect() as con: + with self.assertRaisesRegex(ValueError, "execution is closed"): + db.claim_job(con, self.source, jid, force=True) + with db.connect() as con: + with self.assertRaisesRegex(ValueError, "execution is closed"): + db.claim_rollout(con, self.source, jid, 0) + for call in (lambda: pipeline.expand(self.target), lambda: pipeline.enqueue_e2e_coverage(self.target), + lambda: pipeline.collect_rollouts(self.target, coverage="all", dry_run=True)): + with self.assertRaisesRegex(ValueError, "continuation run"): + call() + + def test_identity_check_catches_substitution_even_when_counts_match(self): + jid = self.job(state="PASS") + self.rollout(jid, 1) + self.rollout(jid, 2) + self.drain() + continuation.apply(self.plan()) + with db.connect() as con: + con.execute("UPDATE rollouts SET kk=3 WHERE run_id=? AND job_id=? AND kk=2", (self.target, jid)) + with self.assertRaisesRegex(ValueError, "identity mismatch"): + continuation.check(self.target) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_coverage.py b/scripts/test_coverage.py index 3824663..08ab8dd 100644 --- a/scripts/test_coverage.py +++ b/scripts/test_coverage.py @@ -94,6 +94,26 @@ class Coverage(unittest.TestCase): self.assertEqual(row["rollout_planned"], "4") self.assertEqual(row["rollout_done"], "2") + def test_all_model_master_preserves_legacy_oracle_evidence(self): + # Existing result/1 rows store gates/stages, without a completion object. + result = {"gates": {"G1": {"hbm_exists": True}, + "G3": {"source": "board", "fp32": 80, "quant": 79, "verdict": "pass-screen"}}, + "stages": {"05-compile": {"state": "OK"}}} + with self.connect() as con: + con.execute("UPDATE jobs SET result=? WHERE model='a'", (json.dumps(result),)) + with patch.object(db, "readonly", self.connect), \ + patch.object(coverage, "model_blockers", lambda *args: []), \ + patch.object(coverage.paths, "CACHE", self.root / "cache"): + path = self.root / "master.csv" + coverage.write_master(str(path)) + rows = {r["model"]: r for r in csv.DictReader(path.open())} + self.assertEqual(set(rows), {"a", "b", "c"}) + for key in ("compile", "hbm_generated", "board", "accuracy", "accuracy_passed"): + self.assertEqual(rows["a"][key], "1") + self.assertEqual(rows["b"][key], "0") + self.assertEqual(rows["a"]["first_glm53_trace"], "") + self.assertEqual(rows["a"]["glm53_exported"], "0") + if __name__ == "__main__": unittest.main() diff --git a/scripts/test_depth_gate.py b/scripts/test_depth_gate.py new file mode 100644 index 0000000..deebfad --- /dev/null +++ b/scripts/test_depth_gate.py @@ -0,0 +1,59 @@ +"""Depth accuracy needs both the original delta1 limit and real alignment.""" +from copy import deepcopy +import unittest + +from harness.gates import g3 + + +class DepthGate(unittest.TestCase): + def setUp(self): + self.cfg = {"metric": "absrel", "kind": "rel_pct", "tol": 5, "require_source": "board", + "extra": {"delta1_abs_pp_tol": -1.0, "scale_alignment": "required"}} + self.measurement = {"metric": "absrel", "fp32": .1, "quant": .104, "source": "board", + "tier": "t1", "n": 1, "extra": { + "delta1": {"status": "measured", "unit": "percent", "fp32": 90., "quant": 89., "delta_pp": -1.}, + "scale_alignment": {"status": "measured", "protocol": {"id": "relative-inverse-ls"}, "n": 1, + "samples": [{"id": "image1", "fp32_scale": 2., "fp32_shift": .1, + "quant_scale": 1.9, "quant_shift": .11}]}}} + + def test_existing_threshold_boundary_passes(self): + self.assertEqual(g3(self.measurement, self.cfg)["verdict"], "pass-screen") + + def test_absrel_pass_does_not_hide_delta1_failure(self): + measurement = deepcopy(self.measurement) + measurement["extra"]["delta1"].update(quant=88.9, delta_pp=-1.1) + result = g3(measurement, self.cfg) + self.assertEqual(result["verdict"], "fail") + self.assertIn("delta1", result["reason"]) + + def test_status_without_measured_alignment_is_unverified(self): + measurement = deepcopy(self.measurement) + del measurement["extra"]["scale_alignment"]["samples"] + self.assertEqual(g3(measurement, self.cfg)["verdict"], "unverified") + + def test_invalid_alignment_and_inconsistent_delta_are_unverified(self): + for field, value in (("quant_scale", float("inf")), ("quant_shift", float("nan"))): + with self.subTest(field=field): + measurement = deepcopy(self.measurement) + measurement["extra"]["scale_alignment"]["samples"][0][field] = value + self.assertEqual(g3(measurement, self.cfg)["verdict"], "unverified") + measurement = deepcopy(self.measurement) + measurement["extra"]["delta1"]["delta_pp"] = 0 + self.assertEqual(g3(measurement, self.cfg)["verdict"], "unverified") + + def test_affine_least_squares_keeps_a_finite_signed_solution(self): + import numpy as np + from harness.eval.mde import aligned_metrics + prediction = np.array([[1., 2.]]) + depth = np.array([[.5, 1.]]) + result = aligned_metrics(prediction, depth, np.ones_like(depth, dtype=bool)) + self.assertEqual(result["scale"], -1.) + self.assertEqual(result["shift"], 3.) + self.assertEqual(result["absrel"], 0.) + measurement = deepcopy(self.measurement) + measurement["extra"]["scale_alignment"]["samples"][0].update(fp32_scale=-1., quant_scale=-1.) + self.assertEqual(g3(measurement, self.cfg)["verdict"], "pass-screen") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_eval_provenance.py b/scripts/test_eval_provenance.py new file mode 100644 index 0000000..ca6ee16 --- /dev/null +++ b/scripts/test_eval_provenance.py @@ -0,0 +1,155 @@ +"""A continued Agent keeps its source evidence without mixing evaluation sets.""" +import copy +import json +import os +import sqlite3 +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from export import reward, to_agentrl +from harness import db, paths, verify + + +def evaluation(n=100): + return {"dataset": "imagenet", "n": n, "content_sha": f"content-{n}", + "manifest_sha": f"archive-{n}", "evaluator_version": "v1"} + + +def verdict(): + return {"terminal": "PASS", "eval_set": evaluation(), + "gates": {"G3": {"verdict": "pass-screen", "kind": "rel_pct", "delta": 1, "tolerance": 5}}, + "oracle_ref": {"G3_kind": "rel_pct", "G3_delta": 3, "eval_set": evaluation()}} + + +class RewardEvaluationIdentity(unittest.TestCase): + def test_oracle_comparison_and_old_review_do_not_affect_reward(self): + v = verdict() + expected = reward.compute({k: value for k, value in v.items() if k != "oracle_ref"}) + for oracle in (None, {}, {"terminal": "PASS", "G3_delta": -99}, + {"terminal": "FAIL", "G3_delta": 99, "latency_ms_mean": 999}, "damaged"): + with self.subTest(oracle=oracle): + candidate = {**v, "oracle_ref": oracle, "review": True, "blocked_correct": True} + self.assertEqual(reward.compute(candidate), expected) + self.assertEqual(expected["recipe"], "v0.6") + self.assertNotIn("acc_rel", expected["terms"]) + self.assertNotIn("lat_rel", expected["terms"]) + self.assertNotIn("blocked", expected["terms"]) + + def test_eval_identity_still_does_not_enable_relative_reward(self): + v = verdict() + expected = reward.compute(v) + for key, value in (("n", 2000), ("content_sha", "different"), ("evaluator_version", "v2")): + v["oracle_ref"]["eval_set"][key] = value + self.assertEqual(reward.compute(v), expected) + v.pop("eval_set") + v["continuation"] = {"source_run_id": "old"} + self.assertEqual(reward.compute(v), expected) + + +class ExportEvaluationIdentity(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + self.addCleanup(patch.stopall) + patch.object(paths, "RUNS", self.root / "runs").start() + patch.object(to_agentrl, "BENCH", self.root).start() + patch.dict(os.environ, {"BENCH_STATE_DIR": str(self.root / "state")}).start() + (self.root / "state").mkdir() + self.database = self.root / "state/jobs.sqlite" + with sqlite3.connect(self.database) as con: + con.executescript(db.SCHEMA) + con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('new','job',2,'repair','teacher','PASS')") + con.execute("INSERT INTO rollouts (run_id,job_id,kk,mode,policy,state) VALUES ('old','job',2,'repair','teacher','TRANSFERRED')") + con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','pending','oracle','PENDING')") + self.job = {"run_id": "new", "job_id": "job", "task": "cls", "march": "nash-m", "backend": "ptq", + "data": {"dataset": "imagenet", "t1": evaluation()}, "evaluator_version": "v1", + "gates": {"required": [], "G1": {}, "G2": {}, "G3": {"kind": "rel_pct", "tol": 5}, "G4": {}}, + "oracle_ref": {"run_id": "old", "job_id": "job"}, + "continuation": {"source_run_id": "old", "source_job_id": "job"}} + self.jd = self.root / "runs/new/job" + self.rd = self.jd / "rollouts/02" + self.rd.mkdir(parents=True) + self.save(self.jd / "job.json", self.job) + old = copy.deepcopy(self.job) + old.update(run_id="old", data={"dataset": "imagenet", "t1": evaluation(2000)}) + old.pop("oracle_ref") + old.pop("continuation") + self.save(self.root / "runs/old/job/job.json", old) + self.save(self.root / "runs/old/job/oracle/result.json", { + "terminal": "PASS", "gates": {"G3": {"kind": "rel_pct", "delta": 3}}}) + self.save(self.rd / "rollout.json", {"state": "DONE", "mode": "repair", "policy": "teacher", "agent_config_version": 1}) + self.save(self.rd / "claim.json", {"terminal": "PASS"}) + self.save(self.rd / "stages/08-eval/meta.json", { + "state": "OK", "measurements": {"G3": {"fp32": 80, "quant": 79.2, "n": 100, "source": "board", "tier": "t1"}}}) + self.save(self.rd / "agent/events.jsonl", {"type": "agent_end", "messages": [ + {"role": "user", "content": "Test this model."}, {"role": "assistant", "content": "Completed."}]}) + self.save(self.rd / "agent/tools.json", [{"name": "bash"}]) + (self.rd / "agent/system.md").write_text("Run the model evaluation.") + + @staticmethod + def save(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + def test_verdict_is_independent_of_absent_pass_fail_or_damaged_oracle(self): + expected = verify.verify_rollout(self.rd, write=False) + expected.pop("verified_at") + old = self.root / "runs/old/job" + # Even a damaged historical job must never be opened by verification. + (old / "job.json").write_text("damaged historical job") + op = old / "oracle/result.json" + for value in (None, '{"terminal":"PASS"}', '{"terminal":"FAIL"}', "damaged"): + with self.subTest(oracle=value): + if value is None: + op.unlink(missing_ok=True) + else: + op.write_text(value) + result = verify.verify_rollout(self.rd, write=False) + result.pop("verified_at") + self.assertEqual(result, expected) + self.assertEqual(result["eval_set"]["n"], 100) + self.assertNotIn("oracle_ref", result) + self.assertFalse(result["review"]) + self.assertEqual(result["schema"], "bench.verdict/3") + self.assertFalse((self.jd / "oracle").exists()) + + def test_export_keeps_source_rollout_and_eval_set_and_excludes_transfer(self): + self.save(self.rd / "verdict.json", verify.verify_rollout(self.rd, write=False)) + manifest = to_agentrl.materialize(["old", "new"], "test", min_agent_version=1) + record = json.loads((self.root / "datasets/test/teacher.jsonl").read_text()) + self.assertEqual(record["rollout"], 2) + self.assertEqual(record["continuation"], {"source_run_id": "old", "source_job_id": "job", "source_rollout": 2}) + self.assertEqual(record["eval_set"]["n"], 100) + self.assertNotIn("oracle_ref", record) + self.assertNotIn("oracle_ref", record["verdict"]) + self.assertNotIn("acc_rel", record["reward_terms"]) + self.assertFalse(record["sft_eligible"]) # historical repair stays analysis + self.assertEqual(manifest["stats"]["scanned"], 1) + self.assertEqual(manifest["stats"]["pipeline_written"], 0) + self.assertEqual(manifest["stats"]["excluded"], 0) + + def test_pipeline_export_records_eval_identity(self): + with sqlite3.connect(self.database) as con: + con.execute("INSERT INTO jobs (run_id,job_id,mode,state) VALUES ('new','job','oracle','FAIL')") + self.save(self.jd / "oracle/result.json", {"terminal": "FAIL"}) + record = next(to_agentrl.pipeline_records(["new"])) + self.assertEqual(record["eval_set"]["n"], 100) + self.assertEqual(record["continuation"]["source_run_id"], "old") + + def test_export_missing_verdict_identity_uses_current_job_identity(self): + result = verify.verify_rollout(self.rd, write=False) + result.pop("eval_set") + result.pop("continuation") + self.save(self.rd / "verdict.json", result) + to_agentrl.materialize(["new"], "missing-identity", min_agent_version=1) + record = json.loads((self.root / "datasets/missing-identity/teacher.jsonl").read_text()) + self.assertEqual(record["eval_set"]["n"], 100) + self.assertNotIn("acc_rel", record["reward_terms"]) + self.assertNotIn("reward_acc_rel_reason", record) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_eval_subset.py b/scripts/test_eval_subset.py new file mode 100644 index 0000000..e841497 --- /dev/null +++ b/scripts/test_eval_subset.py @@ -0,0 +1,225 @@ +"""Small T1 subsets stay reproducible and cannot displace legacy data/cache.""" +from collections import Counter +from contextlib import ExitStack, redirect_stdout +import copy +import hashlib +import io +import json +import os +from pathlib import Path +import random +import tarfile +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from harness import paths +from harness.eval import EVALUATOR_VERSION +from harness.stages.base import StageContext +from harness.stages.s02_baseline import Baseline, baseline_dir +from harness.stages.s08_eval import Eval +from scripts import make_t1 + + +class EvalSubsetTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.pool = self.root / "pool" / "imagenet1k" / "v1" + self.pool.mkdir(parents=True) + (self.pool / "manifest.json").write_text(json.dumps({"sha256": "a" * 64})) + self.stack = ExitStack() + self.addCleanup(self.stack.close) + self.stack.enter_context(patch.object(make_t1, "POOL", self.root / "pool")) + self.stack.enter_context(patch.object(make_t1, "T1", self.root / "t1")) + self.stack.enter_context(patch.object(make_t1, "REG", {"datasets": { + "imagenet1k": {"version": "v1", "t1": {"n": 2000, "seed": 20260914, + "stratify": "label"}}}})) + self.stack.enter_context(patch.object(paths, "CACHE", self.root / "cache")) + + def labels(self, sizes): + labels = {f"{label}-{i}.jpg": label for label, size in enumerate(sizes) for i in range(size)} + (self.pool / "labels.json").write_text(json.dumps(labels)) + return labels + + def images(self, labels): + (self.pool / "images").mkdir() + for filename in labels: + (self.pool / "images" / filename).write_bytes(filename.encode()) + + def test_small_subset_samples_classes_without_sorted_prefix_bias(self): + self.labels([3] * 1000) + items = make_t1.items_imagenet(self.pool, 100, 20260914, "label") + self.assertEqual(len(items), 100) + selected = {item["label"] for item in items} + self.assertEqual(len(selected), 100) + self.assertNotEqual(selected, set(range(100))) + self.assertTrue(any(label >= 900 for label in selected)) + self.assertEqual(items, make_t1.items_imagenet(self.pool, 100, 20260914, "label")) + self.assertNotEqual(selected, {item["label"] for item in + make_t1.items_imagenet(self.pool, 100, 20260915, "label")}) + + def test_cityscapes_keeps_image_label_pairs_and_excludes_training(self): + pool = self.root / "cityscapes" + for i in range(4): + for split in ("val", "train"): + image = pool / f"leftImg8bit/{split}/city/city_{i}_leftImg8bit.png" + label = pool / f"gtFine/{split}/city/city_{i}_gtFine_labelIds.png" + image.parent.mkdir(parents=True, exist_ok=True); image.write_bytes(b"image") + label.parent.mkdir(parents=True, exist_ok=True); label.write_bytes(b"label") + items = make_t1.items_segmentation(pool, "cityscapes", 3, 17) + self.assertEqual(items, make_t1.items_segmentation(pool, "cityscapes", 3, 17)) + self.assertEqual(len(items), 3) + self.assertTrue(all("/val/" in item["src"] and "/val/" in item["label_src"] for item in items)) + self.assertTrue(all(item["label_format"] == "cityscapes_labelIds" for item in items)) + (pool / items[0]["label_src"]).unlink() + with self.assertRaises(FileNotFoundError): + make_t1.items_segmentation(pool, "cityscapes", 3, 17) + + def test_exact_size_with_remainder_and_short_classes(self): + self.labels([1, 2, 9, 9]) + for count in (1, 3, 7, 18, 21): + with self.subTest(count=count): + items = make_t1.items_imagenet(self.pool, count, 37, "label") + self.assertEqual(len(items), count) + self.assertEqual(len({item["id"] for item in items}), count) + self.labels([4] * 5) + counts = Counter(item["label"] for item in make_t1.items_imagenet(self.pool, 13, 37, "label")) + self.assertEqual(sorted(counts.values()), [2, 2, 3, 3, 3]) + + def test_insufficient_samples_fail_instead_of_silently_shrinking(self): + self.labels([2, 3]) + for stratify in ("label", "none"): + with self.assertRaisesRegex(ValueError, "only 5 available"): + make_t1.items_imagenet(self.pool, 6, 1, stratify) + for count in (0, -1): + with self.assertRaisesRegex(ValueError, "positive integer"): + make_t1.items_imagenet(self.pool, count, 1, "label") + (self.pool / "instances_val2017.json").write_text(json.dumps({"images": [{"id": 1}]})) + with self.assertRaisesRegex(ValueError, "only 1 available"): + make_t1.items_coco(self.pool, 2, 1) + + def test_default_2000_selection_is_unchanged(self): + labels = self.labels([3] * 1000) + rng = random.Random(20260914) + expected = [] + for label in range(1000): + filenames = sorted(filename for filename, value in labels.items() if value == label) + rng.shuffle(filenames) + expected.extend(filenames[:2]) + actual = make_t1.items_imagenet(self.pool, 2000, 20260914, "label") + self.assertEqual([item["id"] for item in actual], expected) + + def test_tar_is_deterministic_and_legacy_files_remain_untouched(self): + self.images(self.labels([2] * 1000)) + with redirect_stdout(io.StringIO()): + legacy = make_t1.build("imagenet1k") + legacy_bytes = legacy.read_bytes() + legacy_manifest = legacy.with_name("manifest.json").read_bytes() + subset = make_t1.build("imagenet1k", n=100) + subset_bytes = subset.read_bytes() + subset_manifest = subset.with_name("manifest.json") + manifest_bytes = subset_manifest.read_bytes() + for file in (subset, subset_manifest): + os.utime(file, ns=(1_000_000_000, 1_000_000_000)) + self.assertEqual(make_t1.build("imagenet1k", n=100).read_bytes(), subset_bytes) + self.assertEqual(subset_manifest.read_bytes(), manifest_bytes) + self.assertEqual(subset.stat().st_mtime_ns, 1_000_000_000) + self.assertEqual(subset_manifest.stat().st_mtime_ns, 1_000_000_000) + self.assertEqual(legacy, self.root / "t1/imagenet1k/20260914.tar") + self.assertEqual(subset, self.root / "t1/imagenet1k/n100-s20260914/20260914.tar") + self.assertEqual(legacy.read_bytes(), legacy_bytes) + self.assertEqual(legacy.with_name("manifest.json").read_bytes(), legacy_manifest) + external = json.loads(subset.with_name("manifest.json").read_text()) + self.assertEqual(external["sha256"], hashlib.sha256(subset_bytes).hexdigest()) + with tarfile.open(subset) as archive: + internal = json.load(archive.extractfile("manifest.json")) + self.assertEqual(len(internal["items"]), 100) + self.assertEqual(len(archive.getmembers()), 101) + self.assertEqual(internal["sha256"], external["content_sha256"]) + self.assertEqual(internal["eval_set_id"], external["eval_set_id"]) + with self.assertRaisesRegex(ValueError, "preserve the legacy"): + make_t1.build("imagenet1k", n=100, out_dir=legacy.parent) + with self.assertRaisesRegex(FileExistsError, "different subset"): + make_t1.build("imagenet1k", n=99, out_dir=subset.parent) + self.assertEqual(subset.read_bytes(), subset_bytes) + + def test_existing_custom_package_is_never_repaired_or_overwritten_silently(self): + self.images(self.labels([2, 2, 2])) + with redirect_stdout(io.StringIO()): + subset = make_t1.build("imagenet1k", n=2) + manifest = subset.with_name("manifest.json") + manifest_bytes = manifest.read_bytes() + subset.write_bytes(b"corrupted existing package") + with self.assertRaisesRegex(ValueError, "package hash mismatch"): + make_t1.build("imagenet1k", n=2) + self.assertEqual(subset.read_bytes(), b"corrupted existing package") + subset.unlink() + with self.assertRaisesRegex(ValueError, "package is missing"): + make_t1.build("imagenet1k", n=2) + self.assertFalse(subset.exists()) + self.assertEqual(manifest.read_bytes(), manifest_bytes) + + def test_explicit_cache_uses_full_content_hash_and_version(self): + job = {"model": "resnet18", "evaluator_version": EVALUATOR_VERSION, + "data": {"dataset": "imagenet1k", "t1": {"content_sha": "a" * 64}}} + legacy = baseline_dir(job) + legacy.mkdir(parents=True) + legacy_metrics = {"t1_manifest_sha": "a" * 64, "evaluator_version": EVALUATOR_VERSION, + "metrics": {"top1": 80}} + (legacy / "metrics.json").write_text(json.dumps(legacy_metrics)) + for filename in ("g2_inputs.npz", "g2_fp32.npz"): + (legacy / filename).write_bytes(b"legacy G2") + self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=job))) + subset = copy.deepcopy(job) + subset["data"]["t1"]["eval_set_id"] = "screen-100" + directory = baseline_dir(subset) + self.assertEqual(directory, legacy / ("a" * 64) / EVALUATOR_VERSION) + self.assertIsNone(Baseline().cached(SimpleNamespace(job=subset))) + directory.mkdir(parents=True) + (directory / "metrics.json").write_text(json.dumps(legacy_metrics)) + for filename in ("g2_inputs.npz", "g2_fp32.npz"): + (directory / filename).write_bytes(b"new subset G2") + self.assertIsNotNone(Baseline().cached(SimpleNamespace(job=subset))) + # 同名、同样本量、哈希前缀相同,也不能共用缓存。 + subset["data"]["t1"]["content_sha"] = "a" * 63 + "b" + self.assertNotEqual(baseline_dir(subset), directory) + subset["data"]["t1"]["content_sha"] = "a" * 64 + subset["evaluator_version"] = "0.4.0" + self.assertNotEqual(baseline_dir(subset), directory) + self.assertEqual(json.loads((legacy / "metrics.json").read_text()), legacy_metrics) + for filename in ("g2_inputs.npz", "g2_fp32.npz"): + self.assertEqual((legacy / filename).read_bytes(), b"legacy G2") + subset["data"]["t1"]["content_sha"] = None + with self.assertRaisesRegex(ValueError, "full SHA256"): + baseline_dir(subset) + + def test_fp32_and_board_commands_share_subset_and_g2_cache(self): + job = {"job_id": "resnet18-test", "model": "resnet18", "task": "cls", "backend": "ptq", + "evaluator_version": EVALUATOR_VERSION, + "model_spec": {"source": "torchvision", "input_spec": "1x3x224x224", "resize": 256}, + "gates": {"G2": {"n_samples": 32}}, + "data": {"dataset": "imagenet1k", "t1": { + "path": str(self.root / "t1/imagenet1k/n100-s20260914/20260914.tar"), + "n": 100, "eval_set_id": "screen-100", "content_sha": "a" * 64, + "manifest_sha": "b" * 64}}} + ctx = StageContext("test", job, self.root / "job", self.root / "stages", self.root / "artifacts") + with patch.object(paths, "venv_python", return_value=Path("/test/python")), \ + patch("harness.stages.s02_baseline.weights_file", return_value=self.root / "weights"), \ + patch("harness.stages.s08_eval.onnx_path", return_value=self.root / "model.onnx"), \ + patch("harness.stages.s08_eval.selected_board", return_value={"runtime_dir": "/test/board"}): + fp32, board = Baseline().command(ctx), Eval().command(ctx) + for flag, expected in (("--t1", job["data"]["t1"]["path"]), ("--t1-sha", "b" * 64), + ("--g2-samples", "32"), ("--input-spec", "1x3x224x224"), ("--resize", "256")): + self.assertEqual(fp32[fp32.index(flag) + 1], expected) + self.assertEqual(board[board.index(flag) + 1], expected) + # 评测器将两个 G2 npz 与 metrics.json 都写到 --out;INT8 全部从 --baseline 读取。 + self.assertEqual(fp32[fp32.index("--out") + 1], board[board.index("--baseline") + 1]) + self.assertEqual(Path(board[board.index("--baseline") + 1]), + self.root / "cache/baseline/resnet18--imagenet1k" / ("a" * 64) / EVALUATOR_VERSION) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_explicit_batch.py b/scripts/test_explicit_batch.py new file mode 100644 index 0000000..cf8d739 --- /dev/null +++ b/scripts/test_explicit_batch.py @@ -0,0 +1,41 @@ +import json +from unittest.mock import patch +import unittest +from harness import registry +from scripts.test_collection import Collection + + +class ExplicitBatch(unittest.TestCase): + def test_explicit_admission_keeps_sampling_and_gates_without_changing_csv(self): + before = registry.model_row("yolov8n.seg").copy() + normal = registry.expand_matrix(["oracle", "e2e"]) + self.assertFalse(any(s.model == "yolov8n.seg" for s in normal)) + specs = registry.expand_matrix(["oracle", "e2e"], ["yolov8n.seg"]) + self.assertEqual([(s.mode, s.k, s.policy, s.march) for s in specs], + [("oracle", 1, None, "nash-m"), ("e2e", 4, "teacher", "nash-m")]) + self.assertEqual(registry.gate_snapshot(specs[0])["G3"]["tol"], 5) + self.assertEqual(registry.model_row("yolov8n.seg"), before) + with self.assertRaises(ValueError): + registry.expand_matrix(["oracle"], ["nonexistent"]) + + +class ExplicitRepair(Collection): + def test_admitted_model_does_not_generate_oracle_repairs(self): + from harness import pipeline + self.mx["modes"]["repair"] = {"enabled": True, "when": "oracle_fail", "k": 2, + "policies": ["teacher"], "error_classes": ["accuracy_drop"], "marches": ["nash-m"], "backends": ["ptq"]} + self.job("segment", task="iseg", status="needs_eval", state="FAIL", k=4) + jd = self.root / "runs/run/segment" + jd.mkdir(parents=True) + (jd / "job.json").write_text(json.dumps({"admission": {"method": "explicit_model_batch"}})) + with self.connect() as con: + con.execute("UPDATE jobs SET error_class='accuracy_drop',finished_at=1 WHERE job_id='segment'") + self.assertEqual(pipeline.enqueue_repairs("run"), 0) + self.assertEqual(pipeline.enqueue_repairs("run"), 0) + with self.connect() as con: + rows = db_rows = [dict(r) for r in con.execute("SELECT * FROM rollouts WHERE job_id='segment' ORDER BY kk")] + self.assertEqual([(r["kk"], r["mode"]) for r in rows], [(kk, "e2e") for kk in range(4)]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_generation.py b/scripts/test_generation.py index e4dc47c..1bd9c43 100644 --- a/scripts/test_generation.py +++ b/scripts/test_generation.py @@ -1,4 +1,4 @@ -"""Conditional repair generation preserves the existing matrix and rollout numbering.""" +"""Retired Oracle repair generation preserves historical rows and identities.""" from contextlib import contextmanager from copy import deepcopy from pathlib import Path @@ -56,13 +56,13 @@ class RepairGeneration(unittest.TestCase): with self.connect() as con: for kk in range(4): db.rollout_upsert(con, "run", "job", kk, "e2e", "teacher") - self.assertEqual(pipeline.enqueue_repairs("run"), 2) + self.assertEqual(pipeline.enqueue_repairs("run"), 0) self.assertEqual(pipeline.enqueue_repairs("run"), 0) self.assertEqual([(r["kk"], r["mode"]) for r in self.rollouts()], - [(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e"), (4, "repair"), (5, "repair")]) + [(0, "e2e"), (1, "e2e"), (2, "e2e"), (3, "e2e")]) self.assertEqual(self.mx, original_matrix) - def test_only_finished_eligible_oracles_and_selected_jobs(self): + def test_finished_oracle_failures_do_not_generate_any_repairs(self): self.job("eligible") self.job("not_selected") self.job("passed", state="PASS") @@ -72,9 +72,9 @@ class RepairGeneration(unittest.TestCase): self.job("wrong_backend", backend="qat") self.job("other_run", run="other") chosen = ["eligible", "passed", "unfinished", "unknown_error", "wrong_march", "wrong_backend", "other_run"] - self.assertEqual(pipeline.enqueue_repairs("run", chosen), 2) + self.assertEqual(pipeline.enqueue_repairs("run", chosen), 0) with self.connect() as con: - self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, {"eligible"}) + self.assertEqual({r[0] for r in con.execute("SELECT job_id FROM rollouts")}, set()) def test_mode_enabled_when_and_select_filters(self): self.job() @@ -92,23 +92,27 @@ class RepairGeneration(unittest.TestCase): self.model_rows[0]["status"] = "needs_adapter" self.assertEqual(pipeline.enqueue_repairs("run"), 0) cfg["select"] = {"status": ["needs_adapter"]} - self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 2) + self.assertEqual(pipeline.enqueue_repairs("run", modes=["repair"]), 0) - def test_fill_missing_per_policy_without_replacing_completed_rollouts(self): + def test_historical_rows_and_remaining_quotas_are_not_changed(self): self.job() self.mx["modes"]["repair"]["policies"] = ["teacher", "student"] with self.connect() as con: db.rollout_upsert(con, "run", "job", 7, "repair", "teacher", state="FAIL") db.rollout_upsert(con, "run", "job", 10, "e2e", "teacher", state="PASS") - self.assertEqual(pipeline.enqueue_repairs("run"), 3) + self.assertEqual(pipeline.enqueue_repairs("run"), 0) self.assertEqual(pipeline.enqueue_repairs("run"), 0) rows = self.rollouts() self.assertEqual([(r["kk"], r["mode"], r["policy"], r["state"]) for r in rows], [ (7, "repair", "teacher", "FAIL"), (10, "e2e", "teacher", "PASS"), - (11, "repair", "teacher", "QUEUED"), (12, "repair", "student", "QUEUED"), - (13, "repair", "student", "QUEUED"), ]) + def test_expand_cannot_reenable_repair_through_legacy_matrix(self): + self.mx["modes"]["repair"].update(when="always", enabled=True) + with patch.object(registry, "matrix", return_value=self.mx), \ + patch.object(registry, "models", return_value=self.model_rows): + self.assertEqual(registry.expand_matrix(["repair"]), []) + def test_collect_dry_run_does_not_enqueue(self): self.job() with patch.object(pipeline, "enqueue_repairs") as enqueue: diff --git a/scripts/test_mmseg_static_pool.py b/scripts/test_mmseg_static_pool.py new file mode 100644 index 0000000..a5f7ea6 --- /dev/null +++ b/scripts/test_mmseg_static_pool.py @@ -0,0 +1,31 @@ +"""Non-divisible pooling bins must match PyTorch after actual ONNX export.""" +import io +import unittest + +import numpy as np +import onnxruntime as ort +import torch + +from harness.adapters.mmseg_common import lower_static_pools + + +class StaticPooling(unittest.TestCase): + def test_rectangular_bins_keep_the_official_mean(self): + torch.manual_seed(17) + for shape, output in (((5, 7), 3), ((64, 128), 6), ((8, 12), 2)): + with self.subTest(shape=shape, output=output): + x = torch.randn(1, 3, *shape) + model = torch.nn.Sequential(torch.nn.AdaptiveAvgPool2d(output)).eval() + expected = model(x).numpy() + lower_static_pools(model) + np.testing.assert_array_equal(model(x).numpy(), expected) + stream = io.BytesIO() + torch.onnx.export(model, x, stream, opset_version=11, input_names=['image']) + options = ort.SessionOptions(); options.intra_op_num_threads = 2 + session = ort.InferenceSession(stream.getvalue(), sess_options=options, providers=['CPUExecutionProvider']) + actual = session.run(None, {'image': x.numpy()})[0] + np.testing.assert_allclose(actual, expected, rtol=1e-5, atol=1e-6) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_pose_gate.py b/scripts/test_pose_gate.py new file mode 100644 index 0000000..a38aba9 --- /dev/null +++ b/scripts/test_pose_gate.py @@ -0,0 +1,296 @@ +"""Run in bench-venv: python -m unittest scripts.test_pose_gate scripts.test_coco_extra scripts.test_accuracy_policy. + +Geometry fixtures use actual decoding/letterbox and real COCOeval; no mocked AP. +These are CPU contract tests, not evidence of a board model passing its gate. +""" +import copy +import io +import json +from types import SimpleNamespace +import unittest + +import numpy as np + +from harness import gates +from harness.eval.coco_extra import ( + measure_pose_shift, pose_snapshot, pose_shift_summary, predict, run_fp32, run_quant, +) + + +def detection(x=0, shift=(0, 0), confidence=.9, score=.9): + points = [[x + 10 + i * 2 + shift[0], 10 + i + shift[1], confidence] for i in range(17)] + return {"category_id": 1, "bbox_xyxy": [x, 0, x + 100, 100], "score": score, "keypoints": points} + + +def snapshot(*image_detections): + t1 = SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": i + 1} for i in range(len(image_detections))]) + result = pose_snapshot(t1, "1x3x640x640") + result["images"] = [ + {"image_id": i + 1, "image_size": [640, 480], "scale": 1.0, "pad": [0, 80], "detections": ds} + for i, ds in enumerate(image_detections) + ] + return result + + +def measurement(shift, quant=80): + return {"metric": "oks_ap", "fp32": 80, "quant": quant, "tier": "t1", "source": "board", "n": 1, + "extra": {"keypoint_mean_shift_px": shift["keypoint_mean_shift_px"], "keypoint_shift": pose_shift_summary(shift)}} + + +CFG = {"metric": "oks_ap", "kind": "rel_pct", "tol": 5, "require_source": "board", + "t1_screen_widen_se": 0, "extra": {"keypoint_mean_shift_px_max": 2}} + + +class PoseGeometry(unittest.TestCase): + def test_topdown_uses_actual_affine_without_fabricated_letterbox(self): + reference, measured = snapshot([detection()]), snapshot([detection(shift=(3, 4))]) + for record in (reference, measured): + image = record["images"][0] + image.pop("scale"); image.pop("pad") + image["transform"] = "topdown_affine" + image["detections"][0]["affine"] = [[.5, 0., 7.], [0., .5, 11.]] + result = measure_pose_shift(reference, measured) + self.assertEqual(result["keypoint_mean_shift_px"], 5) + self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "fail") + measured["images"][0]["detections"][0]["affine"][0][2] += 1 + self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified") + measured["images"][0]["detections"][0]["affine"] = [[0., 0., 0.], [0., 0., 0.]] + self.assertEqual(measure_pose_shift(reference, measured)["status"], "unverified") + + def test_euclidean_distance_and_replayable_evidence(self): + result = measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(3, 4))])) + self.assertEqual(result["status"], "measured") + self.assertEqual(result["keypoint_mean_shift_px"], 5) + self.assertEqual((result["n_matched"], result["n_keypoints"]), (1, 17)) + pair = json.loads(json.dumps(result, allow_nan=False))["matches"][0] + diff = np.asarray(pair["quant"]["keypoints"])[:, :2] - np.asarray(pair["fp32"]["keypoints"])[:, :2] + np.testing.assert_allclose(np.linalg.norm(diff, axis=1), pair["distances_px"]) + + def test_metric_is_mean_of_lengths_not_max_rms_or_length_of_mean(self): + a, b = detection(), detection() + b["keypoints"][0][0] += 17 + b["keypoints"][1][0] -= 17 + result = measure_pose_shift(snapshot([a]), snapshot([b])) + self.assertEqual(result["keypoint_mean_shift_px"], 2) + self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "pass-screen") + + def test_order_and_score_changes_do_not_pair_different_people(self): + a = snapshot([detection(), detection(x=200)]) + b = snapshot([detection(x=200, shift=(0, 2), score=.1), detection(shift=(2, 0), score=.99)]) + result = measure_pose_shift(a, b) + self.assertEqual(result["keypoint_mean_shift_px"], 2) + self.assertEqual([(p["fp32_index"], p["quant_index"]) for p in result["matches"]], [(0, 1), (1, 0)]) + + def test_matching_uses_boxes_not_nearest_keypoints(self): + # Deliberately swapped skeletons: matching by keypoints would falsely give zero. + a = snapshot([detection(), detection(x=200)]) + b = snapshot([detection(shift=(200, 0)), detection(x=200, shift=(-200, 0))]) + result = measure_pose_shift(a, b) + self.assertEqual(result["keypoint_mean_shift_px"], 200) + + def test_one_to_one_and_deterministic_iou_tie(self): + result = measure_pose_shift(snapshot([detection(), detection()]), snapshot([detection()])) + self.assertEqual(result["n_matched"], 1) + self.assertEqual(result["matches"][0]["fp32_index"], 0) + self.assertEqual(result["unmatched_fp32"], [{"image_id": 1, "index": 1}]) + + def test_iou_matching_boundary(self): + a, b = detection(), detection() + a["bbox_xyxy"], b["bbox_xyxy"] = [0, 0, 3, 3], [1, 0, 4, 3] + result = measure_pose_shift(snapshot([a]), snapshot([b])) + self.assertEqual(result["matches"][0]["bbox_iou"], .5) + b["bbox_xyxy"] = [1.001, 0, 4.001, 3] + self.assertEqual(measure_pose_shift(snapshot([a]), snapshot([b]))["status"], "unverified") + + def test_no_cross_image_matching(self): + result = measure_pose_shift(snapshot([detection()], []), snapshot([], [detection()])) + self.assertEqual(result["status"], "unverified") + self.assertEqual(result["n_matched"], 0) + self.assertIsNone(result["keypoint_mean_shift_px"]) + + def test_unmatched_detections_are_counted_not_zero_filled(self): + result = measure_pose_shift(snapshot([detection(), detection(x=200)]), + snapshot([detection(shift=(3, 4))])) + self.assertEqual(result["keypoint_mean_shift_px"], 5) + self.assertEqual((result["n_fp32"], result["n_quant"], result["n_matched"]), (2, 1, 1)) + self.assertEqual(gates.g3(measurement(result), CFG)["extra"]["keypoint_shift"]["n_unmatched_fp32"], 1) + + def test_joint_confidence_cannot_hide_shift(self): + result = measure_pose_shift(snapshot([detection(confidence=0)]), + snapshot([detection(shift=(3, 4), confidence=0)])) + self.assertEqual(result["keypoint_mean_shift_px"], 5) + self.assertEqual(result["n_keypoints"], 17) + + def test_empty_missing_and_unmatched_are_unverified(self): + for a, b in ((None, snapshot([])), (snapshot([]), None), (snapshot([]), snapshot([])), + (snapshot([detection()]), snapshot([])), + (snapshot([detection()]), snapshot([detection(x=200)]))): + with self.subTest(fp32=a, quant=b): + result = measure_pose_shift(a, b) + self.assertEqual(result["status"], "unverified") + self.assertEqual(gates.g3(measurement(result), CFG)["verdict"], "unverified") + + def test_invalid_geometry_or_incomplete_evidence_never_measures(self): + reference = snapshot([detection()]) + bad = [] + for value in (float("nan"), float("inf")): + other = copy.deepcopy(reference) + other["images"][0]["detections"][0]["keypoints"][0][0] = value + bad.append(other) + for field, value in (("schema", "older"), ("input_spec", "1x3x320x320"), + ("t1_manifest_sha", "other"), ("image_ids", [2]), ("images", [])): + bad.append({**reference, field: value}) + other = copy.deepcopy(reference) + other["images"][0]["pad"] = [0, 79] + bad.append(other) + other = copy.deepcopy(reference) + other["images"][0]["detections"][0]["keypoints"].pop() + bad.append(other) + other = copy.deepcopy(reference) + other["images"][0]["detections"][0]["category_id"] = 2 + bad.append(other) + for other in bad: + with self.subTest(other=other): + self.assertEqual(measure_pose_shift(reference, other)["status"], "unverified") + + +class PoseGate(unittest.TestCase): + def shifted(self, pixels): + return measurement(measure_pose_shift(snapshot([detection()]), snapshot([detection(shift=(pixels, 0))]))) + + def test_two_pixel_inclusive_boundary(self): + for pixels, verdict in ((0, "pass-screen"), (1.9999, "pass-screen"), (2, "pass-screen"), (2.0001, "fail")): + with self.subTest(pixels=pixels): + result = gates.g3(self.shifted(pixels), CFG) + self.assertEqual(result["verdict"], verdict) + self.assertEqual(result["extra"]["keypoint_shift"]["threshold_px"], 2) + + def test_both_ap_and_geometry_must_pass(self): + for quant, pixels, verdict in ((76, 2, "pass-screen"), (75.999, 0, "fail"), (85, 3, "fail")): + m = self.shifted(pixels) + m["quant"] = quant + self.assertEqual(gates.g3(m, CFG)["verdict"], verdict) + + def test_missing_geometry_cannot_pass_on_oks_alone_even_with_ap_failure(self): + for quant in (80, 0): + m = {"fp32": 80, "quant": quant, "source": "board", "tier": "t1"} + self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") + m["extra"] = {"keypoint_mean_shift_px": 0} + self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") + + def test_bad_counts_values_and_status_only_are_unverified(self): + bad = [] + for value in (None, -1, True, "0", float("nan"), float("inf")): + m = self.shifted(0) + m["extra"]["keypoint_mean_shift_px"] = value + bad.append(m) + for key, value in (("n_matched", 0), ("n_keypoints", 0), ("n_fp32", 99), ("distance_sum_px", None), + ("method", "manual-label"), ("keypoint_mean_shift_px", 1)): + m = self.shifted(0) + m["extra"]["keypoint_shift"][key] = value + bad.append(m) + m = self.shifted(0) + m["extra"]["keypoint_shift"]["distance_sum_px"] = float("nan") + bad.append(m) + for m in bad: + with self.subTest(measurement=m): + self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") + + def test_t1_se_does_not_widen_pixel_threshold(self): + m = self.shifted(2.1) + m["se"] = 100 + self.assertEqual(gates.g3(m, {**CFG, "t1_screen_widen_se": 2})["verdict"], "fail") + + def test_source_na_and_other_task_thresholds_preserved(self): + m = self.shifted(0) + m["source"] = "sim" + self.assertEqual(gates.g3(m, CFG)["verdict"], "unverified") + self.assertEqual(gates.g3(None, {**CFG, "verdict": "na"})["verdict"], "na") + for metric in ("top1", "map_50_95", "mask_ap"): + cfg = {k: v for k, v in CFG.items() if k != "extra"} + cfg["metric"] = metric + for quant, verdict in ((76, "pass-screen"), (75.999, "fail")): + self.assertEqual(gates.g3({"fp32": 80, "quant": quant, "source": "board", "tier": "t1"}, cfg)["verdict"], verdict) + + +def t1_fixture(image_size=(640, 480)): + from PIL import Image + points = detection()["keypoints"] + annotation = {"id": 1, "image_id": 1, "category_id": 1, "bbox": [0, 0, 100, 100], "area": 10000, + "iscrowd": 0, "num_keypoints": 17, "keypoints": [v for x, y, _ in points for v in (x, y, 2)]} + category = {"id": 1, "name": "person", "keypoints": [str(i) for i in range(17)], "skeleton": []} + ann = {"info": {}, "images": [{"id": 1, "width": image_size[0], "height": image_size[1]}], + "annotations": [], "categories": [], "keypoints": {"annotations": [annotation], "categories": [category]}} + return SimpleNamespace(manifest={"sha256": "test-t1"}, items=[{"id": 1}], + image=lambda _: Image.new("RGB", image_size), + tar=SimpleNamespace(extractfile=lambda _: io.BytesIO(json.dumps(ann).encode()))) + + +def head_runner(image_size, input_size, shift=(0, 0), points=None, score=.9): + scale = input_size / max(image_size) + pad = [(input_size - round(s * scale)) // 2 for s in image_size] + head = np.zeros((1, 56, 1), dtype=np.float32) + head[0, :4, 0] = [50 * scale + pad[0], 50 * scale + pad[1], 100 * scale, 100 * scale] + head[0, 4, 0] = score + keypoints = np.asarray(points if points is not None else detection()["keypoints"], dtype=float) + keypoints[:, :2] = (keypoints[:, :2] + np.asarray(shift)) * scale + pad + head[0, 5:, 0] = keypoints.reshape(-1) + return lambda _: [head.copy()] + + +class PoseIntegration(unittest.TestCase): + def test_real_decode_undoes_resize_and_asymmetric_letterbox(self): + t1 = t1_fixture((5, 3)) + points = [[1, 1, .8] for _ in range(17)] + a, b = pose_snapshot(t1, "1x3x8x8"), pose_snapshot(t1, "1x3x8x8") + predict(head_runner((5, 3), 8, points=points), t1, "1x3x8x8", "pose", a) + predict(head_runner((5, 3), 8, shift=(0, 2), points=points), t1, "1x3x8x8", "pose", b) + self.assertEqual(a["images"][0]["pad"], [0, 1]) + self.assertAlmostEqual(a["images"][0]["detections"][0]["keypoints"][0][1], 1, places=6) + self.assertAlmostEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 2, places=6) + + def test_unclipped_evidence_does_not_hide_edge_displacement(self): + t1 = t1_fixture() + points = [[-5, 20, .8] for _ in range(17)] + a, b = pose_snapshot(t1, "1x3x640x640"), pose_snapshot(t1, "1x3x640x640") + pa = predict(head_runner((640, 480), 640, points=points), t1, "1x3x640x640", "pose", a) + pb = predict(head_runner((640, 480), 640, shift=(-3, 0), points=points), t1, "1x3x640x640", "pose", b) + self.assertEqual(pa[0]["keypoints"], pb[0]["keypoints"]) # unchanged AP clipping + self.assertEqual(measure_pose_shift(a, b)["keypoint_mean_shift_px"], 3) + + def test_real_cocoeval_and_json_baseline_quant_wiring(self): + t1 = t1_fixture() + fp = {"task": "pose", "model": "fixture"} + run_fp32(head_runner((640, 480), 640), t1, "1x3x640x640", fp) + self.assertAlmostEqual(fp["metrics"]["oks_ap"], 100) + baseline = json.loads(json.dumps(fp, allow_nan=False)) + quant = {"task": "pose", "model": "fixture"} + run_quant(head_runner((640, 480), 640, shift=(2, 0)), t1, "1x3x640x640", baseline, quant) + self.assertEqual(fp, baseline) # quant cannot mutate the saved reference + self.assertEqual(quant["G3"]["extra"]["keypoint_mean_shift_px"], 2) + self.assertEqual(quant["pose_shift_evidence"]["matches"][0]["distances_px"], [2] * 17) + self.assertNotIn("matches", quant["G3"]["extra"]["keypoint_shift"]) + self.assertGreaterEqual(quant["metrics"]["oks_ap"], 95) + quant["G3"]["source"] = "board" # gate source fixture; runner here is synthetic + self.assertEqual(gates.g3(quant["G3"], CFG)["verdict"], "pass-screen") + json.dumps(quant, allow_nan=False) + + def test_legacy_baseline_keeps_ap_but_shift_is_unverified(self): + t1 = t1_fixture() + out = {"task": "pose", "model": "fixture"} + run_quant(head_runner((640, 480), 640), t1, "1x3x640x640", + {"model": "fixture", "metrics": {"oks_ap": 100}}, out) + self.assertAlmostEqual(out["metrics"]["oks_ap"], 100) + out["G3"]["source"] = "board" + self.assertEqual(gates.g3(out["G3"], CFG)["verdict"], "unverified") + + def test_no_detections_produces_complete_empty_image_evidence(self): + t1 = t1_fixture() + evidence = pose_snapshot(t1, "1x3x640x640") + self.assertEqual(predict(head_runner((640, 480), 640, score=0), t1, "1x3x640x640", "pose", evidence), []) + self.assertEqual(evidence["images"][0]["detections"], []) + self.assertEqual(measure_pose_shift(evidence, evidence)["status"], "unverified") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_reward_board_offline.py b/scripts/test_reward_board_offline.py index 437bc69..1ec48e2 100644 --- a/scripts/test_reward_board_offline.py +++ b/scripts/test_reward_board_offline.py @@ -81,15 +81,19 @@ class BoardOfflineReward(unittest.TestCase): self.assertIsNone(r["unverifiable"]) self.assertEqual((r["terms"]["G1"], r["terms"]["G2"], r["terms"]["G3"]), (2.0, 2.0, 0.0)) - def test_blocked_untouched(self): + def test_blocked_uses_measured_gates_without_oracle_bonus(self): v = regnet_like() v["terminal"] = "BLOCKED"; v["blocked_correct"] = True r = reward.compute(v, max_turns=60) - self.assertEqual(r["terms"]["blocked"], 1.0) - self.assertIsNone(r["unverifiable"]) + self.assertNotIn("blocked", r["terms"]) + self.assertEqual(r["terms"]["G1"], 1.0) + self.assertEqual(r["unverifiable"], "board_offline") + v["blocked_correct"] = False + v["oracle_ref"] = {"terminal": "FAIL"} + self.assertEqual(reward.compute(v, max_turns=60), r) def test_recipe_bumped(self): - self.assertEqual(reward.RECIPE_VERSION, "v0.4") + self.assertEqual(reward.RECIPE_VERSION, "v0.6") if __name__ == "__main__": diff --git a/scripts/test_rollout_backfill.py b/scripts/test_rollout_backfill.py index a384c39..2c672ba 100644 --- a/scripts/test_rollout_backfill.py +++ b/scripts/test_rollout_backfill.py @@ -29,7 +29,8 @@ class RolloutBackfill(unittest.TestCase): self.config = { "paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"}, "perf": {"warmup": 2, "runs": 3}, - "watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60}, + "watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60, + "io_stall_s": 50, "no_progress_ceiling_s": 55}, "pools": {"board": {"workers": 1, "reserve_for_agent": 1}}, } for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"), diff --git a/scripts/test_sync_patch.py b/scripts/test_sync_patch.py new file mode 100644 index 0000000..7222c14 --- /dev/null +++ b/scripts/test_sync_patch.py @@ -0,0 +1,36 @@ +import base64 +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from scripts.sync_patch import REMOTE_APPLY + + +class ScopedSyncTests(unittest.TestCase): + def test_preimage_guard_preserves_unrelated_files_and_backup(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + (root / 'one.py').write_bytes(b'old') + (root / 'other.py').write_bytes(b'remote-only') + payload = {'root': td, 'id': 'test-patch', 'files': [ + {'path': 'new.py', 'expected': None, 'data': base64.b64encode(b'new').decode()}, + {'path': 'one.py', 'expected': hashlib.md5(b'wrong').hexdigest(), + 'data': base64.b64encode(b'changed').decode()}]} + def apply(): + return subprocess.run([sys.executable, '-c', REMOTE_APPLY], input=json.dumps(payload), text=True, capture_output=True) + self.assertNotEqual(apply().returncode, 0) + self.assertFalse((root / 'new.py').exists()) + self.assertEqual((root / 'one.py').read_bytes(), b'old') + payload['files'][1]['expected'] = hashlib.md5(b'old').hexdigest() + result = apply() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((root / 'one.py').read_bytes(), b'changed') + self.assertEqual((root / 'other.py').read_bytes(), b'remote-only') + self.assertEqual((root / '.sync/test-patch/one.py').read_bytes(), b'old') + + +if __name__ == '__main__': + unittest.main()