From c2c52edce92af76bab62ccdd641f7529b7d43add Mon Sep 17 00:00:00 2001 From: "ming01.wang" Date: Thu, 8 Oct 2026 17:52:31 +0800 Subject: [PATCH] bench: freeze the qwen-sft-v1 job list 37 S100P jobs for comparing base Qwen3.8-27B with its GLM-5.3-SFT version, GLM-5.3 as reference: 4 controls GLM passed on the first compile, 26 GLM passed only after 2-9 compiles, 7 harder probes (pose, depth, instance and semantic segmentation). Each model runs 2 rollouts on the same harness; only rollouts started after the 2026-10-08 fixes count, and system interruptions are rerun rather than scored. Co-Authored-By: Claude Opus 5.5 --- benchmarks/qwen-sft-v1.json | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 benchmarks/qwen-sft-v1.json diff --git a/benchmarks/qwen-sft-v1.json b/benchmarks/qwen-sft-v1.json new file mode 100644 index 0000000..dcde54d --- /dev/null +++ b/benchmarks/qwen-sft-v1.json @@ -0,0 +1,66 @@ +{ + "name": "qwen-sft-v1", + "purpose": "Same jobs, same harness, same k for base Qwen3.8-27B and its GLM-5.3-SFT version; GLM-5.3 results are the reference.", + "created": "2026-10-08", + "board": "s100p", + "job_suffix": "--nash-m--ptq--int8", + "protocol": { + "rollouts_per_job": 2, + "count_rollouts_started_after": "2026-10-08T07:30:00+08:00", + "harness_commit": "10b070e", + "agent_version": 15, + "system_interrupted": "not a model failure; excluded and rerun", + "report": [ + "pass rate (agent-finished)", + "compile attempts", + "turns", + "tool error rate", + "wall time minus model wait" + ] + }, + "tiers": { + "control": [ + "cls--densenet121", + "seg--pidnet.s", + "feat--superpoint", + "mde--midas.small" + ], + "medium": [ + "cls--convnext.base", + "cls--convnext.large", + "cls--efficientnet.b0", + "cls--efficientnet.b1", + "cls--mobilenet.v3.large", + "cls--mobilenet.v3.small", + "cls--regnet.y.400mf", + "cls--regnet.y.1.6gf", + "cls--vit.b.32", + "det--foveabox.r50.fpn", + "det--fsaf.r50.fpn", + "det--retinanet.r50.fpn", + "det--retinanet.r101.fpn", + "det--rtmdet.m", + "det--yolo11m", + "det--yolo11n", + "det--yolox.l", + "det--yolox.m", + "det--yolox.x", + "iseg--solov2.r50", + "iseg--yolo11l.seg", + "pose--simcc.resnet50", + "pose--simplebaseline.r50", + "pose--simplebaseline.r152", + "seg--efficientvit.seg.b1", + "seg--ocrnet.hrnet.w48" + ], + "probe": [ + "pose--mspn.r50", + "mde--depth.anything.v1.small", + "mde--depth.anything.v2.small", + "mde--dpt.hybrid", + "iseg--yolov8n.seg", + "iseg--yolo11m.seg", + "seg--fast.scnn" + ] + } +}