data-gen-bpu/config.yaml
ming01.wang e2bdd24711 fix: stop the GLM quota policy from rejecting long contexts
57 of the 68 "incomplete generation" rollouts were never truncated by the
provider: the local broker refused the request with 413
REQUEST_EXCEEDS_INTERNAL_TPM. It reserves UTF-8 body bytes as tokens
(about 3.7x the real count) and rejects any single request above tpm, so
tpm 500000 ended every trajectory once its context passed ~135k tokens,
mostly the long pose/mde/seg ones. GLM-5.3 accepts far longer contexts.

Raise the default GLM tpm to 2000000, as the Qwen group already uses.
Production configs and the stored ledger policy are updated alongside.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 07:36:18 +08:00

173 lines
7.7 KiB
YAML
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# bench/config.yaml — 运行时配置(不进 job.json;job.json 只快照与结果相关的东西)
version: 1
paths:
# 项目根;harness 默认取本文件所在目录的上一级
project_root: null
# 两套 venv。harness 按 backend 选,agent 无权改
venv:
ptq: ~/toolchains/oe-3.7.0/venv # hb_compile / hb_model_info / hb_verifier(无 pip,不装东西)
qat: ~/toolchains/oellm2/venv # plugin + hbdk4 4.12 + hbm_infer(不装东西)
bench: ~/toolchains/bench-venv # 我们自己的:torch/torchvision/ultralytics/pycocotools/pyarrow — 权重、导 ONNX、FP32 基线、评测打分、数据入池
# harness 自身用的解释器(只需 pyyaml),默认 python3
harness_python: python3
# hb_compile 的 working_dir 根目录(中间产物几百 MB/个)。必须放本地盘:
# 实测 JuiceFS 写 1 MB=360 ms vs 本地盘 3 ms,大模型编译会卡在 D 状态被看门狗误判挂死。
# 留空则退回 runs/<run>/<job>/oracle/stages/05-compile/work(JuiceFS 上,只适合小规模)。
work_root: /home/ming01.wang/bench-work
# .horizon skill pack(只读)
horizon: ../.horizon
# agent 运行时(scripts/setup_agent.sh → ~/toolchains/node;pi 是 node 脚本,node 必须在 PATH 或这里给全路径)
pi_bin: ~/toolchains/node/bin/pi
node_bin: ~/toolchains/node/bin/node
pools:
# 机器是共享的:2026-09-16 恢复时 GPU 0–7 均被其他训练占用。
# devices: [-1] 通过 CUDA_VISIBLE_DEVICES 隐藏 GPU,让新增 FP32 基线用 CPU;已有基线照常复用。
# GPU 空闲后可将 devices 恢复为可用卡号,槽位 i → devices[i % len(devices)]。
# reserve_for_agent:只给 rollout 的 `bench stage run` 用的槽位,oracle 拿不到。
# 实测(2026-09-15):两个 oracle 调度进程把 cpu 8/8 占满,第一条 rollout 的编译在队列里饿了 60 分钟。
gpu: { workers: 4, devices: [-1], reserve_for_agent: 1 } # FP32 基线槽;当前 CPU 后端
cpu: { workers: 24, reserve_for_agent: 12 } # export / hb_compile;给真实 Agent 轨迹保留 12 槽
# 2026-09-16 提高采集并发;普通流水线默认 24 个在途任务,共享此 CPU 上限。
# 244 个逻辑 CPU / 1.4 TiB 内存;恢复后以实际完成速度、内存和板卡排队情况检查此上限。
board: { workers: 1, reserve_for_agent: 1 } # 新调度器按 boards.yaml 的实际板数分配;workers 仅兼容旧进程
agent: { workers: 32, rpm: 60 } # 真实 Agent 并发(2026-09-20 12→24→32;acquire 每轮重读 config 热生效,但 collector 线程数启动时算死,扩并发须新起 collector。实测 17 并发 ~15k TPM 无 429;32 并发估算中位 28k / p80 45k TPM、~19rpm);rpm 是端点配额记录,当前调度器未据此限流
watchdog:
stall_s: 900 # 日志不增长 且 进程树 CPU < cpu_idle_pct 持续 stall_s → STALLED
cpu_idle_pct: 5
poll_s: 30
hard_ceiling_s: 86400 # 24h 纯保险
io_stall_s: 21600 # 树内持续有 D 线程(卡在 I/O)这么久 → 也判定异常。
# 别往下调:线程级 D 检测上线前这条是死代码(d_count 恒 0),
# 上线后它第一次生效,收紧会把"链接期正在写盘"换个理由再杀一次。
no_progress_ceiling_s: 28800 # 日志与 CPU 都不动这么久 → 兜底,免疫 D 抖动。
# 必须 > io_stall_s,否则它抢先命中、盖掉更准确的 STALLED 原因。
# 四条阈值单调:stall_s(long 3600) < io_stall_s < 本条 < hard_ceiling_s
soft_alert: p95x3 # 超过历史 p95 × 3 只告警,不杀
perf:
warmup: 20
runs: 200
stats: [mean, p95]
rollouts:
private_home: true # Required by the managed Pi runner.
k: 4
max_turns: 60 # 照抄 oracle 27 轮够;真调优一轮迭代(改配置→编译→看日志)5–8 轮,40 只够两次半
max_tokens: 200000
hard_ceiling_s: 43200 # 12h 只防 agent 进程挂死
jail: true # 用 scripts/agent_jail.sh(unshare -rm)把 oracle/ 兄弟 rollout/ ~/.pi 从文件系统里摘掉。
# path-guard 只是"给句人话的拒绝理由",解析 shell 字符串永远追不平所有绕法。
# 关掉只在调试 agent 行为时用,采数据时必须开。
# --tools 是白名单,同时作用于内置工具与 extension 注册的工具。
# 踩过(2026-09-15 第一条轨迹):claim / oe_doc 没列进来 → claim-card 注册了工具但被白名单关掉,
# agent 全程无法 claim,213 次 bash 调用一次 claim 都没有。extension 工具必须显式列出。
tools: [read, edit, write, bash, grep, claim, oe_doc]
retry:
transient: 2 # oom / board_offline
stalled: 1 # 进 long 车道
deterministic: 0
scheduling:
priority_order: [A, B, C]
c_lane: low # C 类走独立低优先级车道
estimate_source: history_then_params
data:
default_tier: t1
calib_n_default: 128
# LLM 端点。key 不写这里,写 bench/.env.local(gitignore)
endpoints:
teacher:
provider: openai-compatible
base_url: https://open.bigmodel.cn/api/coding/paas/v4
model: glm-5.3
api_key_env: TEACHER_API_KEY
student:
provider: openai-compatible
base_url: null # vLLM 起来后填
model: null
api_key_env: STUDENT_API_KEY
# Shared execution / quota policy; do not fork the account ledger.
collection_v1:
version: collection-v1-integrated
quota_ledger: /home/ming01.wang/.local/state/bench-shared/quota.sqlite
budget_source: internal_conservative_policy_not_provider_limit
provider_limits_status: UNKNOWN
control_coverage: MANAGED_CALLERS_ONLY
credential_groups:
teacher: glm-coding-account
quota_groups:
glm-coding-account:
max_concurrency: 1
rpm: 4
# The broker reserves UTF-8 bytes as tokens (~3.7x real) and rejects any
# single request above tpm; 500000 killed contexts past ~135k tokens.
tpm: 2000000
lifecycle:
version: collection-v1-phase-budgets-2
ordinary_tool_deadline_s: 300
queue_deadline_s: 1800
execution_deadline_s: 43200
suspect_after_s: 600
cancel_grace_s: 15
cleanup_deadline_s: 60
max_auto_resumes: 3
phases:
01-fetch:
queue_deadline_s: 900
execution_deadline_s: 3600
suspect_after_s: 600
cancel_grace_s: 15
cleanup_deadline_s: 60
02-baseline:
queue_deadline_s: 1800
execution_deadline_s: 7200
suspect_after_s: 900
cancel_grace_s: 15
cleanup_deadline_s: 60
03-export:
queue_deadline_s: 1800
execution_deadline_s: 3600
suspect_after_s: 600
cancel_grace_s: 15
cleanup_deadline_s: 60
04-calibrate:
queue_deadline_s: 1800
execution_deadline_s: 7200
suspect_after_s: 900
cancel_grace_s: 15
cleanup_deadline_s: 60
05-compile:
queue_deadline_s: 3600
execution_deadline_s: 14400
suspect_after_s: 900
cancel_grace_s: 15
cleanup_deadline_s: 60
06-hbm:
queue_deadline_s: 900
execution_deadline_s: 1800
suspect_after_s: 600
cancel_grace_s: 15
cleanup_deadline_s: 60
07-board:
queue_deadline_s: 3600
execution_deadline_s: 1800
suspect_after_s: 600
cancel_grace_s: 15
cleanup_deadline_s: 60
08-eval:
queue_deadline_s: 3600
execution_deadline_s: 7200
suspect_after_s: 900
cancel_grace_s: 15
cleanup_deadline_s: 60
terminal:
enabled: false
board_enabled: false