57 of the 68 "incomplete generation" rollouts were never truncated by the provider: the local broker refused the request with 413 REQUEST_EXCEEDS_INTERNAL_TPM. It reserves UTF-8 body bytes as tokens (about 3.7x the real count) and rejects any single request above tpm, so tpm 500000 ended every trajectory once its context passed ~135k tokens, mostly the long pose/mde/seg ones. GLM-5.3 accepts far longer contexts. Raise the default GLM tpm to 2000000, as the Qwen group already uses. Production configs and the stored ledger policy are updated alongside. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
173 lines
7.7 KiB
YAML
173 lines
7.7 KiB
YAML
# bench/config.yaml — 运行时配置(不进 job.json;job.json 只快照与结果相关的东西)
|
||
version: 1
|
||
|
||
paths:
|
||
# 项目根;harness 默认取本文件所在目录的上一级
|
||
project_root: null
|
||
# 两套 venv。harness 按 backend 选,agent 无权改
|
||
venv:
|
||
ptq: ~/toolchains/oe-3.7.0/venv # hb_compile / hb_model_info / hb_verifier(无 pip,不装东西)
|
||
qat: ~/toolchains/oellm2/venv # plugin + hbdk4 4.12 + hbm_infer(不装东西)
|
||
bench: ~/toolchains/bench-venv # 我们自己的:torch/torchvision/ultralytics/pycocotools/pyarrow — 权重、导 ONNX、FP32 基线、评测打分、数据入池
|
||
# harness 自身用的解释器(只需 pyyaml),默认 python3
|
||
harness_python: python3
|
||
# hb_compile 的 working_dir 根目录(中间产物几百 MB/个)。必须放本地盘:
|
||
# 实测 JuiceFS 写 1 MB=360 ms vs 本地盘 3 ms,大模型编译会卡在 D 状态被看门狗误判挂死。
|
||
# 留空则退回 runs/<run>/<job>/oracle/stages/05-compile/work(JuiceFS 上,只适合小规模)。
|
||
work_root: /home/ming01.wang/bench-work
|
||
# .horizon skill pack(只读)
|
||
horizon: ../.horizon
|
||
# agent 运行时(scripts/setup_agent.sh → ~/toolchains/node;pi 是 node 脚本,node 必须在 PATH 或这里给全路径)
|
||
pi_bin: ~/toolchains/node/bin/pi
|
||
node_bin: ~/toolchains/node/bin/node
|
||
|
||
pools:
|
||
# 机器是共享的:2026-09-16 恢复时 GPU 0–7 均被其他训练占用。
|
||
# devices: [-1] 通过 CUDA_VISIBLE_DEVICES 隐藏 GPU,让新增 FP32 基线用 CPU;已有基线照常复用。
|
||
# GPU 空闲后可将 devices 恢复为可用卡号,槽位 i → devices[i % len(devices)]。
|
||
# reserve_for_agent:只给 rollout 的 `bench stage run` 用的槽位,oracle 拿不到。
|
||
# 实测(2026-09-15):两个 oracle 调度进程把 cpu 8/8 占满,第一条 rollout 的编译在队列里饿了 60 分钟。
|
||
gpu: { workers: 4, devices: [-1], reserve_for_agent: 1 } # FP32 基线槽;当前 CPU 后端
|
||
cpu: { workers: 24, reserve_for_agent: 12 } # export / hb_compile;给真实 Agent 轨迹保留 12 槽
|
||
# 2026-09-16 提高采集并发;普通流水线默认 24 个在途任务,共享此 CPU 上限。
|
||
# 244 个逻辑 CPU / 1.4 TiB 内存;恢复后以实际完成速度、内存和板卡排队情况检查此上限。
|
||
board: { workers: 1, reserve_for_agent: 1 } # 新调度器按 boards.yaml 的实际板数分配;workers 仅兼容旧进程
|
||
agent: { workers: 32, rpm: 60 } # 真实 Agent 并发(2026-09-20 12→24→32;acquire 每轮重读 config 热生效,但 collector 线程数启动时算死,扩并发须新起 collector。实测 17 并发 ~15k TPM 无 429;32 并发估算中位 28k / p80 45k TPM、~19rpm);rpm 是端点配额记录,当前调度器未据此限流
|
||
|
||
watchdog:
|
||
stall_s: 900 # 日志不增长 且 进程树 CPU < cpu_idle_pct 持续 stall_s → STALLED
|
||
cpu_idle_pct: 5
|
||
poll_s: 30
|
||
hard_ceiling_s: 86400 # 24h 纯保险
|
||
io_stall_s: 21600 # 树内持续有 D 线程(卡在 I/O)这么久 → 也判定异常。
|
||
# 别往下调:线程级 D 检测上线前这条是死代码(d_count 恒 0),
|
||
# 上线后它第一次生效,收紧会把"链接期正在写盘"换个理由再杀一次。
|
||
no_progress_ceiling_s: 28800 # 日志与 CPU 都不动这么久 → 兜底,免疫 D 抖动。
|
||
# 必须 > io_stall_s,否则它抢先命中、盖掉更准确的 STALLED 原因。
|
||
# 四条阈值单调:stall_s(long 3600) < io_stall_s < 本条 < hard_ceiling_s
|
||
soft_alert: p95x3 # 超过历史 p95 × 3 只告警,不杀
|
||
|
||
perf:
|
||
warmup: 20
|
||
runs: 200
|
||
stats: [mean, p95]
|
||
|
||
rollouts:
|
||
private_home: true # Required by the managed Pi runner.
|
||
k: 4
|
||
max_turns: 60 # 照抄 oracle 27 轮够;真调优一轮迭代(改配置→编译→看日志)5–8 轮,40 只够两次半
|
||
max_tokens: 200000
|
||
hard_ceiling_s: 43200 # 12h 只防 agent 进程挂死
|
||
jail: true # 用 scripts/agent_jail.sh(unshare -rm)把 oracle/ 兄弟 rollout/ ~/.pi 从文件系统里摘掉。
|
||
# path-guard 只是"给句人话的拒绝理由",解析 shell 字符串永远追不平所有绕法。
|
||
# 关掉只在调试 agent 行为时用,采数据时必须开。
|
||
# --tools 是白名单,同时作用于内置工具与 extension 注册的工具。
|
||
# 踩过(2026-09-15 第一条轨迹):claim / oe_doc 没列进来 → claim-card 注册了工具但被白名单关掉,
|
||
# agent 全程无法 claim,213 次 bash 调用一次 claim 都没有。extension 工具必须显式列出。
|
||
tools: [read, edit, write, bash, grep, claim, oe_doc]
|
||
|
||
retry:
|
||
transient: 2 # oom / board_offline
|
||
stalled: 1 # 进 long 车道
|
||
deterministic: 0
|
||
|
||
scheduling:
|
||
priority_order: [A, B, C]
|
||
c_lane: low # C 类走独立低优先级车道
|
||
estimate_source: history_then_params
|
||
|
||
data:
|
||
default_tier: t1
|
||
calib_n_default: 128
|
||
|
||
# LLM 端点。key 不写这里,写 bench/.env.local(gitignore)
|
||
endpoints:
|
||
teacher:
|
||
provider: openai-compatible
|
||
base_url: https://open.bigmodel.cn/api/coding/paas/v4
|
||
model: glm-5.3
|
||
api_key_env: TEACHER_API_KEY
|
||
student:
|
||
provider: openai-compatible
|
||
base_url: null # vLLM 起来后填
|
||
model: null
|
||
api_key_env: STUDENT_API_KEY
|
||
|
||
# Shared execution / quota policy; do not fork the account ledger.
|
||
collection_v1:
|
||
version: collection-v1-integrated
|
||
quota_ledger: /home/ming01.wang/.local/state/bench-shared/quota.sqlite
|
||
budget_source: internal_conservative_policy_not_provider_limit
|
||
provider_limits_status: UNKNOWN
|
||
control_coverage: MANAGED_CALLERS_ONLY
|
||
credential_groups:
|
||
teacher: glm-coding-account
|
||
quota_groups:
|
||
glm-coding-account:
|
||
max_concurrency: 1
|
||
rpm: 4
|
||
# The broker reserves UTF-8 bytes as tokens (~3.7x real) and rejects any
|
||
# single request above tpm; 500000 killed contexts past ~135k tokens.
|
||
tpm: 2000000
|
||
lifecycle:
|
||
version: collection-v1-phase-budgets-2
|
||
ordinary_tool_deadline_s: 300
|
||
queue_deadline_s: 1800
|
||
execution_deadline_s: 43200
|
||
suspect_after_s: 600
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
max_auto_resumes: 3
|
||
phases:
|
||
01-fetch:
|
||
queue_deadline_s: 900
|
||
execution_deadline_s: 3600
|
||
suspect_after_s: 600
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
02-baseline:
|
||
queue_deadline_s: 1800
|
||
execution_deadline_s: 7200
|
||
suspect_after_s: 900
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
03-export:
|
||
queue_deadline_s: 1800
|
||
execution_deadline_s: 3600
|
||
suspect_after_s: 600
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
04-calibrate:
|
||
queue_deadline_s: 1800
|
||
execution_deadline_s: 7200
|
||
suspect_after_s: 900
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
05-compile:
|
||
queue_deadline_s: 3600
|
||
execution_deadline_s: 14400
|
||
suspect_after_s: 900
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
06-hbm:
|
||
queue_deadline_s: 900
|
||
execution_deadline_s: 1800
|
||
suspect_after_s: 600
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
07-board:
|
||
queue_deadline_s: 3600
|
||
execution_deadline_s: 1800
|
||
suspect_after_s: 600
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
08-eval:
|
||
queue_deadline_s: 3600
|
||
execution_deadline_s: 7200
|
||
suspect_after_s: 900
|
||
cancel_grace_s: 15
|
||
cleanup_deadline_s: 60
|
||
terminal:
|
||
enabled: false
|
||
board_enabled: false
|