data-gen-bpu/harness/errors.py
bench a263374839 板上会话目录:失败也删,只在守护进程可能还活着时留空目录;进入时清 6 小时没人动的旧目录;磁盘满归瞬态
10-15 S600 的 /userdata 两次写满(01:31–03:25、06:50–06:56),05 的 07-board/08-eval
报 No space left on device,却记成 unknown。
- 原来任何失败都把 eval-* 目录整个留在板上,没人回头看。现在只有清理没确认时才保留目录和回执,
  因为守护进程可能还在写回执和板级隔离标记;其余内容照删(solo.r50 一块输出 12 GB)。
- 容器被杀时 close() 不会运行:每个新会话先删 6 小时没人动过的 eval-* 目录,并把删了什么写进日志。
- No space left on device → disk_full(瞬态)。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 09:41:29 +08:00

49 lines
2.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""stderr → error_class。正则规则库,unknown 进人工分类队列后在这里补规则。
规则顺序即优先级:先匹配到的赢。瞬态类在 TRANSIENT 里,决定重试策略。
"""
from __future__ import annotations
import re
DETERMINISTIC = ["unsupported_op", "shape_infer", "export_fail", "calib_nan", "verifier_mismatch", "config_error",
# 门失败(gates.classify_gate_failure)
"accuracy_collapse", "accuracy_drop", "numeric_mismatch", "march_mismatch", "perf_over_budget"]
TRANSIENT = ["oom", "board_offline", "board_busy", "disk_full", "stalled"]
ALL = DETERMINISTIC + TRANSIENT + ["unknown"]
# (error_class, compiled regex)。写规则时对着真实 stderr 写,别猜。
RULES: list[tuple[str, re.Pattern]] = [
("board_busy", re.compile(r'"event":\s*"board_busy"|board busy after .*refusing contended', re.I)),
("disk_full", re.compile(r"No space left on device")), # 10-15 S600 /userdata: was "unknown", read as the agent's fault
("config_error", re.compile(r"Key '[^']+' error:|Wrong key '|Missing key|yaml.*(invalid|error)|schema.*error", re.I)), # hb_compile yaml 校验不过:harness/agent 配置写错
("oom", re.compile(r"CUDA out of memory|OutOfMemoryError|std::bad_alloc|Cannot allocate memory|MemoryError|Killed process .* out of memory", re.I)),
("board_offline", re.compile(r"Connection (refused|timed out|reset by peer)|No route to host|ssh: connect to host|board .*offline|hrt_.*: failed to connect", re.I)),
("unsupported_op", re.compile(r"(unsupported|not supported|Unsupported) (op|operator|node|type|layer)|No support for op|op .* is not supported on (nash|bpu)|不支持的算子", re.I)),
("shape_infer", re.compile(r"shape infer|ShapeInference|dynamic (shape|axis|dim)|Cannot infer shape|shape mismatch|unknown dimension", re.I)),
("export_fail", re.compile(r"torch\.onnx|onnx export|symbolic_trace|fx\.trace|TracerWarning: .*failed|Exporting the operator .* is not supported|torch\.export", re.I)),
("calib_nan", re.compile(r"\bnan\b.*(calib|quant|scale)|(calib|quant|scale).*\bnan\b|overflow encountered|inf.*threshold|threshold.*inf", re.I)),
("verifier_mismatch", re.compile(r"hb_verifier.*(fail|mismatch)|cosine.*(below|less than|<)|consistency check failed|verify.*mismatch", re.I)),
]
STALLED_MARK = "__BENCH_WATCHDOG_STALLED__"
def classify(stderr_text: str, stdout_text: str = "", exit_code: int | None = None) -> str:
text = (stderr_text or "") + "\n" + (stdout_text or "")
if STALLED_MARK in text:
return "stalled"
for cls, rx in RULES:
if rx.search(text):
return cls
return "unknown"
def is_transient(error_class: str) -> bool:
return error_class in TRANSIENT
def same_class(a: str | None, b: str | None) -> bool:
"""BLOCKED 判定用:同类错误。unknown 不与任何东西同类(避免误判为正确 BLOCKED)。"""
return bool(a) and a == b and a != "unknown"