10-15 S600 的 /userdata 两次写满(01:31–03:25、06:50–06:56),05 的 07-board/08-eval 报 No space left on device,却记成 unknown。 - 原来任何失败都把 eval-* 目录整个留在板上,没人回头看。现在只有清理没确认时才保留目录和回执, 因为守护进程可能还在写回执和板级隔离标记;其余内容照删(solo.r50 一块输出 12 GB)。 - 容器被杀时 close() 不会运行:每个新会话先删 6 小时没人动过的 eval-* 目录,并把删了什么写进日志。 - No space left on device → disk_full(瞬态)。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
49 lines
2.9 KiB
Python
49 lines
2.9 KiB
Python
"""stderr → error_class。正则规则库,unknown 进人工分类队列后在这里补规则。
|
||
|
||
规则顺序即优先级:先匹配到的赢。瞬态类在 TRANSIENT 里,决定重试策略。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
|
||
DETERMINISTIC = ["unsupported_op", "shape_infer", "export_fail", "calib_nan", "verifier_mismatch", "config_error",
|
||
# 门失败(gates.classify_gate_failure)
|
||
"accuracy_collapse", "accuracy_drop", "numeric_mismatch", "march_mismatch", "perf_over_budget"]
|
||
TRANSIENT = ["oom", "board_offline", "board_busy", "disk_full", "stalled"]
|
||
ALL = DETERMINISTIC + TRANSIENT + ["unknown"]
|
||
|
||
# (error_class, compiled regex)。写规则时对着真实 stderr 写,别猜。
|
||
RULES: list[tuple[str, re.Pattern]] = [
|
||
("board_busy", re.compile(r'"event":\s*"board_busy"|board busy after .*refusing contended', re.I)),
|
||
("disk_full", re.compile(r"No space left on device")), # 10-15 S600 /userdata: was "unknown", read as the agent's fault
|
||
("config_error", re.compile(r"Key '[^']+' error:|Wrong key '|Missing key|yaml.*(invalid|error)|schema.*error", re.I)), # hb_compile yaml 校验不过:harness/agent 配置写错
|
||
("oom", re.compile(r"CUDA out of memory|OutOfMemoryError|std::bad_alloc|Cannot allocate memory|MemoryError|Killed process .* out of memory", re.I)),
|
||
("board_offline", re.compile(r"Connection (refused|timed out|reset by peer)|No route to host|ssh: connect to host|board .*offline|hrt_.*: failed to connect", re.I)),
|
||
("unsupported_op", re.compile(r"(unsupported|not supported|Unsupported) (op|operator|node|type|layer)|No support for op|op .* is not supported on (nash|bpu)|不支持的算子", re.I)),
|
||
("shape_infer", re.compile(r"shape infer|ShapeInference|dynamic (shape|axis|dim)|Cannot infer shape|shape mismatch|unknown dimension", re.I)),
|
||
("export_fail", re.compile(r"torch\.onnx|onnx export|symbolic_trace|fx\.trace|TracerWarning: .*failed|Exporting the operator .* is not supported|torch\.export", re.I)),
|
||
("calib_nan", re.compile(r"\bnan\b.*(calib|quant|scale)|(calib|quant|scale).*\bnan\b|overflow encountered|inf.*threshold|threshold.*inf", re.I)),
|
||
("verifier_mismatch", re.compile(r"hb_verifier.*(fail|mismatch)|cosine.*(below|less than|<)|consistency check failed|verify.*mismatch", re.I)),
|
||
]
|
||
|
||
STALLED_MARK = "__BENCH_WATCHDOG_STALLED__"
|
||
|
||
|
||
def classify(stderr_text: str, stdout_text: str = "", exit_code: int | None = None) -> str:
|
||
text = (stderr_text or "") + "\n" + (stdout_text or "")
|
||
if STALLED_MARK in text:
|
||
return "stalled"
|
||
for cls, rx in RULES:
|
||
if rx.search(text):
|
||
return cls
|
||
return "unknown"
|
||
|
||
|
||
def is_transient(error_class: str) -> bool:
|
||
return error_class in TRANSIENT
|
||
|
||
|
||
def same_class(a: str | None, b: str | None) -> bool:
|
||
"""BLOCKED 判定用:同类错误。unknown 不与任何东西同类(避免误判为正确 BLOCKED)。"""
|
||
return bool(a) and a == b and a != "unknown"
|