data-gen-bpu/harness/errors.py

48 lines
2.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""stderr → error_class。正则规则库,unknown 进人工分类队列后在这里补规则。
规则顺序即优先级:先匹配到的赢。瞬态类在 TRANSIENT 里,决定重试策略。
"""
from __future__ import annotations
import re
DETERMINISTIC = ["unsupported_op", "shape_infer", "export_fail", "calib_nan", "verifier_mismatch", "config_error",
# 门失败(gates.classify_gate_failure)
"accuracy_collapse", "accuracy_drop", "numeric_mismatch", "march_mismatch", "perf_over_budget"]
TRANSIENT = ["oom", "board_offline", "board_busy", "stalled"]
ALL = DETERMINISTIC + TRANSIENT + ["unknown"]
# (error_class, compiled regex)。写规则时对着真实 stderr 写,别猜。
RULES: list[tuple[str, re.Pattern]] = [
("board_busy", re.compile(r'"event":\s*"board_busy"|board busy after .*refusing contended', re.I)),
("config_error", re.compile(r"Key '[^']+' error:|Wrong key '|Missing key|yaml.*(invalid|error)|schema.*error", re.I)), # hb_compile yaml 校验不过:harness/agent 配置写错
("oom", re.compile(r"CUDA out of memory|OutOfMemoryError|std::bad_alloc|Cannot allocate memory|MemoryError|Killed process .* out of memory", re.I)),
("board_offline", re.compile(r"Connection (refused|timed out|reset by peer)|No route to host|ssh: connect to host|board .*offline|hrt_.*: failed to connect", re.I)),
("unsupported_op", re.compile(r"(unsupported|not supported|Unsupported) (op|operator|node|type|layer)|No support for op|op .* is not supported on (nash|bpu)|不支持的算子", re.I)),
("shape_infer", re.compile(r"shape infer|ShapeInference|dynamic (shape|axis|dim)|Cannot infer shape|shape mismatch|unknown dimension", re.I)),
("export_fail", re.compile(r"torch\.onnx|onnx export|symbolic_trace|fx\.trace|TracerWarning: .*failed|Exporting the operator .* is not supported|torch\.export", re.I)),
("calib_nan", re.compile(r"\bnan\b.*(calib|quant|scale)|(calib|quant|scale).*\bnan\b|overflow encountered|inf.*threshold|threshold.*inf", re.I)),
("verifier_mismatch", re.compile(r"hb_verifier.*(fail|mismatch)|cosine.*(below|less than|<)|consistency check failed|verify.*mismatch", re.I)),
]
STALLED_MARK = "__BENCH_WATCHDOG_STALLED__"
def classify(stderr_text: str, stdout_text: str = "", exit_code: int | None = None) -> str:
text = (stderr_text or "") + "\n" + (stdout_text or "")
if STALLED_MARK in text:
return "stalled"
for cls, rx in RULES:
if rx.search(text):
return cls
return "unknown"
def is_transient(error_class: str) -> bool:
return error_class in TRANSIENT
def same_class(a: str | None, b: str | None) -> bool:
"""BLOCKED 判定用:同类错误。unknown 不与任何东西同类(避免误判为正确 BLOCKED)。"""
return bool(a) and a == b and a != "unknown"