Test_Agent/tests/utils/flaky_detector.py
jiawei.chen 6f6053ff51 feat: TestAgent 智能测试平台 v1.0
- CLI + Web 双平台,基于 LLM 的 TDD+DDD 自动化测试生成
- 集成 unified-test-platform 903 个 D-Cloud 用例 + 13 个 Robogo 场景
- Agent 5 步流水线: 扫描 -> DDD -> LLM 生成测试
- 精确回归 + UI 变更检查
- 可视化 HTML 测试报告
- 暗色/亮色主题切换
- 19 个 API 端点 + 6 个 SPA 页面
- 15/15 项 TDD+DDD 合规
2026-07-08 11:27:38 +08:00

305 lines
9.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
Flaky 测试检测系统
功能:
- 检测不稳定的测试
- 记录测试历史执行结果
- 提供智能重试机制
- 生成 Flaky 测试报告
"""
import json
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Set
from dataclasses import dataclass, field, asdict
from collections import defaultdict
@dataclass
class TestExecution:
"""单次测试执行记录"""
timestamp: str
status: str # passed, failed, skipped
duration: float
error_message: Optional[str] = None
@dataclass
class TestHistory:
"""测试历史记录"""
test_id: str
test_name: str
executions: List[TestExecution] = field(default_factory=list)
total_runs: int = 0
passed: int = 0
failed: int = 0
skipped: int = 0
@property
def success_rate(self) -> float:
"""成功率"""
if self.total_runs == 0:
return 0.0
return (self.passed / self.total_runs) * 100
@property
def failure_rate(self) -> float:
"""失败率"""
if self.total_runs == 0:
return 0.0
return (self.failed / self.total_runs) * 100
@property
def is_flaky(self) -> bool:
"""是否为 Flaky 测试
判断标准:
1. 执行次数 >= 5
2. 成功率在 50% - 95% 之间(既不是总是失败,也不是总是通过)
"""
if self.total_runs < 5:
return False
# 如果成功率在 50%-95% 之间,认为是 Flaky
return 50.0 < self.success_rate < 95.0
@property
def flaky_score(self) -> float:
"""
Flaky 分数 (0-100)
计算方式:
- 越接近 50%(最不稳定),分数越高
- 100% 或 0% 最稳定,分数为 0
"""
if self.total_runs < 3:
return 0.0
# 使用二次函数计算,50% 时分数最高
rate = self.success_rate
return 100 - abs(rate - 50) * 2
def add_execution(self, execution: TestExecution):
"""添加执行记录"""
self.executions.append(execution)
self.total_runs += 1
if execution.status == "passed":
self.passed += 1
elif execution.status == "failed":
self.failed += 1
elif execution.status == "skipped":
self.skipped += 1
class FlakyTestRegistry:
"""Flaky 测试注册表"""
def __init__(self, storage_dir: str = "test-results/flaky"):
self.storage_dir = Path(storage_dir)
self.storage_dir.mkdir(parents=True, exist_ok=True)
self.history_file = self.storage_dir / "test_history.json"
self.tests: Dict[str, TestHistory] = {}
self.flaky_tests: Set[str] = set()
# 加载历史数据
self._load_history()
def _load_history(self):
"""加载历史数据"""
if not self.history_file.exists():
return
try:
with open(self.history_file, 'r', encoding='utf-8') as f:
data = json.load(f)
for test_id, test_data in data.items():
history = TestHistory(
test_id=test_id,
test_name=test_data['test_name'],
total_runs=test_data['total_runs'],
passed=test_data['passed'],
failed=test_data['failed'],
skipped=test_data['skipped'],
executions=[
TestExecution(**e) for e in test_data['executions']
]
)
self.tests[test_id] = history
if history.is_flaky:
self.flaky_tests.add(test_id)
except Exception as e:
print(f"⚠️ 加载历史数据失败: {e}")
def _save_history(self):
"""保存历史数据"""
data = {}
for test_id, history in self.tests.items():
data[test_id] = asdict(history)
with open(self.history_file, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def record_execution(
self,
test_id: str,
test_name: str,
status: str,
duration: float,
error_message: str = None
):
"""记录测试执行"""
execution = TestExecution(
timestamp=datetime.now().isoformat(),
status=status,
duration=duration,
error_message=error_message
)
if test_id not in self.tests:
self.tests[test_id] = TestHistory(
test_id=test_id,
test_name=test_name
)
history = self.tests[test_id]
history.add_execution(execution)
# 更新 Flaky 测试列表
if history.is_flaky:
self.flaky_tests.add(test_id)
else:
self.flaky_tests.discard(test_id)
# 保存历史
self._save_history()
def get_flaky_tests(self) -> List[TestHistory]:
"""获取所有 Flaky 测试"""
return [
self.tests[test_id]
for test_id in self.flaky_tests
if test_id in self.tests
]
def get_test_history(self, test_id: str) -> Optional[TestHistory]:
"""获取测试历史"""
return self.tests.get(test_id)
def is_flaky(self, test_id: str) -> bool:
"""判断测试是否为 Flaky"""
return test_id in self.flaky_tests
def should_retry(self, test_id: str, max_retries: int = 2) -> bool:
"""
判断是否应该重试测试
对于 Flaky 测试,建议重试
"""
if test_id in self.flaky_tests:
return True
history = self.tests.get(test_id)
if not history:
return False
# 如果最近 3 次执行中至少有 1 次成功,建议重试
recent_executions = history.executions[-3:]
if any(e.status == "passed" for e in recent_executions):
return True
return False
def get_retry_count(self, test_id: str) -> int:
"""
获取建议的重试次数
基于 Flaky 分数动态计算
"""
history = self.tests.get(test_id)
if not history:
return 1
score = history.flaky_score
if score > 80:
return 3 # 非常不稳定,最多重试 3 次
elif score > 60:
return 2 # 中等不稳定,重试 2 次
elif score > 40:
return 1 # 轻度不稳定,重试 1 次
else:
return 0 # 稳定,无需重试
def generate_report(self) -> str:
"""生成 Flaky 测试报告"""
report = []
report.append("=" * 80)
report.append("Flaky 测试报告")
report.append("=" * 80)
report.append(f"\n生成时间: {datetime.now().isoformat()}")
report.append(f"总测试数: {len(self.tests)}")
report.append(f"Flaky 测试数: {len(self.flaky_tests)}")
report.append("")
if self.flaky_tests:
report.append("## Flaky 测试列表")
report.append("")
# 按分数排序
flaky_list = []
for test_id in self.flaky_tests:
history = self.tests[test_id]
flaky_list.append((test_id, history))
flaky_list.sort(key=lambda x: x[1].flaky_score, reverse=True)
for test_id, history in flaky_list:
report.append(f"### {history.test_name}")
report.append(f"- 测试ID: {test_id}")
report.append(f"- 执行次数: {history.total_runs}")
report.append(f"- 成功: {history.passed} | 失败: {history.failed} | 跳过: {history.skipped}")
report.append(f"- 成功率: {history.success_rate:.2f}%")
report.append(f"- Flaky 分数: {history.flaky_score:.2f}")
report.append(f"- 建议重试次数: {self.get_retry_count(test_id)}")
report.append("")
else:
report.append("✅ 未发现 Flaky 测试")
report.append("")
report.append("=" * 80)
return "\n".join(report)
def save_report(self, output_file: str = None) -> str:
"""保存报告"""
if output_file is None:
output_file = self.storage_dir / "flaky-report.txt"
report = self.generate_report()
Path(output_file).write_text(report, encoding='utf-8')
return str(output_file)
# 全局注册表实例
_registry: Optional[FlakyTestRegistry] = None
def get_registry() -> FlakyTestRegistry:
"""获取全局注册表"""
global _registry
if _registry is None:
_registry = FlakyTestRegistry()
return _registry
def reset_registry():
"""重置注册表"""
global _registry
_registry = None