- CLI + Web 双平台,基于 LLM 的 TDD+DDD 自动化测试生成 - 集成 unified-test-platform 903 个 D-Cloud 用例 + 13 个 Robogo 场景 - Agent 5 步流水线: 扫描 -> DDD -> LLM 生成测试 - 精确回归 + UI 变更检查 - 可视化 HTML 测试报告 - 暗色/亮色主题切换 - 19 个 API 端点 + 6 个 SPA 页面 - 15/15 项 TDD+DDD 合规
305 lines
9.0 KiB
Python
305 lines
9.0 KiB
Python
"""
|
||
Flaky 测试检测系统
|
||
|
||
功能:
|
||
- 检测不稳定的测试
|
||
- 记录测试历史执行结果
|
||
- 提供智能重试机制
|
||
- 生成 Flaky 测试报告
|
||
"""
|
||
|
||
import json
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Dict, List, Optional, Set
|
||
from dataclasses import dataclass, field, asdict
|
||
from collections import defaultdict
|
||
|
||
|
||
@dataclass
|
||
class TestExecution:
|
||
"""单次测试执行记录"""
|
||
timestamp: str
|
||
status: str # passed, failed, skipped
|
||
duration: float
|
||
error_message: Optional[str] = None
|
||
|
||
|
||
@dataclass
|
||
class TestHistory:
|
||
"""测试历史记录"""
|
||
test_id: str
|
||
test_name: str
|
||
executions: List[TestExecution] = field(default_factory=list)
|
||
total_runs: int = 0
|
||
passed: int = 0
|
||
failed: int = 0
|
||
skipped: int = 0
|
||
|
||
@property
|
||
def success_rate(self) -> float:
|
||
"""成功率"""
|
||
if self.total_runs == 0:
|
||
return 0.0
|
||
return (self.passed / self.total_runs) * 100
|
||
|
||
@property
|
||
def failure_rate(self) -> float:
|
||
"""失败率"""
|
||
if self.total_runs == 0:
|
||
return 0.0
|
||
return (self.failed / self.total_runs) * 100
|
||
|
||
@property
|
||
def is_flaky(self) -> bool:
|
||
"""是否为 Flaky 测试
|
||
|
||
判断标准:
|
||
1. 执行次数 >= 5
|
||
2. 成功率在 50% - 95% 之间(既不是总是失败,也不是总是通过)
|
||
"""
|
||
if self.total_runs < 5:
|
||
return False
|
||
|
||
# 如果成功率在 50%-95% 之间,认为是 Flaky
|
||
return 50.0 < self.success_rate < 95.0
|
||
|
||
@property
|
||
def flaky_score(self) -> float:
|
||
"""
|
||
Flaky 分数 (0-100)
|
||
|
||
计算方式:
|
||
- 越接近 50%(最不稳定),分数越高
|
||
- 100% 或 0% 最稳定,分数为 0
|
||
"""
|
||
if self.total_runs < 3:
|
||
return 0.0
|
||
|
||
# 使用二次函数计算,50% 时分数最高
|
||
rate = self.success_rate
|
||
return 100 - abs(rate - 50) * 2
|
||
|
||
def add_execution(self, execution: TestExecution):
|
||
"""添加执行记录"""
|
||
self.executions.append(execution)
|
||
self.total_runs += 1
|
||
|
||
if execution.status == "passed":
|
||
self.passed += 1
|
||
elif execution.status == "failed":
|
||
self.failed += 1
|
||
elif execution.status == "skipped":
|
||
self.skipped += 1
|
||
|
||
|
||
class FlakyTestRegistry:
|
||
"""Flaky 测试注册表"""
|
||
|
||
def __init__(self, storage_dir: str = "test-results/flaky"):
|
||
self.storage_dir = Path(storage_dir)
|
||
self.storage_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
self.history_file = self.storage_dir / "test_history.json"
|
||
self.tests: Dict[str, TestHistory] = {}
|
||
self.flaky_tests: Set[str] = set()
|
||
|
||
# 加载历史数据
|
||
self._load_history()
|
||
|
||
def _load_history(self):
|
||
"""加载历史数据"""
|
||
if not self.history_file.exists():
|
||
return
|
||
|
||
try:
|
||
with open(self.history_file, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
|
||
for test_id, test_data in data.items():
|
||
history = TestHistory(
|
||
test_id=test_id,
|
||
test_name=test_data['test_name'],
|
||
total_runs=test_data['total_runs'],
|
||
passed=test_data['passed'],
|
||
failed=test_data['failed'],
|
||
skipped=test_data['skipped'],
|
||
executions=[
|
||
TestExecution(**e) for e in test_data['executions']
|
||
]
|
||
)
|
||
self.tests[test_id] = history
|
||
|
||
if history.is_flaky:
|
||
self.flaky_tests.add(test_id)
|
||
except Exception as e:
|
||
print(f"⚠️ 加载历史数据失败: {e}")
|
||
|
||
def _save_history(self):
|
||
"""保存历史数据"""
|
||
data = {}
|
||
for test_id, history in self.tests.items():
|
||
data[test_id] = asdict(history)
|
||
|
||
with open(self.history_file, 'w', encoding='utf-8') as f:
|
||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||
|
||
def record_execution(
|
||
self,
|
||
test_id: str,
|
||
test_name: str,
|
||
status: str,
|
||
duration: float,
|
||
error_message: str = None
|
||
):
|
||
"""记录测试执行"""
|
||
execution = TestExecution(
|
||
timestamp=datetime.now().isoformat(),
|
||
status=status,
|
||
duration=duration,
|
||
error_message=error_message
|
||
)
|
||
|
||
if test_id not in self.tests:
|
||
self.tests[test_id] = TestHistory(
|
||
test_id=test_id,
|
||
test_name=test_name
|
||
)
|
||
|
||
history = self.tests[test_id]
|
||
history.add_execution(execution)
|
||
|
||
# 更新 Flaky 测试列表
|
||
if history.is_flaky:
|
||
self.flaky_tests.add(test_id)
|
||
else:
|
||
self.flaky_tests.discard(test_id)
|
||
|
||
# 保存历史
|
||
self._save_history()
|
||
|
||
def get_flaky_tests(self) -> List[TestHistory]:
|
||
"""获取所有 Flaky 测试"""
|
||
return [
|
||
self.tests[test_id]
|
||
for test_id in self.flaky_tests
|
||
if test_id in self.tests
|
||
]
|
||
|
||
def get_test_history(self, test_id: str) -> Optional[TestHistory]:
|
||
"""获取测试历史"""
|
||
return self.tests.get(test_id)
|
||
|
||
def is_flaky(self, test_id: str) -> bool:
|
||
"""判断测试是否为 Flaky"""
|
||
return test_id in self.flaky_tests
|
||
|
||
def should_retry(self, test_id: str, max_retries: int = 2) -> bool:
|
||
"""
|
||
判断是否应该重试测试
|
||
|
||
对于 Flaky 测试,建议重试
|
||
"""
|
||
if test_id in self.flaky_tests:
|
||
return True
|
||
|
||
history = self.tests.get(test_id)
|
||
if not history:
|
||
return False
|
||
|
||
# 如果最近 3 次执行中至少有 1 次成功,建议重试
|
||
recent_executions = history.executions[-3:]
|
||
if any(e.status == "passed" for e in recent_executions):
|
||
return True
|
||
|
||
return False
|
||
|
||
def get_retry_count(self, test_id: str) -> int:
|
||
"""
|
||
获取建议的重试次数
|
||
|
||
基于 Flaky 分数动态计算
|
||
"""
|
||
history = self.tests.get(test_id)
|
||
if not history:
|
||
return 1
|
||
|
||
score = history.flaky_score
|
||
|
||
if score > 80:
|
||
return 3 # 非常不稳定,最多重试 3 次
|
||
elif score > 60:
|
||
return 2 # 中等不稳定,重试 2 次
|
||
elif score > 40:
|
||
return 1 # 轻度不稳定,重试 1 次
|
||
else:
|
||
return 0 # 稳定,无需重试
|
||
|
||
def generate_report(self) -> str:
|
||
"""生成 Flaky 测试报告"""
|
||
report = []
|
||
report.append("=" * 80)
|
||
report.append("Flaky 测试报告")
|
||
report.append("=" * 80)
|
||
report.append(f"\n生成时间: {datetime.now().isoformat()}")
|
||
report.append(f"总测试数: {len(self.tests)}")
|
||
report.append(f"Flaky 测试数: {len(self.flaky_tests)}")
|
||
report.append("")
|
||
|
||
if self.flaky_tests:
|
||
report.append("## Flaky 测试列表")
|
||
report.append("")
|
||
|
||
# 按分数排序
|
||
flaky_list = []
|
||
for test_id in self.flaky_tests:
|
||
history = self.tests[test_id]
|
||
flaky_list.append((test_id, history))
|
||
|
||
flaky_list.sort(key=lambda x: x[1].flaky_score, reverse=True)
|
||
|
||
for test_id, history in flaky_list:
|
||
report.append(f"### {history.test_name}")
|
||
report.append(f"- 测试ID: {test_id}")
|
||
report.append(f"- 执行次数: {history.total_runs}")
|
||
report.append(f"- 成功: {history.passed} | 失败: {history.failed} | 跳过: {history.skipped}")
|
||
report.append(f"- 成功率: {history.success_rate:.2f}%")
|
||
report.append(f"- Flaky 分数: {history.flaky_score:.2f}")
|
||
report.append(f"- 建议重试次数: {self.get_retry_count(test_id)}")
|
||
report.append("")
|
||
else:
|
||
report.append("✅ 未发现 Flaky 测试")
|
||
|
||
report.append("")
|
||
report.append("=" * 80)
|
||
|
||
return "\n".join(report)
|
||
|
||
def save_report(self, output_file: str = None) -> str:
|
||
"""保存报告"""
|
||
if output_file is None:
|
||
output_file = self.storage_dir / "flaky-report.txt"
|
||
|
||
report = self.generate_report()
|
||
Path(output_file).write_text(report, encoding='utf-8')
|
||
|
||
return str(output_file)
|
||
|
||
|
||
# 全局注册表实例
|
||
_registry: Optional[FlakyTestRegistry] = None
|
||
|
||
|
||
def get_registry() -> FlakyTestRegistry:
|
||
"""获取全局注册表"""
|
||
global _registry
|
||
if _registry is None:
|
||
_registry = FlakyTestRegistry()
|
||
return _registry
|
||
|
||
|
||
def reset_registry():
|
||
"""重置注册表"""
|
||
global _registry
|
||
_registry = None
|