""" Flaky 测试检测系统 功能: - 检测不稳定的测试 - 记录测试历史执行结果 - 提供智能重试机制 - 生成 Flaky 测试报告 """ import json from datetime import datetime from pathlib import Path from typing import Dict, List, Optional, Set from dataclasses import dataclass, field, asdict from collections import defaultdict @dataclass class TestExecution: """单次测试执行记录""" timestamp: str status: str # passed, failed, skipped duration: float error_message: Optional[str] = None @dataclass class TestHistory: """测试历史记录""" test_id: str test_name: str executions: List[TestExecution] = field(default_factory=list) total_runs: int = 0 passed: int = 0 failed: int = 0 skipped: int = 0 @property def success_rate(self) -> float: """成功率""" if self.total_runs == 0: return 0.0 return (self.passed / self.total_runs) * 100 @property def failure_rate(self) -> float: """失败率""" if self.total_runs == 0: return 0.0 return (self.failed / self.total_runs) * 100 @property def is_flaky(self) -> bool: """是否为 Flaky 测试 判断标准: 1. 执行次数 >= 5 2. 成功率在 50% - 95% 之间(既不是总是失败,也不是总是通过) """ if self.total_runs < 5: return False # 如果成功率在 50%-95% 之间,认为是 Flaky return 50.0 < self.success_rate < 95.0 @property def flaky_score(self) -> float: """ Flaky 分数 (0-100) 计算方式: - 越接近 50%(最不稳定),分数越高 - 100% 或 0% 最稳定,分数为 0 """ if self.total_runs < 3: return 0.0 # 使用二次函数计算,50% 时分数最高 rate = self.success_rate return 100 - abs(rate - 50) * 2 def add_execution(self, execution: TestExecution): """添加执行记录""" self.executions.append(execution) self.total_runs += 1 if execution.status == "passed": self.passed += 1 elif execution.status == "failed": self.failed += 1 elif execution.status == "skipped": self.skipped += 1 class FlakyTestRegistry: """Flaky 测试注册表""" def __init__(self, storage_dir: str = "test-results/flaky"): self.storage_dir = Path(storage_dir) self.storage_dir.mkdir(parents=True, exist_ok=True) self.history_file = self.storage_dir / "test_history.json" self.tests: Dict[str, TestHistory] = {} self.flaky_tests: Set[str] = set() # 加载历史数据 self._load_history() def _load_history(self): """加载历史数据""" if not self.history_file.exists(): return try: with open(self.history_file, 'r', encoding='utf-8') as f: data = json.load(f) for test_id, test_data in data.items(): history = TestHistory( test_id=test_id, test_name=test_data['test_name'], total_runs=test_data['total_runs'], passed=test_data['passed'], failed=test_data['failed'], skipped=test_data['skipped'], executions=[ TestExecution(**e) for e in test_data['executions'] ] ) self.tests[test_id] = history if history.is_flaky: self.flaky_tests.add(test_id) except Exception as e: print(f"⚠️ 加载历史数据失败: {e}") def _save_history(self): """保存历史数据""" data = {} for test_id, history in self.tests.items(): data[test_id] = asdict(history) with open(self.history_file, 'w', encoding='utf-8') as f: json.dump(data, f, indent=2, ensure_ascii=False) def record_execution( self, test_id: str, test_name: str, status: str, duration: float, error_message: str = None ): """记录测试执行""" execution = TestExecution( timestamp=datetime.now().isoformat(), status=status, duration=duration, error_message=error_message ) if test_id not in self.tests: self.tests[test_id] = TestHistory( test_id=test_id, test_name=test_name ) history = self.tests[test_id] history.add_execution(execution) # 更新 Flaky 测试列表 if history.is_flaky: self.flaky_tests.add(test_id) else: self.flaky_tests.discard(test_id) # 保存历史 self._save_history() def get_flaky_tests(self) -> List[TestHistory]: """获取所有 Flaky 测试""" return [ self.tests[test_id] for test_id in self.flaky_tests if test_id in self.tests ] def get_test_history(self, test_id: str) -> Optional[TestHistory]: """获取测试历史""" return self.tests.get(test_id) def is_flaky(self, test_id: str) -> bool: """判断测试是否为 Flaky""" return test_id in self.flaky_tests def should_retry(self, test_id: str, max_retries: int = 2) -> bool: """ 判断是否应该重试测试 对于 Flaky 测试,建议重试 """ if test_id in self.flaky_tests: return True history = self.tests.get(test_id) if not history: return False # 如果最近 3 次执行中至少有 1 次成功,建议重试 recent_executions = history.executions[-3:] if any(e.status == "passed" for e in recent_executions): return True return False def get_retry_count(self, test_id: str) -> int: """ 获取建议的重试次数 基于 Flaky 分数动态计算 """ history = self.tests.get(test_id) if not history: return 1 score = history.flaky_score if score > 80: return 3 # 非常不稳定,最多重试 3 次 elif score > 60: return 2 # 中等不稳定,重试 2 次 elif score > 40: return 1 # 轻度不稳定,重试 1 次 else: return 0 # 稳定,无需重试 def generate_report(self) -> str: """生成 Flaky 测试报告""" report = [] report.append("=" * 80) report.append("Flaky 测试报告") report.append("=" * 80) report.append(f"\n生成时间: {datetime.now().isoformat()}") report.append(f"总测试数: {len(self.tests)}") report.append(f"Flaky 测试数: {len(self.flaky_tests)}") report.append("") if self.flaky_tests: report.append("## Flaky 测试列表") report.append("") # 按分数排序 flaky_list = [] for test_id in self.flaky_tests: history = self.tests[test_id] flaky_list.append((test_id, history)) flaky_list.sort(key=lambda x: x[1].flaky_score, reverse=True) for test_id, history in flaky_list: report.append(f"### {history.test_name}") report.append(f"- 测试ID: {test_id}") report.append(f"- 执行次数: {history.total_runs}") report.append(f"- 成功: {history.passed} | 失败: {history.failed} | 跳过: {history.skipped}") report.append(f"- 成功率: {history.success_rate:.2f}%") report.append(f"- Flaky 分数: {history.flaky_score:.2f}") report.append(f"- 建议重试次数: {self.get_retry_count(test_id)}") report.append("") else: report.append("✅ 未发现 Flaky 测试") report.append("") report.append("=" * 80) return "\n".join(report) def save_report(self, output_file: str = None) -> str: """保存报告""" if output_file is None: output_file = self.storage_dir / "flaky-report.txt" report = self.generate_report() Path(output_file).write_text(report, encoding='utf-8') return str(output_file) # 全局注册表实例 _registry: Optional[FlakyTestRegistry] = None def get_registry() -> FlakyTestRegistry: """获取全局注册表""" global _registry if _registry is None: _registry = FlakyTestRegistry() return _registry def reset_registry(): """重置注册表""" global _registry _registry = None