droboticflow/tests/release/test_dashboard_stability.py

58 lines
2.9 KiB
Python

import importlib.util
from pathlib import Path
import unittest
import json
from unittest.mock import patch
from types import SimpleNamespace
ROOT = Path(__file__).resolve().parents[2]
def load(name):
spec = importlib.util.spec_from_file_location(name, ROOT / 'scripts' / (name + '.py'))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
health = load('check_dashboard_health')
gate = load('test_dashboard_stability')
class DashboardStabilityTests(unittest.TestCase):
def test_health_verdicts_fail_closed(self):
for body, expected in [
({'status': 'healthy', 'available': True}, 0),
({'status': 'idle', 'available': False}, 0),
({'status': 'degraded', 'available': True}, 1),
({'status': 'unavailable', 'available': False}, 2),
({'status': 'not_built'}, 3), ({'status': 'building'}, 3),
({'status': 'healthy', 'available': False}, 3),
({'status': 'ok'}, 3), ([], 3), (None, 3),
]:
with self.subTest(body=body):
self.assertEqual(health.classify(body), expected)
def test_gate_requires_every_regression_to_execute(self):
events = [{'Action': 'pass', 'Test': name} for name in gate.REQUIRED]
self.assertTrue(gate.verify_events(events, 0)['passed'])
self.assertFalse(gate.verify_events(events[:-1], 0)['passed'])
self.assertFalse(gate.verify_events(events, 1)['passed'])
self.assertFalse(gate.verify_events(events + [{'Action': 'skip', 'Test': 'MongoMissing'}], 0)['passed'])
self.assertFalse(gate.verify_events([], 0)['passed'])
def test_gate_runs_race_and_uninstrumented_timing_and_fails_closed(self):
scale = {"TestAnnotationOutputSnapshotScale", "TestAnnotationOutputSnapshotScale/production-quality-concurrent-warm-reads"}
for fail_timing in (False, True):
calls = []
def run(command, **kwargs):
calls.append(command)
names = gate.REQUIRED | scale if "-race" in command else ({"TestStatisticsPartitionGrowthBeyondGlobalBudget"} if "^TestStatisticsPartitionGrowthBeyondGlobalBudget$" in command else scale)
return SimpleNamespace(returncode=int(fail_timing and "-race" not in command),
stdout="\n".join(json.dumps({"Action": "pass", "Test": name}) for name in names), stderr="")
with patch.object(gate.subprocess, "run", side_effect=run), patch("builtins.print"):
self.assertEqual(gate.main(), int(fail_timing))
self.assertEqual(len(calls), 3)
self.assertIn("-race", calls[0])
self.assertNotIn("-race", calls[1])
self.assertIn("-race=false", calls[1])
self.assertNotIn("-race", calls[2])
self.assertIn("^TestStatisticsPartitionGrowthBeyondGlobalBudget$", calls[2])