droboticflow/scripts/test_dashboard_stability.py

81 lines
4.2 KiB
Python

#!/usr/bin/env python3
"""Release gate: synthetic local Mongo + real loader/API/recovery regressions.
The fixed loopback URI is intentional. Never accepts a production database URI.
"""
import json
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
REQUIRED = {
"TestAnnotationOutputLargeRelatedSetsRemainBoundedAndComplete",
"TestAnnotationOutputNonReportingHistoryDoesNotConsumeBudget",
"TestAnnotationOutputRunLinkIndexIsDeclared",
"TestAnnotationOutputMongoFailureDiagnosticsSurviveRestart",
"TestAnnotationOutputMongoSharedAcrossInstancesAndRestart",
"TestAnnotationOutputHealthIsAdminScopedAndReadOnly",
"TestFailureDiagnosticsSurviveLeaseTakeoverAndClearAfterSuccess",
"TestHealthWarnsBeforeExpiryAndDoesNotTriggerBuild",
"TestAnnotationOutputDatasetPreservesRulesAcrossFilters",
"TestRouteSnapshot",
"TestAnnotationOutputMongoShardedRestartAndFailedBuild",
"TestAnnotationOutputStreamFailureCleansPrivateFiles",
"TestAnnotationOutputStreamMatchesWholeHistoryAcrossLegacyReferences",
"TestShardedGenerationMatchesLegacyAndSurvivesCorruptionAndRestart",
"TestShardedBuildFailureNeverPublishesPartialManifest",
"TestOutputReducerMatchesWholeDataset",
"TestOutputSpoolCleanupRecoversCrashedBuildButKeepsActive",
"TestProductionFactsIndexesAvoidLargeDocumentsAndKeepLegacyLinks",
"TestProductionOverviewBoundsRelatedQueriesAndKeepsLegacyUnion",
"TestLegacyAcceptanceWithoutOperationIDIsNotCoveredByUnrelatedPlan",
}
def verify_events(events, returncode, required=REQUIRED):
passed = {e.get("Test") for e in events if e.get("Action") == "pass"}
skipped = {e.get("Test") for e in events if e.get("Action") == "skip"}
missing = sorted(required - passed)
return {"passed": returncode == 0 and not missing and not skipped,
"missingOrFailed": missing, "skipped": sorted(x for x in skipped if x)}
def main():
env = dict(os.environ, TEST_WORKFORCE_MONGO_URI="mongodb://127.0.0.1:27018", RUN_OUTPUT_HISTORY_SCALE="1", RUN_OUTPUT_SNAPSHOT_BENCHMARK="1")
command = ["go", "test", "-json", "-race", "./internal/outputsnapshot", "./internal/store",
"./internal/workforcestats", "./internal/api", "-run",
"TestAnnotationOutput|TestOutput|TestSharedReaders|TestRefreshFailure|TestLeaseRecovery|"
"TestCorruptCurrent|TestBuildCancellation|TestSnapshotScope|TestFailedBuild|"
"TestFailureDiagnostics|TestHealth|TestRouteSnapshot|TestSharded|TestProductionFactsIndexes|TestProductionOverviewBounds|TestLegacyAcceptanceWithout", "-count=1", "-timeout=120s"]
timing = ["go", "test", "-race=false", "-json", "./internal/api", "-run", "^TestAnnotationOutputSnapshotScale$", "-count=1", "-timeout=120s"]
scale = {"TestAnnotationOutputSnapshotScale", "TestAnnotationOutputSnapshotScale/production-quality-concurrent-warm-reads"}
growth = ["go", "test", "-json", "./internal/store", "-run", "^TestStatisticsPartitionGrowthBeyondGlobalBudget$", "-count=1", "-timeout=120s"]
verdicts = []
for name, args, required in [("race", command, REQUIRED | scale), ("timing", timing, scale), ("partition-growth", growth, {"TestStatisticsPartitionGrowthBeyondGlobalBudget"})]:
try:
result = subprocess.run(args, cwd=ROOT, env=env, capture_output=True, text=True, timeout=240)
except (OSError, subprocess.TimeoutExpired):
print(json.dumps({"passed": False, "phase": name, "error": "stability_runner_failed"}))
return 1
events = []
for line in result.stdout.splitlines():
try:
event = json.loads(line)
except ValueError:
continue
events.append(event)
if event.get("Action") == "output":
sys.stdout.write(event.get("Output", ""))
if result.stderr:
sys.stderr.write(result.stderr)
verdict = verify_events(events, result.returncode, required)
verdict["phase"] = name
verdicts.append(verdict)
print(json.dumps(verdict))
return 0 if all(v["passed"] for v in verdicts) else 1
if __name__ == "__main__":
raise SystemExit(main())