droboticflow/scripts/local/test_analyze_poll_pool_readmix.py

82 lines
4.6 KiB
Python

import importlib.util
from pathlib import Path
import unittest
import json
import tempfile
spec = importlib.util.spec_from_file_location("analyze", Path(__file__).with_name("analyze-poll-pool-readmix.py"))
analyze = importlib.util.module_from_spec(spec)
spec.loader.exec_module(analyze)
class AcceptanceSummaryTests(unittest.TestCase):
def fixture(self):
manifest = {"repetitions": 3, "warmup_seconds": 300, "duration_seconds": 1800,
"variants": [{"label": "baseline"}, {"label": "candidate"}]}
reports = {}
for i in range(1, 4):
for label in ("baseline", "candidate"):
candidate = label == "candidate"
reports[(label, i)] = {"errors": 0, "mongo_write_commands": 0, "process_cpu_seconds": 10,
"actual_seconds": 1800, "go_heap_after_gc": 100,
"samples": [{"Actor": actor, "Key": key, "Hash": "same", "Rows": 1, "Code": 200} for (actor, key), count in analyze._readmix.expected_schedule(1800).items() for _ in range(count)],
"metrics": {**{key: {"p95_ms": 400 if candidate else 1000} for key in analyze.REQUIRED_AVAILABLE},
"annotation.buffer": {"p95_ms": 400 if candidate else 1000,
"gzip_bytes": 40 if candidate else 100, "requests": 1}}}
return manifest, reports
def test_loading_rejects_reports_from_another_run(self):
config = {"repetitions": 1, "duration_seconds": 3, "warmup_seconds": 1,
"authorization_state": "initialized", "run_payload_kib": 256,
"author_history_per_reviewer": 1000,
"variants": [{"label": "candidate", "revision": "fixed", "mode": "summary"}]}
report = {**config, "revision": "fixed", "mode": "summary", "scheduled_seconds": 3,
"errors": 0, "mongo_write_commands": 0,
"resource_samples": [{"phase": "finished"}],
"samples": [{"Actor": actor, "Key": key, "Code": 200}
for (actor, key), count in analyze._readmix.expected_schedule(3).items()
for _ in range(count)]}
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
path = root / "candidate-1.json"
path.write_text(json.dumps(report))
self.assertEqual(analyze.load_reports(root, config)[("candidate", 1)], report)
for field, value in (("revision", "other"), ("mode", "legacy"),
("warmup_seconds", 2), ("scheduled_seconds", 30),
("author_history_per_reviewer", 0), ("run_payload_kib", 0),
("authorization_state", "legacy"), ("resource_samples", [])):
with self.subTest(field=field):
path.write_text(json.dumps(dict(report, **{field: value})))
with self.assertRaises(ValueError):
analyze.load_reports(root, config)
def test_local_thresholds_do_not_imply_capacity_acceptance(self):
m, r = self.fixture()
result = analyze.summarize(m, r)
self.assertTrue(result["variants"]["candidate"]["local_read_thresholds_passed"])
self.assertFalse(result["capacity_accepted"])
def test_short_or_partial_or_regressing_runs_cannot_pass(self):
for case in ("short", "incomplete", "errors", "write", "latency", "cpu", "heap", "identity"):
with self.subTest(case=case):
m, r = self.fixture()
if case == "short":
m["duration_seconds"] = 30
elif case == "incomplete":
r[("candidate", 1)]["samples"] = []
elif case == "errors":
r[("candidate", 1)]["errors"] = 1
elif case == "write":
r[("candidate", 1)]["mongo_write_commands"] = 1
elif case == "latency":
r[("candidate", 1)]["metrics"]["quality.all.available"]["p95_ms"] = 600
elif case == "identity":
r[("candidate", 1)]["samples"] = [dict(s, Hash="different") for s in r[("candidate", 1)]["samples"]]
else:
for i in range(1, 4):
r[("candidate", i)]["process_cpu_seconds" if case == "cpu" else "go_heap_after_gc"] *= 1.2
self.assertFalse(analyze.summarize(m, r)["variants"]["candidate"]["local_read_thresholds_passed"])
if __name__ == "__main__":
unittest.main()