82 lines
4.6 KiB
Python
82 lines
4.6 KiB
Python
import importlib.util
|
|
from pathlib import Path
|
|
import unittest
|
|
import json
|
|
import tempfile
|
|
|
|
spec = importlib.util.spec_from_file_location("analyze", Path(__file__).with_name("analyze-poll-pool-readmix.py"))
|
|
analyze = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(analyze)
|
|
|
|
|
|
class AcceptanceSummaryTests(unittest.TestCase):
|
|
def fixture(self):
|
|
manifest = {"repetitions": 3, "warmup_seconds": 300, "duration_seconds": 1800,
|
|
"variants": [{"label": "baseline"}, {"label": "candidate"}]}
|
|
reports = {}
|
|
for i in range(1, 4):
|
|
for label in ("baseline", "candidate"):
|
|
candidate = label == "candidate"
|
|
reports[(label, i)] = {"errors": 0, "mongo_write_commands": 0, "process_cpu_seconds": 10,
|
|
"actual_seconds": 1800, "go_heap_after_gc": 100,
|
|
"samples": [{"Actor": actor, "Key": key, "Hash": "same", "Rows": 1, "Code": 200} for (actor, key), count in analyze._readmix.expected_schedule(1800).items() for _ in range(count)],
|
|
"metrics": {**{key: {"p95_ms": 400 if candidate else 1000} for key in analyze.REQUIRED_AVAILABLE},
|
|
"annotation.buffer": {"p95_ms": 400 if candidate else 1000,
|
|
"gzip_bytes": 40 if candidate else 100, "requests": 1}}}
|
|
return manifest, reports
|
|
|
|
def test_loading_rejects_reports_from_another_run(self):
|
|
config = {"repetitions": 1, "duration_seconds": 3, "warmup_seconds": 1,
|
|
"authorization_state": "initialized", "run_payload_kib": 256,
|
|
"author_history_per_reviewer": 1000,
|
|
"variants": [{"label": "candidate", "revision": "fixed", "mode": "summary"}]}
|
|
report = {**config, "revision": "fixed", "mode": "summary", "scheduled_seconds": 3,
|
|
"errors": 0, "mongo_write_commands": 0,
|
|
"resource_samples": [{"phase": "finished"}],
|
|
"samples": [{"Actor": actor, "Key": key, "Code": 200}
|
|
for (actor, key), count in analyze._readmix.expected_schedule(3).items()
|
|
for _ in range(count)]}
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
path = root / "candidate-1.json"
|
|
path.write_text(json.dumps(report))
|
|
self.assertEqual(analyze.load_reports(root, config)[("candidate", 1)], report)
|
|
for field, value in (("revision", "other"), ("mode", "legacy"),
|
|
("warmup_seconds", 2), ("scheduled_seconds", 30),
|
|
("author_history_per_reviewer", 0), ("run_payload_kib", 0),
|
|
("authorization_state", "legacy"), ("resource_samples", [])):
|
|
with self.subTest(field=field):
|
|
path.write_text(json.dumps(dict(report, **{field: value})))
|
|
with self.assertRaises(ValueError):
|
|
analyze.load_reports(root, config)
|
|
|
|
def test_local_thresholds_do_not_imply_capacity_acceptance(self):
|
|
m, r = self.fixture()
|
|
result = analyze.summarize(m, r)
|
|
self.assertTrue(result["variants"]["candidate"]["local_read_thresholds_passed"])
|
|
self.assertFalse(result["capacity_accepted"])
|
|
|
|
def test_short_or_partial_or_regressing_runs_cannot_pass(self):
|
|
for case in ("short", "incomplete", "errors", "write", "latency", "cpu", "heap", "identity"):
|
|
with self.subTest(case=case):
|
|
m, r = self.fixture()
|
|
if case == "short":
|
|
m["duration_seconds"] = 30
|
|
elif case == "incomplete":
|
|
r[("candidate", 1)]["samples"] = []
|
|
elif case == "errors":
|
|
r[("candidate", 1)]["errors"] = 1
|
|
elif case == "write":
|
|
r[("candidate", 1)]["mongo_write_commands"] = 1
|
|
elif case == "latency":
|
|
r[("candidate", 1)]["metrics"]["quality.all.available"]["p95_ms"] = 600
|
|
elif case == "identity":
|
|
r[("candidate", 1)]["samples"] = [dict(s, Hash="different") for s in r[("candidate", 1)]["samples"]]
|
|
else:
|
|
for i in range(1, 4):
|
|
r[("candidate", i)]["process_cpu_seconds" if case == "cpu" else "go_heap_after_gc"] *= 1.2
|
|
self.assertFalse(analyze.summarize(m, r)["variants"]["candidate"]["local_read_thresholds_passed"])
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|