droboticflow/scripts/local/analyze-poll-pool-readmix.py

99 lines
5.9 KiB
Python

#!/usr/bin/env python3
"""Summarize a completed local read comparison without granting capacity acceptance."""
import argparse
import importlib.util
from collections import Counter
import json
from pathlib import Path
import statistics
_spec = importlib.util.spec_from_file_location("readmix_schedule", Path(__file__).with_name("poll-pool-readmix.py"))
_readmix = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(_readmix)
REQUIRED_AVAILABLE = {"quality.all.available", "quality.project.available"} | {
f"acceptance.{scope}.{stage}.available" for scope in ("all", "project") for stage in ("acceptance", "prepackage_acceptance")}
def ratio(after, before):
return after / before if before > 0 else None
def summarize(manifest, reports):
rounds = manifest["repetitions"]
variants = [v["label"] for v in manifest["variants"] if v["label"] != "baseline"]
result = {"capacity_accepted": False, "actor_counts": _readmix.actor_counts(manifest),
"claim_counts": manifest.get("claim_counts", False), "duration_complete": manifest["warmup_seconds"] >= 300 and
manifest["duration_seconds"] >= 1800 and rounds >= 3,
"limitations": ["handler read workload only; no renew/media/submission/ingress",
"CPU includes in-process client and gzip; no Mongo or Pod resource metrics",
"heap includes retained benchmark request samples; not standalone service RSS"],
"variants": {}}
for label in variants:
before = [reports[("baseline", i)] for i in range(1, rounds + 1)]
after = [reports[(label, i)] for i in range(1, rounds + 1)]
metrics = {}
for key in sorted(before[0]["metrics"]):
a = [row["metrics"][key]["p95_ms"] for row in before]
b = [row["metrics"][key]["p95_ms"] for row in after]
metrics[key] = {"baseline_p95_ms": a, "candidate_p95_ms": b,
"baseline_median_p95_ms": statistics.median(a), "candidate_median_p95_ms": statistics.median(b),
"each_round_below_2s_and_half_baseline": all(y <= 2000 and y <= x * .5 for x, y in zip(a, b))}
available = [key for key in metrics if key.endswith(".available")]
cpu = [ratio(y["process_cpu_seconds"] / y["actual_seconds"], x["process_cpu_seconds"] / x["actual_seconds"])
for x, y in zip(before, after)]
heap = [ratio(y["go_heap_after_gc"], x["go_heap_after_gc"]) for x, y in zip(before, after)]
gzip = [ratio(y["metrics"]["annotation.buffer"]["gzip_bytes"] / y["metrics"]["annotation.buffer"]["requests"],
x["metrics"]["annotation.buffer"]["gzip_bytes"] / x["metrics"]["annotation.buffer"]["requests"])
for x, y in zip(before, after)]
identity = lambda d: {(s["Actor"], s["Key"], s["Hash"], s["Rows"]) for s in d["samples"]}
equivalent = all(identity(x) == identity(y) for x, y in zip(before, after))
schedule = _readmix.expected_schedule(manifest["duration_seconds"], manifest)
expected = sum(schedule.values())
healthy = all(d["errors"] == 0 and d["mongo_write_commands"] == 0 and len(d["samples"]) == expected and
all(s["Code"] == 200 for s in d["samples"]) and
Counter((s["Actor"], s["Key"]) for s in d["samples"]) == schedule for d in before + after)
sustained_cpu = rounds >= 3 and all(x is not None and x > 1.1 for x in cpu)
sustained_heap = rounds >= 3 and all(x is not None and x > 1.1 for x in heap)
resource_complete = all(x is not None for x in cpu + heap)
available_gate = REQUIRED_AVAILABLE.issubset(metrics) and all(metrics[k]["each_round_below_2s_and_half_baseline"] for k in available)
gzip_gate = all(x is not None and x <= .5 for x in gzip)
result["variants"][label] = {"task_sets_equal": equivalent, "zero_errors_writes_complete_requests": healthy,
"available_latency_gate": available_gate, "buffer_gzip_gate": gzip_gate,
"cpu_per_wall_second_ratios": cpu, "post_gc_heap_ratios": heap, "buffer_gzip_ratios": gzip,
"sustained_cpu_over_10_percent": sustained_cpu, "sustained_heap_over_10_percent": sustained_heap,
"local_read_thresholds_passed": result["duration_complete"] and healthy and equivalent and available_gate and
gzip_gate and resource_complete and not sustained_cpu and not sustained_heap,
"endpoints": metrics}
return result
def load_reports(root, manifest):
reports = {}
for variant in manifest["variants"]:
for i in range(1, manifest["repetitions"] + 1):
report = json.loads((root / f"{variant['label']}-{i}.json").read_text())
_readmix.validate_result(report, variant["revision"], variant["mode"], manifest)
reports[(variant["label"], i)] = report
return reports
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directory", type=Path)
args = parser.parse_args()
root = args.directory.resolve()
status = json.loads((root / "status.json").read_text())
if status.get("state") != "complete":
parser.error("comparison is not complete; retain pending status")
manifest = json.loads((root / "manifest.json").read_text())
reports = load_reports(root, manifest)
summary = summarize(manifest, reports)
(root / "analysis.json").write_text(json.dumps(summary, indent=2) + "\n")
print(json.dumps({"capacity_accepted": False, "duration_complete": summary["duration_complete"],
"variants": {k: {field: value for field, value in v.items() if field != "endpoints"}
for k, v in summary["variants"].items()}}, indent=2))
if __name__ == "__main__":
main()