99 lines
5.9 KiB
Python
99 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Summarize a completed local read comparison without granting capacity acceptance."""
|
|
import argparse
|
|
import importlib.util
|
|
from collections import Counter
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
|
|
|
|
_spec = importlib.util.spec_from_file_location("readmix_schedule", Path(__file__).with_name("poll-pool-readmix.py"))
|
|
_readmix = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(_readmix)
|
|
REQUIRED_AVAILABLE = {"quality.all.available", "quality.project.available"} | {
|
|
f"acceptance.{scope}.{stage}.available" for scope in ("all", "project") for stage in ("acceptance", "prepackage_acceptance")}
|
|
|
|
|
|
def ratio(after, before):
|
|
return after / before if before > 0 else None
|
|
|
|
|
|
def summarize(manifest, reports):
|
|
rounds = manifest["repetitions"]
|
|
variants = [v["label"] for v in manifest["variants"] if v["label"] != "baseline"]
|
|
result = {"capacity_accepted": False, "actor_counts": _readmix.actor_counts(manifest),
|
|
"claim_counts": manifest.get("claim_counts", False), "duration_complete": manifest["warmup_seconds"] >= 300 and
|
|
manifest["duration_seconds"] >= 1800 and rounds >= 3,
|
|
"limitations": ["handler read workload only; no renew/media/submission/ingress",
|
|
"CPU includes in-process client and gzip; no Mongo or Pod resource metrics",
|
|
"heap includes retained benchmark request samples; not standalone service RSS"],
|
|
"variants": {}}
|
|
for label in variants:
|
|
before = [reports[("baseline", i)] for i in range(1, rounds + 1)]
|
|
after = [reports[(label, i)] for i in range(1, rounds + 1)]
|
|
metrics = {}
|
|
for key in sorted(before[0]["metrics"]):
|
|
a = [row["metrics"][key]["p95_ms"] for row in before]
|
|
b = [row["metrics"][key]["p95_ms"] for row in after]
|
|
metrics[key] = {"baseline_p95_ms": a, "candidate_p95_ms": b,
|
|
"baseline_median_p95_ms": statistics.median(a), "candidate_median_p95_ms": statistics.median(b),
|
|
"each_round_below_2s_and_half_baseline": all(y <= 2000 and y <= x * .5 for x, y in zip(a, b))}
|
|
available = [key for key in metrics if key.endswith(".available")]
|
|
cpu = [ratio(y["process_cpu_seconds"] / y["actual_seconds"], x["process_cpu_seconds"] / x["actual_seconds"])
|
|
for x, y in zip(before, after)]
|
|
heap = [ratio(y["go_heap_after_gc"], x["go_heap_after_gc"]) for x, y in zip(before, after)]
|
|
gzip = [ratio(y["metrics"]["annotation.buffer"]["gzip_bytes"] / y["metrics"]["annotation.buffer"]["requests"],
|
|
x["metrics"]["annotation.buffer"]["gzip_bytes"] / x["metrics"]["annotation.buffer"]["requests"])
|
|
for x, y in zip(before, after)]
|
|
identity = lambda d: {(s["Actor"], s["Key"], s["Hash"], s["Rows"]) for s in d["samples"]}
|
|
equivalent = all(identity(x) == identity(y) for x, y in zip(before, after))
|
|
schedule = _readmix.expected_schedule(manifest["duration_seconds"], manifest)
|
|
expected = sum(schedule.values())
|
|
healthy = all(d["errors"] == 0 and d["mongo_write_commands"] == 0 and len(d["samples"]) == expected and
|
|
all(s["Code"] == 200 for s in d["samples"]) and
|
|
Counter((s["Actor"], s["Key"]) for s in d["samples"]) == schedule for d in before + after)
|
|
sustained_cpu = rounds >= 3 and all(x is not None and x > 1.1 for x in cpu)
|
|
sustained_heap = rounds >= 3 and all(x is not None and x > 1.1 for x in heap)
|
|
resource_complete = all(x is not None for x in cpu + heap)
|
|
available_gate = REQUIRED_AVAILABLE.issubset(metrics) and all(metrics[k]["each_round_below_2s_and_half_baseline"] for k in available)
|
|
gzip_gate = all(x is not None and x <= .5 for x in gzip)
|
|
result["variants"][label] = {"task_sets_equal": equivalent, "zero_errors_writes_complete_requests": healthy,
|
|
"available_latency_gate": available_gate, "buffer_gzip_gate": gzip_gate,
|
|
"cpu_per_wall_second_ratios": cpu, "post_gc_heap_ratios": heap, "buffer_gzip_ratios": gzip,
|
|
"sustained_cpu_over_10_percent": sustained_cpu, "sustained_heap_over_10_percent": sustained_heap,
|
|
"local_read_thresholds_passed": result["duration_complete"] and healthy and equivalent and available_gate and
|
|
gzip_gate and resource_complete and not sustained_cpu and not sustained_heap,
|
|
"endpoints": metrics}
|
|
return result
|
|
|
|
|
|
def load_reports(root, manifest):
|
|
reports = {}
|
|
for variant in manifest["variants"]:
|
|
for i in range(1, manifest["repetitions"] + 1):
|
|
report = json.loads((root / f"{variant['label']}-{i}.json").read_text())
|
|
_readmix.validate_result(report, variant["revision"], variant["mode"], manifest)
|
|
reports[(variant["label"], i)] = report
|
|
return reports
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("directory", type=Path)
|
|
args = parser.parse_args()
|
|
root = args.directory.resolve()
|
|
status = json.loads((root / "status.json").read_text())
|
|
if status.get("state") != "complete":
|
|
parser.error("comparison is not complete; retain pending status")
|
|
manifest = json.loads((root / "manifest.json").read_text())
|
|
reports = load_reports(root, manifest)
|
|
summary = summarize(manifest, reports)
|
|
(root / "analysis.json").write_text(json.dumps(summary, indent=2) + "\n")
|
|
print(json.dumps({"capacity_accepted": False, "duration_complete": summary["duration_complete"],
|
|
"variants": {k: {field: value for field, value in v.items() if field != "endpoints"}
|
|
for k, v in summary["variants"].items()}}, indent=2))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|