droboticflow/scripts/local/poll-pool-readmix.py

254 lines
13 KiB
Python

#!/usr/bin/env python3
"""Compare pinned revisions against isolated local Mongo; not capacity acceptance.
Only unique synthetic databases are written. All variants use the same archived
harness. --resume reuses validated completed runs with an identical manifest.
"""
import argparse
from collections import Counter
import hashlib
import fcntl
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import time
ROOT = Path(__file__).resolve().parents[2]
HARNESS = Path("internal/api/poll_pool_load_test.go")
def write_json(path, value):
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, indent=2) + "\n")
temporary.replace(path)
DEFAULT_ACTORS = {"annotation": 60, "quality": 12, "acceptance": 8, "admin": 1}
def actor_counts(config):
counts = config.get("actor_counts", DEFAULT_ACTORS)
if (set(counts) != set(DEFAULT_ACTORS) or counts.get("admin") != 1 or
any(type(v) is not int or v < 1 for v in counts.values()) or sum(counts.values()) > 250):
raise ValueError("invalid actor counts")
return counts
def expected_schedule(seconds, config=None):
config = config or {}
counts = actor_counts(config)
annotation_end = counts["annotation"]
quality_end = annotation_end + counts["quality"]
total = sum(counts.values())
expected = Counter()
for i in range(total):
actor = f"load-user-{i:02d}"
if i < annotation_end:
expected[(actor, "annotation.buffer")] = (seconds + 2) // 3
if config.get("claim_counts", False):
expected[(actor, "annotation.claim-count")] = (seconds + 11) // 12
elif i == total - 1:
expected[(actor, "admin.workforce")] = (seconds + 11) // 12
else:
cycles = (seconds + 11) // 12
for scope, count in (("project", (cycles + 1) // 2), ("all", cycles // 2)):
if not count:
continue
for pool in ("available", "mine", "reviewed"):
if i < quality_end:
expected[(actor, f"quality.{scope}.{pool}")] = count
else:
for stage in ("acceptance", "prepackage_acceptance"):
expected[(actor, f"acceptance.{scope}.{stage}.{pool}")] = count
return expected
def validate_result(data, revision, mode, config):
schedule_expected = expected_schedule(config["duration_seconds"], config)
expected = sum(schedule_expected.values())
if "actor_counts" in config:
if data.get("actor_counts") != actor_counts(config) or data.get("actors") != sum(actor_counts(config).values()):
raise ValueError("result mismatch: actor counts")
if data.get("claim_counts") != config.get("claim_counts", False):
raise ValueError("result mismatch: claim-count workload")
for key, value in {"revision": revision, "mode": mode,
"authorization_state": config["authorization_state"],
"warmup_seconds": config["warmup_seconds"],
"scheduled_seconds": config["duration_seconds"],
"run_payload_kib": config["run_payload_kib"],
"author_history_per_reviewer": config["author_history_per_reviewer"]}.items():
if data.get(key) != value:
raise ValueError(f"result mismatch: {key}")
if len(data.get("samples", [])) != expected:
raise ValueError(f"incomplete requests; expected {expected}")
schedule = Counter((row["Actor"], row["Key"]) for row in data["samples"])
if schedule != schedule_expected:
raise ValueError("actor/endpoint schedule is incomplete or duplicated")
if data.get("errors") != 0 or data.get("mongo_write_commands") != 0:
raise ValueError("read mix had errors or unexpected writes")
if any(row["Code"] != 200 for row in data["samples"]):
raise ValueError("sample contains non-200 response")
if not data.get("resource_samples"):
raise ValueError("resource sampling evidence missing")
def identities(data):
return {(s["Actor"], s["Key"], s["Hash"], s["Rows"]) for s in data["samples"]}
def archive(revision, directory):
directory.mkdir()
process = subprocess.Popen(["git", "archive", revision], cwd=ROOT, stdout=subprocess.PIPE)
try:
subprocess.run(["tar", "-x", "-C", str(directory)], stdin=process.stdout, check=True)
finally:
process.stdout.close()
if process.wait() != 0:
raise RuntimeError(f"archive failed: {revision}")
def acquire_output_lock(output):
handle = (output / ".runner.lock").open("a+")
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
handle.close()
raise RuntimeError("another runner is using this output directory")
handle.seek(0)
handle.truncate()
handle.write(str(os.getpid()) + "\n")
handle.flush()
return handle
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--baseline", default="80bdcee9")
parser.add_argument("--candidate", default="HEAD")
parser.add_argument("--intermediate", help="optional stage-one revision, in summary mode")
parser.add_argument("--baseline-mode", choices=["legacy", "summary"], default="legacy")
parser.add_argument("--authorization-state", choices=["legacy", "initialized"], default="initialized")
parser.add_argument("--author-history", type=int, default=0)
parser.add_argument("--run-payload-kib", type=int, default=0)
parser.add_argument("--warmup", type=int, default=5)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument("--repetitions", type=int, default=1)
parser.add_argument("--annotators", type=int, default=60)
parser.add_argument("--quality-reviewers", type=int, default=12)
parser.add_argument("--acceptance-reviewers", type=int, default=8)
parser.add_argument("--claim-counts", action="store_true", help="also refresh exact annotation counts every 12 seconds")
parser.add_argument("--resume", action="store_true")
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if not (0 <= args.author_history <= 5000 and 0 <= args.run_payload_kib <= 2048 and
1 <= args.warmup <= 7200 and 1 <= args.duration <= 7200 and 1 <= args.repetitions <= 10):
parser.error("invalid fixture size, duration or repetitions")
counts = {"annotation": args.annotators, "quality": args.quality_reviewers,
"acceptance": args.acceptance_reviewers, "admin": 1}
try:
actor_counts({"actor_counts": counts})
except ValueError as error:
parser.error(str(error))
resolve = lambda ref: subprocess.check_output(["git", "rev-parse", "--verify", ref + "^{commit}"], cwd=ROOT, text=True).strip()
variants = [{"label": "baseline", "revision": resolve(args.baseline), "mode": args.baseline_mode}]
if args.intermediate:
variants.append({"label": "stage1", "revision": resolve(args.intermediate), "mode": "summary"})
variants.append({"label": "candidate", "revision": resolve(args.candidate), "mode": "summary"})
harness = (ROOT / HARNESS).read_bytes()
config = {"schema": 3, "actor_counts": counts, "claim_counts": args.claim_counts, "variants": variants, "harness_sha256": hashlib.sha256(harness).hexdigest(),
"authorization_state": args.authorization_state, "run_payload_kib": args.run_payload_kib,
"author_history_per_reviewer": args.author_history, "warmup_seconds": args.warmup,
"duration_seconds": args.duration, "repetitions": args.repetitions}
output = args.output.resolve()
output.mkdir(parents=True, exist_ok=True)
try:
output_lock = acquire_output_lock(output)
except RuntimeError as error:
parser.error(str(error))
manifest = output / "manifest.json"
if args.resume:
if not manifest.exists() or json.loads(manifest.read_text()) != config:
parser.error("resume manifest differs; keep the original revisions, fixture, timing and harness")
elif any(p.name != ".runner.lock" for p in output.iterdir()):
parser.error("output must be empty; use --resume with the original configuration")
else:
write_json(manifest, config)
summary = {**config, "capacity_accepted": False,
"scope": "local handler read mix; no renew/media/submission/ingress/Pod metrics", "pairs": []}
with tempfile.TemporaryDirectory(prefix="poll-pool-pinned-") as temporary:
roots = {}
for variant in variants:
root = Path(temporary) / variant["label"]
archive(variant["revision"], root)
(root / HARNESS).write_bytes(harness)
roots[variant["label"]] = root
for repetition in range(1, args.repetitions + 1):
# Rotate each variant through each execution position in three-way runs.
offset = (repetition - 1) % len(variants)
ordered = variants[offset:] + variants[:offset]
reports = {}
for variant in ordered:
label, revision, mode = variant["label"], variant["revision"], variant["mode"]
name = f"{label}-{repetition}"
destination = output / f"{name}.json"
if args.resume and destination.exists():
try:
data = json.loads(destination.read_text())
validate_result(data, revision, mode, config)
except (ValueError, KeyError, TypeError) as error:
print(f"Retrying incomplete/invalid {name}: {error}", flush=True)
else:
reports[label] = data
print(f"Reusing validated {name}", flush=True)
continue
# Preserve interrupted/failed evidence rather than silently overwriting.
stamp = time.time_ns()
for suffix in (".json", ".log", ".json.resources.jsonl"):
path = output / (name + suffix)
if path.exists():
path.rename(output / (name + suffix + f".previous-{stamp}"))
write_json(output / "status.json", {"state": "running", "run": name, "revision": revision,
"started_unix": time.time(), "completed_pairs": len(summary["pairs"])})
env = dict(os.environ, POLL_POOL_AUTHOR_HISTORY=str(args.author_history),
POLL_POOL_ANNOTATORS=str(args.annotators), POLL_POOL_QUALITY_REVIEWERS=str(args.quality_reviewers),
POLL_POOL_ACCEPTANCE_REVIEWERS=str(args.acceptance_reviewers), POLL_POOL_CLAIM_COUNTS="1" if args.claim_counts else "0",
POLL_POOL_RUN_PAYLOAD_KIB=str(args.run_payload_kib), POLL_POOL_AUTHORIZATION_STATE=args.authorization_state,
POLL_POOL_LOAD="1", POLL_POOL_MODE=mode, POLL_POOL_REVISION=revision,
POLL_POOL_WARMUP_SECONDS=str(args.warmup), POLL_POOL_DURATION_SECONDS=str(args.duration),
POLL_POOL_OUTPUT=str(destination))
print(f"Running {name}: {revision[:8]}, mode={mode}, {args.warmup}s warmup + {args.duration}s measurement", flush=True)
try:
with (output / f"{name}.log").open("w") as log:
result = subprocess.run(["go", "test", "./internal/api", "-run", "^TestPollPoolReadMixLocal$", "-count=1", "-v",
"-timeout", f"{args.warmup + args.duration + 300}s"], cwd=roots[label], env=env,
stdout=log, stderr=subprocess.STDOUT)
if result.returncode:
raise RuntimeError(f"{name} failed; inspect {output / (name + '.log')}")
data = json.loads(destination.read_text())
validate_result(data, revision, mode, config)
except BaseException as error:
write_json(output / "status.json", {"state": "interrupted" if isinstance(error, KeyboardInterrupt) else "failed",
"run": name, "error": str(error), "completed_pairs": len(summary["pairs"])})
raise
reports[label] = data
baseline = reports["baseline"]
for variant in variants[1:]:
other = reports[variant["label"]]
equal = identities(baseline) == identities(other)
summary["pairs"].append({"repetition": repetition, "variant": variant["label"], "task_sets_equal": equal,
"baseline_errors": baseline["errors"], "candidate_errors": other["errors"],
"baseline_metrics": baseline["metrics"], "candidate_metrics": other["metrics"]})
write_json(output / "comparison.json", summary)
if not equal:
write_json(output / "status.json", {"state": "failed", "reason": "task-set mismatch", "repetition": repetition})
raise RuntimeError("task-set equivalence failed")
write_json(output / "status.json", {"state": "complete", "completed_pairs": len(summary["pairs"]), "capacity_accepted": False})
print(f"Read-mix comparison saved: {output / 'comparison.json'}", flush=True)
output_lock.close()
if __name__ == "__main__":
main()