forked from longtao.wu/droboticflow
254 lines
13 KiB
Python
254 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare pinned revisions against isolated local Mongo; not capacity acceptance.
|
|
|
|
Only unique synthetic databases are written. All variants use the same archived
|
|
harness. --resume reuses validated completed runs with an identical manifest.
|
|
"""
|
|
import argparse
|
|
from collections import Counter
|
|
import hashlib
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
HARNESS = Path("internal/api/poll_pool_load_test.go")
|
|
|
|
|
|
def write_json(path, value):
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(value, indent=2) + "\n")
|
|
temporary.replace(path)
|
|
|
|
|
|
DEFAULT_ACTORS = {"annotation": 60, "quality": 12, "acceptance": 8, "admin": 1}
|
|
|
|
|
|
def actor_counts(config):
|
|
counts = config.get("actor_counts", DEFAULT_ACTORS)
|
|
if (set(counts) != set(DEFAULT_ACTORS) or counts.get("admin") != 1 or
|
|
any(type(v) is not int or v < 1 for v in counts.values()) or sum(counts.values()) > 250):
|
|
raise ValueError("invalid actor counts")
|
|
return counts
|
|
|
|
|
|
def expected_schedule(seconds, config=None):
|
|
config = config or {}
|
|
counts = actor_counts(config)
|
|
annotation_end = counts["annotation"]
|
|
quality_end = annotation_end + counts["quality"]
|
|
total = sum(counts.values())
|
|
expected = Counter()
|
|
for i in range(total):
|
|
actor = f"load-user-{i:02d}"
|
|
if i < annotation_end:
|
|
expected[(actor, "annotation.buffer")] = (seconds + 2) // 3
|
|
if config.get("claim_counts", False):
|
|
expected[(actor, "annotation.claim-count")] = (seconds + 11) // 12
|
|
elif i == total - 1:
|
|
expected[(actor, "admin.workforce")] = (seconds + 11) // 12
|
|
else:
|
|
cycles = (seconds + 11) // 12
|
|
for scope, count in (("project", (cycles + 1) // 2), ("all", cycles // 2)):
|
|
if not count:
|
|
continue
|
|
for pool in ("available", "mine", "reviewed"):
|
|
if i < quality_end:
|
|
expected[(actor, f"quality.{scope}.{pool}")] = count
|
|
else:
|
|
for stage in ("acceptance", "prepackage_acceptance"):
|
|
expected[(actor, f"acceptance.{scope}.{stage}.{pool}")] = count
|
|
return expected
|
|
|
|
|
|
def validate_result(data, revision, mode, config):
|
|
schedule_expected = expected_schedule(config["duration_seconds"], config)
|
|
expected = sum(schedule_expected.values())
|
|
if "actor_counts" in config:
|
|
if data.get("actor_counts") != actor_counts(config) or data.get("actors") != sum(actor_counts(config).values()):
|
|
raise ValueError("result mismatch: actor counts")
|
|
if data.get("claim_counts") != config.get("claim_counts", False):
|
|
raise ValueError("result mismatch: claim-count workload")
|
|
for key, value in {"revision": revision, "mode": mode,
|
|
"authorization_state": config["authorization_state"],
|
|
"warmup_seconds": config["warmup_seconds"],
|
|
"scheduled_seconds": config["duration_seconds"],
|
|
"run_payload_kib": config["run_payload_kib"],
|
|
"author_history_per_reviewer": config["author_history_per_reviewer"]}.items():
|
|
if data.get(key) != value:
|
|
raise ValueError(f"result mismatch: {key}")
|
|
if len(data.get("samples", [])) != expected:
|
|
raise ValueError(f"incomplete requests; expected {expected}")
|
|
schedule = Counter((row["Actor"], row["Key"]) for row in data["samples"])
|
|
if schedule != schedule_expected:
|
|
raise ValueError("actor/endpoint schedule is incomplete or duplicated")
|
|
if data.get("errors") != 0 or data.get("mongo_write_commands") != 0:
|
|
raise ValueError("read mix had errors or unexpected writes")
|
|
if any(row["Code"] != 200 for row in data["samples"]):
|
|
raise ValueError("sample contains non-200 response")
|
|
if not data.get("resource_samples"):
|
|
raise ValueError("resource sampling evidence missing")
|
|
|
|
|
|
def identities(data):
|
|
return {(s["Actor"], s["Key"], s["Hash"], s["Rows"]) for s in data["samples"]}
|
|
|
|
|
|
def archive(revision, directory):
|
|
directory.mkdir()
|
|
process = subprocess.Popen(["git", "archive", revision], cwd=ROOT, stdout=subprocess.PIPE)
|
|
try:
|
|
subprocess.run(["tar", "-x", "-C", str(directory)], stdin=process.stdout, check=True)
|
|
finally:
|
|
process.stdout.close()
|
|
if process.wait() != 0:
|
|
raise RuntimeError(f"archive failed: {revision}")
|
|
|
|
|
|
def acquire_output_lock(output):
|
|
handle = (output / ".runner.lock").open("a+")
|
|
try:
|
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
handle.close()
|
|
raise RuntimeError("another runner is using this output directory")
|
|
handle.seek(0)
|
|
handle.truncate()
|
|
handle.write(str(os.getpid()) + "\n")
|
|
handle.flush()
|
|
return handle
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--baseline", default="80bdcee9")
|
|
parser.add_argument("--candidate", default="HEAD")
|
|
parser.add_argument("--intermediate", help="optional stage-one revision, in summary mode")
|
|
parser.add_argument("--baseline-mode", choices=["legacy", "summary"], default="legacy")
|
|
parser.add_argument("--authorization-state", choices=["legacy", "initialized"], default="initialized")
|
|
parser.add_argument("--author-history", type=int, default=0)
|
|
parser.add_argument("--run-payload-kib", type=int, default=0)
|
|
parser.add_argument("--warmup", type=int, default=5)
|
|
parser.add_argument("--duration", type=int, default=30)
|
|
parser.add_argument("--repetitions", type=int, default=1)
|
|
parser.add_argument("--annotators", type=int, default=60)
|
|
parser.add_argument("--quality-reviewers", type=int, default=12)
|
|
parser.add_argument("--acceptance-reviewers", type=int, default=8)
|
|
parser.add_argument("--claim-counts", action="store_true", help="also refresh exact annotation counts every 12 seconds")
|
|
parser.add_argument("--resume", action="store_true")
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if not (0 <= args.author_history <= 5000 and 0 <= args.run_payload_kib <= 2048 and
|
|
1 <= args.warmup <= 7200 and 1 <= args.duration <= 7200 and 1 <= args.repetitions <= 10):
|
|
parser.error("invalid fixture size, duration or repetitions")
|
|
counts = {"annotation": args.annotators, "quality": args.quality_reviewers,
|
|
"acceptance": args.acceptance_reviewers, "admin": 1}
|
|
try:
|
|
actor_counts({"actor_counts": counts})
|
|
except ValueError as error:
|
|
parser.error(str(error))
|
|
resolve = lambda ref: subprocess.check_output(["git", "rev-parse", "--verify", ref + "^{commit}"], cwd=ROOT, text=True).strip()
|
|
variants = [{"label": "baseline", "revision": resolve(args.baseline), "mode": args.baseline_mode}]
|
|
if args.intermediate:
|
|
variants.append({"label": "stage1", "revision": resolve(args.intermediate), "mode": "summary"})
|
|
variants.append({"label": "candidate", "revision": resolve(args.candidate), "mode": "summary"})
|
|
harness = (ROOT / HARNESS).read_bytes()
|
|
config = {"schema": 3, "actor_counts": counts, "claim_counts": args.claim_counts, "variants": variants, "harness_sha256": hashlib.sha256(harness).hexdigest(),
|
|
"authorization_state": args.authorization_state, "run_payload_kib": args.run_payload_kib,
|
|
"author_history_per_reviewer": args.author_history, "warmup_seconds": args.warmup,
|
|
"duration_seconds": args.duration, "repetitions": args.repetitions}
|
|
output = args.output.resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
output_lock = acquire_output_lock(output)
|
|
except RuntimeError as error:
|
|
parser.error(str(error))
|
|
manifest = output / "manifest.json"
|
|
if args.resume:
|
|
if not manifest.exists() or json.loads(manifest.read_text()) != config:
|
|
parser.error("resume manifest differs; keep the original revisions, fixture, timing and harness")
|
|
elif any(p.name != ".runner.lock" for p in output.iterdir()):
|
|
parser.error("output must be empty; use --resume with the original configuration")
|
|
else:
|
|
write_json(manifest, config)
|
|
summary = {**config, "capacity_accepted": False,
|
|
"scope": "local handler read mix; no renew/media/submission/ingress/Pod metrics", "pairs": []}
|
|
with tempfile.TemporaryDirectory(prefix="poll-pool-pinned-") as temporary:
|
|
roots = {}
|
|
for variant in variants:
|
|
root = Path(temporary) / variant["label"]
|
|
archive(variant["revision"], root)
|
|
(root / HARNESS).write_bytes(harness)
|
|
roots[variant["label"]] = root
|
|
for repetition in range(1, args.repetitions + 1):
|
|
# Rotate each variant through each execution position in three-way runs.
|
|
offset = (repetition - 1) % len(variants)
|
|
ordered = variants[offset:] + variants[:offset]
|
|
reports = {}
|
|
for variant in ordered:
|
|
label, revision, mode = variant["label"], variant["revision"], variant["mode"]
|
|
name = f"{label}-{repetition}"
|
|
destination = output / f"{name}.json"
|
|
if args.resume and destination.exists():
|
|
try:
|
|
data = json.loads(destination.read_text())
|
|
validate_result(data, revision, mode, config)
|
|
except (ValueError, KeyError, TypeError) as error:
|
|
print(f"Retrying incomplete/invalid {name}: {error}", flush=True)
|
|
else:
|
|
reports[label] = data
|
|
print(f"Reusing validated {name}", flush=True)
|
|
continue
|
|
# Preserve interrupted/failed evidence rather than silently overwriting.
|
|
stamp = time.time_ns()
|
|
for suffix in (".json", ".log", ".json.resources.jsonl"):
|
|
path = output / (name + suffix)
|
|
if path.exists():
|
|
path.rename(output / (name + suffix + f".previous-{stamp}"))
|
|
write_json(output / "status.json", {"state": "running", "run": name, "revision": revision,
|
|
"started_unix": time.time(), "completed_pairs": len(summary["pairs"])})
|
|
env = dict(os.environ, POLL_POOL_AUTHOR_HISTORY=str(args.author_history),
|
|
POLL_POOL_ANNOTATORS=str(args.annotators), POLL_POOL_QUALITY_REVIEWERS=str(args.quality_reviewers),
|
|
POLL_POOL_ACCEPTANCE_REVIEWERS=str(args.acceptance_reviewers), POLL_POOL_CLAIM_COUNTS="1" if args.claim_counts else "0",
|
|
POLL_POOL_RUN_PAYLOAD_KIB=str(args.run_payload_kib), POLL_POOL_AUTHORIZATION_STATE=args.authorization_state,
|
|
POLL_POOL_LOAD="1", POLL_POOL_MODE=mode, POLL_POOL_REVISION=revision,
|
|
POLL_POOL_WARMUP_SECONDS=str(args.warmup), POLL_POOL_DURATION_SECONDS=str(args.duration),
|
|
POLL_POOL_OUTPUT=str(destination))
|
|
print(f"Running {name}: {revision[:8]}, mode={mode}, {args.warmup}s warmup + {args.duration}s measurement", flush=True)
|
|
try:
|
|
with (output / f"{name}.log").open("w") as log:
|
|
result = subprocess.run(["go", "test", "./internal/api", "-run", "^TestPollPoolReadMixLocal$", "-count=1", "-v",
|
|
"-timeout", f"{args.warmup + args.duration + 300}s"], cwd=roots[label], env=env,
|
|
stdout=log, stderr=subprocess.STDOUT)
|
|
if result.returncode:
|
|
raise RuntimeError(f"{name} failed; inspect {output / (name + '.log')}")
|
|
data = json.loads(destination.read_text())
|
|
validate_result(data, revision, mode, config)
|
|
except BaseException as error:
|
|
write_json(output / "status.json", {"state": "interrupted" if isinstance(error, KeyboardInterrupt) else "failed",
|
|
"run": name, "error": str(error), "completed_pairs": len(summary["pairs"])})
|
|
raise
|
|
reports[label] = data
|
|
baseline = reports["baseline"]
|
|
for variant in variants[1:]:
|
|
other = reports[variant["label"]]
|
|
equal = identities(baseline) == identities(other)
|
|
summary["pairs"].append({"repetition": repetition, "variant": variant["label"], "task_sets_equal": equal,
|
|
"baseline_errors": baseline["errors"], "candidate_errors": other["errors"],
|
|
"baseline_metrics": baseline["metrics"], "candidate_metrics": other["metrics"]})
|
|
write_json(output / "comparison.json", summary)
|
|
if not equal:
|
|
write_json(output / "status.json", {"state": "failed", "reason": "task-set mismatch", "repetition": repetition})
|
|
raise RuntimeError("task-set equivalence failed")
|
|
write_json(output / "status.json", {"state": "complete", "completed_pairs": len(summary["pairs"]), "capacity_accepted": False})
|
|
print(f"Read-mix comparison saved: {output / 'comparison.json'}", flush=True)
|
|
output_lock.close()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|