droboticflow-new/scripts/source_index_capacity_plan.py
2026-10-17 13:11:00 +08:00

142 lines
10 KiB
Python

#!/usr/bin/env python3
"""Render an offline resource/concurrency review plan. Never calls kubectl or applies it."""
import argparse
import copy
import hashlib
import json
from pathlib import Path
import re
ROOT = Path(__file__).resolve().parents[1]
PROFILE = ROOT / "docs/capacity/source-index-100.json"
def render(snapshot, candidate, profile, package_image=None, load_report=None, lock_stage="barrier"):
if not re.fullmatch(r"[a-f0-9]{40}", candidate):
raise ValueError("A full candidate commit is required")
if lock_stage not in {"barrier", "striped"}:
raise ValueError("Lock stage must be barrier or striped")
namespace = profile["namespace"]
deployments = {d["metadata"]["name"]: d for d in snapshot["items"] if d.get("kind", "Deployment") == "Deployment"}
blockers, patches = [], []
if (not load_report or load_report.get("activeOperators") != 100 or load_report.get("completed") != 100
or load_report.get("sourceFramesPerCamera") != 27000
or load_report.get("projectWriteLockMode") != "striped"
or load_report.get("requests", {}).get("draft-save", {}).get("p95Ms", float("inf")) > 750):
blockers.append("100-operator production-scale local HTTP acceptance with draft-save p95 <=750ms is required")
if package_image is None:
blockers.append("New source-index package operator image digest and provenance are required")
elif not re.fullmatch(r"[^\s@]+@sha256:[a-f0-9]{64}", package_image):
raise ValueError("Package operator image must be pinned by digest")
for name, desired in profile["deployments"].items():
if name == "droboticflow-annotation-validator" and name not in deployments:
if lock_stage == "striped":
blockers.append(name + ": bridge validator must already exist before enabling striped writes")
continue
d = deployments.get(name)
if d is None or d["metadata"].get("namespace") != namespace:
raise ValueError("Missing deployment or wrong namespace: " + name)
containers = d["spec"]["template"]["spec"]["containers"]
matches = [(i, c) for i, c in enumerate(containers) if c["name"] == desired["container"]]
if len(matches) != 1:
raise ValueError("Unexpected container layout")
index, container = matches[0]
env = container.get("env", [])
values = {e["name"]: e.get("value") for e in env}
if values.get("DROBOTICFLOW_BUILD_COMMIT") != candidate:
blockers.append(name + ": candidate image/build stamp must be deployed before enabling this profile")
if lock_stage == "striped":
status = d.get("status", {})
desired_replicas = d["spec"]["replicas"]
generation = d["metadata"].get("generation")
complete = (desired_replicas > 0 and generation is not None
and status.get("observedGeneration", 0) >= generation
and all(status.get(k, 0) == desired_replicas for k in ("replicas", "updatedReplicas", "readyReplicas", "availableReplicas"))
and status.get("unavailableReplicas", 0) == 0 and status.get("terminatingReplicas", 0) == 0)
if values.get("DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE") not in {"barrier", "striped"} or not complete:
blockers.append(name + ": completed barrier-capable rollout is required before striped activation")
base = f"/spec/template/spec/containers/{index}"
patch = [
{"op": "test", "path": "/metadata/uid", "value": d["metadata"]["uid"]},
{"op": "test", "path": "/metadata/resourceVersion", "value": d["metadata"]["resourceVersion"]},
{"op": "test", "path": base + "/image", "value": container["image"]},
{"op": "replace", "path": "/spec/replicas", "value": desired["replicas"]},
{"op": "add", "path": base + "/resources", "value": {k: desired[k] for k in ("requests", "limits")}},
]
desired_env = dict(desired["env"])
desired_env["DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE"] = lock_stage
if name == "droboticflow-worker" and package_image:
desired_env["ANNOTATION_SOURCE_PACKAGE_IMAGE"] = package_image
for key, value in desired_env.items():
indices = [i for i, e in enumerate(env) if e["name"] == key]
if len(indices) > 1:
raise ValueError("Duplicate relevant environment variable")
patch.append({"op": "replace" if indices else "add", "path": base + "/env/" + (str(indices[0]) if indices else "-"), "value": {"name": key, "value": value}})
patches.append({"deployment": name, "patch": patch})
processor = deployments["droboticflow-annotation-processor"]
desired = profile["deployments"]["droboticflow-annotation-validator"]
validator = None
if "droboticflow-annotation-validator" not in deployments:
pod = processor["spec"]["template"]["spec"]
source = next(c for c in pod["containers"] if c["name"] == "processor")
allowed = {"DROSYNC_COORDINATOR_PORT", "MONGO_DB", "MONGO_URI", "BOS_ENDPOINT", "WORKSPACE_MOUNT_PATH", "ANNOTATION_WORKSPACE_CONNECTION_ID", "ANNOTATION_WORKSPACE_BUCKET", "ANNOTATION_WORKSPACE_MAPPINGS", "MONGO_CONNECT_RETRIES", "MONGO_CONNECT_RETRY_DELAY_MS"}
env = [copy.deepcopy(e) for e in source.get("env", []) if e["name"] in allowed]
ports = [e for e in env if e["name"] == "DROSYNC_COORDINATOR_PORT"]
if len(ports) != 1 or not (ports[0].get("valueFrom", {}).get("configMapKeyRef")
or re.fullmatch(r"[0-9]+", ports[0].get("value", "")) and 0 < int(ports[0]["value"]) <= 65535):
raise ValueError("Validator requires one explicit integer coordinator port or ConfigMap reference; service-link tcp:// values cannot be parsed")
mongo = next((e for e in env if e["name"] == "MONGO_URI"), {})
if "secretKeyRef" not in mongo.get("valueFrom", {}):
raise ValueError("Mongo must remain a Secret reference; no literal credentials can be rendered")
env += [{"name": k, "value": v} for k, v in {**desired["env"], "DROBOTICFLOW_DATABASE_BOOTSTRAP": "verify-existing"}.items()]
for variable in env:
if variable["name"] == "DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE":
variable["value"] = lock_stage
env += [{"name": "ANNOTATION_PROCESSOR_MODE", "value": "processor"}]
env.append({"name": "DROBOTICFLOW_BUILD_COMMIT", "value": candidate})
label = {"app": "droboticflow-annotation-validator"}
validator = {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"name": label["app"], "namespace": namespace}, "spec": {
"replicas": desired["replicas"], "strategy": {"type": "RollingUpdate", "rollingUpdate": {"maxSurge": 0, "maxUnavailable": 1}}, "selector": {"matchLabels": label},
"template": {"metadata": {"labels": label}, "spec": {
"automountServiceAccountToken": False, "terminationGracePeriodSeconds": 30,
"imagePullSecrets": pod.get("imagePullSecrets", []),
"containers": [{"name": "processor", "image": source["image"], "command": source.get("command", ["/usr/local/bin/droboticflow-api"]), "env": env,
"resources": {k: desired[k] for k in ("requests", "limits")}, "volumeMounts": source["volumeMounts"], "ports": source.get("ports", []), "readinessProbe": source.get("readinessProbe", {})}],
"volumes": pod["volumes"],
}}}}
result = {"schemaVersion": 1, "profile": profile["profile"], "candidate": candidate, "namespace": namespace, "lockStage": lock_stage,
"state": "blocked_until_candidate_and_review" if blockers else "review_only_not_authorized",
"blockers": blockers, "patches": patches, "createValidator": validator, "budget": profile["budget"], "constraints": profile["constraints"],
"approvalRequired": "Separate explicit approval for final images, resource/config changes, new validators, source-copy writes, startup indexes and read/write verification scope",
"writerInventoryReviewRequired": "Deployment status is not a full writer inventory: review Pods, Jobs, CronJobs, maintenance scripts and external Mongo writers; no old gate-only writer may remain before striped activation"}
cpu = sum(float(d["limits"]["cpu"]) * d["replicas"] for d in profile["deployments"].values())
memory = sum(int(d["limits"]["memory"].removesuffix("Gi")) * 2**30 * d["replicas"] for d in profile["deployments"].values())
allowance = profile["legacyExportBudget"]
if cpu + allowance["cpu"] > profile["budget"]["cpu"] or memory + allowance["memoryBytes"] > profile["budget"]["memoryBytes"]:
raise ValueError("Proposed limits exceed the approved budget")
result["resourceTotals"] = {"steadyLimitCPU": cpu, "steadyLimitMemoryBytes": memory, "exportAllowanceEnforced": False,
"unallocatedCPUAfterExportAllowance": profile["budget"]["cpu"] - cpu - allowance["cpu"],
"unallocatedMemoryBytesAfterExportAllowance": profile["budget"]["memoryBytes"] - memory - allowance["memoryBytes"]}
result["localLoadEvidence"] = load_report
result["planHash"] = hashlib.sha256(json.dumps(result, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--snapshot", required=True, type=Path)
parser.add_argument("--candidate", required=True)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--package-image")
parser.add_argument("--load-report", type=Path)
parser.add_argument("--lock-stage", choices=("barrier", "striped"), default="barrier")
args = parser.parse_args()
result = render(json.loads(args.snapshot.read_text()), args.candidate, json.loads(PROFILE.read_text()), args.package_image,
json.loads(args.load_report.read_text()) if args.load_report else None, args.lock_stage)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps({"state": result["state"], "planHash": result["planHash"], "blockers": result["blockers"]}))
if __name__ == "__main__":
main()