droboticflow/scripts/poc_strict_preflight_result.py
2026-08-20 19:47:43 +08:00

557 lines
24 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import ast
import datetime as dt
import hashlib
import json
import os
import pathlib
import re
import subprocess
from typing import Any
ROOT = pathlib.Path(__file__).resolve().parents[1]
UNMIGRATED_HISTORICAL_OPERATORS = {"temporal_causal_eval"}
RESULT_PREFIX = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="
DEFAULT_REQUIRED_EXTERNALS = "bos,mongo"
STAGE_HANDLERS = {
"fixture-readiness": "fixture_readiness",
"deployment-readiness": "deployment_readiness",
"runner-image-readiness": "runner_image_readiness",
"strict-preflight-readiness": "strict_preflight_readiness",
}
REPOSITORY_POLICY_VIOLATION_KEYS = (
"repositoryManifestErrors",
"operatorImageContractErrors",
"duplicateOperatorIds",
"duplicateOperatorNames",
"legacyOperators",
"builtinHelperCallOperators",
"hardcodedExternalApiSources",
"helperBuiltinKeys",
)
def load_json(path: pathlib.Path) -> dict[str, Any]:
text = path.read_text(encoding="utf-8").strip()
if not text:
return {}
payload = json.loads(text)
if not isinstance(payload, dict):
raise SystemExit(f"{path} must contain a JSON object")
return payload
def now_id() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y%m%d%H%M%S")
def now_iso() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z")
def source_commit() -> str:
for env_name in ("POC_ACCEPTANCE_SOURCE_COMMIT", "DROBOTICFLOW_BUILD_COMMIT"):
value = os.getenv(env_name, "").strip()
if value:
return value
try:
return subprocess.run(
["git", "-C", str(ROOT), "rev-parse", "HEAD"],
check=True,
capture_output=True,
text=True,
).stdout.strip() or "unknown"
except (OSError, subprocess.CalledProcessError):
return "unknown"
def builtin_operator_catalog_metadata() -> tuple[str, int]:
catalog_path = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
try:
catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise SystemExit(f"builtin operator catalog is not readable: {catalog_path}") from error
if not isinstance(catalog, list) or not catalog:
raise SystemExit("builtin operator catalog must be a non-empty JSON list")
canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
# Go's encoding/json keeps UTF-8 but escapes HTML-sensitive and line-separator runes.
for raw, escaped in (
("&", "\\u0026"),
("<", "\\u003c"),
(">", "\\u003e"),
("\u2028", "\\u2028"),
("\u2029", "\\u2029"),
):
canonical_text = canonical_text.replace(raw, escaped)
canonical = canonical_text.encode("utf-8")
return hashlib.sha256(canonical).hexdigest(), len(catalog)
def acceptance_provenance(completed_at: str | None = None) -> dict[str, Any]:
catalog_hash, catalog_count = builtin_operator_catalog_metadata()
return {
"sourceCommit": source_commit(),
"catalogHash": catalog_hash,
"catalogCount": catalog_count,
"completedAt": completed_at or now_iso(),
}
def required_externals(raw: str) -> list[str]:
return sorted({item.strip().lower() for item in raw.split(",") if item.strip()})
def operator_inventory() -> list[str]:
operators: list[str] = []
builtin_root = ROOT / "backend" / "operators" / "builtin"
for metadata_path in sorted(builtin_root.glob("*/operator.json")):
slug = metadata_path.parent.name
if slug in UNMIGRATED_HISTORICAL_OPERATORS:
continue
metadata: dict[str, Any] = {}
try:
loaded_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
except Exception:
loaded_metadata = {}
if isinstance(loaded_metadata, dict):
metadata = loaded_metadata
if metadata.get("type") not in ("builtin", None):
continue
if not slug:
continue
operators.append(slug)
return sorted(set(operators))
def project_relative(path: pathlib.Path) -> str:
try:
return str(path.relative_to(ROOT))
except ValueError:
return str(path)
def operator_helper_builtin_keys() -> list[str]:
runner_path = ROOT / "backend" / "operator-runner" / "main.py"
if not runner_path.exists():
return []
try:
tree = ast.parse(runner_path.read_text(encoding="utf-8"))
except SyntaxError as error:
return [f"parse-error:{error.msg}"]
keys: list[str] = []
for node in tree.body:
if not isinstance(node, ast.Assign):
continue
if not any(isinstance(target, ast.Name) and target.id == "OPERATOR_HELPERS" for target in node.targets):
continue
if not isinstance(node.value, ast.Dict):
continue
for key in node.value.keys:
if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.startswith("builtin_"):
keys.append(key.value)
return sorted(set(keys))
def static_operator_layout() -> dict[str, Any]:
operator_root = ROOT / "backend" / "operators" / "builtin"
catalog_path = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
# Historical source retained outside the active builtin catalog must not
# inflate strict current-operator coverage requirements.
metadata_paths = sorted(path for path in operator_root.glob("*/operator.json") if path.parent.name not in UNMIGRATED_HISTORICAL_OPERATORS)
sources: list[str] = []
repository_manifests: list[str] = []
repository_manifest_errors: list[str] = []
operator_image_contract_errors: list[str] = []
legacy: list[str] = []
builtin_helper_calls: list[str] = []
hardcoded_external_api_sources: list[str] = []
seen_operator_ids: dict[str, str] = {}
duplicate_operator_ids: list[str] = []
seen_operator_names: dict[str, str] = {}
duplicate_operator_names: list[str] = []
catalog_items: dict[str, dict[str, Any]] = {}
try:
loaded_catalog = json.loads(catalog_path.read_text(encoding="utf-8"))
if isinstance(loaded_catalog, list):
catalog_items = {
str(item.get("id", "")).strip(): item
for item in loaded_catalog
if isinstance(item, dict) and item.get("type") == "builtin"
}
else:
operator_image_contract_errors.append("builtin catalog must be a JSON list")
except Exception as error:
operator_image_contract_errors.append(f"builtin catalog is not readable ({type(error).__name__}: {error})")
for metadata_path in metadata_paths:
slug = metadata_path.parent.name
operator_path = metadata_path.with_name("operator.py")
repository_path = metadata_path.with_name("repository.json")
if operator_path.exists():
sources.append(project_relative(operator_path))
source = operator_path.read_text(encoding="utf-8")
else:
source = ""
sources.append(project_relative(operator_path))
repository_manifest_errors.append(f"{slug}: missing operator.py")
try:
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
except Exception as error:
repository_manifest_errors.append(f"{slug}: invalid operator.json ({type(error).__name__}: {error})")
metadata = {}
if not isinstance(metadata, dict):
repository_manifest_errors.append(f"{slug}: operator.json must be an object")
metadata = {}
operator_id = str(metadata.get("id", "")).strip()
operator_name = str(metadata.get("name", "")).strip()
if not operator_id:
repository_manifest_errors.append(f"{slug}: operator.json id is required")
elif operator_id in seen_operator_ids:
duplicate_operator_ids.append(operator_id)
else:
seen_operator_ids[operator_id] = slug
if not operator_name:
repository_manifest_errors.append(f"{slug}: operator.json name is required")
elif operator_name in seen_operator_names:
duplicate_operator_names.append(operator_name)
else:
seen_operator_names[operator_name] = slug
if slug == "human_review" and metadata.get("runtimeHandler") != "builtin_human_review":
repository_manifest_errors.append("human_review: runtimeHandler must be builtin_human_review")
if "LEGACY_HANDLER" in source:
legacy.append(slug)
if 'helpers["builtin_' in source or "helpers['builtin_" in source:
builtin_helper_calls.append(slug)
if not repository_path.exists():
repository_manifest_errors.append(f"{slug}: missing repository.json")
continue
try:
repository_manifest = json.loads(repository_path.read_text(encoding="utf-8"))
except Exception as error:
repository_manifest_errors.append(f"{slug}: invalid repository.json ({type(error).__name__}: {error})")
continue
if not isinstance(repository_manifest, dict):
repository_manifest_errors.append(f"{slug}: repository.json must be an object")
continue
expected_fields = {
"schemaVersion": "droboticflow.operator.repository/v1",
"repositoryId": f"builtin/{slug}",
"operatorId": metadata.get("id"),
"operatorSlug": slug,
"sourceRoot": f"backend/operators/builtin/{slug}",
"entrypoint": "operator.py",
"metadata": "operator.json",
"runtimeSource": metadata.get("runtimeSource"),
"runtimeHandler": metadata.get("runtimeHandler"),
"containerEntrypoint": "backend/operator-runner/main.py",
}
for key, expected in expected_fields.items():
if repository_manifest.get(key) != expected:
repository_manifest_errors.append(
f"{slug}: repository.json {key}={repository_manifest.get(key)!r}, expected {expected!r}"
)
if "externalApi" in metadata and repository_manifest.get("externalApi") != metadata.get("externalApi"):
repository_manifest_errors.append(f"{slug}: repository.json externalApi must match operator.json")
catalog_item = catalog_items.get(operator_id)
if not catalog_item:
operator_image_contract_errors.append(f"{slug}: missing builtin catalog item")
catalog_item = {}
image = str(catalog_item.get("image") or repository_manifest.get("image") or "").strip()
image_id = str(catalog_item.get("imageId") or repository_manifest.get("imageId") or "").strip()
runtime_contract = catalog_item.get("runtimeContract") or repository_manifest.get("runtimeContract")
if not image:
operator_image_contract_errors.append(f"{slug}: builtin image is required")
if "droboticflow-operator-runner" in image:
operator_image_contract_errors.append(f"{slug}: builtin image must not use shared operator-runner final image")
if "droboticflow-operator-" not in image:
operator_image_contract_errors.append(f"{slug}: builtin image must use an independent operator image tag")
if not re.search(r"@sha256:[a-fA-F0-9]{64}$", image_id):
operator_image_contract_errors.append(f"{slug}: imageId must be digest-pinned")
if catalog_item.get("containerCommand") != ["python", "/app/backend/operator-runner/main.py"]:
operator_image_contract_errors.append(f"{slug}: containerCommand must point at the operator runner entrypoint")
if catalog_item.get("containerArgs", []) != []:
operator_image_contract_errors.append(f"{slug}: containerArgs must default to []")
if not isinstance(runtime_contract, dict):
operator_image_contract_errors.append(f"{slug}: runtimeContract is required")
elif not runtime_contract.get("imagePolicy", {}).get("independentFinalImage"):
operator_image_contract_errors.append(f"{slug}: runtimeContract must require independent final image")
command_keys = {
str(param.get("key", "")).strip().upper()
for param in catalog_item.get("hyperparams", [])
if isinstance(param, dict)
}
if operator_id != "operator-builtin-shell-command" and command_keys.intersection({"COMMAND", "SCRIPT"}):
operator_image_contract_errors.append(f"{slug}: non-shell builtin must not expose COMMAND/SCRIPT")
acceptance = repository_manifest.get("acceptance") if isinstance(repository_manifest.get("acceptance"), dict) else {}
if not acceptance.get("requiresRealInputs"):
repository_manifest_errors.append(f"{slug}: repository.json must require real inputs")
for key in ("mockOnlyAccepted", "simulationOnlyAccepted", "hardcodedOutputAccepted", "shallowOnlyAccepted"):
if acceptance.get(key) is not False:
repository_manifest_errors.append(f"{slug}: repository.json acceptance.{key} must be false")
if acceptance.get("requiresPersistentWorkspaceEvidence") is not True:
repository_manifest_errors.append(f"{slug}: repository.json must require persistent workspace evidence")
dependencies = repository_manifest.get("runtimeDependencies")
if not isinstance(dependencies, list) or not dependencies:
repository_manifest_errors.append(f"{slug}: repository.json must list runtime dependencies")
repository_manifests.append(project_relative(repository_path))
return {
"operatorFileCount": len(metadata_paths),
"repositoryManifestCount": len(repository_manifests),
"repositoryManifestErrors": repository_manifest_errors,
"operatorImageContractErrors": operator_image_contract_errors,
"duplicateOperatorIds": sorted(duplicate_operator_ids),
"duplicateOperatorNames": sorted(duplicate_operator_names),
"legacyOperators": legacy,
"builtinHelperCallOperators": builtin_helper_calls,
"hardcodedExternalApiSources": hardcoded_external_api_sources,
"helperBuiltinKeys": operator_helper_builtin_keys(),
"runtimeSources": sources,
"repositoryManifests": repository_manifests,
}
def operator_repository_summary(static_payload: dict[str, Any]) -> dict[str, Any]:
violations: list[str] = []
for key in REPOSITORY_POLICY_VIOLATION_KEYS:
for item in static_payload.get(key, []) or []:
text = str(item).strip()
if text:
violations.append(f"{key}:{text}")
required_count = int(static_payload.get("operatorFileCount") or 0)
repository_count = int(static_payload.get("repositoryManifestCount") or 0)
return {
"operatorRepositoryRequiredCount": required_count,
"operatorRepositoryCount": repository_count,
"operatorRepositoryCoverageComplete": required_count > 0 and repository_count == required_count,
"operatorRepositoryPolicyClean": len(violations) == 0,
"operatorRepositoryPolicyViolations": violations,
}
def readiness_stage(check_name: str) -> str:
if check_name == "fixture-manifest":
return "fixture-readiness"
if check_name.startswith("deployment:"):
return "deployment-readiness"
if check_name == "operator-runner-image":
return "runner-image-readiness"
return "strict-preflight-readiness"
def readiness_issues(payload: dict[str, Any]) -> list[tuple[str, str, dict[str, Any]]]:
issues: list[tuple[str, str, dict[str, Any]]] = []
for check in payload.get("checks") or []:
if not isinstance(check, dict):
continue
if check.get("status") == "passed" or check.get("required") is False:
continue
name = str(check.get("name") or "unknown")
stage = readiness_stage(name)
message = str(check.get("message") or f"{name} did not pass")
evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {}
issues.append((stage, message, {"checkName": name, **evidence}))
if issues:
return issues
status = str(payload.get("status") or "").strip()
if status and status != "ready":
message = "POC strict readiness did not pass"
missing = payload.get("missing")
if isinstance(missing, list) and missing:
message += ": " + ", ".join(str(item.get("message") if isinstance(item, dict) else item) for item in missing)
issues.append(("strict-preflight-readiness", message, {}))
return issues
def readiness_fixture_sources(payload: dict[str, Any]) -> dict[str, Any]:
for check in payload.get("checks") or []:
if not isinstance(check, dict):
continue
if check.get("name") != "fixture-manifest" or check.get("status") != "passed":
continue
evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {}
raw_fixtures = evidence.get("fixtures") if isinstance(evidence.get("fixtures"), dict) else {}
items: dict[str, dict[str, Any]] = {}
for name, raw_path in sorted(raw_fixtures.items()):
path = str(raw_path or "").strip()
if not path:
continue
items[str(name)] = {"path": path, "generated": False}
if not items or evidence.get("shellCommandConfigured") is not True:
return {}
return {
"allowGeneratedFixtures": False,
"fixtureManifestPath": str(evidence.get("path") or "").strip(),
"items": items,
}
return {}
def fixture_source_counts(fixture_sources: dict[str, Any]) -> dict[str, Any]:
items = fixture_sources.get("items") if isinstance(fixture_sources.get("items"), dict) else {}
generated_count = 0
real_count = 0
for item in items.values():
if not isinstance(item, dict):
continue
if item.get("generated") is True:
generated_count += 1
else:
real_count += 1
return {
"generatedFixtureCount": generated_count,
"realFixtureCount": real_count,
"allFixturesReal": bool(items) and generated_count == 0 and fixture_sources.get("allowGeneratedFixtures") is not True,
}
def compact_probe_evidence(payload: dict[str, Any]) -> dict[str, Any]:
evidence: dict[str, Any] = {
"source": payload.get("source"),
"namespace": payload.get("namespace"),
"secretName": payload.get("secretName"),
"status": payload.get("status"),
}
context = payload.get("context")
if isinstance(context, dict):
evidence["context"] = context
probes: list[dict[str, Any]] = []
for item in payload.get("probes") or []:
if not isinstance(item, dict):
continue
response = item.get("response") if isinstance(item.get("response"), dict) else {}
probes.append(
{
"slug": item.get("slug"),
"status": item.get("status"),
"httpStatus": response.get("status"),
"bodyKind": response.get("bodyKind"),
"accepted": response.get("accepted"),
}
)
if probes:
evidence["probes"] = probes
return {key: value for key, value in evidence.items() if value not in (None, "", [], {})}
def issue_missing_coverage(stage: str, all_operators: list[str]) -> list[str]:
return all_operators
def safe_handler(stage: str) -> str:
return STAGE_HANDLERS.get(stage, re.sub(r"[^a-z0-9]+", "_", stage.lower()).strip("_") or "strict_preflight")
def build_result(args: argparse.Namespace) -> dict[str, Any]:
all_operators = operator_inventory()
static_payload = static_operator_layout()
repository_summary = operator_repository_summary(static_payload)
required = required_externals(args.required_externals)
source = args.source
if source == "readiness":
payload = load_json(pathlib.Path(args.input_json))
issues = readiness_issues(payload)
namespace = str(payload.get("namespace") or args.namespace)
workspace_mount = str(payload.get("workspaceMountPath") or args.workspace_mount_path).rstrip("/") or "/workspace"
required = required_externals(",".join(payload.get("requiredExternals") or required))
fixture_sources = readiness_fixture_sources(payload)
else:
raise SystemExit(f"unsupported source: {source}")
if not issues:
raise SystemExit("preflight result requires at least one blocking issue")
started_at = now_iso()
run_id = args.run_id.strip() or f"poc-strict-preflight-{now_id()}"
records: list[dict[str, Any]] = []
missing_coverage_set: set[str] = set()
failures: list[str] = []
stages: list[str] = []
for stage, reason, evidence in issues:
stages.append(stage)
failures.append(reason)
missing_coverage = issue_missing_coverage(stage, all_operators)
missing_coverage_set.update(missing_coverage)
record_evidence = dict(evidence)
record_evidence["missingOperatorCoverage"] = missing_coverage
records.append(
{
"operator": "poc_acceptance_preflight",
"handler": safe_handler(stage),
"status": "failed",
"error": reason,
"evidence": record_evidence,
"preflightStage": stage,
}
)
unique_stages = list(dict.fromkeys(stages))
stage = unique_stages[0] if len(unique_stages) == 1 else "strict-preflight-readiness"
missing_coverage = sorted(missing_coverage_set)
required_external_systems = sorted({"bos", "mongo"}.intersection(set(required)))
fixture_counts = fixture_source_counts(fixture_sources)
result = {
"runId": run_id,
"workspace": f"{workspace_mount}/acceptance/{run_id}",
"preflightFailure": True,
"preflightStage": stage,
"preflightStages": unique_stages,
"acceptanceScope": "full" if {"bos", "mongo"}.issubset(set(required)) else "dependency-isolation",
"fullExternalAcceptance": False,
"operatorCount": len(all_operators),
"operatorPassedCount": 0,
"coveredOperatorCount": 0,
"allOperatorCoverageComplete": False,
"blockingOperatorCoverageComplete": False,
"missingCoverage": missing_coverage,
"blockingMissingCoverage": missing_coverage,
"requiredExternals": required,
"statusCounts": {"failed": len(records)},
"passedCount": 0,
"externalVerificationPassed": [],
"externalVerificationMissing": required_external_systems,
"externalVerificationPassedCount": 0,
"externalVerificationRequiredCount": len(required_external_systems),
**fixture_counts,
"failedCount": len(records),
"skippedCount": 0,
"requiredSkippedCount": 0,
"optionalSkippedCount": 0,
"static": static_payload,
**repository_summary,
"records": records,
"failures": failures,
"status": "failed",
"namespace": namespace,
"startedAt": started_at,
"endedAt": started_at,
**acceptance_provenance(started_at),
}
if fixture_sources:
result["fixtureSources"] = fixture_sources
return result
def main() -> int:
parser = argparse.ArgumentParser(description="Build a standard POC acceptance result for strict preflight blockers.")
parser.add_argument("--source", choices=("readiness",), required=True)
parser.add_argument("--input-json", required=True)
parser.add_argument("--run-id", default=os.getenv("POC_OPERATOR_ACCEPTANCE_JOB_NAME") or "")
parser.add_argument("--namespace", default=os.getenv("K8S_NAMESPACE") or "droboticflow")
parser.add_argument("--workspace-mount-path", default=os.getenv("WORKSPACE_MOUNT_PATH") or "/workspace")
parser.add_argument("--required-externals", default=os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS") or DEFAULT_REQUIRED_EXTERNALS)
args = parser.parse_args()
print(RESULT_PREFIX + json.dumps(build_result(args), ensure_ascii=False, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())