droboticflow/scripts/poc_strict_preflight_result.py
2026-06-23 21:23:22 +08:00

460 lines
20 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import ast
import datetime as dt
import json
import os
import pathlib
import re
from typing import Any
ROOT = pathlib.Path(__file__).resolve().parents[1]
RESULT_PREFIX = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="
DEFAULT_REQUIRED_EXTERNALS = "bos,dcloud,mongo"
STAGE_HANDLERS = {
"dcloud-secret-readiness": "dcloud_secret_readiness",
"dcloud-fat-probe": "dcloud_fat_probe",
"fixture-readiness": "fixture_readiness",
"deployment-readiness": "deployment_readiness",
"runner-image-readiness": "runner_image_readiness",
"strict-preflight-readiness": "strict_preflight_readiness",
}
REPOSITORY_POLICY_VIOLATION_KEYS = (
"repositoryManifestErrors",
"duplicateOperatorIds",
"duplicateOperatorNames",
"legacyOperators",
"builtinHelperCallOperators",
"hardcodedExternalApiSources",
"helperBuiltinKeys",
)
def load_json(path: pathlib.Path) -> dict[str, Any]:
text = path.read_text(encoding="utf-8").strip()
if not text:
return {}
payload = json.loads(text)
if not isinstance(payload, dict):
raise SystemExit(f"{path} must contain a JSON object")
return payload
def now_id() -> str:
return dt.datetime.now(dt.UTC).strftime("%Y%m%d%H%M%S")
def now_iso() -> str:
return dt.datetime.now(dt.UTC).isoformat().replace("+00:00", "Z")
def required_externals(raw: str) -> list[str]:
return sorted({item.strip().lower() for item in raw.split(",") if item.strip()})
def operator_inventory() -> tuple[list[str], list[str]]:
operators: list[str] = []
dcloud_operators: list[str] = []
builtin_root = ROOT / "backend" / "operators" / "builtin"
for metadata_path in sorted(builtin_root.glob("*/operator.json")):
slug = metadata_path.parent.name
metadata: dict[str, Any] = {}
try:
loaded_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
except Exception:
loaded_metadata = {}
if isinstance(loaded_metadata, dict):
metadata = loaded_metadata
if metadata.get("type") not in ("builtin", None):
continue
if not slug:
continue
operators.append(slug)
strict_externals: set[str] = set()
repository_path = metadata_path.with_name("repository.json")
try:
repository = json.loads(repository_path.read_text(encoding="utf-8"))
except Exception:
repository = {}
if isinstance(repository, dict):
strict_externals = {
str(item).strip().lower()
for item in repository.get("strictPocAcceptanceExternals") or []
if str(item).strip()
}
external_api = metadata.get("externalApi") if isinstance(metadata.get("externalApi"), dict) else {}
if "dcloud" in strict_externals or slug.startswith("dcloud_") or external_api:
dcloud_operators.append(slug)
return sorted(set(operators)), sorted(set(dcloud_operators))
def project_relative(path: pathlib.Path) -> str:
try:
return str(path.relative_to(ROOT))
except ValueError:
return str(path)
def operator_helper_builtin_keys() -> list[str]:
runner_path = ROOT / "backend" / "operator-runner" / "main.py"
if not runner_path.exists():
return []
try:
tree = ast.parse(runner_path.read_text(encoding="utf-8"))
except SyntaxError as error:
return [f"parse-error:{error.msg}"]
keys: list[str] = []
for node in tree.body:
if not isinstance(node, ast.Assign):
continue
if not any(isinstance(target, ast.Name) and target.id == "OPERATOR_HELPERS" for target in node.targets):
continue
if not isinstance(node.value, ast.Dict):
continue
for key in node.value.keys:
if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.startswith("builtin_"):
keys.append(key.value)
return sorted(set(keys))
def static_operator_layout() -> dict[str, Any]:
operator_root = ROOT / "backend" / "operators" / "builtin"
metadata_paths = sorted(operator_root.glob("*/operator.json"))
sources: list[str] = []
repository_manifests: list[str] = []
repository_manifest_errors: list[str] = []
legacy: list[str] = []
builtin_helper_calls: list[str] = []
hardcoded_external_api_sources: list[str] = []
seen_operator_ids: dict[str, str] = {}
duplicate_operator_ids: list[str] = []
seen_operator_names: dict[str, str] = {}
duplicate_operator_names: list[str] = []
for metadata_path in metadata_paths:
slug = metadata_path.parent.name
operator_path = metadata_path.with_name("operator.py")
repository_path = metadata_path.with_name("repository.json")
if operator_path.exists():
sources.append(project_relative(operator_path))
source = operator_path.read_text(encoding="utf-8")
else:
source = ""
sources.append(project_relative(operator_path))
repository_manifest_errors.append(f"{slug}: missing operator.py")
try:
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
except Exception as error:
repository_manifest_errors.append(f"{slug}: invalid operator.json ({type(error).__name__}: {error})")
metadata = {}
if not isinstance(metadata, dict):
repository_manifest_errors.append(f"{slug}: operator.json must be an object")
metadata = {}
operator_id = str(metadata.get("id", "")).strip()
operator_name = str(metadata.get("name", "")).strip()
if not operator_id:
repository_manifest_errors.append(f"{slug}: operator.json id is required")
elif operator_id in seen_operator_ids:
duplicate_operator_ids.append(operator_id)
else:
seen_operator_ids[operator_id] = slug
if not operator_name:
repository_manifest_errors.append(f"{slug}: operator.json name is required")
elif operator_name in seen_operator_names:
duplicate_operator_names.append(operator_name)
else:
seen_operator_names[operator_name] = slug
if slug == "human_review" and metadata.get("runtimeHandler") != "builtin_human_review":
repository_manifest_errors.append("human_review: runtimeHandler must be builtin_human_review")
if slug.startswith("dcloud_") and (
"https://gitea.d-robotics.cc" in source or "https://cloud-fat.d-robotics.cc" in source
):
hardcoded_external_api_sources.append(slug)
if "LEGACY_HANDLER" in source:
legacy.append(slug)
if 'helpers["builtin_' in source or "helpers['builtin_" in source:
builtin_helper_calls.append(slug)
if not repository_path.exists():
repository_manifest_errors.append(f"{slug}: missing repository.json")
continue
try:
repository_manifest = json.loads(repository_path.read_text(encoding="utf-8"))
except Exception as error:
repository_manifest_errors.append(f"{slug}: invalid repository.json ({type(error).__name__}: {error})")
continue
if not isinstance(repository_manifest, dict):
repository_manifest_errors.append(f"{slug}: repository.json must be an object")
continue
expected_fields = {
"schemaVersion": "droboticflow.operator.repository/v1",
"repositoryId": f"builtin/{slug}",
"operatorId": metadata.get("id"),
"operatorSlug": slug,
"sourceRoot": f"backend/operators/builtin/{slug}",
"entrypoint": "operator.py",
"metadata": "operator.json",
"runtimeSource": metadata.get("runtimeSource"),
"runtimeHandler": metadata.get("runtimeHandler"),
"containerEntrypoint": "backend/operator-runner/main.py",
}
for key, expected in expected_fields.items():
if repository_manifest.get(key) != expected:
repository_manifest_errors.append(
f"{slug}: repository.json {key}={repository_manifest.get(key)!r}, expected {expected!r}"
)
if "externalApi" in metadata and repository_manifest.get("externalApi") != metadata.get("externalApi"):
repository_manifest_errors.append(f"{slug}: repository.json externalApi must match operator.json")
acceptance = repository_manifest.get("acceptance") if isinstance(repository_manifest.get("acceptance"), dict) else {}
if not acceptance.get("requiresRealInputs"):
repository_manifest_errors.append(f"{slug}: repository.json must require real inputs")
for key in ("mockOnlyAccepted", "simulationOnlyAccepted", "hardcodedOutputAccepted", "shallowOnlyAccepted"):
if acceptance.get(key) is not False:
repository_manifest_errors.append(f"{slug}: repository.json acceptance.{key} must be false")
if acceptance.get("requiresPersistentWorkspaceEvidence") is not True:
repository_manifest_errors.append(f"{slug}: repository.json must require persistent workspace evidence")
dependencies = repository_manifest.get("runtimeDependencies")
if not isinstance(dependencies, list) or not dependencies:
repository_manifest_errors.append(f"{slug}: repository.json must list runtime dependencies")
repository_manifests.append(project_relative(repository_path))
return {
"operatorFileCount": len(metadata_paths),
"repositoryManifestCount": len(repository_manifests),
"repositoryManifestErrors": repository_manifest_errors,
"duplicateOperatorIds": sorted(duplicate_operator_ids),
"duplicateOperatorNames": sorted(duplicate_operator_names),
"legacyOperators": legacy,
"builtinHelperCallOperators": builtin_helper_calls,
"hardcodedExternalApiSources": hardcoded_external_api_sources,
"helperBuiltinKeys": operator_helper_builtin_keys(),
"runtimeSources": sources,
"repositoryManifests": repository_manifests,
}
def operator_repository_summary(static_payload: dict[str, Any]) -> dict[str, Any]:
violations: list[str] = []
for key in REPOSITORY_POLICY_VIOLATION_KEYS:
for item in static_payload.get(key, []) or []:
text = str(item).strip()
if text:
violations.append(f"{key}:{text}")
required_count = int(static_payload.get("operatorFileCount") or 0)
repository_count = int(static_payload.get("repositoryManifestCount") or 0)
return {
"operatorRepositoryRequiredCount": required_count,
"operatorRepositoryCount": repository_count,
"operatorRepositoryCoverageComplete": required_count > 0 and repository_count == required_count,
"operatorRepositoryPolicyClean": len(violations) == 0,
"operatorRepositoryPolicyViolations": violations,
}
def readiness_stage(check_name: str) -> str:
if check_name == "dcloud-secret":
return "dcloud-secret-readiness"
if check_name == "fixture-manifest":
return "fixture-readiness"
if check_name.startswith("deployment:"):
return "deployment-readiness"
if check_name == "operator-runner-image":
return "runner-image-readiness"
return "strict-preflight-readiness"
def dcloud_probe_reason(payload: dict[str, Any]) -> str:
status = str(payload.get("status") or "failed")
missing = payload.get("missing") if isinstance(payload.get("missing"), list) else []
if missing:
return "D-Cloud FAT probe is blocked: " + ", ".join(str(item) for item in missing)
error = str(payload.get("error") or "").strip()
if error:
return "D-Cloud FAT probe failed: " + error
failed_probes: list[str] = []
for item in payload.get("probes") or []:
if not isinstance(item, dict):
continue
if item.get("status") != "passed":
slug = str(item.get("slug") or "unknown")
response = item.get("response") if isinstance(item.get("response"), dict) else {}
http_status = response.get("status")
suffix = f" HTTP {http_status}" if http_status is not None else ""
failed_probes.append(slug + suffix)
if failed_probes:
return "D-Cloud FAT probe did not pass: " + ", ".join(failed_probes)
return f"D-Cloud FAT probe status is {status}"
def readiness_issues(payload: dict[str, Any]) -> list[tuple[str, str, dict[str, Any]]]:
issues: list[tuple[str, str, dict[str, Any]]] = []
for check in payload.get("checks") or []:
if not isinstance(check, dict):
continue
if check.get("status") == "passed" or check.get("required") is False:
continue
name = str(check.get("name") or "unknown")
stage = readiness_stage(name)
message = str(check.get("message") or f"{name} did not pass")
evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {}
issues.append((stage, message, {"checkName": name, **evidence}))
if issues:
return issues
status = str(payload.get("status") or "").strip()
if status and status != "ready":
message = "POC strict readiness did not pass"
missing = payload.get("missing")
if isinstance(missing, list) and missing:
message += ": " + ", ".join(str(item.get("message") if isinstance(item, dict) else item) for item in missing)
issues.append(("strict-preflight-readiness", message, {}))
return issues
def compact_probe_evidence(payload: dict[str, Any]) -> dict[str, Any]:
evidence: dict[str, Any] = {
"source": payload.get("source"),
"namespace": payload.get("namespace"),
"secretName": payload.get("secretName"),
"status": payload.get("status"),
}
context = payload.get("context")
if isinstance(context, dict):
evidence["context"] = context
probes: list[dict[str, Any]] = []
for item in payload.get("probes") or []:
if not isinstance(item, dict):
continue
response = item.get("response") if isinstance(item.get("response"), dict) else {}
probes.append(
{
"slug": item.get("slug"),
"status": item.get("status"),
"httpStatus": response.get("status"),
"bodyKind": response.get("bodyKind"),
"accepted": response.get("accepted"),
}
)
if probes:
evidence["probes"] = probes
return {key: value for key, value in evidence.items() if value not in (None, "", [], {})}
def issue_missing_coverage(stage: str, all_operators: list[str], dcloud_operators: list[str]) -> list[str]:
if stage in {"dcloud-secret-readiness", "dcloud-fat-probe"}:
return dcloud_operators
return all_operators
def safe_handler(stage: str) -> str:
return STAGE_HANDLERS.get(stage, re.sub(r"[^a-z0-9]+", "_", stage.lower()).strip("_") or "strict_preflight")
def build_result(args: argparse.Namespace) -> dict[str, Any]:
all_operators, dcloud_operators = operator_inventory()
static_payload = static_operator_layout()
repository_summary = operator_repository_summary(static_payload)
required = required_externals(args.required_externals)
source = args.source
if source == "readiness":
payload = load_json(pathlib.Path(args.input_json))
issues = readiness_issues(payload)
namespace = str(payload.get("namespace") or args.namespace)
workspace_mount = str(payload.get("workspaceMountPath") or args.workspace_mount_path).rstrip("/") or "/workspace"
required = required_externals(",".join(payload.get("requiredExternals") or required))
elif source == "dcloud-probe":
payload = load_json(pathlib.Path(args.input_json))
issues = [("dcloud-fat-probe", dcloud_probe_reason(payload), compact_probe_evidence(payload))]
namespace = str(payload.get("namespace") or args.namespace)
workspace_mount = args.workspace_mount_path.rstrip("/") or "/workspace"
else:
raise SystemExit(f"unsupported source: {source}")
if not issues:
raise SystemExit("preflight result requires at least one blocking issue")
started_at = now_iso()
run_id = args.run_id.strip() or f"poc-strict-preflight-{now_id()}"
records: list[dict[str, Any]] = []
missing_coverage_set: set[str] = set()
failures: list[str] = []
stages: list[str] = []
for stage, reason, evidence in issues:
stages.append(stage)
failures.append(reason)
missing_coverage = issue_missing_coverage(stage, all_operators, dcloud_operators)
missing_coverage_set.update(missing_coverage)
record_evidence = dict(evidence)
record_evidence["missingOperatorCoverage"] = missing_coverage
records.append(
{
"operator": "poc_acceptance_preflight",
"handler": safe_handler(stage),
"status": "failed",
"error": reason,
"evidence": record_evidence,
"preflightStage": stage,
}
)
unique_stages = list(dict.fromkeys(stages))
stage = unique_stages[0] if len(unique_stages) == 1 else "strict-preflight-readiness"
missing_coverage = sorted(missing_coverage_set)
required_external_systems = sorted({"bos", "dcloud", "mongo"}.intersection(set(required)))
return {
"runId": run_id,
"workspace": f"{workspace_mount}/acceptance/{run_id}",
"preflightFailure": True,
"preflightStage": stage,
"preflightStages": unique_stages,
"acceptanceScope": "full" if {"bos", "dcloud", "mongo"}.issubset(set(required)) else "dependency-isolation",
"fullExternalAcceptance": False,
"operatorCount": len(all_operators),
"operatorPassedCount": 0,
"coveredOperatorCount": 0,
"allOperatorCoverageComplete": False,
"blockingOperatorCoverageComplete": False,
"missingCoverage": missing_coverage,
"blockingMissingCoverage": missing_coverage,
"requiredExternals": required,
"statusCounts": {"failed": len(records)},
"passedCount": 0,
"externalVerificationPassed": [],
"externalVerificationMissing": required_external_systems,
"externalVerificationPassedCount": 0,
"externalVerificationRequiredCount": len(required_external_systems),
"failedCount": len(records),
"skippedCount": 0,
"requiredSkippedCount": 0,
"optionalSkippedCount": 0,
"static": static_payload,
**repository_summary,
"records": records,
"failures": failures,
"status": "failed",
"namespace": namespace,
"startedAt": started_at,
"endedAt": started_at,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Build a standard POC acceptance result for strict preflight blockers.")
parser.add_argument("--source", choices=("readiness", "dcloud-probe"), required=True)
parser.add_argument("--input-json", required=True)
parser.add_argument("--run-id", default=os.getenv("POC_OPERATOR_ACCEPTANCE_JOB_NAME") or "")
parser.add_argument("--namespace", default=os.getenv("K8S_NAMESPACE") or "droboticflow")
parser.add_argument("--workspace-mount-path", default=os.getenv("WORKSPACE_MOUNT_PATH") or "/workspace")
parser.add_argument("--required-externals", default=os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS") or DEFAULT_REQUIRED_EXTERNALS)
args = parser.parse_args()
print(RESULT_PREFIX + json.dumps(build_result(args), ensure_ascii=False, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())