forked from longtao.wu/droboticflow
460 lines
20 KiB
Python
460 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import ast
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
from typing import Any
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
RESULT_PREFIX = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="
|
|
DEFAULT_REQUIRED_EXTERNALS = "bos,dcloud,mongo"
|
|
STAGE_HANDLERS = {
|
|
"dcloud-secret-readiness": "dcloud_secret_readiness",
|
|
"dcloud-fat-probe": "dcloud_fat_probe",
|
|
"fixture-readiness": "fixture_readiness",
|
|
"deployment-readiness": "deployment_readiness",
|
|
"runner-image-readiness": "runner_image_readiness",
|
|
"strict-preflight-readiness": "strict_preflight_readiness",
|
|
}
|
|
REPOSITORY_POLICY_VIOLATION_KEYS = (
|
|
"repositoryManifestErrors",
|
|
"duplicateOperatorIds",
|
|
"duplicateOperatorNames",
|
|
"legacyOperators",
|
|
"builtinHelperCallOperators",
|
|
"hardcodedExternalApiSources",
|
|
"helperBuiltinKeys",
|
|
)
|
|
|
|
|
|
def load_json(path: pathlib.Path) -> dict[str, Any]:
|
|
text = path.read_text(encoding="utf-8").strip()
|
|
if not text:
|
|
return {}
|
|
payload = json.loads(text)
|
|
if not isinstance(payload, dict):
|
|
raise SystemExit(f"{path} must contain a JSON object")
|
|
return payload
|
|
|
|
|
|
def now_id() -> str:
|
|
return dt.datetime.now(dt.UTC).strftime("%Y%m%d%H%M%S")
|
|
|
|
|
|
def now_iso() -> str:
|
|
return dt.datetime.now(dt.UTC).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def required_externals(raw: str) -> list[str]:
|
|
return sorted({item.strip().lower() for item in raw.split(",") if item.strip()})
|
|
|
|
|
|
def operator_inventory() -> tuple[list[str], list[str]]:
|
|
operators: list[str] = []
|
|
dcloud_operators: list[str] = []
|
|
builtin_root = ROOT / "backend" / "operators" / "builtin"
|
|
for metadata_path in sorted(builtin_root.glob("*/operator.json")):
|
|
slug = metadata_path.parent.name
|
|
metadata: dict[str, Any] = {}
|
|
try:
|
|
loaded_metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
loaded_metadata = {}
|
|
if isinstance(loaded_metadata, dict):
|
|
metadata = loaded_metadata
|
|
if metadata.get("type") not in ("builtin", None):
|
|
continue
|
|
if not slug:
|
|
continue
|
|
operators.append(slug)
|
|
strict_externals: set[str] = set()
|
|
repository_path = metadata_path.with_name("repository.json")
|
|
try:
|
|
repository = json.loads(repository_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
repository = {}
|
|
if isinstance(repository, dict):
|
|
strict_externals = {
|
|
str(item).strip().lower()
|
|
for item in repository.get("strictPocAcceptanceExternals") or []
|
|
if str(item).strip()
|
|
}
|
|
external_api = metadata.get("externalApi") if isinstance(metadata.get("externalApi"), dict) else {}
|
|
if "dcloud" in strict_externals or slug.startswith("dcloud_") or external_api:
|
|
dcloud_operators.append(slug)
|
|
return sorted(set(operators)), sorted(set(dcloud_operators))
|
|
|
|
|
|
def project_relative(path: pathlib.Path) -> str:
|
|
try:
|
|
return str(path.relative_to(ROOT))
|
|
except ValueError:
|
|
return str(path)
|
|
|
|
|
|
def operator_helper_builtin_keys() -> list[str]:
|
|
runner_path = ROOT / "backend" / "operator-runner" / "main.py"
|
|
if not runner_path.exists():
|
|
return []
|
|
try:
|
|
tree = ast.parse(runner_path.read_text(encoding="utf-8"))
|
|
except SyntaxError as error:
|
|
return [f"parse-error:{error.msg}"]
|
|
keys: list[str] = []
|
|
for node in tree.body:
|
|
if not isinstance(node, ast.Assign):
|
|
continue
|
|
if not any(isinstance(target, ast.Name) and target.id == "OPERATOR_HELPERS" for target in node.targets):
|
|
continue
|
|
if not isinstance(node.value, ast.Dict):
|
|
continue
|
|
for key in node.value.keys:
|
|
if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.startswith("builtin_"):
|
|
keys.append(key.value)
|
|
return sorted(set(keys))
|
|
|
|
|
|
def static_operator_layout() -> dict[str, Any]:
|
|
operator_root = ROOT / "backend" / "operators" / "builtin"
|
|
metadata_paths = sorted(operator_root.glob("*/operator.json"))
|
|
sources: list[str] = []
|
|
repository_manifests: list[str] = []
|
|
repository_manifest_errors: list[str] = []
|
|
legacy: list[str] = []
|
|
builtin_helper_calls: list[str] = []
|
|
hardcoded_external_api_sources: list[str] = []
|
|
seen_operator_ids: dict[str, str] = {}
|
|
duplicate_operator_ids: list[str] = []
|
|
seen_operator_names: dict[str, str] = {}
|
|
duplicate_operator_names: list[str] = []
|
|
|
|
for metadata_path in metadata_paths:
|
|
slug = metadata_path.parent.name
|
|
operator_path = metadata_path.with_name("operator.py")
|
|
repository_path = metadata_path.with_name("repository.json")
|
|
if operator_path.exists():
|
|
sources.append(project_relative(operator_path))
|
|
source = operator_path.read_text(encoding="utf-8")
|
|
else:
|
|
source = ""
|
|
sources.append(project_relative(operator_path))
|
|
repository_manifest_errors.append(f"{slug}: missing operator.py")
|
|
try:
|
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
except Exception as error:
|
|
repository_manifest_errors.append(f"{slug}: invalid operator.json ({type(error).__name__}: {error})")
|
|
metadata = {}
|
|
if not isinstance(metadata, dict):
|
|
repository_manifest_errors.append(f"{slug}: operator.json must be an object")
|
|
metadata = {}
|
|
|
|
operator_id = str(metadata.get("id", "")).strip()
|
|
operator_name = str(metadata.get("name", "")).strip()
|
|
if not operator_id:
|
|
repository_manifest_errors.append(f"{slug}: operator.json id is required")
|
|
elif operator_id in seen_operator_ids:
|
|
duplicate_operator_ids.append(operator_id)
|
|
else:
|
|
seen_operator_ids[operator_id] = slug
|
|
if not operator_name:
|
|
repository_manifest_errors.append(f"{slug}: operator.json name is required")
|
|
elif operator_name in seen_operator_names:
|
|
duplicate_operator_names.append(operator_name)
|
|
else:
|
|
seen_operator_names[operator_name] = slug
|
|
if slug == "human_review" and metadata.get("runtimeHandler") != "builtin_human_review":
|
|
repository_manifest_errors.append("human_review: runtimeHandler must be builtin_human_review")
|
|
if slug.startswith("dcloud_") and (
|
|
"https://gitea.d-robotics.cc" in source or "https://cloud-fat.d-robotics.cc" in source
|
|
):
|
|
hardcoded_external_api_sources.append(slug)
|
|
if "LEGACY_HANDLER" in source:
|
|
legacy.append(slug)
|
|
if 'helpers["builtin_' in source or "helpers['builtin_" in source:
|
|
builtin_helper_calls.append(slug)
|
|
if not repository_path.exists():
|
|
repository_manifest_errors.append(f"{slug}: missing repository.json")
|
|
continue
|
|
try:
|
|
repository_manifest = json.loads(repository_path.read_text(encoding="utf-8"))
|
|
except Exception as error:
|
|
repository_manifest_errors.append(f"{slug}: invalid repository.json ({type(error).__name__}: {error})")
|
|
continue
|
|
if not isinstance(repository_manifest, dict):
|
|
repository_manifest_errors.append(f"{slug}: repository.json must be an object")
|
|
continue
|
|
|
|
expected_fields = {
|
|
"schemaVersion": "droboticflow.operator.repository/v1",
|
|
"repositoryId": f"builtin/{slug}",
|
|
"operatorId": metadata.get("id"),
|
|
"operatorSlug": slug,
|
|
"sourceRoot": f"backend/operators/builtin/{slug}",
|
|
"entrypoint": "operator.py",
|
|
"metadata": "operator.json",
|
|
"runtimeSource": metadata.get("runtimeSource"),
|
|
"runtimeHandler": metadata.get("runtimeHandler"),
|
|
"containerEntrypoint": "backend/operator-runner/main.py",
|
|
}
|
|
for key, expected in expected_fields.items():
|
|
if repository_manifest.get(key) != expected:
|
|
repository_manifest_errors.append(
|
|
f"{slug}: repository.json {key}={repository_manifest.get(key)!r}, expected {expected!r}"
|
|
)
|
|
if "externalApi" in metadata and repository_manifest.get("externalApi") != metadata.get("externalApi"):
|
|
repository_manifest_errors.append(f"{slug}: repository.json externalApi must match operator.json")
|
|
acceptance = repository_manifest.get("acceptance") if isinstance(repository_manifest.get("acceptance"), dict) else {}
|
|
if not acceptance.get("requiresRealInputs"):
|
|
repository_manifest_errors.append(f"{slug}: repository.json must require real inputs")
|
|
for key in ("mockOnlyAccepted", "simulationOnlyAccepted", "hardcodedOutputAccepted", "shallowOnlyAccepted"):
|
|
if acceptance.get(key) is not False:
|
|
repository_manifest_errors.append(f"{slug}: repository.json acceptance.{key} must be false")
|
|
if acceptance.get("requiresPersistentWorkspaceEvidence") is not True:
|
|
repository_manifest_errors.append(f"{slug}: repository.json must require persistent workspace evidence")
|
|
dependencies = repository_manifest.get("runtimeDependencies")
|
|
if not isinstance(dependencies, list) or not dependencies:
|
|
repository_manifest_errors.append(f"{slug}: repository.json must list runtime dependencies")
|
|
repository_manifests.append(project_relative(repository_path))
|
|
|
|
return {
|
|
"operatorFileCount": len(metadata_paths),
|
|
"repositoryManifestCount": len(repository_manifests),
|
|
"repositoryManifestErrors": repository_manifest_errors,
|
|
"duplicateOperatorIds": sorted(duplicate_operator_ids),
|
|
"duplicateOperatorNames": sorted(duplicate_operator_names),
|
|
"legacyOperators": legacy,
|
|
"builtinHelperCallOperators": builtin_helper_calls,
|
|
"hardcodedExternalApiSources": hardcoded_external_api_sources,
|
|
"helperBuiltinKeys": operator_helper_builtin_keys(),
|
|
"runtimeSources": sources,
|
|
"repositoryManifests": repository_manifests,
|
|
}
|
|
|
|
|
|
def operator_repository_summary(static_payload: dict[str, Any]) -> dict[str, Any]:
|
|
violations: list[str] = []
|
|
for key in REPOSITORY_POLICY_VIOLATION_KEYS:
|
|
for item in static_payload.get(key, []) or []:
|
|
text = str(item).strip()
|
|
if text:
|
|
violations.append(f"{key}:{text}")
|
|
required_count = int(static_payload.get("operatorFileCount") or 0)
|
|
repository_count = int(static_payload.get("repositoryManifestCount") or 0)
|
|
return {
|
|
"operatorRepositoryRequiredCount": required_count,
|
|
"operatorRepositoryCount": repository_count,
|
|
"operatorRepositoryCoverageComplete": required_count > 0 and repository_count == required_count,
|
|
"operatorRepositoryPolicyClean": len(violations) == 0,
|
|
"operatorRepositoryPolicyViolations": violations,
|
|
}
|
|
|
|
|
|
def readiness_stage(check_name: str) -> str:
|
|
if check_name == "dcloud-secret":
|
|
return "dcloud-secret-readiness"
|
|
if check_name == "fixture-manifest":
|
|
return "fixture-readiness"
|
|
if check_name.startswith("deployment:"):
|
|
return "deployment-readiness"
|
|
if check_name == "operator-runner-image":
|
|
return "runner-image-readiness"
|
|
return "strict-preflight-readiness"
|
|
|
|
|
|
def dcloud_probe_reason(payload: dict[str, Any]) -> str:
|
|
status = str(payload.get("status") or "failed")
|
|
missing = payload.get("missing") if isinstance(payload.get("missing"), list) else []
|
|
if missing:
|
|
return "D-Cloud FAT probe is blocked: " + ", ".join(str(item) for item in missing)
|
|
error = str(payload.get("error") or "").strip()
|
|
if error:
|
|
return "D-Cloud FAT probe failed: " + error
|
|
failed_probes: list[str] = []
|
|
for item in payload.get("probes") or []:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
if item.get("status") != "passed":
|
|
slug = str(item.get("slug") or "unknown")
|
|
response = item.get("response") if isinstance(item.get("response"), dict) else {}
|
|
http_status = response.get("status")
|
|
suffix = f" HTTP {http_status}" if http_status is not None else ""
|
|
failed_probes.append(slug + suffix)
|
|
if failed_probes:
|
|
return "D-Cloud FAT probe did not pass: " + ", ".join(failed_probes)
|
|
return f"D-Cloud FAT probe status is {status}"
|
|
|
|
|
|
def readiness_issues(payload: dict[str, Any]) -> list[tuple[str, str, dict[str, Any]]]:
|
|
issues: list[tuple[str, str, dict[str, Any]]] = []
|
|
for check in payload.get("checks") or []:
|
|
if not isinstance(check, dict):
|
|
continue
|
|
if check.get("status") == "passed" or check.get("required") is False:
|
|
continue
|
|
name = str(check.get("name") or "unknown")
|
|
stage = readiness_stage(name)
|
|
message = str(check.get("message") or f"{name} did not pass")
|
|
evidence = check.get("evidence") if isinstance(check.get("evidence"), dict) else {}
|
|
issues.append((stage, message, {"checkName": name, **evidence}))
|
|
if issues:
|
|
return issues
|
|
status = str(payload.get("status") or "").strip()
|
|
if status and status != "ready":
|
|
message = "POC strict readiness did not pass"
|
|
missing = payload.get("missing")
|
|
if isinstance(missing, list) and missing:
|
|
message += ": " + ", ".join(str(item.get("message") if isinstance(item, dict) else item) for item in missing)
|
|
issues.append(("strict-preflight-readiness", message, {}))
|
|
return issues
|
|
|
|
|
|
def compact_probe_evidence(payload: dict[str, Any]) -> dict[str, Any]:
|
|
evidence: dict[str, Any] = {
|
|
"source": payload.get("source"),
|
|
"namespace": payload.get("namespace"),
|
|
"secretName": payload.get("secretName"),
|
|
"status": payload.get("status"),
|
|
}
|
|
context = payload.get("context")
|
|
if isinstance(context, dict):
|
|
evidence["context"] = context
|
|
probes: list[dict[str, Any]] = []
|
|
for item in payload.get("probes") or []:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
response = item.get("response") if isinstance(item.get("response"), dict) else {}
|
|
probes.append(
|
|
{
|
|
"slug": item.get("slug"),
|
|
"status": item.get("status"),
|
|
"httpStatus": response.get("status"),
|
|
"bodyKind": response.get("bodyKind"),
|
|
"accepted": response.get("accepted"),
|
|
}
|
|
)
|
|
if probes:
|
|
evidence["probes"] = probes
|
|
return {key: value for key, value in evidence.items() if value not in (None, "", [], {})}
|
|
|
|
|
|
def issue_missing_coverage(stage: str, all_operators: list[str], dcloud_operators: list[str]) -> list[str]:
|
|
if stage in {"dcloud-secret-readiness", "dcloud-fat-probe"}:
|
|
return dcloud_operators
|
|
return all_operators
|
|
|
|
|
|
def safe_handler(stage: str) -> str:
|
|
return STAGE_HANDLERS.get(stage, re.sub(r"[^a-z0-9]+", "_", stage.lower()).strip("_") or "strict_preflight")
|
|
|
|
|
|
def build_result(args: argparse.Namespace) -> dict[str, Any]:
|
|
all_operators, dcloud_operators = operator_inventory()
|
|
static_payload = static_operator_layout()
|
|
repository_summary = operator_repository_summary(static_payload)
|
|
required = required_externals(args.required_externals)
|
|
source = args.source
|
|
if source == "readiness":
|
|
payload = load_json(pathlib.Path(args.input_json))
|
|
issues = readiness_issues(payload)
|
|
namespace = str(payload.get("namespace") or args.namespace)
|
|
workspace_mount = str(payload.get("workspaceMountPath") or args.workspace_mount_path).rstrip("/") or "/workspace"
|
|
required = required_externals(",".join(payload.get("requiredExternals") or required))
|
|
elif source == "dcloud-probe":
|
|
payload = load_json(pathlib.Path(args.input_json))
|
|
issues = [("dcloud-fat-probe", dcloud_probe_reason(payload), compact_probe_evidence(payload))]
|
|
namespace = str(payload.get("namespace") or args.namespace)
|
|
workspace_mount = args.workspace_mount_path.rstrip("/") or "/workspace"
|
|
else:
|
|
raise SystemExit(f"unsupported source: {source}")
|
|
|
|
if not issues:
|
|
raise SystemExit("preflight result requires at least one blocking issue")
|
|
|
|
started_at = now_iso()
|
|
run_id = args.run_id.strip() or f"poc-strict-preflight-{now_id()}"
|
|
records: list[dict[str, Any]] = []
|
|
missing_coverage_set: set[str] = set()
|
|
failures: list[str] = []
|
|
stages: list[str] = []
|
|
for stage, reason, evidence in issues:
|
|
stages.append(stage)
|
|
failures.append(reason)
|
|
missing_coverage = issue_missing_coverage(stage, all_operators, dcloud_operators)
|
|
missing_coverage_set.update(missing_coverage)
|
|
record_evidence = dict(evidence)
|
|
record_evidence["missingOperatorCoverage"] = missing_coverage
|
|
records.append(
|
|
{
|
|
"operator": "poc_acceptance_preflight",
|
|
"handler": safe_handler(stage),
|
|
"status": "failed",
|
|
"error": reason,
|
|
"evidence": record_evidence,
|
|
"preflightStage": stage,
|
|
}
|
|
)
|
|
|
|
unique_stages = list(dict.fromkeys(stages))
|
|
stage = unique_stages[0] if len(unique_stages) == 1 else "strict-preflight-readiness"
|
|
missing_coverage = sorted(missing_coverage_set)
|
|
required_external_systems = sorted({"bos", "dcloud", "mongo"}.intersection(set(required)))
|
|
return {
|
|
"runId": run_id,
|
|
"workspace": f"{workspace_mount}/acceptance/{run_id}",
|
|
"preflightFailure": True,
|
|
"preflightStage": stage,
|
|
"preflightStages": unique_stages,
|
|
"acceptanceScope": "full" if {"bos", "dcloud", "mongo"}.issubset(set(required)) else "dependency-isolation",
|
|
"fullExternalAcceptance": False,
|
|
"operatorCount": len(all_operators),
|
|
"operatorPassedCount": 0,
|
|
"coveredOperatorCount": 0,
|
|
"allOperatorCoverageComplete": False,
|
|
"blockingOperatorCoverageComplete": False,
|
|
"missingCoverage": missing_coverage,
|
|
"blockingMissingCoverage": missing_coverage,
|
|
"requiredExternals": required,
|
|
"statusCounts": {"failed": len(records)},
|
|
"passedCount": 0,
|
|
"externalVerificationPassed": [],
|
|
"externalVerificationMissing": required_external_systems,
|
|
"externalVerificationPassedCount": 0,
|
|
"externalVerificationRequiredCount": len(required_external_systems),
|
|
"failedCount": len(records),
|
|
"skippedCount": 0,
|
|
"requiredSkippedCount": 0,
|
|
"optionalSkippedCount": 0,
|
|
"static": static_payload,
|
|
**repository_summary,
|
|
"records": records,
|
|
"failures": failures,
|
|
"status": "failed",
|
|
"namespace": namespace,
|
|
"startedAt": started_at,
|
|
"endedAt": started_at,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Build a standard POC acceptance result for strict preflight blockers.")
|
|
parser.add_argument("--source", choices=("readiness", "dcloud-probe"), required=True)
|
|
parser.add_argument("--input-json", required=True)
|
|
parser.add_argument("--run-id", default=os.getenv("POC_OPERATOR_ACCEPTANCE_JOB_NAME") or "")
|
|
parser.add_argument("--namespace", default=os.getenv("K8S_NAMESPACE") or "droboticflow")
|
|
parser.add_argument("--workspace-mount-path", default=os.getenv("WORKSPACE_MOUNT_PATH") or "/workspace")
|
|
parser.add_argument("--required-externals", default=os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS") or DEFAULT_REQUIRED_EXTERNALS)
|
|
args = parser.parse_args()
|
|
|
|
print(RESULT_PREFIX + json.dumps(build_result(args), ensure_ascii=False, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|