diff --git a/README.md b/README.md index 0967e305..907f4c18 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ Drobodata is a workflow orchestration platform for embodied AI and vision pipeli - `npm run reconcile:data -- `: remove legacy demo data, reset settings, and bootstrap built-in operators through the real API; honors `DROBOTICFLOW_API_TOKEN` and optional session login env vars for protected deployments - `npm run acceptance:object-storage`: browser-driven object-storage acceptance against a configured live storage connection; requires `OBJECT_STORAGE_CONNECTION_ID`, `OBJECT_STORAGE_CONNECTION_NAME`, or `OBJECT_STORAGE_BUCKET` - `npm run acceptance:poc-operators`: run the POC Kubernetes Job that executes the deployed builtin operator modules with real files, video, MongoDB, BOS, and D-Cloud FAT calls when the required POC external credentials are configured +- `npm run acceptance:poc-readiness`: preflight POC deployment image, D-Cloud FAT secret, and real fixture manifest readiness as JSON without launching an acceptance Job or printing secret values - `npm run acceptance:poc-fixtures`: copy caller-provided real POC acceptance fixture files/directories into the shared workspace PVC and write the fixture manifest consumed by `acceptance:poc-operators`; it does not generate mock, simulated, or hardcoded fixtures - `npm run acceptance:xplatform`: X-platform operator-chain acceptance against an explicit real dataset; requires `XPLATFORM_SOURCE_PATH`, explicit `XPLATFORM_RULES_JSON`, all `XPLATFORM_MIN_*` counts, quality/slicing thresholds, and a storage connection selector when the source or upload path is `bos://...`; count and segment values must be exact safe positive integers - `python3 scripts/wjzh_operator_regression.py --preflight-only`: preflight the production-style WJZH real-data operator regression against a live API; requires explicit `SOURCE_PATH`, `UPLOAD_ROOT_PREFIX`, storage connection selector, target video filenames, route/filter values, real Robotics dataset paths, incremental seed metadata, quality thresholds, video timing, and `SAMPLING_RULES_JSON`; integer timing values must be exact safe positive integers diff --git a/docs/operations.md b/docs/operations.md index aae30e57..8e9867e5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -194,6 +194,14 @@ Run the deployed POC operator-runner full builtin-operator acceptance Job: npm run acceptance:poc-operators ``` +Before launching the Job, run the read-only readiness gate: + +```bash +DEPLOY_ENV=poc npm run acceptance:poc-readiness +``` + +The readiness gate emits JSON and exits non-zero when the deployed API/worker images are not ready, the configured operator-runner image is missing, `droboticflow-dcloud-secret` is absent or incomplete, or the real fixture manifest is not readable on the shared `/workspace` PVC. It decodes secret keys only to check that required values are non-empty and never prints token, cookie, password, AK, SK, or kubeconfig data. A `ready` result is only a precondition for final acceptance; the strict pass signal still comes from `npm run acceptance:poc-operators` returning `fullExternalAcceptance=true`. + `scripts/poc-operator-full-acceptance.sh` loads the `poc` deployment environment on `dev`, reads the current `OPERATOR_RUNNER_IMAGE`, workspace PVC, Mongo secret, and BOS workspace secret references from Kubernetes, mounts `scripts/poc_operator_full_acceptance_job.py` into a temporary Job, and prints one `DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=` result. The Job executes builtin modules from the deployed runner image and writes evidence under `/workspace/acceptance//`; if the PVC is misconfigured and the runner cannot write there, the run fails by default instead of silently accepting temporary evidence. Set `POC_ACCEPTANCE_ALLOW_TMP_FALLBACK=true` only for diagnostic runs that intentionally write to `/tmp/droboticflow-acceptance//` and report `workspaceFallbackReason`. Final POC operator acceptance requires real input fixtures that already exist inside the Job container, usually on the shared `/workspace` PVC. Prefer a single `POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH=/workspace/fixtures/poc-operator-acceptance.json` file so the platform run records one reproducible fixture contract: diff --git a/package.json b/package.json index 85f87f16..7c6e2a49 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "reconcile:data": "./scripts/reconcile-system-data.sh", "acceptance:operators-preview": "node scripts/playwright-preview-operators-check.mjs", "acceptance:poc-operators": "./scripts/poc-operator-full-acceptance.sh", + "acceptance:poc-readiness": "./scripts/poc-acceptance-readiness.sh", "acceptance:poc-fixtures": "./scripts/configure-poc-acceptance-fixtures.sh", "acceptance:object-storage": "node scripts/object-storage-acceptance.mjs", "acceptance:xplatform": "python3 scripts/xplatform-live-acceptance.py" diff --git a/scripts/poc-acceptance-readiness.sh b/scripts/poc-acceptance-readiness.sh new file mode 100755 index 00000000..0853118f --- /dev/null +++ b/scripts/poc-acceptance-readiness.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DEPLOY_ENV="${DEPLOY_ENV:-}" + +if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then + DEPLOY_ENV="$1" + shift +fi + +export DEPLOY_ENV +# shellcheck source=scripts/load-deploy-env.sh +source "$ROOT_DIR/scripts/load-deploy-env.sh" + +python3 "$ROOT_DIR/scripts/poc_acceptance_readiness.py" "$@" diff --git a/scripts/poc_acceptance_readiness.py b/scripts/poc_acceptance_readiness.py new file mode 100755 index 00000000..5bcc57fd --- /dev/null +++ b/scripts/poc_acceptance_readiness.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from typing import Any + + +DEFAULT_NAMESPACE = "droboticflow" +DEFAULT_DCLOUD_SECRET = "droboticflow-dcloud-secret" +DEFAULT_FIXTURE_MANIFEST = "/workspace/fixtures/poc-operator-acceptance/poc-operator-acceptance.json" + + +@dataclass +class Check: + name: str + status: str + message: str + evidence: dict[str, Any] + required: bool = True + + +def env_bool(name: str, default: bool = False) -> bool: + raw = os.getenv(name, "").strip().lower() + if not raw: + return default + if raw in {"1", "true", "yes", "y", "on"}: + return True + if raw in {"0", "false", "no", "n", "off"}: + return False + raise SystemExit(f"{name} must be a boolean value") + + +def run_kubectl(args: list[str], *, input_text: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["kubectl", *args], + input=input_text, + text=True, + capture_output=True, + check=False, + ) + + +def kubectl_json(args: list[str]) -> dict[str, Any] | None: + result = run_kubectl([*args, "-o", "json"]) + if result.returncode != 0: + return None + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def decode_secret_value(secret: dict[str, Any], *keys: str) -> str: + data = secret.get("data") + if not isinstance(data, dict): + return "" + for key in keys: + raw = data.get(key) + if not isinstance(raw, str) or not raw: + continue + try: + value = base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip() + except Exception: + continue + if value: + return value + return "" + + +def deployment_check(namespace: str, name: str, expected_tag: str) -> Check: + payload = kubectl_json(["-n", namespace, "get", "deploy", name]) + if payload is None: + return Check("deployment:" + name, "failed", f"deployment {namespace}/{name} is missing or unreadable", {}) + + status = payload.get("status") if isinstance(payload.get("status"), dict) else {} + spec = payload.get("spec") if isinstance(payload.get("spec"), dict) else {} + template = spec.get("template") if isinstance(spec.get("template"), dict) else {} + pod_spec = template.get("spec") if isinstance(template.get("spec"), dict) else {} + containers = pod_spec.get("containers") if isinstance(pod_spec.get("containers"), list) else [] + images = [item.get("image", "") for item in containers if isinstance(item, dict)] + ready = int(status.get("readyReplicas") or 0) + replicas = int(status.get("replicas") or spec.get("replicas") or 0) + + issues: list[str] = [] + if replicas <= 0 or ready < replicas: + issues.append(f"ready replicas {ready}/{replicas}") + if expected_tag and not all(str(image).endswith(":" + expected_tag) for image in images): + issues.append(f"image tag does not match {expected_tag}") + if issues: + return Check("deployment:" + name, "failed", "; ".join(issues), {"images": images, "readyReplicas": ready, "replicas": replicas}) + return Check("deployment:" + name, "passed", "deployment is ready", {"images": images, "readyReplicas": ready, "replicas": replicas}) + + +def runner_image_check(namespace: str, expected_tag: str) -> Check: + payload = kubectl_json(["-n", namespace, "get", "configmap", "droboticflow-config"]) + if payload is None: + return Check("operator-runner-image", "failed", "configmap droboticflow-config is missing or unreadable", {}) + data = payload.get("data") if isinstance(payload.get("data"), dict) else {} + image = str(data.get("OPERATOR_RUNNER_IMAGE") or "") + if not image: + return Check("operator-runner-image", "failed", "OPERATOR_RUNNER_IMAGE is not configured", {}) + if expected_tag and not image.endswith(":" + expected_tag): + return Check("operator-runner-image", "failed", f"operator runner image tag does not match {expected_tag}", {"image": image}) + return Check("operator-runner-image", "passed", "operator runner image is configured", {"image": image}) + + +def dcloud_secret_check(namespace: str, secret_name: str, required: bool) -> Check: + payload = kubectl_json(["-n", namespace, "get", "secret", secret_name]) + if payload is None: + status = "failed" if required else "skipped" + return Check("dcloud-secret", status, f"D-Cloud FAT secret {namespace}/{secret_name} is missing", {"secretName": secret_name}, required=required) + + missing: list[str] = [] + base_url = decode_secret_value(payload, "DCLOUD_FAT_BASE_URL", "DCLOUD_BASE_URL") + token = decode_secret_value(payload, "DCLOUD_FAT_AUTH_TOKEN", "DCLOUD_API_TOKEN") + cookie = decode_secret_value(payload, "DCLOUD_FAT_COOKIE", "DCLOUD_COOKIE") + username = decode_secret_value(payload, "DCLOUD_FAT_USERNAME", "DCLOUD_USERNAME") + password = decode_secret_value(payload, "DCLOUD_FAT_PASSWORD", "DCLOUD_PASSWORD") + org = decode_secret_value(payload, "DCLOUD_ORG_SYMBOL") + sourceapp = decode_secret_value(payload, "DCLOUD_SOURCEAPP") + + if not base_url: + missing.append("DCLOUD_FAT_BASE_URL") + if not (token or cookie or (username and password)): + missing.append("D-Cloud auth token/cookie or username/password") + if not org: + missing.append("DCLOUD_ORG_SYMBOL") + if not sourceapp: + missing.append("DCLOUD_SOURCEAPP") + + if missing: + return Check( + "dcloud-secret", + "failed" if required else "skipped", + "D-Cloud FAT secret is incomplete: " + ", ".join(missing), + {"secretName": secret_name, "missing": missing}, + required=required, + ) + return Check( + "dcloud-secret", + "passed", + "D-Cloud FAT secret has the required key set", + { + "secretName": secret_name, + "baseUrlConfigured": bool(base_url), + "tokenConfigured": bool(token), + "cookieConfigured": bool(cookie), + "usernamePasswordConfigured": bool(username and password), + "orgSymbolConfigured": bool(org), + "sourceappConfigured": bool(sourceapp), + }, + required=required, + ) + + +def api_pod_name(namespace: str) -> str: + result = run_kubectl(["-n", namespace, "get", "pod", "-l", "app=droboticflow-api", "-o", "jsonpath={.items[0].metadata.name}"]) + return result.stdout.strip() if result.returncode == 0 else "" + + +def workspace_path_check(namespace: str, path: str, kind: str, label: str) -> Check: + if not path.startswith("/workspace/") and path != "/workspace": + return Check(label, "failed", f"{label} must point under /workspace", {"path": path}) + pod = api_pod_name(namespace) + if not pod: + return Check(label, "failed", "no droboticflow-api pod is available for workspace validation", {"path": path}) + test_script = { + "file": 'test -f "$1" && test -r "$1" && test -s "$1"', + "dir": 'test -d "$1" && test -r "$1"', + "any": 'test -e "$1" && test -r "$1"', + }[kind] + result = run_kubectl(["-n", namespace, "exec", pod, "--", "sh", "-c", test_script, "--", path]) + if result.returncode != 0: + detail = "readable non-empty file" if kind == "file" else f"readable {kind}" + return Check(label, "failed", f"{label} is not a {detail} inside the shared workspace PVC", {"path": path}) + return Check(label, "passed", f"{label} is readable in the shared workspace PVC", {"path": path}) + + +def fixture_manifest_check(namespace: str, manifest_path: str) -> Check: + return workspace_path_check(namespace, manifest_path, "file", "fixture-manifest") + + +def required_external_enabled(target: str) -> bool: + raw = os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS", "bos,dcloud,mongo") + return target in {part.strip().lower() for part in raw.split(",") if part.strip()} + + +def main() -> None: + namespace = os.getenv("K8S_NAMESPACE", DEFAULT_NAMESPACE) + expected_tag = os.getenv("IMAGE_TAG", "").strip() + dcloud_secret = os.getenv("POC_ACCEPTANCE_DCLOUD_SECRET_NAME", DEFAULT_DCLOUD_SECRET) + manifest_path = os.getenv("POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH", DEFAULT_FIXTURE_MANIFEST) + allow_generated = env_bool("POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES", False) + require_dcloud = required_external_enabled("dcloud") + + checks = [ + deployment_check(namespace, "droboticflow-api", expected_tag), + deployment_check(namespace, "droboticflow-worker", expected_tag), + runner_image_check(namespace, expected_tag), + dcloud_secret_check(namespace, dcloud_secret, required=require_dcloud), + ] + if allow_generated: + checks.append(Check("fixture-manifest", "skipped", "generated fixtures are enabled for diagnostics; this is not final acceptance evidence", {"allowGeneratedFixtures": True}, required=True)) + else: + checks.append(fixture_manifest_check(namespace, manifest_path)) + + failed_required = [check for check in checks if check.required and check.status == "failed"] + payload = { + "status": "ready" if not failed_required else "blocked", + "namespace": namespace, + "expectedImageTag": expected_tag, + "requiredExternals": sorted({part.strip().lower() for part in os.getenv("POC_ACCEPTANCE_REQUIRED_EXTERNALS", "bos,dcloud,mongo").split(",") if part.strip()}), + "checks": [check.__dict__ for check in checks], + "missing": [{"name": check.name, "message": check.message, "evidence": check.evidence} for check in failed_required], + "nextActions": [], + } + if any(check.name == "dcloud-secret" and check.status == "failed" for check in checks): + payload["nextActions"].append("Configure droboticflow-dcloud-secret with scripts/configure-poc-dcloud-secret.sh using real D-Cloud FAT credentials.") + if any(check.name == "fixture-manifest" and check.status == "failed" for check in checks): + payload["nextActions"].append("Create a real fixture manifest with npm run acceptance:poc-fixtures using non-generated local or /workspace fixture data.") + + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + raise SystemExit(0 if payload["status"] == "ready" else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/backend/poc_acceptance_readiness_test.py b/tests/backend/poc_acceptance_readiness_test.py new file mode 100644 index 00000000..99af2b4a --- /dev/null +++ b/tests/backend/poc_acceptance_readiness_test.py @@ -0,0 +1,134 @@ +import base64 +import json +import os +import pathlib +import subprocess + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "poc-acceptance-readiness.sh" + + +def b64(value: str) -> str: + return base64.b64encode(value.encode("utf-8")).decode("ascii") + + +def write_fake_kubectl(tmp_path: pathlib.Path, *, dcloud_secret: dict[str, str] | None, manifest_exists: bool, image_tag: str = "tag-1") -> pathlib.Path: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + kubectl = fake_bin / "kubectl" + secret_literal = json.dumps({"data": {key: b64(value) for key, value in (dcloud_secret or {}).items()}}) if dcloud_secret is not None else "" + kubectl.write_text( + "#!/usr/bin/env python3\n" + "import json\n" + "import pathlib\n" + "import sys\n" + f"secret_literal = {secret_literal!r}\n" + f"manifest_exists = {manifest_exists!r}\n" + f"image_tag = {image_tag!r}\n" + "args = sys.argv[1:]\n" + "def emit(obj):\n" + " print(json.dumps(obj))\n" + "if args[:4] == ['-n', 'droboticflow', 'get', 'deploy']:\n" + " name = args[4]\n" + " image_name = 'droboticflow-api' if name == 'droboticflow-api' else 'droboticflow-worker'\n" + " emit({'spec': {'replicas': 1, 'template': {'spec': {'containers': [{'image': 'registry.example/dataflow/' + image_name + ':' + image_tag}]}}}, 'status': {'replicas': 1, 'readyReplicas': 1}})\n" + " raise SystemExit(0)\n" + "if args[:5] == ['-n', 'droboticflow', 'get', 'configmap', 'droboticflow-config']:\n" + " emit({'data': {'OPERATOR_RUNNER_IMAGE': 'registry.example/dataflow/droboticflow-operator-runner:' + image_tag}})\n" + " raise SystemExit(0)\n" + "if args[:5] == ['-n', 'droboticflow', 'get', 'secret', 'droboticflow-dcloud-secret']:\n" + " if not secret_literal:\n" + " raise SystemExit(1)\n" + " print(secret_literal)\n" + " raise SystemExit(0)\n" + "if args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n" + " print('droboticflow-api-test')\n" + " raise SystemExit(0)\n" + "if args[:3] == ['-n', 'droboticflow', 'exec']:\n" + " raise SystemExit(0 if manifest_exists else 1)\n" + "print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n" + "raise SystemExit(97)\n", + encoding="utf-8", + ) + kubectl.chmod(0o755) + return fake_bin + + +def run_script(tmp_path: pathlib.Path, *, dcloud_secret: dict[str, str] | None, manifest_exists: bool, extra_env: dict[str, str] | None = None): + fake_bin = write_fake_kubectl(tmp_path, dcloud_secret=dcloud_secret, manifest_exists=manifest_exists) + env = os.environ.copy() + for key in ( + "DEPLOY_ENV", + "DROBOTICFLOW_DEPLOY_ENV_LOADED", + "IMAGE_TAG", + "K8S_NAMESPACE", + "POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES", + "POC_ACCEPTANCE_DCLOUD_SECRET_NAME", + "POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH", + "POC_ACCEPTANCE_REQUIRED_EXTERNALS", + ): + env.pop(key, None) + env.update( + { + "PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}", + "DROBOTICFLOW_DEPLOY_ENV_LOADED": "test", + "IMAGE_TAG": "tag-1", + "K8S_NAMESPACE": "droboticflow", + } + ) + if extra_env: + env.update(extra_env) + result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) + return result + + +def parse_payload(result: subprocess.CompletedProcess) -> dict: + return json.loads(result.stdout) + + +def complete_secret() -> dict[str, str]: + return { + "DCLOUD_FAT_BASE_URL": "https://cloud-fat.example", + "DCLOUD_FAT_AUTH_TOKEN": "token", + "DCLOUD_ORG_SYMBOL": "org", + "DCLOUD_SOURCEAPP": "dataCopilot", + } + + +def test_poc_acceptance_readiness_reports_missing_secret_and_manifest(tmp_path): + result = run_script(tmp_path, dcloud_secret=None, manifest_exists=False) + + assert result.returncode == 1 + payload = parse_payload(result) + assert payload["status"] == "blocked" + missing_names = {item["name"] for item in payload["missing"]} + assert missing_names == {"dcloud-secret", "fixture-manifest"} + assert "token" not in result.stdout + result.stderr + assert payload["nextActions"] + + +def test_poc_acceptance_readiness_passes_with_complete_secret_and_manifest(tmp_path): + result = run_script(tmp_path, dcloud_secret=complete_secret(), manifest_exists=True) + + assert result.returncode == 0, result.stderr + result.stdout + payload = parse_payload(result) + assert payload["status"] == "ready" + assert payload["missing"] == [] + assert all(check["status"] == "passed" for check in payload["checks"]) + + +def test_poc_acceptance_readiness_allows_dependency_isolation_without_dcloud(tmp_path): + result = run_script( + tmp_path, + dcloud_secret=None, + manifest_exists=True, + extra_env={"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,mongo"}, + ) + + assert result.returncode == 0, result.stderr + result.stdout + payload = parse_payload(result) + assert payload["status"] == "ready" + dcloud_check = next(check for check in payload["checks"] if check["name"] == "dcloud-secret") + assert dcloud_check["required"] is False + assert dcloud_check["status"] == "skipped"