droboticflow/tests/backend/poc_operator_acceptance_script_test.py

773 lines
34 KiB
Python

import base64
import hashlib
import json
import os
import pathlib
import subprocess
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "poc-operator-full-acceptance.sh"
JOB_SCRIPT = ROOT / "scripts" / "poc_operator_full_acceptance_job.py"
def go_catalog_canonical_bytes(catalog):
canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
for raw, escaped in (
("&", "\\u0026"),
("<", "\\u003c"),
(">", "\\u003e"),
("\u2028", "\\u2028"),
("\u2029", "\\u2029"),
):
canonical_text = canonical_text.replace(raw, escaped)
return canonical_text.encode("utf-8")
def unsigned_jwt(exp: int) -> str:
def segment(payload: dict) -> str:
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
return segment({"alg": "none", "typ": "JWT"}) + "." + segment({"exp": exp}) + ".signature"
def run_script(extra_env: dict[str, str]) -> subprocess.CompletedProcess:
env = os.environ.copy()
for key in (
"BOS_ACCESS_KEY_ID",
"API_BASE_URL",
"BASE_URL",
"BOS_ENDPOINT",
"BOS_SECRET_ACCESS_KEY",
"DROBOTICFLOW_API_TOKEN",
"DROBOTICFLOW_BUILD_COMMIT",
"DCLOUD_API_TOKEN",
"DCLOUD_FAT_BASE_URL",
"DCLOUD_FAT_COOKIE",
"DCLOUD_FAT_AUTH_TOKEN",
"DCLOUD_FAT_USERNAME",
"DCLOUD_FAT_PASSWORD",
"DCLOUD_FAT_SIGNIN_URL",
"DCLOUD_FAT_SIGNIN_ENDPOINT",
"DCLOUD_COOKIE",
"DCLOUD_USERNAME",
"DCLOUD_PASSWORD",
"DEPLOY_ENV",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_CATALOG_COUNT",
"POC_ACCEPTANCE_CATALOG_HASH",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_SOURCE_COMMIT",
"POC_ACCEPTANCE_ALLOW_TMP_FALLBACK",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_JSON_FILTER_FIELD",
"POC_ACCEPTANCE_JSON_FILTER_EXPECTED",
"POC_ACCEPTANCE_JSON_EXTRACT_FIELD",
"POC_ACCEPTANCE_JSON_FIELD_MAP_JSON",
"POC_ACCEPTANCE_TEMPLATE_TEXT",
"POC_ACCEPTANCE_MIN_CLARITY",
"POC_ACCEPTANCE_MAX_SHAKE",
"POC_ACCEPTANCE_MAX_HAND_OCCLUSION",
"POC_ACCEPTANCE_CLARITY_SCALE",
"POC_ACCEPTANCE_XPLATFORM_DATE_START",
"POC_ACCEPTANCE_XPLATFORM_DATE_END",
"POC_ACCEPTANCE_XPLATFORM_PLACE",
"POC_ACCEPTANCE_XPLATFORM_TYPE",
"POC_ACCEPTANCE_SEGMENT_DURATION_SECONDS",
"POC_ACCEPTANCE_FRAME_INTERVAL_SECONDS",
"POC_ACCEPTANCE_AUDIT_RULES_JSON",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_BOS_BUCKET",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_IMPORT_API_BASE_URL",
"POC_ACCEPTANCE_IMPORT_RESULT",
):
env.pop(key, None)
env.update(
{
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "true",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_ALLOW_TMP_FALLBACK": "false",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES": "false",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
}
)
env.update(extra_env)
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def run_script_with_poc_preset(tmp_path: pathlib.Path, extra_env: dict[str, str]) -> subprocess.CompletedProcess:
env = os.environ.copy()
for key in (
"BOS_ENDPOINT",
"DEPLOY_ENV",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"HOME",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_IMPORT_RESULT",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"XDG_CONFIG_HOME",
):
env.pop(key, None)
env.update(
{
"DEPLOY_ENV": "poc",
"HOME": str(tmp_path / "home"),
"XDG_CONFIG_HOME": str(tmp_path / "xdg"),
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "true",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_IMPORT_RESULT": "false",
}
)
env.update(extra_env)
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def write_fake_kubectl_missing_dcloud_secret(tmp_path: pathlib.Path, calls_path: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import os\n"
"import pathlib\n"
"import sys\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"if len(args) >= 5 and args[:4] == ['-n', 'droboticflow', 'get', 'secret']:\n"
" raise SystemExit(1)\n"
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
" print('droboticflow-api-test')\n"
" raise SystemExit(0)\n"
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
" command = args[args.index('--') + 1:] if '--' in args else []\n"
" if len(command) >= 3 and command[:2] == ['sh', '-c']:\n"
" script = command[2]\n"
" if 'find \"$1\"' in script:\n"
" raise SystemExit(0)\n"
" if 'head -c \"$2\" \"$1\"' in script:\n"
" raise SystemExit(1)\n"
" existing = {item for item in os.environ.get('FAKE_WORKSPACE_EXISTING_PATHS', '').split(os.pathsep) if item}\n"
" raise SystemExit(0 if args[-1] in existing else 1)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def write_fake_kubectl_incomplete_dcloud_secret(tmp_path: pathlib.Path, calls_path: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import base64\n"
"import json\n"
"import os\n"
"import pathlib\n"
"import sys\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"if len(args) >= 5 and args[:4] == ['-n', 'droboticflow', 'get', 'secret']:\n"
" def enc(value):\n"
" return base64.b64encode(value.encode('utf-8')).decode('ascii')\n"
" print(json.dumps({'data': {\n"
" 'DCLOUD_FAT_AUTH_TOKEN': enc('token'),\n"
" 'DCLOUD_ORG_SYMBOL': enc('org-1'),\n"
" 'DCLOUD_SOURCEAPP': enc('dataCopilot'),\n"
" }}))\n"
" raise SystemExit(0)\n"
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
" print('droboticflow-api-test')\n"
" raise SystemExit(0)\n"
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
" command = args[args.index('--') + 1:] if '--' in args else []\n"
" if len(command) >= 3 and command[:2] == ['sh', '-c']:\n"
" script = command[2]\n"
" if 'find \"$1\"' in script:\n"
" raise SystemExit(0)\n"
" if 'head -c \"$2\" \"$1\"' in script:\n"
" raise SystemExit(1)\n"
" existing = {item for item in os.environ.get('FAKE_WORKSPACE_EXISTING_PATHS', '').split(os.pathsep) if item}\n"
" raise SystemExit(0 if args[-1] in existing else 1)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def write_fake_kubectl_complete_dcloud_secret(tmp_path: pathlib.Path, calls_path: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import base64\n"
"import json\n"
"import os\n"
"import pathlib\n"
"import sys\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"if len(args) >= 5 and args[:4] == ['-n', 'droboticflow', 'get', 'secret']:\n"
" def enc(value):\n"
" return base64.b64encode(value.encode('utf-8')).decode('ascii')\n"
" data = {\n"
" 'DCLOUD_FAT_BASE_URL': enc('https://cloud-fat.example'),\n"
" 'DCLOUD_ORG_SYMBOL': enc('org-1'),\n"
" 'DCLOUD_SOURCEAPP': enc('dataCopilot'),\n"
" }\n"
" if os.environ.get('FAKE_DCLOUD_DIRECT_HEADER_ONLY') == 'true':\n"
" data['DCLOUD_D_USER_ID'] = enc('user-1')\n"
" else:\n"
" data['DCLOUD_FAT_AUTH_TOKEN'] = enc(os.environ.get('FAKE_DCLOUD_AUTH_TOKEN', 'token'))\n"
" print(json.dumps({'data': data}))\n"
" raise SystemExit(0)\n"
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
" print('droboticflow-api-test')\n"
" raise SystemExit(0)\n"
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
" command = args[args.index('--') + 1:] if '--' in args else []\n"
" if len(command) >= 3 and command[:2] == ['sh', '-c']:\n"
" script = command[2]\n"
" if 'find \"$1\"' in script:\n"
" raise SystemExit(0)\n"
" if 'head -c \"$2\" \"$1\"' in script:\n"
" raise SystemExit(1)\n"
" existing = {item for item in os.environ.get('FAKE_WORKSPACE_EXISTING_PATHS', '').split(os.pathsep) if item}\n"
" raise SystemExit(0 if args[-1] in existing else 1)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def run_script_with_missing_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
fake_bin = write_fake_kubectl_missing_dcloud_secret(tmp_path, calls_path)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"API_BASE_URL",
"BASE_URL",
"DCLOUD_FAT_BASE_URL",
"DCLOUD_FAT_SIGNIN_URL",
"DCLOUD_FAT_SIGNIN_ENDPOINT",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"DROBOTICFLOW_API_TOKEN",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_IMPORT_API_BASE_URL",
"POC_ACCEPTANCE_IMPORT_RESULT",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "false",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,dcloud,mongo",
"POC_ACCEPTANCE_DISCOVERY_ROOTS": "/workspace/datasets/fixtures",
}
)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path
def run_script_with_incomplete_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
fake_bin = write_fake_kubectl_incomplete_dcloud_secret(tmp_path, calls_path)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"DCLOUD_FAT_BASE_URL",
"DCLOUD_FAT_SIGNIN_URL",
"DCLOUD_FAT_SIGNIN_ENDPOINT",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "false",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,dcloud,mongo",
"POC_ACCEPTANCE_IMPORT_RESULT": "false",
"POC_ACCEPTANCE_DISCOVERY_ROOTS": "/workspace/datasets/fixtures",
}
)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path
def run_script_with_complete_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
fake_bin = write_fake_kubectl_complete_dcloud_secret(tmp_path, calls_path)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"DCLOUD_FAT_BASE_URL",
"DCLOUD_FAT_SIGNIN_URL",
"DCLOUD_FAT_SIGNIN_ENDPOINT",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "false",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,dcloud,mongo",
"POC_ACCEPTANCE_IMPORT_RESULT": "false",
"POC_ACCEPTANCE_DISCOVERY_ROOTS": "/workspace/datasets/fixtures",
}
)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path
def preflight_payload(result: subprocess.CompletedProcess) -> dict[str, object]:
marker = next(line for line in result.stdout.splitlines() if line.startswith("DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="))
return json.loads(marker.split("=", 1)[1])
def assert_fixture_discovery_summary(payload: dict[str, object], record_index: int = 0) -> None:
records = payload["records"]
assert isinstance(records, list)
record = records[record_index]
assert isinstance(record, dict)
evidence = record["evidence"]
assert isinstance(evidence, dict)
discovery = evidence["fixtureDiscovery"]
assert isinstance(discovery, dict)
assert discovery["status"] == "incomplete"
assert discovery["roots"] == ["/workspace/datasets/fixtures"]
assert discovery["excludedRoots"] == [
"/workspace/system/acceptance/general",
"/workspace/system/acceptance/droboticflow",
"/workspace/system/builds/contexts",
"/workspace/system/builds/images",
"/workspace/poc-workflow-acceptance-*",
"/workspace/system/runs",
]
assert discovery["candidateCounts"] == {
"csv": 0,
"json": 0,
"lerobot": 0,
"model": 0,
"rosbag": 0,
"xplatform": 0,
}
assert sorted(discovery["missing"]) == ["csv", "json", "lerobot", "model", "rosbag", "shellCommand", "xplatform"]
def assert_fixture_discovery_was_called(calls_path: pathlib.Path) -> None:
calls = calls_path.read_text(encoding="utf-8").splitlines()
assert any("get pod -l app=droboticflow-api" in call for call in calls)
assert any('find "$1"' in call for call in calls)
assert not any(" apply " in f" {call} " for call in calls)
assert not any(" create " in f" {call} " for call in calls)
def test_poc_operator_acceptance_rejects_invalid_image_before_kubectl():
result = run_script({"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/operator runner:test"})
assert result.returncode == 1
assert "OPERATOR_RUNNER_IMAGE must be a non-empty container image reference" in result.stderr
def test_poc_operator_acceptance_rejects_invalid_wait_timeout_before_kubectl():
result = run_script({"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "soon"})
assert result.returncode == 1
assert "POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT must be a positive Go-style duration" in result.stderr
def test_poc_operator_acceptance_renders_strict_poc_job_manifest():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"DROBOTICFLOW_BUILD_COMMIT": "build-commit-from-release",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert 'name: "droboticflow-operator-acceptance-test"' in result.stdout
assert 'namespace: "droboticflow"' in result.stdout
assert 'image: "registry.example.com/dataflow/droboticflow-operator-runner:test"' in result.stdout
assert "name: POC_ACCEPTANCE_RUN_ID" in result.stdout
assert 'value: "droboticflow-operator-acceptance-test"' in result.stdout
assert "name: POC_ACCEPTANCE_SOURCE_COMMIT" in result.stdout
assert 'value: "build-commit-from-release"' in result.stdout
catalog = json.loads((ROOT / "internal" / "api" / "builtin_operators_catalog.json").read_text(encoding="utf-8"))
canonical = go_catalog_canonical_bytes(catalog)
assert "name: POC_ACCEPTANCE_CATALOG_HASH" in result.stdout
assert f'value: "{hashlib.sha256(canonical).hexdigest()}"' in result.stdout
assert "name: POC_ACCEPTANCE_CATALOG_COUNT" in result.stdout
assert f'value: "{len(catalog)}"' in result.stdout
assert 'value: "bos,mongo"' in result.stdout
assert 'value: "false"' in result.stdout
assert "name: POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES" in result.stdout
assert "name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stdout
assert 'value: "/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"' in result.stdout
assert "name: POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_CSV_INPUT_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_JSON_INPUT_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_LEROBOT_SOURCE_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_ROSBAG_SOURCE_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_MODEL_JSON_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_SHELL_COMMAND" in result.stdout
assert 'value: "poc-bucket"' in result.stdout
assert 'value: "https://bj.bcebos.com"' in result.stdout
assert "name: droboticflow-bos-workspace" in result.stdout
assert 'claimName: "droboticflow-workspace"' in result.stdout
def test_poc_operator_acceptance_rejects_invalid_generated_fixture_flag_before_kubectl():
result = run_script({"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES": "maybe"})
assert result.returncode == 1
assert "POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES must be a boolean value" in result.stderr
def test_poc_operator_acceptance_renders_real_fixture_inputs():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH": "/workspace/datasets/fixtures/xplatform",
"POC_ACCEPTANCE_CSV_INPUT_PATH": "/workspace/datasets/fixtures/records.csv",
"POC_ACCEPTANCE_JSON_INPUT_PATH": "/workspace/datasets/fixtures/records.json",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH": "/workspace/datasets/fixtures/lerobot",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH": "/workspace/datasets/fixtures/rosbag2",
"POC_ACCEPTANCE_MODEL_JSON_PATH": "/workspace/datasets/fixtures/model.json",
"POC_ACCEPTANCE_SHELL_COMMAND": "python3 /workspace/datasets/fixtures/shell_acceptance.py",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert 'value: "/workspace/datasets/fixtures/xplatform"' in result.stdout
assert 'value: "/workspace/datasets/fixtures/records.csv"' in result.stdout
assert 'value: "/workspace/datasets/fixtures/records.json"' in result.stdout
assert 'value: "/workspace/datasets/fixtures/lerobot"' in result.stdout
assert 'value: "/workspace/datasets/fixtures/rosbag2"' in result.stdout
assert 'value: "/workspace/datasets/fixtures/model.json"' in result.stdout
assert 'value: "python3 /workspace/datasets/fixtures/shell_acceptance.py"' in result.stdout
def test_poc_operator_acceptance_renders_fixture_manifest_path():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/datasets/fixtures/poc-acceptance.json",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert "name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stdout
assert 'value: "/workspace/datasets/fixtures/poc-acceptance.json"' in result.stdout
def test_poc_operator_acceptance_imports_failed_job_marker_before_wait_status_exit():
source = SCRIPT.read_text(encoding="utf-8")
marker_index = source.index("result_marker_found=false")
import_index = source.index("if [[ \"$result_marker_found\" == \"true\" ]]")
wait_failure_index = source.index("if [[ $wait_status -ne 0 ]]")
assert marker_index < import_index < wait_failure_index
def test_poc_operator_acceptance_loads_poc_bos_endpoint_default(tmp_path):
result = run_script_with_poc_preset(tmp_path, {})
assert result.returncode == 0, result.stderr + result.stdout
assert 'value: "https://bj.bcebos.com"' in result.stdout
def test_poc_operator_acceptance_does_not_render_removed_dcloud_fat_env():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"DCLOUD_FAT_BASE_URL": "https://cloud-fat.override.example",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert "DCLOUD_FAT_BASE_URL" not in result.stdout
assert "droboticflow-dcloud-secret" not in result.stdout
def test_poc_operator_acceptance_rejects_fixture_manifest_outside_workspace_before_job(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/tmp/poc-operator-acceptance.json",
},
)
assert result.returncode == 1
assert "POC acceptance fixture manifest must point under WORKSPACE_MOUNT_PATH /workspace" in result.stderr
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "fixture-readiness"
assert payload["records"][0]["handler"] == "fixture_readiness"
assert_fixture_discovery_summary(payload)
assert_fixture_discovery_was_called(calls_path)
def test_poc_operator_acceptance_rejects_missing_fixture_manifest_inside_workspace_before_job(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/datasets/fixtures/missing-poc-operator-acceptance.json",
},
)
assert result.returncode == 1
assert "POC acceptance fixture manifest is not a readable non-empty file inside the POC workspace mount" in result.stderr
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "fixture-readiness"
assert payload["records"][0]["handler"] == "fixture_readiness"
assert_fixture_discovery_summary(payload)
calls = calls_path.read_text(encoding="utf-8").splitlines()
assert '-n droboticflow exec droboticflow-api-test -- sh -c test -f "$1" && test -r "$1" && test -s "$1" -- /workspace/datasets/fixtures/missing-poc-operator-acceptance.json' in calls
assert_fixture_discovery_was_called(calls_path)
def test_poc_operator_acceptance_rejects_missing_default_fixture_manifest_before_job(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{"POC_ACCEPTANCE_SOURCE_COMMIT": "inline-preflight-source"},
)
assert result.returncode == 1
assert "POC acceptance fixture manifest is not a readable non-empty file inside the POC workspace mount" in result.stderr
assert "/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json" in result.stderr
assert "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" in result.stdout
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "fixture-readiness"
assert payload["acceptanceScope"] == "full"
assert payload["fullExternalAcceptance"] is False
assert payload["operatorCount"] > 20
assert payload["blockingMissingCoverage"] == payload["missingCoverage"]
assert len(payload["blockingMissingCoverage"]) == payload["operatorCount"]
assert payload["records"][0]["handler"] == "fixture_readiness"
catalog = json.loads((ROOT / "internal" / "api" / "builtin_operators_catalog.json").read_text(encoding="utf-8"))
canonical = go_catalog_canonical_bytes(catalog)
assert payload["sourceCommit"] == "inline-preflight-source"
assert payload["catalogHash"] == hashlib.sha256(canonical).hexdigest()
assert payload["catalogCount"] == len(catalog)
assert payload["completedAt"] == payload["endedAt"]
assert_fixture_discovery_summary(payload)
assert_fixture_discovery_was_called(calls_path)
def test_poc_operator_acceptance_skips_fixture_preflight_when_manifest_is_configured(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "true",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance.json",
},
)
assert result.returncode == 0, result.stderr + result.stdout
assert "fixture-readiness" not in result.stdout
assert not calls_path.exists()
assert "name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stdout
assert 'value: "/workspace/datasets/fixtures/poc-operator-acceptance.json"' in result.stdout
def test_poc_operator_acceptance_job_script_compiles():
result = subprocess.run(["python3", "-m", "py_compile", str(JOB_SCRIPT)], cwd=ROOT, text=True, capture_output=True, check=False)
assert result.returncode == 0, result.stderr