droboticflow/tests/backend/poc_operator_acceptance_script_test.py
2026-06-23 14:39:17 +08:00

890 lines
40 KiB
Python

import base64
import json
import os
import pathlib
import subprocess
import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "poc-operator-full-acceptance.sh"
JOB_SCRIPT = ROOT / "scripts" / "poc_operator_full_acceptance_job.py"
def unsigned_jwt(exp: int) -> str:
def segment(payload: dict) -> str:
raw = json.dumps(payload, separators=(",", ":")).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
return segment({"alg": "none", "typ": "JWT"}) + "." + segment({"exp": exp}) + ".signature"
def run_script(extra_env: dict[str, str]) -> subprocess.CompletedProcess:
env = os.environ.copy()
for key in (
"BOS_ACCESS_KEY_ID",
"API_BASE_URL",
"BASE_URL",
"BOS_ENDPOINT",
"BOS_SECRET_ACCESS_KEY",
"DROBOTICFLOW_API_TOKEN",
"DCLOUD_API_TOKEN",
"DCLOUD_FAT_BASE_URL",
"DCLOUD_FAT_COOKIE",
"DCLOUD_FAT_AUTH_TOKEN",
"DCLOUD_FAT_USERNAME",
"DCLOUD_FAT_PASSWORD",
"DCLOUD_COOKIE",
"DCLOUD_USERNAME",
"DCLOUD_PASSWORD",
"DEPLOY_ENV",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_TMP_FALLBACK",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_JSON_FILTER_FIELD",
"POC_ACCEPTANCE_JSON_FILTER_EXPECTED",
"POC_ACCEPTANCE_JSON_EXTRACT_FIELD",
"POC_ACCEPTANCE_JSON_FIELD_MAP_JSON",
"POC_ACCEPTANCE_TEMPLATE_TEXT",
"POC_ACCEPTANCE_MIN_CLARITY",
"POC_ACCEPTANCE_MAX_SHAKE",
"POC_ACCEPTANCE_MAX_HAND_OCCLUSION",
"POC_ACCEPTANCE_CLARITY_SCALE",
"POC_ACCEPTANCE_XPLATFORM_DATE_START",
"POC_ACCEPTANCE_XPLATFORM_DATE_END",
"POC_ACCEPTANCE_XPLATFORM_PLACE",
"POC_ACCEPTANCE_XPLATFORM_TYPE",
"POC_ACCEPTANCE_SEGMENT_DURATION_SECONDS",
"POC_ACCEPTANCE_FRAME_INTERVAL_SECONDS",
"POC_ACCEPTANCE_AUDIT_RULES_JSON",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_BOS_BUCKET",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_IMPORT_API_BASE_URL",
"POC_ACCEPTANCE_IMPORT_RESULT",
):
env.pop(key, None)
env.update(
{
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "true",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_ALLOW_TMP_FALLBACK": "false",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES": "false",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
}
)
env.update(extra_env)
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def run_script_with_poc_preset(tmp_path: pathlib.Path, extra_env: dict[str, str]) -> subprocess.CompletedProcess:
env = os.environ.copy()
for key in (
"BOS_ENDPOINT",
"DEPLOY_ENV",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"HOME",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_IMPORT_RESULT",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"XDG_CONFIG_HOME",
):
env.pop(key, None)
env.update(
{
"DEPLOY_ENV": "poc",
"HOME": str(tmp_path / "home"),
"XDG_CONFIG_HOME": str(tmp_path / "xdg"),
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "true",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_IMPORT_RESULT": "false",
}
)
env.update(extra_env)
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def write_fake_kubectl_missing_dcloud_secret(tmp_path: pathlib.Path, calls_path: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import os\n"
"import pathlib\n"
"import sys\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"if len(args) >= 5 and args[:4] == ['-n', 'droboticflow', 'get', 'secret']:\n"
" raise SystemExit(1)\n"
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
" print('droboticflow-api-test')\n"
" raise SystemExit(0)\n"
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
" command = args[args.index('--') + 1:] if '--' in args else []\n"
" if len(command) >= 3 and command[:2] == ['sh', '-c']:\n"
" script = command[2]\n"
" if 'find \"$1\"' in script:\n"
" raise SystemExit(0)\n"
" if 'head -c \"$2\" \"$1\"' in script:\n"
" raise SystemExit(1)\n"
" existing = {item for item in os.environ.get('FAKE_WORKSPACE_EXISTING_PATHS', '').split(os.pathsep) if item}\n"
" raise SystemExit(0 if args[-1] in existing else 1)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def write_fake_kubectl_incomplete_dcloud_secret(tmp_path: pathlib.Path, calls_path: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import base64\n"
"import json\n"
"import os\n"
"import pathlib\n"
"import sys\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"if len(args) >= 5 and args[:4] == ['-n', 'droboticflow', 'get', 'secret']:\n"
" def enc(value):\n"
" return base64.b64encode(value.encode('utf-8')).decode('ascii')\n"
" print(json.dumps({'data': {\n"
" 'DCLOUD_FAT_AUTH_TOKEN': enc('token'),\n"
" 'DCLOUD_ORG_SYMBOL': enc('org-1'),\n"
" 'DCLOUD_SOURCEAPP': enc('dataCopilot'),\n"
" }}))\n"
" raise SystemExit(0)\n"
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
" print('droboticflow-api-test')\n"
" raise SystemExit(0)\n"
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
" command = args[args.index('--') + 1:] if '--' in args else []\n"
" if len(command) >= 3 and command[:2] == ['sh', '-c']:\n"
" script = command[2]\n"
" if 'find \"$1\"' in script:\n"
" raise SystemExit(0)\n"
" if 'head -c \"$2\" \"$1\"' in script:\n"
" raise SystemExit(1)\n"
" existing = {item for item in os.environ.get('FAKE_WORKSPACE_EXISTING_PATHS', '').split(os.pathsep) if item}\n"
" raise SystemExit(0 if args[-1] in existing else 1)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def write_fake_kubectl_complete_dcloud_secret(tmp_path: pathlib.Path, calls_path: pathlib.Path) -> pathlib.Path:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import base64\n"
"import json\n"
"import os\n"
"import pathlib\n"
"import sys\n"
f"calls_path = pathlib.Path({str(calls_path)!r})\n"
"args = sys.argv[1:]\n"
"with calls_path.open('a', encoding='utf-8') as handle:\n"
" handle.write(' '.join(args) + '\\n')\n"
"if len(args) >= 5 and args[:4] == ['-n', 'droboticflow', 'get', 'secret']:\n"
" def enc(value):\n"
" return base64.b64encode(value.encode('utf-8')).decode('ascii')\n"
" print(json.dumps({'data': {\n"
" 'DCLOUD_FAT_BASE_URL': enc('https://cloud-fat.example'),\n"
" 'DCLOUD_FAT_AUTH_TOKEN': enc(os.environ.get('FAKE_DCLOUD_AUTH_TOKEN', 'token')),\n"
" 'DCLOUD_ORG_SYMBOL': enc('org-1'),\n"
" 'DCLOUD_SOURCEAPP': enc('dataCopilot'),\n"
" }}))\n"
" raise SystemExit(0)\n"
"if len(args) >= 8 and args[:4] == ['-n', 'droboticflow', 'get', 'pod']:\n"
" print('droboticflow-api-test')\n"
" raise SystemExit(0)\n"
"if len(args) >= 5 and args[:3] == ['-n', 'droboticflow', 'exec']:\n"
" command = args[args.index('--') + 1:] if '--' in args else []\n"
" if len(command) >= 3 and command[:2] == ['sh', '-c']:\n"
" script = command[2]\n"
" if 'find \"$1\"' in script:\n"
" raise SystemExit(0)\n"
" if 'head -c \"$2\" \"$1\"' in script:\n"
" raise SystemExit(1)\n"
" existing = {item for item in os.environ.get('FAKE_WORKSPACE_EXISTING_PATHS', '').split(os.pathsep) if item}\n"
" raise SystemExit(0 if args[-1] in existing else 1)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
return fake_bin
def run_script_with_missing_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
fake_bin = write_fake_kubectl_missing_dcloud_secret(tmp_path, calls_path)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"API_BASE_URL",
"BASE_URL",
"DCLOUD_FAT_BASE_URL",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"DROBOTICFLOW_API_TOKEN",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_IMPORT_API_BASE_URL",
"POC_ACCEPTANCE_IMPORT_RESULT",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "false",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,dcloud,mongo",
"POC_ACCEPTANCE_DISCOVERY_ROOTS": "/workspace/fixtures",
}
)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path
def run_script_with_incomplete_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
fake_bin = write_fake_kubectl_incomplete_dcloud_secret(tmp_path, calls_path)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"DCLOUD_FAT_BASE_URL",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "false",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,dcloud,mongo",
"POC_ACCEPTANCE_IMPORT_RESULT": "false",
"POC_ACCEPTANCE_DISCOVERY_ROOTS": "/workspace/fixtures",
}
)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path
def run_script_with_complete_dcloud_secret(
tmp_path: pathlib.Path, extra_env: dict[str, str] | None = None
) -> tuple[subprocess.CompletedProcess, pathlib.Path]:
calls_path = tmp_path / "kubectl-calls.log"
fake_bin = write_fake_kubectl_complete_dcloud_secret(tmp_path, calls_path)
env = os.environ.copy()
for key in (
"DEPLOY_ENV",
"DCLOUD_FAT_BASE_URL",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_IMAGE_PULL_SECRET_NAME",
"K8S_NAMESPACE",
"OPERATOR_RUNNER_IMAGE",
"POC_ACCEPTANCE_BOS_BUCKET",
"POC_ACCEPTANCE_BOS_ENDPOINT",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH",
"POC_ACCEPTANCE_DISCOVERY_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_EXCLUDE_ROOTS",
"POC_ACCEPTANCE_DISCOVERY_MAX_RESULTS",
"POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH",
"POC_ACCEPTANCE_CSV_INPUT_PATH",
"POC_ACCEPTANCE_JSON_INPUT_PATH",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH",
"POC_ACCEPTANCE_MODEL_JSON_PATH",
"POC_ACCEPTANCE_SHELL_COMMAND",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT",
"WORKSPACE_MOUNT_PATH",
"WORKSPACE_PVC_NAME",
):
env.pop(key, None)
env.update(
{
"PATH": f"{fake_bin}{os.pathsep}{env.get('PATH', '')}",
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
"WORKSPACE_PVC_NAME": "droboticflow-workspace",
"WORKSPACE_MOUNT_PATH": "/workspace",
"K8S_IMAGE_PULL_SECRET_NAME": "droboticflow-registry",
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "false",
"POC_OPERATOR_ACCEPTANCE_JOB_NAME": "droboticflow-operator-acceptance-test",
"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "300s",
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME": "droboticflow-dcloud-secret",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos,dcloud,mongo",
"POC_ACCEPTANCE_IMPORT_RESULT": "false",
"POC_ACCEPTANCE_DISCOVERY_ROOTS": "/workspace/fixtures",
}
)
if extra_env:
env.update(extra_env)
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
return result, calls_path
def preflight_payload(result: subprocess.CompletedProcess) -> dict[str, object]:
marker = next(line for line in result.stdout.splitlines() if line.startswith("DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="))
return json.loads(marker.split("=", 1)[1])
def assert_fixture_discovery_summary(payload: dict[str, object], record_index: int = 0) -> None:
records = payload["records"]
assert isinstance(records, list)
record = records[record_index]
assert isinstance(record, dict)
evidence = record["evidence"]
assert isinstance(evidence, dict)
discovery = evidence["fixtureDiscovery"]
assert isinstance(discovery, dict)
assert discovery["status"] == "incomplete"
assert discovery["roots"] == ["/workspace/fixtures"]
assert discovery["excludedRoots"] == [
"/workspace/acceptance",
"/workspace/droboticflow-acceptance",
"/workspace/build-contexts",
"/workspace/image-builds",
"/workspace/poc-workflow-acceptance-*",
"/workspace/runs",
]
assert discovery["candidateCounts"] == {
"csv": 0,
"json": 0,
"lerobot": 0,
"model": 0,
"rosbag": 0,
"xplatform": 0,
}
assert sorted(discovery["missing"]) == ["csv", "json", "lerobot", "model", "rosbag", "shellCommand", "xplatform"]
def assert_fixture_discovery_was_called(calls_path: pathlib.Path) -> None:
calls = calls_path.read_text(encoding="utf-8").splitlines()
assert any("get pod -l app=droboticflow-api" in call for call in calls)
assert any('find "$1"' in call for call in calls)
assert not any(" apply " in f" {call} " for call in calls)
assert not any(" create " in f" {call} " for call in calls)
def test_poc_operator_acceptance_rejects_invalid_image_before_kubectl():
result = run_script({"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/operator runner:test"})
assert result.returncode == 1
assert "OPERATOR_RUNNER_IMAGE must be a non-empty container image reference" in result.stderr
def test_poc_operator_acceptance_rejects_invalid_wait_timeout_before_kubectl():
result = run_script({"POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT": "soon"})
assert result.returncode == 1
assert "POC_OPERATOR_ACCEPTANCE_WAIT_TIMEOUT must be a positive Go-style duration" in result.stderr
def test_poc_operator_acceptance_renders_strict_poc_job_manifest():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert 'name: "droboticflow-operator-acceptance-test"' in result.stdout
assert 'namespace: "droboticflow"' in result.stdout
assert 'image: "registry.example.com/dataflow/droboticflow-operator-runner:test"' in result.stdout
assert 'value: "bos,dcloud,mongo"' in result.stdout
assert 'value: "false"' in result.stdout
assert "name: POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES" in result.stdout
assert "name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stdout
assert 'value: "/workspace/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"' in result.stdout
assert "name: POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_CSV_INPUT_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_JSON_INPUT_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_LEROBOT_SOURCE_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_ROSBAG_SOURCE_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_MODEL_JSON_PATH" in result.stdout
assert "name: POC_ACCEPTANCE_SHELL_COMMAND" in result.stdout
assert 'value: "poc-bucket"' in result.stdout
assert 'value: "https://bj.bcebos.com"' in result.stdout
assert "name: droboticflow-bos-workspace" in result.stdout
assert "name: \"droboticflow-dcloud-secret\"" in result.stdout
assert "key: DCLOUD_FAT_BASE_URL" in result.stdout
assert "key: DCLOUD_FAT_AUTH_TOKEN" in result.stdout
assert "key: DCLOUD_FAT_COOKIE" in result.stdout
assert "key: DCLOUD_FAT_USERNAME" in result.stdout
assert "key: DCLOUD_FAT_PASSWORD" in result.stdout
assert "key: DCLOUD_D_USER_ID" in result.stdout
assert "key: DCLOUD_ORG_SYMBOL" in result.stdout
assert "key: DCLOUD_SOURCEAPP" in result.stdout
assert 'claimName: "droboticflow-workspace"' in result.stdout
def test_poc_operator_acceptance_rejects_invalid_generated_fixture_flag_before_kubectl():
result = run_script({"POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES": "maybe"})
assert result.returncode == 1
assert "POC_ACCEPTANCE_ALLOW_GENERATED_FIXTURES must be a boolean value" in result.stderr
def test_poc_operator_acceptance_renders_real_fixture_inputs():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH": "/workspace/fixtures/xplatform",
"POC_ACCEPTANCE_CSV_INPUT_PATH": "/workspace/fixtures/records.csv",
"POC_ACCEPTANCE_JSON_INPUT_PATH": "/workspace/fixtures/records.json",
"POC_ACCEPTANCE_LEROBOT_SOURCE_PATH": "/workspace/fixtures/lerobot",
"POC_ACCEPTANCE_ROSBAG_SOURCE_PATH": "/workspace/fixtures/rosbag2",
"POC_ACCEPTANCE_MODEL_JSON_PATH": "/workspace/fixtures/model.json",
"POC_ACCEPTANCE_SHELL_COMMAND": "python3 /workspace/fixtures/shell_acceptance.py",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert 'value: "/workspace/fixtures/xplatform"' in result.stdout
assert 'value: "/workspace/fixtures/records.csv"' in result.stdout
assert 'value: "/workspace/fixtures/records.json"' in result.stdout
assert 'value: "/workspace/fixtures/lerobot"' in result.stdout
assert 'value: "/workspace/fixtures/rosbag2"' in result.stdout
assert 'value: "/workspace/fixtures/model.json"' in result.stdout
assert 'value: "python3 /workspace/fixtures/shell_acceptance.py"' in result.stdout
def test_poc_operator_acceptance_renders_fixture_manifest_path():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/poc-acceptance.json",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert "name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stdout
assert 'value: "/workspace/fixtures/poc-acceptance.json"' in result.stdout
def test_poc_operator_acceptance_imports_failed_job_marker_before_wait_status_exit():
source = SCRIPT.read_text(encoding="utf-8")
marker_index = source.index("result_marker_found=false")
import_index = source.index("if [[ \"$result_marker_found\" == \"true\" ]]")
wait_failure_index = source.index("if [[ $wait_status -ne 0 ]]")
assert marker_index < import_index < wait_failure_index
def test_poc_operator_acceptance_loads_poc_bos_endpoint_default(tmp_path):
result = run_script_with_poc_preset(tmp_path, {})
assert result.returncode == 0, result.stderr + result.stdout
assert 'value: "https://bj.bcebos.com"' in result.stdout
def test_poc_operator_acceptance_renders_explicit_dcloud_base_url_override():
result = run_script(
{
"POC_ACCEPTANCE_BOS_BUCKET": "poc-bucket",
"POC_ACCEPTANCE_BOS_ENDPOINT": "https://bj.bcebos.com",
"DCLOUD_FAT_BASE_URL": "https://cloud-fat.override.example",
}
)
assert result.returncode == 0, result.stderr + result.stdout
assert 'name: DCLOUD_FAT_BASE_URL\n value: "https://cloud-fat.override.example"' in result.stdout
assert "key: DCLOUD_FAT_BASE_URL" not in result.stdout
def test_poc_operator_acceptance_rejects_missing_required_dcloud_secret_before_job(tmp_path):
result, calls_path = run_script_with_missing_dcloud_secret(
tmp_path,
{
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/poc-operator-acceptance.json",
"FAKE_WORKSPACE_EXISTING_PATHS": "/workspace/fixtures/poc-operator-acceptance.json",
},
)
assert result.returncode == 1
assert "D-Cloud FAT secret droboticflow/droboticflow-dcloud-secret is missing" in result.stderr
assert "configure-poc-dcloud-secret.sh" in result.stderr
assert "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" in result.stdout
assert '"preflightFailure": true' in result.stdout
assert '"acceptanceScope": "full"' in result.stdout
assert '"fullExternalAcceptance": false' in result.stdout
assert calls_path.read_text(encoding="utf-8").splitlines() == [
"-n droboticflow get secret droboticflow-dcloud-secret -o json",
"-n droboticflow get pod -l app=droboticflow-api -o jsonpath={.items[0].metadata.name}",
'-n droboticflow exec droboticflow-api-test -- sh -c test -f "$1" && test -r "$1" && test -s "$1" -- /workspace/fixtures/poc-operator-acceptance.json',
]
def test_poc_operator_acceptance_rejects_fixture_manifest_outside_workspace_before_job(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/tmp/poc-operator-acceptance.json",
},
)
assert result.returncode == 1
assert "POC acceptance fixture manifest must point under WORKSPACE_MOUNT_PATH /workspace" in result.stderr
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "fixture-readiness"
assert payload["records"][0]["handler"] == "fixture_readiness"
assert_fixture_discovery_summary(payload)
assert_fixture_discovery_was_called(calls_path)
def test_poc_operator_acceptance_rejects_missing_fixture_manifest_inside_workspace_before_job(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/missing-poc-operator-acceptance.json",
},
)
assert result.returncode == 1
assert "POC acceptance fixture manifest is not a readable non-empty file inside the POC workspace mount" in result.stderr
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "fixture-readiness"
assert payload["records"][0]["handler"] == "fixture_readiness"
assert_fixture_discovery_summary(payload)
calls = calls_path.read_text(encoding="utf-8").splitlines()
assert '-n droboticflow exec droboticflow-api-test -- sh -c test -f "$1" && test -r "$1" && test -s "$1" -- /workspace/fixtures/missing-poc-operator-acceptance.json' in calls
assert_fixture_discovery_was_called(calls_path)
def test_poc_operator_acceptance_reports_all_readiness_blockers_before_job(tmp_path):
result, calls_path = run_script_with_missing_dcloud_secret(tmp_path)
assert result.returncode == 1
assert "D-Cloud FAT secret droboticflow/droboticflow-dcloud-secret is missing" in result.stderr
assert "POC acceptance fixture manifest is not a readable non-empty file inside the POC workspace mount" in result.stderr
assert "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" in result.stdout
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "preflight-readiness"
assert payload["preflightStages"] == ["dcloud-secret-readiness", "fixture-readiness"]
assert payload["acceptanceScope"] == "full"
assert payload["fullExternalAcceptance"] is False
assert payload["failedCount"] == 2
assert payload["statusCounts"] == {"failed": 2}
assert payload["operatorCount"] > 20
assert len(payload["blockingMissingCoverage"]) == payload["operatorCount"]
assert [record["handler"] for record in payload["records"]] == [
"dcloud_secret_readiness",
"fixture_readiness",
]
assert [record["preflightStage"] for record in payload["records"]] == [
"dcloud-secret-readiness",
"fixture-readiness",
]
assert_fixture_discovery_summary(payload, record_index=1)
assert_fixture_discovery_was_called(calls_path)
def test_poc_operator_acceptance_rejects_dcloud_secret_without_base_url_before_job(tmp_path):
result, calls_path = run_script_with_incomplete_dcloud_secret(
tmp_path,
{
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/poc-operator-acceptance.json",
"FAKE_WORKSPACE_EXISTING_PATHS": "/workspace/fixtures/poc-operator-acceptance.json",
},
)
assert result.returncode == 1
assert "D-Cloud FAT secret droboticflow/droboticflow-dcloud-secret is incomplete" in result.stderr
assert "DCLOUD_FAT_BASE_URL" in result.stderr
assert "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" in result.stdout
assert '"preflightFailure": true' in result.stdout
assert calls_path.read_text(encoding="utf-8").splitlines() == [
"-n droboticflow get secret droboticflow-dcloud-secret -o json",
"-n droboticflow get pod -l app=droboticflow-api -o jsonpath={.items[0].metadata.name}",
'-n droboticflow exec droboticflow-api-test -- sh -c test -f "$1" && test -r "$1" && test -s "$1" -- /workspace/fixtures/poc-operator-acceptance.json',
]
def test_poc_operator_acceptance_rejects_expired_dcloud_jwt_before_job(tmp_path):
token = unsigned_jwt(int(time.time()) - 60)
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"FAKE_DCLOUD_AUTH_TOKEN": token,
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/poc-operator-acceptance.json",
"FAKE_WORKSPACE_EXISTING_PATHS": "/workspace/fixtures/poc-operator-acceptance.json",
},
)
assert result.returncode == 1
assert "D-Cloud FAT secret droboticflow/droboticflow-dcloud-secret is incomplete" in result.stderr
assert "JWT is expired" in result.stderr
assert token not in result.stdout + result.stderr
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "dcloud-secret-readiness"
assert calls_path.read_text(encoding="utf-8").splitlines() == [
"-n droboticflow get secret droboticflow-dcloud-secret -o json",
"-n droboticflow get pod -l app=droboticflow-api -o jsonpath={.items[0].metadata.name}",
'-n droboticflow exec droboticflow-api-test -- sh -c test -f "$1" && test -r "$1" && test -s "$1" -- /workspace/fixtures/poc-operator-acceptance.json',
]
def test_poc_operator_acceptance_rejects_missing_default_fixture_manifest_before_job(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(tmp_path)
assert result.returncode == 1
assert "POC acceptance fixture manifest is not a readable non-empty file inside the POC workspace mount" in result.stderr
assert "/workspace/fixtures/poc-operator-acceptance/poc-operator-acceptance.json" in result.stderr
assert "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" in result.stdout
payload = preflight_payload(result)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "fixture-readiness"
assert payload["acceptanceScope"] == "full"
assert payload["fullExternalAcceptance"] is False
assert payload["operatorCount"] > 20
assert payload["blockingMissingCoverage"] == payload["missingCoverage"]
assert len(payload["blockingMissingCoverage"]) == payload["operatorCount"]
assert payload["records"][0]["handler"] == "fixture_readiness"
assert_fixture_discovery_summary(payload)
assert_fixture_discovery_was_called(calls_path)
def test_poc_operator_acceptance_skips_fixture_preflight_when_manifest_is_configured(tmp_path):
result, calls_path = run_script_with_complete_dcloud_secret(
tmp_path,
{
"POC_OPERATOR_ACCEPTANCE_DRY_RUN": "true",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/poc-operator-acceptance.json",
},
)
assert result.returncode == 0, result.stderr + result.stdout
assert "fixture-readiness" not in result.stdout
assert not calls_path.exists()
assert "name: POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH" in result.stdout
assert 'value: "/workspace/fixtures/poc-operator-acceptance.json"' in result.stdout
def test_poc_operator_acceptance_imports_missing_dcloud_secret_preflight_failure(tmp_path):
received: dict[str, object] = {}
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
length = int(self.headers.get("Content-Length", "0"))
received["path"] = self.path
received["body"] = json.loads(self.rfile.read(length).decode("utf-8"))
self.send_response(201)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps({"run": {"id": "run-preflight-failure"}}).encode("utf-8"))
def log_message(self, *_args):
return
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
result, calls_path = run_script_with_missing_dcloud_secret(
tmp_path,
{
"POC_ACCEPTANCE_IMPORT_API_BASE_URL": f"http://127.0.0.1:{server.server_port}",
"POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH": "/workspace/fixtures/poc-operator-acceptance.json",
"FAKE_WORKSPACE_EXISTING_PATHS": "/workspace/fixtures/poc-operator-acceptance.json",
"NO_PROXY": "127.0.0.1,localhost",
"no_proxy": "127.0.0.1,localhost",
},
)
finally:
server.shutdown()
thread.join(timeout=5)
assert result.returncode == 1
assert json.loads(result.stdout.splitlines()[-1]) == {"imported": True, "runId": "run-preflight-failure"}
assert calls_path.read_text(encoding="utf-8").splitlines() == [
"-n droboticflow get secret droboticflow-dcloud-secret -o json",
"-n droboticflow get pod -l app=droboticflow-api -o jsonpath={.items[0].metadata.name}",
'-n droboticflow exec droboticflow-api-test -- sh -c test -f "$1" && test -r "$1" && test -s "$1" -- /workspace/fixtures/poc-operator-acceptance.json',
]
assert received["path"] == "/api/acceptance/poc-operator-result"
body = received["body"]
assert isinstance(body, dict)
result_payload = body["result"]
assert result_payload["status"] == "failed"
assert result_payload["preflightFailure"] is True
assert result_payload["preflightStage"] == "dcloud-secret-readiness"
assert result_payload["acceptanceScope"] == "full"
assert result_payload["fullExternalAcceptance"] is False
assert result_payload["operatorCount"] > 0
assert result_payload["operatorPassedCount"] == 0
assert sorted(result_payload["blockingMissingCoverage"]) == [
"dcloud_data_generation",
"dcloud_embodied_annotation",
]
assert body["source"]["jobName"] == "droboticflow-operator-acceptance-test"
assert body["source"]["image"] == "registry.example.com/dataflow/droboticflow-operator-runner:test"
def test_poc_operator_acceptance_job_script_compiles():
result = subprocess.run(["python3", "-m", "py_compile", str(JOB_SCRIPT)], cwd=ROOT, text=True, capture_output=True, check=False)
assert result.returncode == 0, result.stderr