droboticflow/tests/backend/poc_strict_full_acceptance_script_test.py

340 lines
14 KiB
Python

import hashlib
import os
import pathlib
import subprocess
import json
ROOT = pathlib.Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "poc-strict-full-acceptance.sh"
PREFLIGHT_RESULT_SCRIPT = ROOT / "scripts" / "poc_strict_preflight_result.py"
def write_executable(path: pathlib.Path, content: str) -> pathlib.Path:
path.write_text(content, encoding="utf-8")
path.chmod(0o755)
return path
def run_script(
tmp_path: pathlib.Path,
*,
readiness_content: str,
dcloud_probe_content: str = "print('probe should be ignored')\n",
operator_content: str,
extra_env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess:
readiness = write_executable(tmp_path / "readiness.sh", readiness_content)
dcloud_probe = tmp_path / "dcloud_probe.py"
dcloud_probe.write_text(dcloud_probe_content, encoding="utf-8")
operator = write_executable(tmp_path / "operator.sh", operator_content)
env = os.environ.copy()
for key in (
"API_BASE_URL",
"API_PUBLIC_SERVICE_NAME",
"BASE_URL",
"DEPLOY_ENV",
"DROBOTICFLOW_ACCEPTANCE_EMAIL",
"DROBOTICFLOW_ACCEPTANCE_PASSWORD",
"DROBOTICFLOW_API_TOKEN",
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
"K8S_NAMESPACE",
"POC_ACCEPTANCE_API_TOKEN_SECRET_KEY",
"POC_ACCEPTANCE_API_TOKEN_SECRET_NAME",
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
"POC_ACCEPTANCE_IMPORT_API_BASE_URL",
"POC_ACCEPTANCE_IMPORT_RESULT",
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
"POC_STRICT_DCLOUD_PROBE_SCRIPT_PATH",
"POC_STRICT_OPERATOR_ACCEPTANCE_SCRIPT_PATH",
"POC_STRICT_READINESS_SCRIPT_PATH",
):
env.pop(key, None)
env.update(
{
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
"K8S_NAMESPACE": "droboticflow",
"POC_STRICT_READINESS_SCRIPT_PATH": str(readiness),
"POC_STRICT_DCLOUD_PROBE_SCRIPT_PATH": str(dcloud_probe),
"POC_STRICT_OPERATOR_ACCEPTANCE_SCRIPT_PATH": str(operator),
"STRICT_TEST_TMP": str(tmp_path),
}
)
if extra_env:
env.update(extra_env)
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def marker_payload(stdout: str) -> dict:
prefix = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="
line = next(item for item in stdout.splitlines() if item.startswith(prefix))
return json.loads(line[len(prefix) :])
def go_catalog_canonical_bytes(catalog):
canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
for raw, escaped in (
("&", "\\u0026"),
("<", "\\u003c"),
(">", "\\u003e"),
("\u2028", "\\u2028"),
("\u2029", "\\u2029"),
):
canonical_text = canonical_text.replace(raw, escaped)
return canonical_text.encode("utf-8")
def test_preflight_result_generates_default_run_id_on_current_python(tmp_path):
readiness = tmp_path / "readiness.json"
readiness.write_text(
json.dumps(
{
"status": "blocked",
"namespace": "droboticflow",
"checks": [
{
"name": "fixture-manifest",
"status": "failed",
"required": True,
"message": "fixture-manifest is missing",
"evidence": {"path": "/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"},
}
],
}
),
encoding="utf-8",
)
env = os.environ.copy()
env["POC_ACCEPTANCE_SOURCE_COMMIT"] = "strict-preflight-source"
env["DROBOTICFLOW_BUILD_COMMIT"] = "ignored-build-source"
result = subprocess.run(
[
"python3",
str(PREFLIGHT_RESULT_SCRIPT),
"--source",
"readiness",
"--input-json",
str(readiness),
"--namespace",
"droboticflow",
"--required-externals",
"bos,mongo",
],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
check=False,
)
assert result.returncode == 0, result.stderr + result.stdout
payload = marker_payload(result.stdout)
assert payload["runId"].startswith("poc-strict-preflight-")
assert payload["startedAt"].endswith("Z")
assert payload["preflightStage"] == "fixture-readiness"
assert payload["preflightFailure"] is True
catalog = json.loads((ROOT / "internal" / "api" / "builtin_operators_catalog.json").read_text(encoding="utf-8"))
canonical = go_catalog_canonical_bytes(catalog)
assert payload["sourceCommit"] == "strict-preflight-source"
assert payload["catalogHash"] == hashlib.sha256(canonical).hexdigest()
assert payload["catalogCount"] == len(catalog)
assert payload["completedAt"] == payload["endedAt"]
def test_strict_acceptance_stops_when_readiness_is_blocked(tmp_path):
result = run_script(
tmp_path,
readiness_content=(
"#!/usr/bin/env bash\n"
"printf '%s\\n' '{\"status\":\"blocked\",\"namespace\":\"droboticflow\",\"workspaceMountPath\":\"/workspace\","
"\"requiredExternals\":[\"bos\",\"mongo\"],\"checks\":["
"{\"name\":\"operator-runner-image\",\"status\":\"failed\",\"required\":true,"
"\"message\":\"OPERATOR_RUNNER_IMAGE is not configured\",\"evidence\":{}},"
"{\"name\":\"fixture-manifest\",\"status\":\"passed\",\"required\":true,"
"\"message\":\"fixture-manifest and real fixture paths are ready\","
"\"evidence\":{\"path\":\"/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json\","
"\"fixtures\":{\"xplatform\":\"/workspace/datasets/fixtures/poc-operator-acceptance/xplatform\","
"\"csv\":\"/workspace/datasets/fixtures/poc-operator-acceptance/csv/records.csv\","
"\"json\":\"/workspace/datasets/fixtures/poc-operator-acceptance/json/records.json\","
"\"lerobot\":\"/workspace/datasets/fixtures/poc-operator-acceptance/lerobot\","
"\"rosbag\":\"/workspace/datasets/fixtures/poc-operator-acceptance/rosbag\","
"\"model\":\"/workspace/datasets/fixtures/poc-operator-acceptance/model/quality-gate.json\"},"
"\"shellCommandConfigured\":true}}]}'\n"
"exit 1\n"
),
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
)
assert result.returncode == 1
payload = marker_payload(result.stdout)
assert payload["preflightFailure"] is True
assert payload["preflightStage"] == "runner-image-readiness"
assert payload["acceptanceScope"] == "full"
assert payload["fullExternalAcceptance"] is False
assert payload["operatorPassedCount"] == 0
assert len(payload["blockingMissingCoverage"]) == payload["operatorRepositoryRequiredCount"]
assert payload["operatorRepositoryRequiredCount"] > 0
assert payload["operatorRepositoryCount"] == payload["operatorRepositoryRequiredCount"]
assert payload["operatorRepositoryCoverageComplete"] is True
assert payload["operatorRepositoryPolicyClean"] is True
assert payload["operatorRepositoryPolicyViolations"] == []
assert payload["static"]["operatorFileCount"] == payload["operatorRepositoryRequiredCount"]
assert payload["static"]["repositoryManifestCount"] == payload["operatorRepositoryCount"]
assert payload["static"]["repositoryManifestErrors"] == []
assert payload["realFixtureCount"] == 6
assert payload["generatedFixtureCount"] == 0
assert payload["allFixturesReal"] is True
assert payload["fixtureSources"]["allowGeneratedFixtures"] is False
assert payload["fixtureSources"]["fixtureManifestPath"] == "/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"
assert sorted(payload["fixtureSources"]["items"]) == ["csv", "json", "lerobot", "model", "rosbag", "xplatform"]
assert payload["fixtureSources"]["items"]["csv"] == {
"path": "/workspace/datasets/fixtures/poc-operator-acceptance/csv/records.csv",
"generated": False,
}
assert not (tmp_path / "operator-called").exists()
def test_strict_acceptance_ignores_removed_dcloud_probe(tmp_path):
result = run_script(
tmp_path,
readiness_content='#!/usr/bin/env bash\nprintf \'{"status":"ready"}\\n\'\n',
dcloud_probe_content=(
"import json, os, pathlib, sys\n"
"pathlib.Path(os.environ['STRICT_TEST_TMP'], 'dcloud-called').write_text(' '.join(sys.argv))\n"
"print(json.dumps({'status':'blocked','missing':['DCLOUD_FAT_BASE_URL']}))\n"
"raise SystemExit(1)\n"
),
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
)
assert result.returncode == 0, result.stderr + result.stdout
assert not (tmp_path / "dcloud-called").exists()
assert (tmp_path / "operator-called").exists()
def test_strict_acceptance_runs_operator_with_full_external_scope(tmp_path):
result = run_script(
tmp_path,
readiness_content='#!/usr/bin/env bash\nprintf \'{"status":"ready"}\\n\'\n',
operator_content='#!/usr/bin/env bash\nprintf "externals=%s\\n" "$POC_ACCEPTANCE_REQUIRED_EXTERNALS"\n',
)
assert result.returncode == 0, result.stderr + result.stdout
assert result.stdout == "externals=bos,mongo\n"
assert "POC strict preflight passed: readiness" in result.stderr
assert "dcloud-fat-probe" not in result.stderr
def test_strict_acceptance_rejects_dependency_isolation_scope(tmp_path):
result = run_script(
tmp_path,
readiness_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/readiness-called\"\n",
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
extra_env={"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos"},
)
assert result.returncode == 1
assert "requires POC_ACCEPTANCE_REQUIRED_EXTERNALS to include bos,mongo" in result.stderr
assert "missing: mongo" in result.stderr
assert not (tmp_path / "readiness-called").exists()
assert not (tmp_path / "operator-called").exists()
def test_strict_acceptance_imports_preflight_blocker_when_platform_url_is_configured(tmp_path):
import_script = tmp_path / "import.py"
import_script.write_text(
"import json, os, pathlib, sys\n"
"result_file = pathlib.Path(sys.argv[sys.argv.index('--result-file') + 1])\n"
"pathlib.Path(os.environ['STRICT_TEST_TMP'], 'imported.txt').write_text(result_file.read_text())\n"
"print(json.dumps({'imported': True, 'runId': 'run-preflight'}))\n",
encoding="utf-8",
)
result = run_script(
tmp_path,
readiness_content=(
"#!/usr/bin/env bash\n"
"printf '%s\\n' '{\"status\":\"blocked\",\"checks\":["
"{\"name\":\"operator-runner-image\",\"status\":\"failed\",\"required\":true,"
"\"message\":\"OPERATOR_RUNNER_IMAGE is not configured\",\"evidence\":{}}]}'\n"
"exit 1\n"
),
dcloud_probe_content="print('should not run')\n",
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
extra_env={
"POC_STRICT_IMPORT_SCRIPT_PATH": str(import_script),
"POC_ACCEPTANCE_IMPORT_API_BASE_URL": "http://127.0.0.1:3001",
},
)
assert result.returncode == 1
assert '{"imported": true, "runId": "run-preflight"}' in result.stdout
imported = marker_payload((tmp_path / "imported.txt").read_text(encoding="utf-8"))
assert imported["preflightStage"] == "runner-image-readiness"
assert imported["blockingMissingCoverage"]
assert not (tmp_path / "operator-called").exists()
def test_strict_acceptance_auto_discovers_platform_import_endpoint_and_token(tmp_path):
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
kubectl = fake_bin / "kubectl"
kubectl.write_text(
"#!/usr/bin/env python3\n"
"import json\n"
"import sys\n"
"args = sys.argv[1:]\n"
"if args == ['-n', 'droboticflow', 'get', 'svc', 'droboticflow-api-public', '-o', 'json']:\n"
" print(json.dumps({'status': {'loadBalancer': {'ingress': [{'ip': '192.168.0.74'}, {'ip': '120.48.164.19'}]}}}))\n"
" raise SystemExit(0)\n"
"if args == ['-n', 'droboticflow', 'get', 'secret', 'droboticflow-auth-secret', '-o', 'json']:\n"
" print(json.dumps({'data': {'DROBOTICFLOW_API_TOKEN': 'cG9jLXRva2Vu'}}))\n"
" raise SystemExit(0)\n"
"if args[:5] == ['-n', 'droboticflow', 'get', 'configmap', 'droboticflow-config']:\n"
" print('registry.example.com/dataflow-poc/droboticflow-operator-runner:test')\n"
" raise SystemExit(0)\n"
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
"raise SystemExit(97)\n",
encoding="utf-8",
)
kubectl.chmod(0o755)
import_script = tmp_path / "import.py"
import_script.write_text(
"import json, os, pathlib, sys\n"
"payload = {\n"
" 'apiBaseUrl': os.environ.get('API_BASE_URL'),\n"
" 'apiToken': os.environ.get('DROBOTICFLOW_API_TOKEN'),\n"
" 'argApiBaseUrl': sys.argv[sys.argv.index('--api-base-url') + 1],\n"
"}\n"
"pathlib.Path(os.environ['STRICT_TEST_TMP'], 'import-env.json').write_text(json.dumps(payload))\n"
"print(json.dumps({'imported': True, 'runId': 'run-auto-preflight'}))\n",
encoding="utf-8",
)
result = run_script(
tmp_path,
readiness_content=(
"#!/usr/bin/env bash\n"
"printf '%s\\n' '{\"status\":\"blocked\",\"checks\":["
"{\"name\":\"operator-runner-image\",\"status\":\"failed\",\"required\":true,"
"\"message\":\"OPERATOR_RUNNER_IMAGE is not configured\",\"evidence\":{}}]}'\n"
"exit 1\n"
),
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
extra_env={
"PATH": f"{fake_bin}{os.pathsep}{os.environ.get('PATH', '')}",
"POC_STRICT_IMPORT_SCRIPT_PATH": str(import_script),
},
)
assert result.returncode == 1
assert '{"imported": true, "runId": "run-auto-preflight"}' in result.stdout
import_env = json.loads((tmp_path / "import-env.json").read_text(encoding="utf-8"))
assert import_env == {
"apiBaseUrl": "http://120.48.164.19",
"apiToken": "poc-token",
"argApiBaseUrl": "http://120.48.164.19",
}
assert not (tmp_path / "operator-called").exists()