forked from longtao.wu/droboticflow
340 lines
14 KiB
Python
340 lines
14 KiB
Python
import hashlib
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
import json
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "poc-strict-full-acceptance.sh"
|
|
PREFLIGHT_RESULT_SCRIPT = ROOT / "scripts" / "poc_strict_preflight_result.py"
|
|
|
|
|
|
def write_executable(path: pathlib.Path, content: str) -> pathlib.Path:
|
|
path.write_text(content, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
return path
|
|
|
|
|
|
def run_script(
|
|
tmp_path: pathlib.Path,
|
|
*,
|
|
readiness_content: str,
|
|
dcloud_probe_content: str = "print('probe should be ignored')\n",
|
|
operator_content: str,
|
|
extra_env: dict[str, str] | None = None,
|
|
) -> subprocess.CompletedProcess:
|
|
readiness = write_executable(tmp_path / "readiness.sh", readiness_content)
|
|
dcloud_probe = tmp_path / "dcloud_probe.py"
|
|
dcloud_probe.write_text(dcloud_probe_content, encoding="utf-8")
|
|
operator = write_executable(tmp_path / "operator.sh", operator_content)
|
|
env = os.environ.copy()
|
|
for key in (
|
|
"API_BASE_URL",
|
|
"API_PUBLIC_SERVICE_NAME",
|
|
"BASE_URL",
|
|
"DEPLOY_ENV",
|
|
"DROBOTICFLOW_ACCEPTANCE_EMAIL",
|
|
"DROBOTICFLOW_ACCEPTANCE_PASSWORD",
|
|
"DROBOTICFLOW_API_TOKEN",
|
|
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
|
|
"K8S_NAMESPACE",
|
|
"POC_ACCEPTANCE_API_TOKEN_SECRET_KEY",
|
|
"POC_ACCEPTANCE_API_TOKEN_SECRET_NAME",
|
|
"POC_ACCEPTANCE_DCLOUD_SECRET_NAME",
|
|
"POC_ACCEPTANCE_IMPORT_API_BASE_URL",
|
|
"POC_ACCEPTANCE_IMPORT_RESULT",
|
|
"POC_ACCEPTANCE_REQUIRED_EXTERNALS",
|
|
"POC_STRICT_DCLOUD_PROBE_SCRIPT_PATH",
|
|
"POC_STRICT_OPERATOR_ACCEPTANCE_SCRIPT_PATH",
|
|
"POC_STRICT_READINESS_SCRIPT_PATH",
|
|
):
|
|
env.pop(key, None)
|
|
env.update(
|
|
{
|
|
"DROBOTICFLOW_DEPLOY_ENV_LOADED": "test",
|
|
"K8S_NAMESPACE": "droboticflow",
|
|
"POC_STRICT_READINESS_SCRIPT_PATH": str(readiness),
|
|
"POC_STRICT_DCLOUD_PROBE_SCRIPT_PATH": str(dcloud_probe),
|
|
"POC_STRICT_OPERATOR_ACCEPTANCE_SCRIPT_PATH": str(operator),
|
|
"STRICT_TEST_TMP": str(tmp_path),
|
|
}
|
|
)
|
|
if extra_env:
|
|
env.update(extra_env)
|
|
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
|
|
|
|
def marker_payload(stdout: str) -> dict:
|
|
prefix = "DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE="
|
|
line = next(item for item in stdout.splitlines() if item.startswith(prefix))
|
|
return json.loads(line[len(prefix) :])
|
|
|
|
|
|
def go_catalog_canonical_bytes(catalog):
|
|
canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
for raw, escaped in (
|
|
("&", "\\u0026"),
|
|
("<", "\\u003c"),
|
|
(">", "\\u003e"),
|
|
("\u2028", "\\u2028"),
|
|
("\u2029", "\\u2029"),
|
|
):
|
|
canonical_text = canonical_text.replace(raw, escaped)
|
|
return canonical_text.encode("utf-8")
|
|
|
|
|
|
def test_preflight_result_generates_default_run_id_on_current_python(tmp_path):
|
|
readiness = tmp_path / "readiness.json"
|
|
readiness.write_text(
|
|
json.dumps(
|
|
{
|
|
"status": "blocked",
|
|
"namespace": "droboticflow",
|
|
"checks": [
|
|
{
|
|
"name": "fixture-manifest",
|
|
"status": "failed",
|
|
"required": True,
|
|
"message": "fixture-manifest is missing",
|
|
"evidence": {"path": "/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"},
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
env = os.environ.copy()
|
|
env["POC_ACCEPTANCE_SOURCE_COMMIT"] = "strict-preflight-source"
|
|
env["DROBOTICFLOW_BUILD_COMMIT"] = "ignored-build-source"
|
|
result = subprocess.run(
|
|
[
|
|
"python3",
|
|
str(PREFLIGHT_RESULT_SCRIPT),
|
|
"--source",
|
|
"readiness",
|
|
"--input-json",
|
|
str(readiness),
|
|
"--namespace",
|
|
"droboticflow",
|
|
"--required-externals",
|
|
"bos,mongo",
|
|
],
|
|
cwd=ROOT,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
payload = marker_payload(result.stdout)
|
|
assert payload["runId"].startswith("poc-strict-preflight-")
|
|
assert payload["startedAt"].endswith("Z")
|
|
assert payload["preflightStage"] == "fixture-readiness"
|
|
assert payload["preflightFailure"] is True
|
|
catalog = json.loads((ROOT / "internal" / "api" / "builtin_operators_catalog.json").read_text(encoding="utf-8"))
|
|
canonical = go_catalog_canonical_bytes(catalog)
|
|
assert payload["sourceCommit"] == "strict-preflight-source"
|
|
assert payload["catalogHash"] == hashlib.sha256(canonical).hexdigest()
|
|
assert payload["catalogCount"] == len(catalog)
|
|
assert payload["completedAt"] == payload["endedAt"]
|
|
|
|
|
|
def test_strict_acceptance_stops_when_readiness_is_blocked(tmp_path):
|
|
result = run_script(
|
|
tmp_path,
|
|
readiness_content=(
|
|
"#!/usr/bin/env bash\n"
|
|
"printf '%s\\n' '{\"status\":\"blocked\",\"namespace\":\"droboticflow\",\"workspaceMountPath\":\"/workspace\","
|
|
"\"requiredExternals\":[\"bos\",\"mongo\"],\"checks\":["
|
|
"{\"name\":\"operator-runner-image\",\"status\":\"failed\",\"required\":true,"
|
|
"\"message\":\"OPERATOR_RUNNER_IMAGE is not configured\",\"evidence\":{}},"
|
|
"{\"name\":\"fixture-manifest\",\"status\":\"passed\",\"required\":true,"
|
|
"\"message\":\"fixture-manifest and real fixture paths are ready\","
|
|
"\"evidence\":{\"path\":\"/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json\","
|
|
"\"fixtures\":{\"xplatform\":\"/workspace/datasets/fixtures/poc-operator-acceptance/xplatform\","
|
|
"\"csv\":\"/workspace/datasets/fixtures/poc-operator-acceptance/csv/records.csv\","
|
|
"\"json\":\"/workspace/datasets/fixtures/poc-operator-acceptance/json/records.json\","
|
|
"\"lerobot\":\"/workspace/datasets/fixtures/poc-operator-acceptance/lerobot\","
|
|
"\"rosbag\":\"/workspace/datasets/fixtures/poc-operator-acceptance/rosbag\","
|
|
"\"model\":\"/workspace/datasets/fixtures/poc-operator-acceptance/model/quality-gate.json\"},"
|
|
"\"shellCommandConfigured\":true}}]}'\n"
|
|
"exit 1\n"
|
|
),
|
|
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
payload = marker_payload(result.stdout)
|
|
assert payload["preflightFailure"] is True
|
|
assert payload["preflightStage"] == "runner-image-readiness"
|
|
assert payload["acceptanceScope"] == "full"
|
|
assert payload["fullExternalAcceptance"] is False
|
|
assert payload["operatorPassedCount"] == 0
|
|
assert len(payload["blockingMissingCoverage"]) == payload["operatorRepositoryRequiredCount"]
|
|
assert payload["operatorRepositoryRequiredCount"] > 0
|
|
assert payload["operatorRepositoryCount"] == payload["operatorRepositoryRequiredCount"]
|
|
assert payload["operatorRepositoryCoverageComplete"] is True
|
|
assert payload["operatorRepositoryPolicyClean"] is True
|
|
assert payload["operatorRepositoryPolicyViolations"] == []
|
|
assert payload["static"]["operatorFileCount"] == payload["operatorRepositoryRequiredCount"]
|
|
assert payload["static"]["repositoryManifestCount"] == payload["operatorRepositoryCount"]
|
|
assert payload["static"]["repositoryManifestErrors"] == []
|
|
assert payload["realFixtureCount"] == 6
|
|
assert payload["generatedFixtureCount"] == 0
|
|
assert payload["allFixturesReal"] is True
|
|
assert payload["fixtureSources"]["allowGeneratedFixtures"] is False
|
|
assert payload["fixtureSources"]["fixtureManifestPath"] == "/workspace/datasets/fixtures/poc-operator-acceptance/poc-operator-acceptance.json"
|
|
assert sorted(payload["fixtureSources"]["items"]) == ["csv", "json", "lerobot", "model", "rosbag", "xplatform"]
|
|
assert payload["fixtureSources"]["items"]["csv"] == {
|
|
"path": "/workspace/datasets/fixtures/poc-operator-acceptance/csv/records.csv",
|
|
"generated": False,
|
|
}
|
|
assert not (tmp_path / "operator-called").exists()
|
|
|
|
|
|
def test_strict_acceptance_ignores_removed_dcloud_probe(tmp_path):
|
|
result = run_script(
|
|
tmp_path,
|
|
readiness_content='#!/usr/bin/env bash\nprintf \'{"status":"ready"}\\n\'\n',
|
|
dcloud_probe_content=(
|
|
"import json, os, pathlib, sys\n"
|
|
"pathlib.Path(os.environ['STRICT_TEST_TMP'], 'dcloud-called').write_text(' '.join(sys.argv))\n"
|
|
"print(json.dumps({'status':'blocked','missing':['DCLOUD_FAT_BASE_URL']}))\n"
|
|
"raise SystemExit(1)\n"
|
|
),
|
|
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert not (tmp_path / "dcloud-called").exists()
|
|
assert (tmp_path / "operator-called").exists()
|
|
|
|
|
|
def test_strict_acceptance_runs_operator_with_full_external_scope(tmp_path):
|
|
result = run_script(
|
|
tmp_path,
|
|
readiness_content='#!/usr/bin/env bash\nprintf \'{"status":"ready"}\\n\'\n',
|
|
operator_content='#!/usr/bin/env bash\nprintf "externals=%s\\n" "$POC_ACCEPTANCE_REQUIRED_EXTERNALS"\n',
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert result.stdout == "externals=bos,mongo\n"
|
|
assert "POC strict preflight passed: readiness" in result.stderr
|
|
assert "dcloud-fat-probe" not in result.stderr
|
|
|
|
|
|
def test_strict_acceptance_rejects_dependency_isolation_scope(tmp_path):
|
|
result = run_script(
|
|
tmp_path,
|
|
readiness_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/readiness-called\"\n",
|
|
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
|
|
extra_env={"POC_ACCEPTANCE_REQUIRED_EXTERNALS": "bos"},
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert "requires POC_ACCEPTANCE_REQUIRED_EXTERNALS to include bos,mongo" in result.stderr
|
|
assert "missing: mongo" in result.stderr
|
|
assert not (tmp_path / "readiness-called").exists()
|
|
assert not (tmp_path / "operator-called").exists()
|
|
|
|
|
|
def test_strict_acceptance_imports_preflight_blocker_when_platform_url_is_configured(tmp_path):
|
|
import_script = tmp_path / "import.py"
|
|
import_script.write_text(
|
|
"import json, os, pathlib, sys\n"
|
|
"result_file = pathlib.Path(sys.argv[sys.argv.index('--result-file') + 1])\n"
|
|
"pathlib.Path(os.environ['STRICT_TEST_TMP'], 'imported.txt').write_text(result_file.read_text())\n"
|
|
"print(json.dumps({'imported': True, 'runId': 'run-preflight'}))\n",
|
|
encoding="utf-8",
|
|
)
|
|
result = run_script(
|
|
tmp_path,
|
|
readiness_content=(
|
|
"#!/usr/bin/env bash\n"
|
|
"printf '%s\\n' '{\"status\":\"blocked\",\"checks\":["
|
|
"{\"name\":\"operator-runner-image\",\"status\":\"failed\",\"required\":true,"
|
|
"\"message\":\"OPERATOR_RUNNER_IMAGE is not configured\",\"evidence\":{}}]}'\n"
|
|
"exit 1\n"
|
|
),
|
|
dcloud_probe_content="print('should not run')\n",
|
|
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
|
|
extra_env={
|
|
"POC_STRICT_IMPORT_SCRIPT_PATH": str(import_script),
|
|
"POC_ACCEPTANCE_IMPORT_API_BASE_URL": "http://127.0.0.1:3001",
|
|
},
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert '{"imported": true, "runId": "run-preflight"}' in result.stdout
|
|
imported = marker_payload((tmp_path / "imported.txt").read_text(encoding="utf-8"))
|
|
assert imported["preflightStage"] == "runner-image-readiness"
|
|
assert imported["blockingMissingCoverage"]
|
|
assert not (tmp_path / "operator-called").exists()
|
|
|
|
|
|
def test_strict_acceptance_auto_discovers_platform_import_endpoint_and_token(tmp_path):
|
|
fake_bin = tmp_path / "bin"
|
|
fake_bin.mkdir()
|
|
kubectl = fake_bin / "kubectl"
|
|
kubectl.write_text(
|
|
"#!/usr/bin/env python3\n"
|
|
"import json\n"
|
|
"import sys\n"
|
|
"args = sys.argv[1:]\n"
|
|
"if args == ['-n', 'droboticflow', 'get', 'svc', 'droboticflow-api-public', '-o', 'json']:\n"
|
|
" print(json.dumps({'status': {'loadBalancer': {'ingress': [{'ip': '192.168.0.74'}, {'ip': '120.48.164.19'}]}}}))\n"
|
|
" raise SystemExit(0)\n"
|
|
"if args == ['-n', 'droboticflow', 'get', 'secret', 'droboticflow-auth-secret', '-o', 'json']:\n"
|
|
" print(json.dumps({'data': {'DROBOTICFLOW_API_TOKEN': 'cG9jLXRva2Vu'}}))\n"
|
|
" raise SystemExit(0)\n"
|
|
"if args[:5] == ['-n', 'droboticflow', 'get', 'configmap', 'droboticflow-config']:\n"
|
|
" print('registry.example.com/dataflow-poc/droboticflow-operator-runner:test')\n"
|
|
" raise SystemExit(0)\n"
|
|
"print('unexpected kubectl call: ' + ' '.join(args), file=sys.stderr)\n"
|
|
"raise SystemExit(97)\n",
|
|
encoding="utf-8",
|
|
)
|
|
kubectl.chmod(0o755)
|
|
|
|
import_script = tmp_path / "import.py"
|
|
import_script.write_text(
|
|
"import json, os, pathlib, sys\n"
|
|
"payload = {\n"
|
|
" 'apiBaseUrl': os.environ.get('API_BASE_URL'),\n"
|
|
" 'apiToken': os.environ.get('DROBOTICFLOW_API_TOKEN'),\n"
|
|
" 'argApiBaseUrl': sys.argv[sys.argv.index('--api-base-url') + 1],\n"
|
|
"}\n"
|
|
"pathlib.Path(os.environ['STRICT_TEST_TMP'], 'import-env.json').write_text(json.dumps(payload))\n"
|
|
"print(json.dumps({'imported': True, 'runId': 'run-auto-preflight'}))\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = run_script(
|
|
tmp_path,
|
|
readiness_content=(
|
|
"#!/usr/bin/env bash\n"
|
|
"printf '%s\\n' '{\"status\":\"blocked\",\"checks\":["
|
|
"{\"name\":\"operator-runner-image\",\"status\":\"failed\",\"required\":true,"
|
|
"\"message\":\"OPERATOR_RUNNER_IMAGE is not configured\",\"evidence\":{}}]}'\n"
|
|
"exit 1\n"
|
|
),
|
|
operator_content="#!/usr/bin/env bash\ntouch \"$STRICT_TEST_TMP/operator-called\"\n",
|
|
extra_env={
|
|
"PATH": f"{fake_bin}{os.pathsep}{os.environ.get('PATH', '')}",
|
|
"POC_STRICT_IMPORT_SCRIPT_PATH": str(import_script),
|
|
},
|
|
)
|
|
|
|
assert result.returncode == 1
|
|
assert '{"imported": true, "runId": "run-auto-preflight"}' in result.stdout
|
|
import_env = json.loads((tmp_path / "import-env.json").read_text(encoding="utf-8"))
|
|
assert import_env == {
|
|
"apiBaseUrl": "http://120.48.164.19",
|
|
"apiToken": "poc-token",
|
|
"argApiBaseUrl": "http://120.48.164.19",
|
|
}
|
|
assert not (tmp_path / "operator-called").exists()
|