droboticflow/tests/backend/reconcile_system_data_test.py
2026-08-17 16:04:44 +08:00

353 lines
15 KiB
Python

import os
import pathlib
import sys
import tempfile
import unittest
from typing import Any
from unittest import mock
ROOT = pathlib.Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "scripts"))
import reconcile_system_data as reconcile
class FakeClient:
def __init__(self, responses: dict[tuple[str, str], Any]):
self.responses = responses
self.requests: list[tuple[str, str, Any, tuple[int, ...], int, int]] = []
def request_json(
self,
method,
path,
payload=None,
*,
expected=(200, 201, 202, 204),
timeout=60,
retries=1,
retry_delay_seconds=1.0,
):
self.requests.append((method, path, payload, expected, timeout, retries))
key = (method.upper(), path)
if key not in self.responses:
raise AssertionError(f"unexpected request {key}")
response = self.responses[key]
if callable(response):
return response(payload)
return response
class ReconcileSystemDataTest(unittest.TestCase):
def test_legacy_operator_cleanup_never_deletes_builtin_latest_images(self):
self.assertFalse(
reconcile.is_legacy_operator(
{
"id": "operator-builtin-real-source",
"type": "builtin",
"image": "ccr.example/droboticflow-operator-runner:latest",
}
)
)
self.assertTrue(reconcile.is_legacy_operator({"id": "op-old", "type": "custom", "image": "runner:stable"}))
self.assertTrue(reconcile.is_legacy_operator({"id": "custom-operator", "type": "custom", "image": "runner:latest"}))
def test_build_settings_payload_preserves_worker_and_defaults(self):
payload = reconcile.build_settings_payload(
{
"worker": {"pollIntervalMs": 2500, "maxConcurrentRuns": 3},
"defaults": {"timeoutSeconds": 1800},
},
"https://poc.example:6443",
"droboticflow-dev",
)
self.assertEqual(payload["k8s"]["endpoint"], "https://poc.example:6443")
self.assertEqual(payload["k8s"]["namespace"], "droboticflow-dev")
self.assertEqual(payload["worker"], {"pollIntervalMs": 2500, "maxConcurrentRuns": 3})
self.assertEqual(payload["defaults"], {"timeoutSeconds": 1800})
def test_reconcile_recognizes_live_acceptance_temporary_projects(self):
self.assertTrue(reconcile.is_legacy_project({"description": "Created by scripts/live-workflow-acceptance.sh"}))
self.assertFalse(reconcile.is_legacy_project({"description": "customer project"}))
def test_sync_builtin_operators_uses_live_api_catalog(self):
client = FakeClient(
{
(
"GET",
"/api/operators",
): {
"items": [
{"id": "custom-1", "name": "Custom", "type": "custom", "image": "old"},
{
"id": "operator-builtin-mongodb-write",
"name": "MongoDB 写入",
"type": "builtin",
"category": "DATA",
"description": "write business data",
"inputs": [{"id": "input", "name": "input", "type": "json"}],
"outputs": [{"id": "result", "name": "result", "type": "json"}],
"image": "runner:old",
"skillDefinition": "Write JSON documents",
"hyperparams": [{"key": "MONGO_COLLECTION", "value": ""}],
},
]
},
("PUT", "/api/operators/operator-builtin-mongodb-write"): {"id": "operator-builtin-mongodb-write"},
}
)
updated = reconcile.sync_builtin_operators(client, "runner:new")
self.assertEqual(updated, 1)
put_request = client.requests[1]
self.assertEqual(put_request[1], "/api/operators/operator-builtin-mongodb-write")
self.assertEqual(put_request[2]["name"], "MongoDB 写入")
self.assertEqual(put_request[2]["image"], "runner:new")
self.assertEqual(put_request[2]["imageId"], "runner:new")
self.assertEqual(put_request[5], 6)
def test_sync_builtin_operators_preserves_independent_operator_images(self):
release_image, release_image_id = reconcile.release_image_reference("real_source")
client = FakeClient(
{
("GET", "/api/operators"): {
"items": [
{
"id": "operator-builtin-real-source",
"name": "真实数据源",
"type": "builtin",
"category": "IO",
"description": "source",
"inputs": [],
"outputs": [],
"image": release_image,
"imageId": release_image_id,
"containerCommand": ["python", "/app/backend/operator-runner/main.py"],
"containerArgs": [],
"runtimeContract": {"imagePolicy": {"independentFinalImage": True}},
}
]
},
}
)
updated = reconcile.sync_builtin_operators(client, "runner:new")
self.assertEqual(updated, 0)
self.assertEqual(len(client.requests), 1)
def test_sync_builtin_operators_uses_release_manifest_for_independent_images(self):
release_image, release_image_id = reconcile.release_image_reference("real_source")
client = FakeClient(
{
("GET", "/api/operators"): {
"items": [
{
"id": "operator-builtin-real-source",
"name": "真实数据源",
"type": "builtin",
"category": "IO",
"description": "source",
"inputs": [],
"outputs": [],
"image": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
"imageId": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
"containerCommand": ["python", "/app/backend/operator-runner/main.py"],
"containerArgs": [],
"runtimeContract": {"imagePolicy": {"independentFinalImage": True}},
}
]
},
("PUT", "/api/operators/operator-builtin-real-source"): {"id": "operator-builtin-real-source"},
}
)
updated = reconcile.sync_builtin_operators(
client,
"ccr.example/dataflow-poc/droboticflow-operator-runner:live",
image_digest_resolver=lambda image: release_image_id,
)
self.assertEqual(updated, 1)
put_request = client.requests[1]
self.assertEqual(put_request[2]["image"], release_image)
self.assertEqual(put_request[2]["imageId"], release_image_id)
def test_sync_builtin_operators_rejects_release_registry_digest_drift(self):
registry_digest = "sha256:" + "b" * 64
client = FakeClient(
{
("GET", "/api/operators"): {
"items": [
{
"id": "operator-builtin-real-source",
"name": "真实数据源",
"type": "builtin",
"category": "IO",
"description": "source",
"inputs": [],
"outputs": [],
"image": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
"imageId": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
"containerCommand": ["python", "/app/backend/operator-runner/main.py"],
"containerArgs": [],
"runtimeContract": {"imagePolicy": {"independentFinalImage": True}},
}
]
},
}
)
with self.assertRaisesRegex(reconcile.ReconcileError, "release digest drift"):
reconcile.sync_builtin_operators(
client,
"ccr.example/dataflow-poc/droboticflow-operator-runner:live",
image_digest_resolver=lambda image: reconcile.image_ref_with_digest(image, registry_digest),
)
def test_sync_builtin_operators_can_target_one_operator(self):
release_image, release_image_id = reconcile.release_image_reference("real_source")
client = FakeClient(
{
("GET", "/api/operators"): {
"items": [
{
"id": "operator-builtin-real-source",
"name": "真实数据源",
"type": "builtin",
"category": "IO",
"description": "source",
"inputs": [],
"outputs": [],
"image": release_image,
"imageId": release_image,
"containerCommand": [],
},
{
"id": "operator-builtin-mongodb-write",
"name": "MongoDB 写入",
"type": "builtin",
"category": "DATA",
"description": "mongo",
"inputs": [],
"outputs": [],
"image": "runner:old",
"imageId": "runner:old",
"containerCommand": [],
},
]
},
("PUT", "/api/operators/operator-builtin-real-source"): {"id": "operator-builtin-real-source"},
}
)
updated = reconcile.sync_builtin_operators(
client,
"runner:new",
image_digest_resolver=lambda image: release_image_id,
operator_ids={"operator-builtin-real-source"},
)
self.assertEqual(updated, 1)
self.assertEqual(len(client.requests), 2)
self.assertEqual(client.requests[1][1], "/api/operators/operator-builtin-real-source")
self.assertEqual(client.requests[1][2]["image"], release_image)
def test_image_ref_with_digest_replaces_tag_with_digest(self):
self.assertEqual(
reconcile.image_ref_with_digest("ccr.example/ns/droboticflow-operator-yolo-train:v1", "sha256:" + "c" * 64),
"ccr.example/ns/droboticflow-operator-yolo-train@sha256:" + "c" * 64,
)
def test_parse_bearer_challenge(self):
self.assertEqual(
reconcile.parse_bearer_challenge(
'Bearer realm="https://registry.example/token",service="registry.example",scope="repository:ns/image:pull"'
),
{
"realm": "https://registry.example/token",
"service": "registry.example",
"scope": "repository:ns/image:pull",
},
)
def test_registry_digest_resolver_auto_enables_when_auth_file_is_present(self):
with tempfile.NamedTemporaryFile("w", encoding="utf-8") as auth_file:
auth_file.write('{"auths":{"registry.example":{"auth":"dXNlcjpwYXNz"}}}')
auth_file.flush()
with mock.patch.dict(
os.environ,
{"REGISTRY_AUTH_FILE": auth_file.name, "RECONCILE_OPERATOR_IMAGE_DIGESTS": ""},
):
self.assertIsNotNone(reconcile.registry_digest_resolver_from_env())
def test_registry_digest_resolver_can_be_disabled_explicitly(self):
with tempfile.NamedTemporaryFile("w", encoding="utf-8") as auth_file:
auth_file.write('{"auths":{"registry.example":{"auth":"dXNlcjpwYXNz"}}}')
auth_file.flush()
with mock.patch.dict(
os.environ,
{"REGISTRY_AUTH_FILE": auth_file.name, "RECONCILE_OPERATOR_IMAGE_DIGESTS": "false"},
):
self.assertIsNone(reconcile.registry_digest_resolver_from_env())
def test_reconcile_deletes_legacy_records_and_keeps_builtin_catalog_entries(self):
client = FakeClient(
{
("GET", "/api/users"): {
"items": [
{"id": "1", "email": "demo@emboflow.ai"},
{"id": "operator", "email": "operator@example.test"},
]
},
("GET", "/api/projects"): {"items": [{"id": "3", "description": "old"}, {"id": "live"}]},
("GET", "/api/templates"): {"items": [{"id": "tpl-old"}, {"id": "template-live"}]},
("GET", "/api/operators"): {
"items": [
{"id": "op-old", "type": "custom", "image": "runner:latest"},
{
"id": "operator-builtin-real-source",
"name": "真实数据源",
"type": "builtin",
"category": "IO",
"description": "source",
"inputs": [],
"outputs": [],
"image": "runner:latest",
},
]
},
("GET", "/api/settings"): {
"worker": {"pollIntervalMs": 1000, "maxConcurrentRuns": 2},
"defaults": {"runTimeoutSeconds": 3600},
},
("DELETE", "/api/users/1"): None,
("DELETE", "/api/projects/3"): None,
("DELETE", "/api/templates/tpl-old"): None,
("DELETE", "/api/operators/op-old"): None,
("PUT", "/api/settings"): {},
("PUT", "/api/operators/operator-builtin-real-source"): {},
}
)
reconcile.reconcile(
client,
k8s_endpoint="https://poc.example:6443",
k8s_namespace="droboticflow",
operator_runner_image="runner:new",
)
deleted_paths = [request[1] for request in client.requests if request[0] == "DELETE"]
self.assertEqual(
deleted_paths,
["/api/users/1", "/api/projects/3", "/api/templates/tpl-old", "/api/operators/op-old"],
)
self.assertNotIn("/api/operators/operator-builtin-real-source", deleted_paths)
if __name__ == "__main__":
unittest.main()