forked from longtao.wu/droboticflow
353 lines
15 KiB
Python
353 lines
15 KiB
Python
import os
|
|
import pathlib
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from typing import Any
|
|
from unittest import mock
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
import reconcile_system_data as reconcile
|
|
|
|
|
|
class FakeClient:
|
|
def __init__(self, responses: dict[tuple[str, str], Any]):
|
|
self.responses = responses
|
|
self.requests: list[tuple[str, str, Any, tuple[int, ...], int, int]] = []
|
|
|
|
def request_json(
|
|
self,
|
|
method,
|
|
path,
|
|
payload=None,
|
|
*,
|
|
expected=(200, 201, 202, 204),
|
|
timeout=60,
|
|
retries=1,
|
|
retry_delay_seconds=1.0,
|
|
):
|
|
self.requests.append((method, path, payload, expected, timeout, retries))
|
|
key = (method.upper(), path)
|
|
if key not in self.responses:
|
|
raise AssertionError(f"unexpected request {key}")
|
|
response = self.responses[key]
|
|
if callable(response):
|
|
return response(payload)
|
|
return response
|
|
|
|
|
|
class ReconcileSystemDataTest(unittest.TestCase):
|
|
def test_legacy_operator_cleanup_never_deletes_builtin_latest_images(self):
|
|
self.assertFalse(
|
|
reconcile.is_legacy_operator(
|
|
{
|
|
"id": "operator-builtin-real-source",
|
|
"type": "builtin",
|
|
"image": "ccr.example/droboticflow-operator-runner:latest",
|
|
}
|
|
)
|
|
)
|
|
self.assertTrue(reconcile.is_legacy_operator({"id": "op-old", "type": "custom", "image": "runner:stable"}))
|
|
self.assertTrue(reconcile.is_legacy_operator({"id": "custom-operator", "type": "custom", "image": "runner:latest"}))
|
|
|
|
def test_build_settings_payload_preserves_worker_and_defaults(self):
|
|
payload = reconcile.build_settings_payload(
|
|
{
|
|
"worker": {"pollIntervalMs": 2500, "maxConcurrentRuns": 3},
|
|
"defaults": {"timeoutSeconds": 1800},
|
|
},
|
|
"https://poc.example:6443",
|
|
"droboticflow-dev",
|
|
)
|
|
|
|
self.assertEqual(payload["k8s"]["endpoint"], "https://poc.example:6443")
|
|
self.assertEqual(payload["k8s"]["namespace"], "droboticflow-dev")
|
|
self.assertEqual(payload["worker"], {"pollIntervalMs": 2500, "maxConcurrentRuns": 3})
|
|
self.assertEqual(payload["defaults"], {"timeoutSeconds": 1800})
|
|
|
|
def test_reconcile_recognizes_live_acceptance_temporary_projects(self):
|
|
self.assertTrue(reconcile.is_legacy_project({"description": "Created by scripts/live-workflow-acceptance.sh"}))
|
|
self.assertFalse(reconcile.is_legacy_project({"description": "customer project"}))
|
|
|
|
def test_sync_builtin_operators_uses_live_api_catalog(self):
|
|
client = FakeClient(
|
|
{
|
|
(
|
|
"GET",
|
|
"/api/operators",
|
|
): {
|
|
"items": [
|
|
{"id": "custom-1", "name": "Custom", "type": "custom", "image": "old"},
|
|
{
|
|
"id": "operator-builtin-mongodb-write",
|
|
"name": "MongoDB 写入",
|
|
"type": "builtin",
|
|
"category": "DATA",
|
|
"description": "write business data",
|
|
"inputs": [{"id": "input", "name": "input", "type": "json"}],
|
|
"outputs": [{"id": "result", "name": "result", "type": "json"}],
|
|
"image": "runner:old",
|
|
"skillDefinition": "Write JSON documents",
|
|
"hyperparams": [{"key": "MONGO_COLLECTION", "value": ""}],
|
|
},
|
|
]
|
|
},
|
|
("PUT", "/api/operators/operator-builtin-mongodb-write"): {"id": "operator-builtin-mongodb-write"},
|
|
}
|
|
)
|
|
|
|
updated = reconcile.sync_builtin_operators(client, "runner:new")
|
|
|
|
self.assertEqual(updated, 1)
|
|
put_request = client.requests[1]
|
|
self.assertEqual(put_request[1], "/api/operators/operator-builtin-mongodb-write")
|
|
self.assertEqual(put_request[2]["name"], "MongoDB 写入")
|
|
self.assertEqual(put_request[2]["image"], "runner:new")
|
|
self.assertEqual(put_request[2]["imageId"], "runner:new")
|
|
self.assertEqual(put_request[5], 6)
|
|
|
|
def test_sync_builtin_operators_preserves_independent_operator_images(self):
|
|
release_image, release_image_id = reconcile.release_image_reference("real_source")
|
|
client = FakeClient(
|
|
{
|
|
("GET", "/api/operators"): {
|
|
"items": [
|
|
{
|
|
"id": "operator-builtin-real-source",
|
|
"name": "真实数据源",
|
|
"type": "builtin",
|
|
"category": "IO",
|
|
"description": "source",
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"image": release_image,
|
|
"imageId": release_image_id,
|
|
"containerCommand": ["python", "/app/backend/operator-runner/main.py"],
|
|
"containerArgs": [],
|
|
"runtimeContract": {"imagePolicy": {"independentFinalImage": True}},
|
|
}
|
|
]
|
|
},
|
|
}
|
|
)
|
|
|
|
updated = reconcile.sync_builtin_operators(client, "runner:new")
|
|
|
|
self.assertEqual(updated, 0)
|
|
self.assertEqual(len(client.requests), 1)
|
|
|
|
def test_sync_builtin_operators_uses_release_manifest_for_independent_images(self):
|
|
release_image, release_image_id = reconcile.release_image_reference("real_source")
|
|
client = FakeClient(
|
|
{
|
|
("GET", "/api/operators"): {
|
|
"items": [
|
|
{
|
|
"id": "operator-builtin-real-source",
|
|
"name": "真实数据源",
|
|
"type": "builtin",
|
|
"category": "IO",
|
|
"description": "source",
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"image": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
|
|
"imageId": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
|
|
"containerCommand": ["python", "/app/backend/operator-runner/main.py"],
|
|
"containerArgs": [],
|
|
"runtimeContract": {"imagePolicy": {"independentFinalImage": True}},
|
|
}
|
|
]
|
|
},
|
|
("PUT", "/api/operators/operator-builtin-real-source"): {"id": "operator-builtin-real-source"},
|
|
}
|
|
)
|
|
|
|
updated = reconcile.sync_builtin_operators(
|
|
client,
|
|
"ccr.example/dataflow-poc/droboticflow-operator-runner:live",
|
|
image_digest_resolver=lambda image: release_image_id,
|
|
)
|
|
|
|
self.assertEqual(updated, 1)
|
|
put_request = client.requests[1]
|
|
self.assertEqual(put_request[2]["image"], release_image)
|
|
self.assertEqual(put_request[2]["imageId"], release_image_id)
|
|
|
|
def test_sync_builtin_operators_rejects_release_registry_digest_drift(self):
|
|
registry_digest = "sha256:" + "b" * 64
|
|
client = FakeClient(
|
|
{
|
|
("GET", "/api/operators"): {
|
|
"items": [
|
|
{
|
|
"id": "operator-builtin-real-source",
|
|
"name": "真实数据源",
|
|
"type": "builtin",
|
|
"category": "IO",
|
|
"description": "source",
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"image": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
|
|
"imageId": "registry.example/dataflow/droboticflow-operator-real-source:builtin",
|
|
"containerCommand": ["python", "/app/backend/operator-runner/main.py"],
|
|
"containerArgs": [],
|
|
"runtimeContract": {"imagePolicy": {"independentFinalImage": True}},
|
|
}
|
|
]
|
|
},
|
|
}
|
|
)
|
|
|
|
with self.assertRaisesRegex(reconcile.ReconcileError, "release digest drift"):
|
|
reconcile.sync_builtin_operators(
|
|
client,
|
|
"ccr.example/dataflow-poc/droboticflow-operator-runner:live",
|
|
image_digest_resolver=lambda image: reconcile.image_ref_with_digest(image, registry_digest),
|
|
)
|
|
|
|
def test_sync_builtin_operators_can_target_one_operator(self):
|
|
release_image, release_image_id = reconcile.release_image_reference("real_source")
|
|
client = FakeClient(
|
|
{
|
|
("GET", "/api/operators"): {
|
|
"items": [
|
|
{
|
|
"id": "operator-builtin-real-source",
|
|
"name": "真实数据源",
|
|
"type": "builtin",
|
|
"category": "IO",
|
|
"description": "source",
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"image": release_image,
|
|
"imageId": release_image,
|
|
"containerCommand": [],
|
|
},
|
|
{
|
|
"id": "operator-builtin-mongodb-write",
|
|
"name": "MongoDB 写入",
|
|
"type": "builtin",
|
|
"category": "DATA",
|
|
"description": "mongo",
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"image": "runner:old",
|
|
"imageId": "runner:old",
|
|
"containerCommand": [],
|
|
},
|
|
]
|
|
},
|
|
("PUT", "/api/operators/operator-builtin-real-source"): {"id": "operator-builtin-real-source"},
|
|
}
|
|
)
|
|
|
|
updated = reconcile.sync_builtin_operators(
|
|
client,
|
|
"runner:new",
|
|
image_digest_resolver=lambda image: release_image_id,
|
|
operator_ids={"operator-builtin-real-source"},
|
|
)
|
|
|
|
self.assertEqual(updated, 1)
|
|
self.assertEqual(len(client.requests), 2)
|
|
self.assertEqual(client.requests[1][1], "/api/operators/operator-builtin-real-source")
|
|
self.assertEqual(client.requests[1][2]["image"], release_image)
|
|
|
|
def test_image_ref_with_digest_replaces_tag_with_digest(self):
|
|
self.assertEqual(
|
|
reconcile.image_ref_with_digest("ccr.example/ns/droboticflow-operator-yolo-train:v1", "sha256:" + "c" * 64),
|
|
"ccr.example/ns/droboticflow-operator-yolo-train@sha256:" + "c" * 64,
|
|
)
|
|
|
|
def test_parse_bearer_challenge(self):
|
|
self.assertEqual(
|
|
reconcile.parse_bearer_challenge(
|
|
'Bearer realm="https://registry.example/token",service="registry.example",scope="repository:ns/image:pull"'
|
|
),
|
|
{
|
|
"realm": "https://registry.example/token",
|
|
"service": "registry.example",
|
|
"scope": "repository:ns/image:pull",
|
|
},
|
|
)
|
|
|
|
def test_registry_digest_resolver_auto_enables_when_auth_file_is_present(self):
|
|
with tempfile.NamedTemporaryFile("w", encoding="utf-8") as auth_file:
|
|
auth_file.write('{"auths":{"registry.example":{"auth":"dXNlcjpwYXNz"}}}')
|
|
auth_file.flush()
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{"REGISTRY_AUTH_FILE": auth_file.name, "RECONCILE_OPERATOR_IMAGE_DIGESTS": ""},
|
|
):
|
|
self.assertIsNotNone(reconcile.registry_digest_resolver_from_env())
|
|
|
|
def test_registry_digest_resolver_can_be_disabled_explicitly(self):
|
|
with tempfile.NamedTemporaryFile("w", encoding="utf-8") as auth_file:
|
|
auth_file.write('{"auths":{"registry.example":{"auth":"dXNlcjpwYXNz"}}}')
|
|
auth_file.flush()
|
|
with mock.patch.dict(
|
|
os.environ,
|
|
{"REGISTRY_AUTH_FILE": auth_file.name, "RECONCILE_OPERATOR_IMAGE_DIGESTS": "false"},
|
|
):
|
|
self.assertIsNone(reconcile.registry_digest_resolver_from_env())
|
|
|
|
def test_reconcile_deletes_legacy_records_and_keeps_builtin_catalog_entries(self):
|
|
client = FakeClient(
|
|
{
|
|
("GET", "/api/users"): {
|
|
"items": [
|
|
{"id": "1", "email": "demo@emboflow.ai"},
|
|
{"id": "operator", "email": "operator@example.test"},
|
|
]
|
|
},
|
|
("GET", "/api/projects"): {"items": [{"id": "3", "description": "old"}, {"id": "live"}]},
|
|
("GET", "/api/templates"): {"items": [{"id": "tpl-old"}, {"id": "template-live"}]},
|
|
("GET", "/api/operators"): {
|
|
"items": [
|
|
{"id": "op-old", "type": "custom", "image": "runner:latest"},
|
|
{
|
|
"id": "operator-builtin-real-source",
|
|
"name": "真实数据源",
|
|
"type": "builtin",
|
|
"category": "IO",
|
|
"description": "source",
|
|
"inputs": [],
|
|
"outputs": [],
|
|
"image": "runner:latest",
|
|
},
|
|
]
|
|
},
|
|
("GET", "/api/settings"): {
|
|
"worker": {"pollIntervalMs": 1000, "maxConcurrentRuns": 2},
|
|
"defaults": {"runTimeoutSeconds": 3600},
|
|
},
|
|
("DELETE", "/api/users/1"): None,
|
|
("DELETE", "/api/projects/3"): None,
|
|
("DELETE", "/api/templates/tpl-old"): None,
|
|
("DELETE", "/api/operators/op-old"): None,
|
|
("PUT", "/api/settings"): {},
|
|
("PUT", "/api/operators/operator-builtin-real-source"): {},
|
|
}
|
|
)
|
|
|
|
reconcile.reconcile(
|
|
client,
|
|
k8s_endpoint="https://poc.example:6443",
|
|
k8s_namespace="droboticflow",
|
|
operator_runner_image="runner:new",
|
|
)
|
|
|
|
deleted_paths = [request[1] for request in client.requests if request[0] == "DELETE"]
|
|
self.assertEqual(
|
|
deleted_paths,
|
|
["/api/users/1", "/api/projects/3", "/api/templates/tpl-old", "/api/operators/op-old"],
|
|
)
|
|
self.assertNotIn("/api/operators/operator-builtin-real-source", deleted_paths)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|