droboticflow/tests/backend/operator_repository_layout_test.py
2026-08-26 21:04:23 +08:00

378 lines
21 KiB
Python

import json
import hashlib
import pathlib
import subprocess
import unittest
import sys
ROOT = pathlib.Path(__file__).resolve().parents[2]
CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin"
OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin"
MANIFEST_SCRIPT = ROOT / "scripts" / "generate_operator_repository_manifests.py"
IMAGE_SCRIPT = ROOT / "scripts" / "generate_operator_images.py"
UNMIGRATED_HISTORICAL_REPOSITORIES = {"temporal_causal_eval"}
sys.path.insert(0, str(ROOT / "scripts"))
from operator_release_manifest import RELEASE_SCHEMA_VERSION, release_environment, source_tree_sha256
def operator_slug(operator_id: str) -> str:
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
def current_operator_metadata_paths() -> list[pathlib.Path]:
return sorted(
path
for path in OPERATOR_REPO_ROOT.glob("*/operator.json")
if path.parent.name not in UNMIGRATED_HISTORICAL_REPOSITORIES
)
class OperatorRepositoryLayoutTest(unittest.TestCase):
def test_dcloud_fat_operators_are_not_registered(self):
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8")) if item.get("type") == "builtin"}
self.assertNotIn("operator-builtin-dcloud-embodied-annotation", catalog)
self.assertNotIn("operator-builtin-dcloud-data-generation", catalog)
self.assertFalse((OPERATOR_ROOT / "dcloud_embodied_annotation").exists())
self.assertFalse((OPERATOR_ROOT / "dcloud_data_generation").exists())
def test_every_builtin_operator_has_repository_metadata(self):
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
builtin_items = [item for item in catalog if item.get("type") == "builtin"]
operator_metadata_paths = current_operator_metadata_paths()
self.assertEqual(len(builtin_items), len(operator_metadata_paths))
for item in builtin_items:
operator_id = str(item["id"])
preferred_slug = operator_slug(operator_id)
matching_paths = [
path
for path in operator_metadata_paths
if json.loads(path.read_text(encoding="utf-8")).get("id") == operator_id
]
self.assertTrue(matching_paths, f"{operator_id} must have operator-repos/builtin metadata")
metadata_path = matching_paths[0]
operator_dir = metadata_path.parent
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
self.assertEqual(metadata["id"], operator_id)
self.assertEqual(metadata["runtimeHandler"], item["runtimeHandler"])
self.assertEqual(metadata["migrationStatus"], "migrated")
self.assertEqual(item["runtimeSource"], metadata["runtimeSource"])
self.assertEqual(metadata["runtimeSource"], f"operator-repos/builtin/{operator_dir.name}/src/operator.py")
self.assertTrue(metadata["acceptance"]["requiresRealInputs"])
self.assertFalse(metadata["acceptance"]["mockOnlyAccepted"])
self.assertFalse(metadata["acceptance"]["simulationOnlyAccepted"])
self.assertFalse(metadata["acceptance"]["hardcodedOutputAccepted"])
self.assertFalse(metadata["acceptance"]["shallowOnlyAccepted"])
self.assertTrue(operator_dir.name == preferred_slug or metadata["migrationStatus"] == "migrated")
self.assertTrue((operator_dir / "src" / "operator.py").exists())
self.assertTrue((operator_dir / "Dockerfile").exists())
self.assertTrue((operator_dir / "runtime_contract.json").exists())
self.assertTrue((operator_dir / "tests" / "acceptance" / "fixture_contract.json").exists())
self.assertTrue((operator_dir / "release.json").exists())
def test_operator_sources_exist(self):
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8")) if item.get("type") == "builtin"}
for metadata_path in current_operator_metadata_paths():
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
catalog_item = catalog[metadata["id"]]
self.assertEqual(metadata.get("migrationStatus"), "migrated")
self.assertEqual(catalog_item["runtimeSource"], metadata["runtimeSource"])
runtime_source = ROOT / metadata["runtimeSource"]
self.assertTrue(runtime_source.exists(), f"{metadata['id']} runtime source is missing")
self.assertNotEqual(metadata["runtimeSource"], "backend/operator-runner/main.py")
def test_all_operator_sources_do_not_call_legacy_builtin_handlers(self):
for metadata_path in current_operator_metadata_paths():
slug = metadata_path.parent.name
with self.subTest(operator=slug):
source = (OPERATOR_REPO_ROOT / slug / "src" / "operator.py").read_text(encoding="utf-8")
self.assertNotIn("LEGACY_HANDLER", source)
self.assertNotIn('helpers["builtin_', source)
self.assertNotIn("helpers['builtin_", source)
self.assertNotIn("command_model_operator", source)
self.assertNotIn("run_command_model_operator", source)
self.assertFalse((OPERATOR_ROOT / "command_model_operator.py").exists())
def test_human_review_uses_independent_runtime_handler(self):
metadata = json.loads((OPERATOR_REPO_ROOT / "human_review" / "operator.json").read_text(encoding="utf-8"))
catalog = {
item["id"]: item
for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
if item.get("type") == "builtin"
}
manifest = json.loads((OPERATOR_REPO_ROOT / "human_review" / "repository.json").read_text(encoding="utf-8"))
self.assertEqual(metadata["runtimeHandler"], "builtin_human_review")
self.assertEqual(catalog["operator-builtin-human-review"]["runtimeHandler"], "builtin_human_review")
self.assertEqual(manifest["runtimeHandler"], "builtin_human_review")
self.assertNotEqual(metadata["runtimeHandler"], "builtin_condition_router")
def test_every_builtin_operator_has_independent_repository_manifest(self):
for metadata_path in current_operator_metadata_paths():
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
slug = metadata_path.parent.name
manifest_path = metadata_path.with_name("repository.json")
with self.subTest(operator=slug):
self.assertTrue(manifest_path.exists(), f"{slug} must have repository.json")
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
self.assertEqual(manifest["schemaVersion"], "droboticflow.operator.repository/v1")
self.assertEqual(manifest["repositoryId"], f"builtin/{slug}")
self.assertEqual(manifest["operatorId"], metadata["id"])
self.assertEqual(manifest["operatorSlug"], slug)
self.assertEqual(manifest["sourceRoot"], f"operator-repos/builtin/{slug}")
self.assertEqual(manifest["entrypoint"], "src/operator.py")
self.assertEqual(manifest["metadata"], "operator.json")
self.assertEqual(manifest["runtimeSource"], metadata["runtimeSource"])
self.assertEqual(manifest["runtimeHandler"], metadata["runtimeHandler"])
self.assertEqual(manifest["containerEntrypoint"], "backend/operator-runner/main.py")
self.assertEqual(manifest["releaseManifest"], "release.json")
self.assertEqual(manifest["releaseSchemaVersion"], RELEASE_SCHEMA_VERSION)
self.assertEqual(manifest["sourceTreeSha256"], source_tree_sha256(slug))
self.assertTrue(manifest["image"])
self.assertIn("droboticflow-operator-", manifest["image"])
self.assertNotIn("droboticflow-operator-runner", manifest["image"])
self.assertRegex(manifest["imageId"], r"@sha256:[a-f0-9]{64}$")
self.assertEqual(manifest["containerCommand"], ["python", "/app/backend/operator-runner/main.py"])
self.assertEqual(manifest["containerArgs"], [])
self.assertTrue(manifest["runtimeContract"]["imagePolicy"]["independentFinalImage"])
self.assertTrue(manifest["runtimeContract"]["imagePolicy"]["forbidsSharedOperatorRunnerFinalImage"])
self.assertTrue(manifest["runtimeDependencies"])
self.assertTrue(manifest["acceptance"]["requiresRealInputs"])
self.assertFalse(manifest["acceptance"]["mockOnlyAccepted"])
self.assertFalse(manifest["acceptance"]["simulationOnlyAccepted"])
self.assertFalse(manifest["acceptance"]["hardcodedOutputAccepted"])
self.assertFalse(manifest["acceptance"]["shallowOnlyAccepted"])
self.assertTrue(manifest["acceptance"]["requiresPersistentWorkspaceEvidence"])
if "externalApi" in metadata:
self.assertEqual(manifest["externalApi"], metadata["externalApi"])
def test_every_builtin_operator_has_release_manifest_as_image_truth(self):
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8")) if item.get("type") == "builtin"}
self.assertEqual(len(catalog), len(current_operator_metadata_paths()))
for metadata_path in current_operator_metadata_paths():
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
slug = metadata_path.parent.name
release_path = metadata_path.with_name("release.json")
repository_path = metadata_path.with_name("repository.json")
with self.subTest(operator=slug):
self.assertTrue(release_path.exists(), f"{slug} must have release.json")
release = json.loads(release_path.read_text(encoding="utf-8"))
repository = json.loads(repository_path.read_text(encoding="utf-8"))
catalog_item = catalog[metadata["id"]]
self.assertEqual(release["schemaVersion"], RELEASE_SCHEMA_VERSION)
self.assertEqual(release["operatorId"], metadata["id"])
self.assertEqual(release["slug"], slug)
self.assertRegex(release["imageId"], r"@sha256:[a-f0-9]{64}$")
if catalog_item.get("publicationStatus") == "draft":
self.assertNotEqual(release["sourceTreeSha256"], source_tree_sha256(slug))
else:
self.assertEqual(release["sourceTreeSha256"], source_tree_sha256(slug))
self.assertIn("poc", release["environmentReleases"])
poc_release = release_environment(release, "poc")
self.assertEqual(poc_release["image"], catalog_item["image"])
self.assertEqual(poc_release["imageId"], catalog_item["imageId"])
self.assertEqual(poc_release["image"], repository["image"])
self.assertEqual(poc_release["imageId"], repository["imageId"])
def test_operator_repository_manifests_are_generated(self):
result = subprocess.run(
["python3", str(MANIFEST_SCRIPT), "--check"],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
self.assertEqual(result.returncode, 0, result.stderr + result.stdout)
def test_every_builtin_operator_has_independent_image_contract(self):
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
builtin_items = [item for item in catalog if item.get("type") == "builtin"]
self.assertEqual(len(builtin_items), len(current_operator_metadata_paths()))
for item in builtin_items:
with self.subTest(operator=item["id"]):
self.assertTrue(item.get("image"))
self.assertIn("droboticflow-operator-", item["image"])
self.assertNotIn("droboticflow-operator-runner", item["image"])
self.assertRegex(item.get("imageId", ""), r"@sha256:[a-f0-9]{64}$")
self.assertEqual(item.get("containerCommand"), ["python", "/app/backend/operator-runner/main.py"])
self.assertEqual(item.get("containerArgs"), [])
contract = item.get("runtimeContract")
self.assertIsInstance(contract, dict)
self.assertTrue(contract["imagePolicy"]["independentFinalImage"])
self.assertTrue(contract["imagePolicy"]["forbidsSharedOperatorRunnerFinalImage"])
self.assertTrue(contract["artifacts"]["requiresLineage"])
command_keys = {
str(param.get("key", "")).strip().upper()
for param in item.get("hyperparams", [])
}
if item["id"] == "operator-builtin-shell-command":
self.assertFalse(command_keys & {"COMMAND", "SCRIPT"})
self.assertIn("SCRIPT_ID", command_keys)
self.assertIn("SCRIPT_ARGS_JSON", command_keys)
else:
self.assertFalse(command_keys & {"COMMAND", "SCRIPT"})
def test_remote_gpu_service_contracts_are_preserved(self):
catalog = {
item["id"]: item
for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
if item.get("type") == "builtin"
}
expected_gpu_types = {
"operator-builtin-hand-removal": "NVIDIA A800 80GB",
"operator-builtin-wilor-hand-pose": "NVIDIA L20 48GB",
}
for operator_id, expected_gpu_type in expected_gpu_types.items():
with self.subTest(operator=operator_id):
contract = catalog[operator_id]["runtimeContract"]
self.assertFalse(contract["resources"]["requiresGpu"])
self.assertTrue(contract["resources"]["externalGpuService"])
self.assertEqual(contract["resources"]["recommendedGpuType"], expected_gpu_type)
self.assertTrue(contract["acceptance"]["requiresGpuEvidence"])
def test_qwen3vl_operators_are_long_running(self):
catalog = {
item["id"]: item
for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
if item.get("type") == "builtin"
}
for operator_id in (
"operator-builtin-qwen-behavior-rough-cut",
"operator-builtin-qwen-temporal-localization",
"operator-builtin-wilor-valid-event-detection",
"operator-builtin-qwen-label-crop-classification",
):
with self.subTest(operator=operator_id):
self.assertTrue(catalog[operator_id]["runtimeContract"].get("longRunning"))
def test_builtin_operator_dockerfiles_are_generated(self):
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
builtin_items = [item for item in catalog if item.get("type") == "builtin"]
for item in builtin_items:
slug = operator_slug(item["id"])
dockerfile = ROOT / "operator-repos" / "builtin" / slug / "Dockerfile"
with self.subTest(operator=item["id"]):
self.assertTrue(dockerfile.exists(), f"{item['id']} must have an independent Dockerfile")
text = dockerfile.read_text(encoding="utf-8")
self.assertIn(f"DROBOTICFLOW_OPERATOR_ID={item['id']}", text)
self.assertIn(f"DROBOTICFLOW_OPERATOR_HANDLER={item['runtimeHandler']}", text)
self.assertIn(f"DROBOTICFLOW_OPERATOR_SOURCE=/app/operator-repos/builtin/{slug}/src/operator.py", text)
self.assertIn('ENTRYPOINT ["python", "/app/backend/operator-runner/main.py"]', text)
def test_operator_image_contracts_are_generated(self):
result = subprocess.run(
["python3", str(IMAGE_SCRIPT), "--check"],
cwd=ROOT,
text=True,
capture_output=True,
check=False,
)
self.assertEqual(result.returncode, 0, result.stderr + result.stdout)
def test_video_face_blur_image_pins_verified_yunet_asset(self):
dockerfile = (OPERATOR_REPO_ROOT / "video_face_blur" / "Dockerfile").read_text(encoding="utf-8")
asset_root = OPERATOR_REPO_ROOT / "video_face_blur" / "assets"
model = asset_root / "face_detection_yunet_2023mar.onnx"
manifest = json.loads((asset_root / "model_manifest.json").read_text(encoding="utf-8"))
self.assertIn("opencv-python-headless==4.11.0.86", dockerfile)
self.assertIn("--force-reinstall --no-deps", dockerfile)
self.assertIn("USER root", dockerfile)
self.assertIn("USER 10001:10001", dockerfile)
self.assertIn("runtimeDownloadsAllowed", dockerfile)
self.assertNotIn("curl ", dockerfile)
self.assertNotIn("wget ", dockerfile)
self.assertTrue(model.is_file())
self.assertEqual(model.stat().st_size, 232589)
self.assertEqual(hashlib.sha256(model.read_bytes()).hexdigest(), manifest["sha256"])
self.assertEqual(manifest["licenseSpdx"], "MIT")
self.assertTrue((asset_root / manifest["licenseFile"]).is_file())
def test_orbbec_undistort_image_pins_opencv_runtime(self):
dockerfile = (OPERATOR_REPO_ROOT / "orbbec_left_undistort" / "Dockerfile").read_text(encoding="utf-8")
self.assertIn("opencv-python-headless==4.11.0.86", dockerfile)
self.assertIn("DROBOTICFLOW_OPERATOR_HANDLER=builtin_orbbec_left_undistort", dockerfile)
def test_annotation_gate_image_overlays_native_runner_handler(self):
dockerfile = (OPERATOR_REPO_ROOT / "annotation_gate" / "Dockerfile").read_text(encoding="utf-8")
self.assertIn("COPY backend ./backend", dockerfile)
self.assertIn("DROBOTICFLOW_OPERATOR_HANDLER=builtin_annotation_gate", dockerfile)
def test_qwen_behavior_rough_cut_image_can_materialize_bos_inputs(self):
dockerfile = (OPERATOR_REPO_ROOT / "qwen_behavior_rough_cut" / "Dockerfile").read_text(encoding="utf-8")
self.assertIn("bce-python-sdk", dockerfile)
def test_holobrain_base_uses_resilient_ubuntu_mirror(self):
dockerfile = (ROOT / "operator-images" / "builtin" / "holobrain-base.Dockerfile").read_text(
encoding="utf-8"
)
self.assertIn("ARG UBUNTU_MIRROR=https://mirrors.aliyun.com/ubuntu", dockerfile)
self.assertIn("http://archive.ubuntu.com/ubuntu|$UBUNTU_MIRROR", dockerfile)
self.assertIn("http://security.ubuntu.com/ubuntu|$UBUNTU_MIRROR", dockerfile)
self.assertIn("Acquire::Retries=5", dockerfile)
self.assertIn("HOLOBRAIN_OFFLINE_PARENT_IMAGE=", dockerfile)
self.assertIn("@sha256:cc23ad2eb5d7515f2135ef9380ca69e9b5578be3b74044e1e6d708c47f08d127", dockerfile)
self.assertNotIn("snapshot_download", dockerfile)
self.assertIn("find /opt/models/HoloBrain_v0.0_Qwen -name '*.safetensors'", dockerfile)
def test_holobrain_robotwin_images_pin_a_single_numpy_runtime(self):
for slug in ("holobrain_inference_test", "holobrain_robotwin_eval"):
with self.subTest(operator=slug):
dockerfile = (OPERATOR_REPO_ROOT / slug / "Dockerfile").read_text(encoding="utf-8")
self.assertIn(
'RUN "$(command -v python)" -m venv --system-site-packages '
"/opt/holobrain-robotwin-venv",
dockerfile,
)
self.assertIn("PATH=/opt/holobrain-robotwin-venv/bin:${PATH}", dockerfile)
self.assertIn("/opt/holobrain-robotwin-venv/bin/python -m pip install", dockerfile)
self.assertIn("numpy==1.26.4", dockerfile)
self.assertIn("-c /tmp/holobrain-robotwin-constraints.txt", dockerfile)
self.assertIn('np.__version__ == metadata.version("numpy") == "1.26.4"', dockerfile)
self.assertIn("Path(sys.prefix).resolve() == venv", dockerfile)
self.assertIn("Path(sys.executable).is_relative_to(venv)", dockerfile)
self.assertIn("package_path.is_relative_to(venv)", dockerfile)
self.assertIn("distribution_path.is_relative_to(venv)", dockerfile)
self.assertIn("import accelerate", dockerfile)
self.assertNotIn("NUMPY_SITE", dockerfile)
self.assertNotIn("rm -rf", dockerfile)
self.assertNotIn("metadata.distributions()", dockerfile)
self.assertNotIn('"numpy>=2.0,<2.3"', dockerfile)
def test_holobrain_train_and_export_keep_the_shared_runtime(self):
for slug in ("holobrain_train", "holobrain_export"):
with self.subTest(operator=slug):
dockerfile = (OPERATOR_REPO_ROOT / slug / "Dockerfile").read_text(encoding="utf-8")
self.assertIn('"numpy>=2.0,<2.3"', dockerfile)
self.assertNotIn("holobrain-robotwin-venv", dockerfile)
def test_holobrain_long_running_contracts_cover_data_prepare_and_train(self):
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))}
for operator_id in (
"operator-builtin-holobrain-data-prepare",
"operator-builtin-holobrain-train",
):
with self.subTest(operator=operator_id):
contract = catalog[operator_id]["runtimeContract"]
self.assertIs(contract["longRunning"], True)
if __name__ == "__main__":
unittest.main()