378 lines
21 KiB
Python
378 lines
21 KiB
Python
import json
|
|
import hashlib
|
|
import pathlib
|
|
import subprocess
|
|
import unittest
|
|
import sys
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
|
|
OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin"
|
|
OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin"
|
|
MANIFEST_SCRIPT = ROOT / "scripts" / "generate_operator_repository_manifests.py"
|
|
IMAGE_SCRIPT = ROOT / "scripts" / "generate_operator_images.py"
|
|
UNMIGRATED_HISTORICAL_REPOSITORIES = {"temporal_causal_eval"}
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from operator_release_manifest import RELEASE_SCHEMA_VERSION, release_environment, source_tree_sha256
|
|
|
|
|
|
def operator_slug(operator_id: str) -> str:
|
|
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
|
|
|
|
|
|
def current_operator_metadata_paths() -> list[pathlib.Path]:
|
|
return sorted(
|
|
path
|
|
for path in OPERATOR_REPO_ROOT.glob("*/operator.json")
|
|
if path.parent.name not in UNMIGRATED_HISTORICAL_REPOSITORIES
|
|
)
|
|
|
|
|
|
class OperatorRepositoryLayoutTest(unittest.TestCase):
|
|
def test_dcloud_fat_operators_are_not_registered(self):
|
|
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8")) if item.get("type") == "builtin"}
|
|
|
|
self.assertNotIn("operator-builtin-dcloud-embodied-annotation", catalog)
|
|
self.assertNotIn("operator-builtin-dcloud-data-generation", catalog)
|
|
self.assertFalse((OPERATOR_ROOT / "dcloud_embodied_annotation").exists())
|
|
self.assertFalse((OPERATOR_ROOT / "dcloud_data_generation").exists())
|
|
|
|
def test_every_builtin_operator_has_repository_metadata(self):
|
|
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
|
|
builtin_items = [item for item in catalog if item.get("type") == "builtin"]
|
|
operator_metadata_paths = current_operator_metadata_paths()
|
|
|
|
self.assertEqual(len(builtin_items), len(operator_metadata_paths))
|
|
for item in builtin_items:
|
|
operator_id = str(item["id"])
|
|
preferred_slug = operator_slug(operator_id)
|
|
matching_paths = [
|
|
path
|
|
for path in operator_metadata_paths
|
|
if json.loads(path.read_text(encoding="utf-8")).get("id") == operator_id
|
|
]
|
|
self.assertTrue(matching_paths, f"{operator_id} must have operator-repos/builtin metadata")
|
|
metadata_path = matching_paths[0]
|
|
operator_dir = metadata_path.parent
|
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(metadata["id"], operator_id)
|
|
self.assertEqual(metadata["runtimeHandler"], item["runtimeHandler"])
|
|
self.assertEqual(metadata["migrationStatus"], "migrated")
|
|
self.assertEqual(item["runtimeSource"], metadata["runtimeSource"])
|
|
self.assertEqual(metadata["runtimeSource"], f"operator-repos/builtin/{operator_dir.name}/src/operator.py")
|
|
self.assertTrue(metadata["acceptance"]["requiresRealInputs"])
|
|
self.assertFalse(metadata["acceptance"]["mockOnlyAccepted"])
|
|
self.assertFalse(metadata["acceptance"]["simulationOnlyAccepted"])
|
|
self.assertFalse(metadata["acceptance"]["hardcodedOutputAccepted"])
|
|
self.assertFalse(metadata["acceptance"]["shallowOnlyAccepted"])
|
|
self.assertTrue(operator_dir.name == preferred_slug or metadata["migrationStatus"] == "migrated")
|
|
self.assertTrue((operator_dir / "src" / "operator.py").exists())
|
|
self.assertTrue((operator_dir / "Dockerfile").exists())
|
|
self.assertTrue((operator_dir / "runtime_contract.json").exists())
|
|
self.assertTrue((operator_dir / "tests" / "acceptance" / "fixture_contract.json").exists())
|
|
self.assertTrue((operator_dir / "release.json").exists())
|
|
|
|
def test_operator_sources_exist(self):
|
|
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8")) if item.get("type") == "builtin"}
|
|
for metadata_path in current_operator_metadata_paths():
|
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
catalog_item = catalog[metadata["id"]]
|
|
self.assertEqual(metadata.get("migrationStatus"), "migrated")
|
|
self.assertEqual(catalog_item["runtimeSource"], metadata["runtimeSource"])
|
|
runtime_source = ROOT / metadata["runtimeSource"]
|
|
self.assertTrue(runtime_source.exists(), f"{metadata['id']} runtime source is missing")
|
|
self.assertNotEqual(metadata["runtimeSource"], "backend/operator-runner/main.py")
|
|
|
|
def test_all_operator_sources_do_not_call_legacy_builtin_handlers(self):
|
|
for metadata_path in current_operator_metadata_paths():
|
|
slug = metadata_path.parent.name
|
|
with self.subTest(operator=slug):
|
|
source = (OPERATOR_REPO_ROOT / slug / "src" / "operator.py").read_text(encoding="utf-8")
|
|
self.assertNotIn("LEGACY_HANDLER", source)
|
|
self.assertNotIn('helpers["builtin_', source)
|
|
self.assertNotIn("helpers['builtin_", source)
|
|
self.assertNotIn("command_model_operator", source)
|
|
self.assertNotIn("run_command_model_operator", source)
|
|
self.assertFalse((OPERATOR_ROOT / "command_model_operator.py").exists())
|
|
|
|
def test_human_review_uses_independent_runtime_handler(self):
|
|
metadata = json.loads((OPERATOR_REPO_ROOT / "human_review" / "operator.json").read_text(encoding="utf-8"))
|
|
catalog = {
|
|
item["id"]: item
|
|
for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
|
|
if item.get("type") == "builtin"
|
|
}
|
|
manifest = json.loads((OPERATOR_REPO_ROOT / "human_review" / "repository.json").read_text(encoding="utf-8"))
|
|
|
|
self.assertEqual(metadata["runtimeHandler"], "builtin_human_review")
|
|
self.assertEqual(catalog["operator-builtin-human-review"]["runtimeHandler"], "builtin_human_review")
|
|
self.assertEqual(manifest["runtimeHandler"], "builtin_human_review")
|
|
self.assertNotEqual(metadata["runtimeHandler"], "builtin_condition_router")
|
|
|
|
def test_every_builtin_operator_has_independent_repository_manifest(self):
|
|
for metadata_path in current_operator_metadata_paths():
|
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
slug = metadata_path.parent.name
|
|
manifest_path = metadata_path.with_name("repository.json")
|
|
with self.subTest(operator=slug):
|
|
self.assertTrue(manifest_path.exists(), f"{slug} must have repository.json")
|
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
self.assertEqual(manifest["schemaVersion"], "droboticflow.operator.repository/v1")
|
|
self.assertEqual(manifest["repositoryId"], f"builtin/{slug}")
|
|
self.assertEqual(manifest["operatorId"], metadata["id"])
|
|
self.assertEqual(manifest["operatorSlug"], slug)
|
|
self.assertEqual(manifest["sourceRoot"], f"operator-repos/builtin/{slug}")
|
|
self.assertEqual(manifest["entrypoint"], "src/operator.py")
|
|
self.assertEqual(manifest["metadata"], "operator.json")
|
|
self.assertEqual(manifest["runtimeSource"], metadata["runtimeSource"])
|
|
self.assertEqual(manifest["runtimeHandler"], metadata["runtimeHandler"])
|
|
self.assertEqual(manifest["containerEntrypoint"], "backend/operator-runner/main.py")
|
|
self.assertEqual(manifest["releaseManifest"], "release.json")
|
|
self.assertEqual(manifest["releaseSchemaVersion"], RELEASE_SCHEMA_VERSION)
|
|
self.assertEqual(manifest["sourceTreeSha256"], source_tree_sha256(slug))
|
|
self.assertTrue(manifest["image"])
|
|
self.assertIn("droboticflow-operator-", manifest["image"])
|
|
self.assertNotIn("droboticflow-operator-runner", manifest["image"])
|
|
self.assertRegex(manifest["imageId"], r"@sha256:[a-f0-9]{64}$")
|
|
self.assertEqual(manifest["containerCommand"], ["python", "/app/backend/operator-runner/main.py"])
|
|
self.assertEqual(manifest["containerArgs"], [])
|
|
self.assertTrue(manifest["runtimeContract"]["imagePolicy"]["independentFinalImage"])
|
|
self.assertTrue(manifest["runtimeContract"]["imagePolicy"]["forbidsSharedOperatorRunnerFinalImage"])
|
|
self.assertTrue(manifest["runtimeDependencies"])
|
|
self.assertTrue(manifest["acceptance"]["requiresRealInputs"])
|
|
self.assertFalse(manifest["acceptance"]["mockOnlyAccepted"])
|
|
self.assertFalse(manifest["acceptance"]["simulationOnlyAccepted"])
|
|
self.assertFalse(manifest["acceptance"]["hardcodedOutputAccepted"])
|
|
self.assertFalse(manifest["acceptance"]["shallowOnlyAccepted"])
|
|
self.assertTrue(manifest["acceptance"]["requiresPersistentWorkspaceEvidence"])
|
|
if "externalApi" in metadata:
|
|
self.assertEqual(manifest["externalApi"], metadata["externalApi"])
|
|
|
|
def test_every_builtin_operator_has_release_manifest_as_image_truth(self):
|
|
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8")) if item.get("type") == "builtin"}
|
|
|
|
self.assertEqual(len(catalog), len(current_operator_metadata_paths()))
|
|
for metadata_path in current_operator_metadata_paths():
|
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
slug = metadata_path.parent.name
|
|
release_path = metadata_path.with_name("release.json")
|
|
repository_path = metadata_path.with_name("repository.json")
|
|
with self.subTest(operator=slug):
|
|
self.assertTrue(release_path.exists(), f"{slug} must have release.json")
|
|
release = json.loads(release_path.read_text(encoding="utf-8"))
|
|
repository = json.loads(repository_path.read_text(encoding="utf-8"))
|
|
catalog_item = catalog[metadata["id"]]
|
|
self.assertEqual(release["schemaVersion"], RELEASE_SCHEMA_VERSION)
|
|
self.assertEqual(release["operatorId"], metadata["id"])
|
|
self.assertEqual(release["slug"], slug)
|
|
self.assertRegex(release["imageId"], r"@sha256:[a-f0-9]{64}$")
|
|
if catalog_item.get("publicationStatus") == "draft":
|
|
self.assertNotEqual(release["sourceTreeSha256"], source_tree_sha256(slug))
|
|
else:
|
|
self.assertEqual(release["sourceTreeSha256"], source_tree_sha256(slug))
|
|
self.assertIn("poc", release["environmentReleases"])
|
|
poc_release = release_environment(release, "poc")
|
|
self.assertEqual(poc_release["image"], catalog_item["image"])
|
|
self.assertEqual(poc_release["imageId"], catalog_item["imageId"])
|
|
self.assertEqual(poc_release["image"], repository["image"])
|
|
self.assertEqual(poc_release["imageId"], repository["imageId"])
|
|
|
|
def test_operator_repository_manifests_are_generated(self):
|
|
result = subprocess.run(
|
|
["python3", str(MANIFEST_SCRIPT), "--check"],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr + result.stdout)
|
|
|
|
def test_every_builtin_operator_has_independent_image_contract(self):
|
|
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
|
|
builtin_items = [item for item in catalog if item.get("type") == "builtin"]
|
|
|
|
self.assertEqual(len(builtin_items), len(current_operator_metadata_paths()))
|
|
for item in builtin_items:
|
|
with self.subTest(operator=item["id"]):
|
|
self.assertTrue(item.get("image"))
|
|
self.assertIn("droboticflow-operator-", item["image"])
|
|
self.assertNotIn("droboticflow-operator-runner", item["image"])
|
|
self.assertRegex(item.get("imageId", ""), r"@sha256:[a-f0-9]{64}$")
|
|
self.assertEqual(item.get("containerCommand"), ["python", "/app/backend/operator-runner/main.py"])
|
|
self.assertEqual(item.get("containerArgs"), [])
|
|
contract = item.get("runtimeContract")
|
|
self.assertIsInstance(contract, dict)
|
|
self.assertTrue(contract["imagePolicy"]["independentFinalImage"])
|
|
self.assertTrue(contract["imagePolicy"]["forbidsSharedOperatorRunnerFinalImage"])
|
|
self.assertTrue(contract["artifacts"]["requiresLineage"])
|
|
command_keys = {
|
|
str(param.get("key", "")).strip().upper()
|
|
for param in item.get("hyperparams", [])
|
|
}
|
|
if item["id"] == "operator-builtin-shell-command":
|
|
self.assertFalse(command_keys & {"COMMAND", "SCRIPT"})
|
|
self.assertIn("SCRIPT_ID", command_keys)
|
|
self.assertIn("SCRIPT_ARGS_JSON", command_keys)
|
|
else:
|
|
self.assertFalse(command_keys & {"COMMAND", "SCRIPT"})
|
|
|
|
def test_remote_gpu_service_contracts_are_preserved(self):
|
|
catalog = {
|
|
item["id"]: item
|
|
for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
|
|
if item.get("type") == "builtin"
|
|
}
|
|
|
|
expected_gpu_types = {
|
|
"operator-builtin-hand-removal": "NVIDIA A800 80GB",
|
|
"operator-builtin-wilor-hand-pose": "NVIDIA L20 48GB",
|
|
}
|
|
for operator_id, expected_gpu_type in expected_gpu_types.items():
|
|
with self.subTest(operator=operator_id):
|
|
contract = catalog[operator_id]["runtimeContract"]
|
|
self.assertFalse(contract["resources"]["requiresGpu"])
|
|
self.assertTrue(contract["resources"]["externalGpuService"])
|
|
self.assertEqual(contract["resources"]["recommendedGpuType"], expected_gpu_type)
|
|
self.assertTrue(contract["acceptance"]["requiresGpuEvidence"])
|
|
|
|
def test_qwen3vl_operators_are_long_running(self):
|
|
catalog = {
|
|
item["id"]: item
|
|
for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
|
|
if item.get("type") == "builtin"
|
|
}
|
|
|
|
for operator_id in (
|
|
"operator-builtin-qwen-behavior-rough-cut",
|
|
"operator-builtin-qwen-temporal-localization",
|
|
"operator-builtin-wilor-valid-event-detection",
|
|
"operator-builtin-qwen-label-crop-classification",
|
|
):
|
|
with self.subTest(operator=operator_id):
|
|
self.assertTrue(catalog[operator_id]["runtimeContract"].get("longRunning"))
|
|
|
|
def test_builtin_operator_dockerfiles_are_generated(self):
|
|
catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8"))
|
|
builtin_items = [item for item in catalog if item.get("type") == "builtin"]
|
|
for item in builtin_items:
|
|
slug = operator_slug(item["id"])
|
|
dockerfile = ROOT / "operator-repos" / "builtin" / slug / "Dockerfile"
|
|
with self.subTest(operator=item["id"]):
|
|
self.assertTrue(dockerfile.exists(), f"{item['id']} must have an independent Dockerfile")
|
|
text = dockerfile.read_text(encoding="utf-8")
|
|
self.assertIn(f"DROBOTICFLOW_OPERATOR_ID={item['id']}", text)
|
|
self.assertIn(f"DROBOTICFLOW_OPERATOR_HANDLER={item['runtimeHandler']}", text)
|
|
self.assertIn(f"DROBOTICFLOW_OPERATOR_SOURCE=/app/operator-repos/builtin/{slug}/src/operator.py", text)
|
|
self.assertIn('ENTRYPOINT ["python", "/app/backend/operator-runner/main.py"]', text)
|
|
|
|
def test_operator_image_contracts_are_generated(self):
|
|
result = subprocess.run(
|
|
["python3", str(IMAGE_SCRIPT), "--check"],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr + result.stdout)
|
|
|
|
def test_video_face_blur_image_pins_verified_yunet_asset(self):
|
|
dockerfile = (OPERATOR_REPO_ROOT / "video_face_blur" / "Dockerfile").read_text(encoding="utf-8")
|
|
asset_root = OPERATOR_REPO_ROOT / "video_face_blur" / "assets"
|
|
model = asset_root / "face_detection_yunet_2023mar.onnx"
|
|
manifest = json.loads((asset_root / "model_manifest.json").read_text(encoding="utf-8"))
|
|
|
|
self.assertIn("opencv-python-headless==4.11.0.86", dockerfile)
|
|
self.assertIn("--force-reinstall --no-deps", dockerfile)
|
|
self.assertIn("USER root", dockerfile)
|
|
self.assertIn("USER 10001:10001", dockerfile)
|
|
self.assertIn("runtimeDownloadsAllowed", dockerfile)
|
|
self.assertNotIn("curl ", dockerfile)
|
|
self.assertNotIn("wget ", dockerfile)
|
|
self.assertTrue(model.is_file())
|
|
self.assertEqual(model.stat().st_size, 232589)
|
|
self.assertEqual(hashlib.sha256(model.read_bytes()).hexdigest(), manifest["sha256"])
|
|
self.assertEqual(manifest["licenseSpdx"], "MIT")
|
|
self.assertTrue((asset_root / manifest["licenseFile"]).is_file())
|
|
|
|
def test_orbbec_undistort_image_pins_opencv_runtime(self):
|
|
dockerfile = (OPERATOR_REPO_ROOT / "orbbec_left_undistort" / "Dockerfile").read_text(encoding="utf-8")
|
|
|
|
self.assertIn("opencv-python-headless==4.11.0.86", dockerfile)
|
|
self.assertIn("DROBOTICFLOW_OPERATOR_HANDLER=builtin_orbbec_left_undistort", dockerfile)
|
|
|
|
def test_annotation_gate_image_overlays_native_runner_handler(self):
|
|
dockerfile = (OPERATOR_REPO_ROOT / "annotation_gate" / "Dockerfile").read_text(encoding="utf-8")
|
|
|
|
self.assertIn("COPY backend ./backend", dockerfile)
|
|
self.assertIn("DROBOTICFLOW_OPERATOR_HANDLER=builtin_annotation_gate", dockerfile)
|
|
|
|
def test_qwen_behavior_rough_cut_image_can_materialize_bos_inputs(self):
|
|
dockerfile = (OPERATOR_REPO_ROOT / "qwen_behavior_rough_cut" / "Dockerfile").read_text(encoding="utf-8")
|
|
|
|
self.assertIn("bce-python-sdk", dockerfile)
|
|
|
|
def test_holobrain_base_uses_resilient_ubuntu_mirror(self):
|
|
dockerfile = (ROOT / "operator-images" / "builtin" / "holobrain-base.Dockerfile").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
self.assertIn("ARG UBUNTU_MIRROR=https://mirrors.aliyun.com/ubuntu", dockerfile)
|
|
self.assertIn("http://archive.ubuntu.com/ubuntu|$UBUNTU_MIRROR", dockerfile)
|
|
self.assertIn("http://security.ubuntu.com/ubuntu|$UBUNTU_MIRROR", dockerfile)
|
|
self.assertIn("Acquire::Retries=5", dockerfile)
|
|
self.assertIn("HOLOBRAIN_OFFLINE_PARENT_IMAGE=", dockerfile)
|
|
self.assertIn("@sha256:cc23ad2eb5d7515f2135ef9380ca69e9b5578be3b74044e1e6d708c47f08d127", dockerfile)
|
|
self.assertNotIn("snapshot_download", dockerfile)
|
|
self.assertIn("find /opt/models/HoloBrain_v0.0_Qwen -name '*.safetensors'", dockerfile)
|
|
|
|
def test_holobrain_robotwin_images_pin_a_single_numpy_runtime(self):
|
|
for slug in ("holobrain_inference_test", "holobrain_robotwin_eval"):
|
|
with self.subTest(operator=slug):
|
|
dockerfile = (OPERATOR_REPO_ROOT / slug / "Dockerfile").read_text(encoding="utf-8")
|
|
|
|
self.assertIn(
|
|
'RUN "$(command -v python)" -m venv --system-site-packages '
|
|
"/opt/holobrain-robotwin-venv",
|
|
dockerfile,
|
|
)
|
|
self.assertIn("PATH=/opt/holobrain-robotwin-venv/bin:${PATH}", dockerfile)
|
|
self.assertIn("/opt/holobrain-robotwin-venv/bin/python -m pip install", dockerfile)
|
|
self.assertIn("numpy==1.26.4", dockerfile)
|
|
self.assertIn("-c /tmp/holobrain-robotwin-constraints.txt", dockerfile)
|
|
self.assertIn('np.__version__ == metadata.version("numpy") == "1.26.4"', dockerfile)
|
|
self.assertIn("Path(sys.prefix).resolve() == venv", dockerfile)
|
|
self.assertIn("Path(sys.executable).is_relative_to(venv)", dockerfile)
|
|
self.assertIn("package_path.is_relative_to(venv)", dockerfile)
|
|
self.assertIn("distribution_path.is_relative_to(venv)", dockerfile)
|
|
self.assertIn("import accelerate", dockerfile)
|
|
self.assertNotIn("NUMPY_SITE", dockerfile)
|
|
self.assertNotIn("rm -rf", dockerfile)
|
|
self.assertNotIn("metadata.distributions()", dockerfile)
|
|
self.assertNotIn('"numpy>=2.0,<2.3"', dockerfile)
|
|
|
|
def test_holobrain_train_and_export_keep_the_shared_runtime(self):
|
|
for slug in ("holobrain_train", "holobrain_export"):
|
|
with self.subTest(operator=slug):
|
|
dockerfile = (OPERATOR_REPO_ROOT / slug / "Dockerfile").read_text(encoding="utf-8")
|
|
|
|
self.assertIn('"numpy>=2.0,<2.3"', dockerfile)
|
|
self.assertNotIn("holobrain-robotwin-venv", dockerfile)
|
|
|
|
def test_holobrain_long_running_contracts_cover_data_prepare_and_train(self):
|
|
catalog = {item["id"]: item for item in json.loads(CATALOG_PATH.read_text(encoding="utf-8"))}
|
|
|
|
for operator_id in (
|
|
"operator-builtin-holobrain-data-prepare",
|
|
"operator-builtin-holobrain-train",
|
|
):
|
|
with self.subTest(operator=operator_id):
|
|
contract = catalog[operator_id]["runtimeContract"]
|
|
self.assertIs(contract["longRunning"], True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|