#!/usr/bin/env python3 from __future__ import annotations import argparse import json import pathlib import sys from typing import Any ROOT = pathlib.Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from operator_release_manifest import RELEASE_SCHEMA_VERSION, load_release, release_image_reference, source_tree_sha256 CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json" OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin" OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin" UNMIGRATED_HISTORICAL_REPOSITORIES = {"temporal_causal_eval"} RUNTIME_DEPENDENCIES: dict[str, list[str]] = { "action_state_quality_validate": ["json-jsonl-parquet-files", "pyarrow"], "act_policy_train": ["huggingface-lerobot-act", "lerobot-dataset-or-local-root", "gpu-a800"], "agibot_world2026_validate": ["agibotworld2026-meta-data-video-files", "pyarrow", "parquet-files"], "aloha_hdf5_validate": ["aloha-act-hdf5-files", "h5py", "numpy"], "annotation_gate": ["video-artifacts", "annotation-revisions"], "audit_sample": ["json-artifacts"], "bop_dataset_validate": ["bop-scenewise-dataset-files", "json-files", "rgb-depth-mask-image-files", "ply-model-files"], "bos_incremental_scan": ["baidu-bos"], "bridgedata_tfds_manifest_validate": ["bridgedata-tfds-rlds-directory", "tfrecord-files", "crc32c-structural-validation"], "camera_calibration_validate": ["camera-calibration-files", "json-files", "optional-yaml-files"], "orbbec_left_undistort": ["orbbec-calibration-files", "pyyaml", "numpy", "opencv-python-headless", "ffmpeg", "video-artifacts"], "calvin_episode_validate": ["calvin-episode-npz-files", "numpy", "language-annotation-files"], "coco_annotation_validate": ["coco-json", "image-files"], "conditional_route": ["json-artifacts"], "csv_to_json": ["csv-files"], "dataset_schema_contract_validate": ["csv-json-jsonl-parquet-hdf5-npz-files", "pyarrow", "h5py", "numpy"], "dataset_split_leakage_validate": ["split-manifest-csv-json-jsonl-parquet-files", "pyarrow"], "dataset_split_manifest": ["json-artifacts", "csv-files"], "dataset_provenance_audit": ["dataset-card-files", "license-files", "pyyaml"], "delivery_options": ["json-artifacts"], "dexycb_dataset_validate": ["dexycb-dataset-files", "numpy", "npz-label-files", "rgb-depth-image-files"], "diffusion_policy_train": ["real-stanford-diffusion-policy", "zarr-or-lerobot-trajectory-files", "gpu-a800"], "directory_scan": ["filesystem"], "droid_raw_episode_validate": ["droid-raw-episode-hdf5-metadata-video-files", "h5py", "numpy"], "embodiment_metadata_validate": ["csv-json-jsonl-parquet-metadata-files", "pyarrow"], "episode_boundary_validate": ["json-jsonl-parquet-files", "pyarrow"], "force_torque_sensor_validate": ["csv-json-jsonl-parquet-hdf5-force-torque-files", "pyarrow", "h5py", "numpy"], "furniturebench_pkl_demonstration_validate": ["furniturebench-pkl-files", "pickle", "numpy"], "graspnet_dataset_validate": ["graspnet-1billion-dataset-files", "numpy", "npz-npy-files", "rgb-depth-mask-png-files", "xml-annotations"], "hdf5_demonstration_summary": ["hdf5-files", "h5py"], "hf_dataset_probe": ["huggingface-dataset-viewer-api"], "hand_object_pose_validate": ["json-jsonl-csv-parquet-files", "image-files", "mask-files", "pyarrow"], "hand_removal": ["propainter-inference-service", "sam3-mask-artifacts", "image-video-files", "http-api", "gpu-a800-service"], "head_pose": ["6drepnet-service", "video-artifacts", "http-api"], "ho3d_dataset_validate": ["ho3d-h2o3d-dataset-files", "pickle", "numpy", "rgb-depth-image-files"], "holobrain_data_prepare": ["official-robotwin2-dataset", "huggingface-hub", "real-robotwin-files"], "holobrain_export": ["roboorchardlab-holobrain-export", "holobrain-checkpoint", "gpu-a800"], "holobrain_inference_test": ["roboorchardlab-holobrain-pipeline", "real-sample-path", "gpu-a800"], "holobrain_robotwin_eval": ["roboorchardlab-robotwin-eval", "robotwin-benchmark", "gpu-a800"], "holobrain_train": ["roboorchardlab-holobrain-train", "official-robotwin2-dataset", "qwen-vl-checkpoint", "gpu-a800"], "human_review": ["json-artifacts"], "json_field_extract": ["json-artifacts"], "json_filter": ["json-artifacts"], "json_map_rename": ["json-artifacts"], "json_to_csv": ["json-artifacts"], "joint_limit_validate": ["urdf-files", "json-jsonl-csv-parquet-files", "pyarrow"], "language_instruction_validate": ["json-jsonl-csv-parquet-files", "pyarrow"], "lerobot_dataset_validate": ["lerobot-dataset-files", "pyarrow", "parquet-files"], "lerobot_metadata_consistency_validate": ["lerobot-v3-meta-files", "pyarrow", "parquet-files"], "lerobot_v3_episode_index_validate": ["lerobot-v3-meta-data-files", "pyarrow", "parquet-files"], "lerobot_video_alignment_validate": ["lerobot-v3-data-files", "pyarrow", "parquet-files", "video-files", "opencv-python-headless"], "lerobot_summary": ["lerobot-dataset-files"], "libero_hdf5_task_validate": ["libero-hdf5-demo-files", "libero-bddl-files", "h5py", "numpy"], "maniskill_trajectory_validate": ["maniskill-hdf5-json-files", "h5py", "numpy"], "media_reference_validate": ["csv-json-jsonl-parquet-manifests", "image-video-depth-pointcloud-files", "opencv-python-headless", "numpy", "pyarrow", "ffprobe"], "mcap_topic_summary": ["mcap-files", "mcap-python"], "merge": ["json-artifacts", "bounded-upstream-manifests"], "mjcf_actuator_validate": ["mjcf-files", "xml-parser"], "model_inference": ["json-model-or-command"], "mongodb_write": ["mongodb"], "ms_swift_dataset_export": ["accepted-annotation-revisions", "real-video-files", "ffmpeg", "jsonl"], "package": ["ffmpeg", "video-files", "accepted-annotation-partitions", "headpose-manifests", "optional-handpose-manifests"], "openvla_finetune": ["openvla-lora-finetune", "rlds-vla-dataset", "peft", "gpu-a800"], "openvla_inference": ["openvla-transformers-inference", "openvla-model-or-upstream-adapter", "gpu-a800"], "orbbec_data_normalize": ["json-artifacts", "orbbec-ego-layout"], "partnet_mobility_asset_validate": ["partnet-mobility-sapien-assets", "urdf-files", "json-files", "semantics-txt", "mesh-obj-files"], "parquet_table_summary": ["parquet-files", "pyarrow"], "pointcloud_summary": ["pcd-files", "ply-files"], "real_source": ["filesystem"], "rgbd_frame_pair_validate": ["rgb-images", "depth-images", "opencv-python-headless", "numpy", "pyyaml"], "rlds_episode_summary": ["rlds-json-or-tfds-files"], "rh20t_scene_validate": ["rh20t-scene-directory", "numpy", "npy-files", "image-video-files"], "robomind_hdf5_quality_validate": ["robomind-trajectory-hdf5-files", "h5py", "numpy"], "robomimic_hdf5_validate": ["robomimic-hdf5-files", "h5py", "numpy"], "robocasa_hdf5_demonstration_validate": ["robocasa-hdf5-demo-files", "h5py", "numpy"], "roboturk_real_hdf5_validate": ["roboturk-real-aligned-hdf5-files", "h5py", "numpy"], "roboset_hdf5_validate": ["roboset-robohive-hdf5-trajectory-files", "h5py", "numpy"], "robot_model_asset_validate": ["urdf-files", "mjcf-files", "mesh-assets"], "rlbench_dataset_validate": ["rlbench-episode-files", "pickle-files", "image-files"], "ros_joint_state_validate": ["ros-joint-state-files", "json-jsonl-csv-parquet-files", "pyarrow"], "rosbag_summary": ["rosbag2-files", "sqlite"], "rtx_action_schema_validate": ["csv-json-jsonl-parquet-hdf5-action-files", "pyarrow", "h5py", "numpy"], "sensor_timestamp_sync_validate": ["timestamp-csv-jsonl"], "segmentation_mask_validate": ["segmentation-mask-images", "opencv-python-headless", "numpy"], "semantic_prelabel": ["json-artifacts", "annotation-schema"], "shell_command": ["container-shell"], "sim3d_asset_generate": ["json-artifacts", "obj-mtl-gltf-files"], "template_render": ["json-artifacts"], "tfrecord_structural_validate": ["tfrecord-files", "crc32c-structural-validation"], "tf_transform_tree_validate": ["json-jsonl-csv-yaml-files", "optional-pyyaml"], "trajectory_integrity_validate": ["trajectory-json", "local-media-files"], "umi_zarr_dataset_validate": ["umi-zarr-dataset", "zarr", "numpy"], "upload": ["filesystem", "optional-baidu-bos"], "video_annotate": ["video-files", "json-artifacts"], "video_face_blur": ["ffmpeg", "opencv-python-headless", "opencv-yunet-onnx", "verified-local-model-manifest", "video-files"], "video_filter": ["json-artifacts"], "video_quality": ["ffprobe", "video-files"], "video_slice": ["ffmpeg", "video-files"], "video_target_clip": ["opencv-python-headless", "numpy", "ffmpeg", "video-files", "optional-opencv-dnn-model"], "vla_model_evaluate": ["vla-inference-result-json", "vla-sample-action-manifest"], "vla_model_inference": ["vla-policy-model-json", "vla-sample-action-manifest"], "vla_model_train": ["vla-sample-action-manifest", "json-jsonl-csv-files"], "vla_sample_manifest_validate": ["json-jsonl-csv-parquet-files", "image-files", "pyarrow"], "wilor_hand_pose": ["wilor-inference-service", "image-video-files", "http-api", "gpu-l20-service"], "wilor_valid_event_detection": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files"], "webdataset_shard_summary": ["webdataset-tar-shards", "python-tarfile"], "xplatform_dataset_index": ["xplatform-dataset-files"], "xplatform_dataset_validate": ["xplatform-dataset-index"], "qwen_temporal_localization": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files"], "qwen_behavior_rough_cut": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "bce-python-sdk", "real-video-files", "orbbec-pts-csv"], "qwen_label_crop_classification": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "ms-swift", "peft", "torch-cuda", "accepted-annotation-dataset"], "yolo_detection_validate": ["yolo-label-files", "image-files", "pyyaml"], "yolo_embodied_detection_train": ["real-yolo-training-command", "yolo-or-coco-dataset", "gpu-a800"], "yolo_video_target_clip": ["yolo-onnx-model", "opencv-python-headless", "numpy", "ffmpeg", "video-files", "gpu-a800"], "zarr_trajectory_summary": ["zarr-v2-v3-metadata", "filesystem"], } STRICT_EXTERNALS: dict[str, list[str]] = { "bos_incremental_scan": ["bos"], "mongodb_write": ["mongo"], "upload": ["bos"], } def load_json(path: pathlib.Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) def dump_json(payload: Any) -> str: return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" def operator_slug(operator_id: str) -> str: return operator_id.removeprefix("operator-builtin-").replace("-", "_") def legacy_manifest( *, slug: str, metadata: dict[str, Any], catalog_item: dict[str, Any], image: str, image_id: str, dependencies: list[str], ) -> dict[str, Any]: acceptance = dict(metadata["acceptance"]) manifest: dict[str, Any] = { "schemaVersion": "droboticflow.operator.repository/v1", "repositoryId": f"builtin/{slug}", "operatorId": metadata["id"], "operatorSlug": slug, "preferredSlug": operator_slug(metadata["id"]), "name": metadata["name"], "category": metadata["category"], "type": metadata["type"], "sourceRoot": f"backend/operators/builtin/{slug}", "entrypoint": "operator.py", "metadata": "operator.json", "runtimeSource": metadata["runtimeSource"], "runtimeHandler": metadata["runtimeHandler"], "containerEntrypoint": "backend/operator-runner/main.py", "image": image, "imageId": image_id, "containerCommand": catalog_item.get("containerCommand", []), "containerArgs": catalog_item.get("containerArgs", []), "runtimeContract": catalog_item.get("runtimeContract", {}), "pythonImport": f"backend.operators.builtin.{slug}.operator:run", "runtimeDependencies": dependencies, "strictPocAcceptanceExternals": STRICT_EXTERNALS.get(slug, []), "acceptance": { **acceptance, "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", "requiresPersistentWorkspaceEvidence": True, }, } if "externalApi" in metadata: manifest["externalApi"] = metadata["externalApi"] return manifest def expected_manifests(slugs: set[str] | None = None) -> dict[pathlib.Path, dict[str, Any]]: catalog_items = { item["id"]: item for item in load_json(CATALOG_PATH) if item.get("type") == "builtin" } manifests: dict[pathlib.Path, dict[str, Any]] = {} split_operator_ids: set[str] = set() metadata_paths = sorted(OPERATOR_REPO_ROOT.glob("*/operator.json")) if slugs is not None: available = {path.parent.name for path in metadata_paths} - UNMIGRATED_HISTORICAL_REPOSITORIES if not slugs or slugs - available: raise RuntimeError("unknown or empty operator repository selection") metadata_paths = [path for path in metadata_paths if path.parent.name in slugs] for metadata_path in metadata_paths: if metadata_path.parent.name in UNMIGRATED_HISTORICAL_REPOSITORIES: continue metadata = load_json(metadata_path) operator_id = metadata["id"] if operator_id not in catalog_items: raise RuntimeError(f"{metadata_path} operator id is missing from catalog: {operator_id}") catalog_item = catalog_items[operator_id] slug = metadata_path.parent.name runtime_source = f"operator-repos/builtin/{slug}/src/operator.py" if metadata.get("runtimeSource") != runtime_source: raise RuntimeError(f"{metadata_path} runtimeSource must be {runtime_source}") dependencies = RUNTIME_DEPENDENCIES.get(slug) if dependencies is None: raise RuntimeError(f"{slug} is missing runtime dependency metadata") acceptance = dict(metadata["acceptance"]) is_draft = str(catalog_item.get("publicationStatus", "")).strip().lower() == "draft" try: release = load_release(slug) image, image_id = release_image_reference(slug) except RuntimeError: if not is_draft: raise release = {} image = str(catalog_item.get("image", "")).strip() image_id = str(catalog_item.get("imageId", "")).strip() manifest: dict[str, Any] = { "schemaVersion": "droboticflow.operator.repository/v1", "repositoryId": f"builtin/{slug}", "operatorId": operator_id, "operatorSlug": slug, "preferredSlug": operator_slug(operator_id), "name": metadata["name"], "category": metadata["category"], "type": metadata["type"], "sourceRoot": f"operator-repos/builtin/{slug}", "entrypoint": "src/operator.py", "metadata": "operator.json", "runtimeSource": runtime_source, "runtimeHandler": metadata["runtimeHandler"], "containerEntrypoint": "backend/operator-runner/main.py", "releaseManifest": "release.json", "releaseSchemaVersion": release.get("schemaVersion", RELEASE_SCHEMA_VERSION), "sourceTreeSha256": source_tree_sha256(slug), "image": image, "imageId": image_id, "containerCommand": catalog_item.get("containerCommand", []), "containerArgs": catalog_item.get("containerArgs", []), "runtimeContract": catalog_item.get("runtimeContract", {}), "pythonImport": f"operator-repos/builtin/{slug}/src/operator.py:run", "runtimeDependencies": dependencies, "strictPocAcceptanceExternals": STRICT_EXTERNALS.get(slug, []), "acceptance": { **acceptance, "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", "requiresPersistentWorkspaceEvidence": True, "requiresChineseWorkflowAcceptance": True, "requiresChainedWorkflow": True, "fixtureContract": "tests/acceptance/fixture_contract.json", }, } if "externalApi" in metadata: manifest["externalApi"] = metadata["externalApi"] manifests[metadata_path.with_name("repository.json")] = manifest split_operator_ids.add(operator_id) legacy_metadata_path = OPERATOR_ROOT / slug / "operator.json" if not legacy_metadata_path.exists(): raise RuntimeError(f"{slug} is missing legacy runtime metadata") legacy_metadata = load_json(legacy_metadata_path) if legacy_metadata.get("id") != operator_id: raise RuntimeError( f"{legacy_metadata_path} operator id {legacy_metadata.get('id')!r} does not match {operator_id!r}" ) manifests[legacy_metadata_path.with_name("repository.json")] = legacy_manifest( slug=slug, metadata=legacy_metadata, catalog_item=catalog_item, image=image, image_id=image_id, dependencies=dependencies, ) missing_catalog = sorted(set(catalog_items) - split_operator_ids) if slugs is None else [] if missing_catalog: raise RuntimeError(f"catalog operators missing repository manifests: {', '.join(missing_catalog)}") return manifests def write_manifests(manifests: dict[pathlib.Path, dict[str, Any]]) -> None: for path, manifest in manifests.items(): path.write_text(dump_json(manifest), encoding="utf-8") def check_manifests(manifests: dict[pathlib.Path, dict[str, Any]]) -> int: mismatches: list[str] = [] for path, manifest in manifests.items(): expected = dump_json(manifest) if not path.exists(): mismatches.append(f"missing {path.relative_to(ROOT)}") continue actual = path.read_text(encoding="utf-8") if actual != expected: mismatches.append(f"outdated {path.relative_to(ROOT)}") if mismatches: for mismatch in mismatches: print(mismatch, file=sys.stderr) return 1 return 0 def main() -> int: parser = argparse.ArgumentParser(description="Generate or check builtin operator repository manifests.") mode = parser.add_mutually_exclusive_group() mode.add_argument("--write", action="store_true", help="write repository.json files") mode.add_argument("--check", action="store_true", help="check repository.json files without writing") parser.add_argument("--slug", action="append", help="publish/check only these independent repositories; default checks every operator") args = parser.parse_args() manifests = expected_manifests(set(args.slug) if args.slug is not None else None) if args.check: return check_manifests(manifests) write_manifests(manifests) return 0 if __name__ == "__main__": raise SystemExit(main())