droboticflow/scripts/generate_operator_repository_manifests.py

340 lines
19 KiB
Python

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import pathlib
import sys
from typing import Any
ROOT = pathlib.Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from operator_release_manifest import RELEASE_SCHEMA_VERSION, load_release, release_image_reference, source_tree_sha256
CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin"
OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin"
UNMIGRATED_HISTORICAL_REPOSITORIES = {"temporal_causal_eval"}
RUNTIME_DEPENDENCIES: dict[str, list[str]] = {
"action_state_quality_validate": ["json-jsonl-parquet-files", "pyarrow"],
"act_policy_train": ["huggingface-lerobot-act", "lerobot-dataset-or-local-root", "gpu-a800"],
"agibot_world2026_validate": ["agibotworld2026-meta-data-video-files", "pyarrow", "parquet-files"],
"aloha_hdf5_validate": ["aloha-act-hdf5-files", "h5py", "numpy"],
"annotation_gate": ["video-artifacts", "annotation-revisions"],
"audit_sample": ["json-artifacts"],
"bop_dataset_validate": ["bop-scenewise-dataset-files", "json-files", "rgb-depth-mask-image-files", "ply-model-files"],
"bos_incremental_scan": ["baidu-bos"],
"bridgedata_tfds_manifest_validate": ["bridgedata-tfds-rlds-directory", "tfrecord-files", "crc32c-structural-validation"],
"camera_calibration_validate": ["camera-calibration-files", "json-files", "optional-yaml-files"],
"orbbec_left_undistort": ["orbbec-calibration-files", "pyyaml", "numpy", "opencv-python-headless", "ffmpeg", "video-artifacts"],
"calvin_episode_validate": ["calvin-episode-npz-files", "numpy", "language-annotation-files"],
"coco_annotation_validate": ["coco-json", "image-files"],
"conditional_route": ["json-artifacts"],
"csv_to_json": ["csv-files"],
"dataset_schema_contract_validate": ["csv-json-jsonl-parquet-hdf5-npz-files", "pyarrow", "h5py", "numpy"],
"dataset_split_leakage_validate": ["split-manifest-csv-json-jsonl-parquet-files", "pyarrow"],
"dataset_split_manifest": ["json-artifacts", "csv-files"],
"dataset_provenance_audit": ["dataset-card-files", "license-files", "pyyaml"],
"delivery_options": ["json-artifacts"],
"dexycb_dataset_validate": ["dexycb-dataset-files", "numpy", "npz-label-files", "rgb-depth-image-files"],
"diffusion_policy_train": ["real-stanford-diffusion-policy", "zarr-or-lerobot-trajectory-files", "gpu-a800"],
"directory_scan": ["filesystem"],
"droid_raw_episode_validate": ["droid-raw-episode-hdf5-metadata-video-files", "h5py", "numpy"],
"embodiment_metadata_validate": ["csv-json-jsonl-parquet-metadata-files", "pyarrow"],
"episode_boundary_validate": ["json-jsonl-parquet-files", "pyarrow"],
"force_torque_sensor_validate": ["csv-json-jsonl-parquet-hdf5-force-torque-files", "pyarrow", "h5py", "numpy"],
"furniturebench_pkl_demonstration_validate": ["furniturebench-pkl-files", "pickle", "numpy"],
"graspnet_dataset_validate": ["graspnet-1billion-dataset-files", "numpy", "npz-npy-files", "rgb-depth-mask-png-files", "xml-annotations"],
"hdf5_demonstration_summary": ["hdf5-files", "h5py"],
"hf_dataset_probe": ["huggingface-dataset-viewer-api"],
"hand_object_pose_validate": ["json-jsonl-csv-parquet-files", "image-files", "mask-files", "pyarrow"],
"hand_removal": ["propainter-inference-service", "sam3-mask-artifacts", "image-video-files", "http-api", "gpu-a800-service"],
"head_pose": ["6drepnet-service", "video-artifacts", "http-api"],
"ho3d_dataset_validate": ["ho3d-h2o3d-dataset-files", "pickle", "numpy", "rgb-depth-image-files"],
"holobrain_data_prepare": ["official-robotwin2-dataset", "huggingface-hub", "real-robotwin-files"],
"holobrain_export": ["roboorchardlab-holobrain-export", "holobrain-checkpoint", "gpu-a800"],
"holobrain_inference_test": ["roboorchardlab-holobrain-pipeline", "real-sample-path", "gpu-a800"],
"holobrain_robotwin_eval": ["roboorchardlab-robotwin-eval", "robotwin-benchmark", "gpu-a800"],
"holobrain_train": ["roboorchardlab-holobrain-train", "official-robotwin2-dataset", "qwen-vl-checkpoint", "gpu-a800"],
"human_review": ["json-artifacts"],
"json_field_extract": ["json-artifacts"],
"json_filter": ["json-artifacts"],
"json_map_rename": ["json-artifacts"],
"json_to_csv": ["json-artifacts"],
"joint_limit_validate": ["urdf-files", "json-jsonl-csv-parquet-files", "pyarrow"],
"language_instruction_validate": ["json-jsonl-csv-parquet-files", "pyarrow"],
"lerobot_dataset_validate": ["lerobot-dataset-files", "pyarrow", "parquet-files"],
"lerobot_metadata_consistency_validate": ["lerobot-v3-meta-files", "pyarrow", "parquet-files"],
"lerobot_v3_episode_index_validate": ["lerobot-v3-meta-data-files", "pyarrow", "parquet-files"],
"lerobot_video_alignment_validate": ["lerobot-v3-data-files", "pyarrow", "parquet-files", "video-files", "opencv-python-headless"],
"lerobot_summary": ["lerobot-dataset-files"],
"libero_hdf5_task_validate": ["libero-hdf5-demo-files", "libero-bddl-files", "h5py", "numpy"],
"maniskill_trajectory_validate": ["maniskill-hdf5-json-files", "h5py", "numpy"],
"media_reference_validate": ["csv-json-jsonl-parquet-manifests", "image-video-depth-pointcloud-files", "opencv-python-headless", "numpy", "pyarrow", "ffprobe"],
"mcap_topic_summary": ["mcap-files", "mcap-python"],
"merge": ["json-artifacts", "bounded-upstream-manifests"],
"mjcf_actuator_validate": ["mjcf-files", "xml-parser"],
"model_inference": ["json-model-or-command"],
"mongodb_write": ["mongodb"],
"ms_swift_dataset_export": ["accepted-annotation-revisions", "real-video-files", "ffmpeg", "jsonl"],
"package": ["ffmpeg", "video-files", "accepted-annotation-partitions", "headpose-manifests", "optional-handpose-manifests"],
"openvla_finetune": ["openvla-lora-finetune", "rlds-vla-dataset", "peft", "gpu-a800"],
"openvla_inference": ["openvla-transformers-inference", "openvla-model-or-upstream-adapter", "gpu-a800"],
"orbbec_data_normalize": ["json-artifacts", "orbbec-ego-layout"],
"partnet_mobility_asset_validate": ["partnet-mobility-sapien-assets", "urdf-files", "json-files", "semantics-txt", "mesh-obj-files"],
"parquet_table_summary": ["parquet-files", "pyarrow"],
"pointcloud_summary": ["pcd-files", "ply-files"],
"real_source": ["filesystem"],
"rgbd_frame_pair_validate": ["rgb-images", "depth-images", "opencv-python-headless", "numpy", "pyyaml"],
"rlds_episode_summary": ["rlds-json-or-tfds-files"],
"rh20t_scene_validate": ["rh20t-scene-directory", "numpy", "npy-files", "image-video-files"],
"robomind_hdf5_quality_validate": ["robomind-trajectory-hdf5-files", "h5py", "numpy"],
"robomimic_hdf5_validate": ["robomimic-hdf5-files", "h5py", "numpy"],
"robocasa_hdf5_demonstration_validate": ["robocasa-hdf5-demo-files", "h5py", "numpy"],
"roboturk_real_hdf5_validate": ["roboturk-real-aligned-hdf5-files", "h5py", "numpy"],
"roboset_hdf5_validate": ["roboset-robohive-hdf5-trajectory-files", "h5py", "numpy"],
"robot_model_asset_validate": ["urdf-files", "mjcf-files", "mesh-assets"],
"rlbench_dataset_validate": ["rlbench-episode-files", "pickle-files", "image-files"],
"ros_joint_state_validate": ["ros-joint-state-files", "json-jsonl-csv-parquet-files", "pyarrow"],
"rosbag_summary": ["rosbag2-files", "sqlite"],
"rtx_action_schema_validate": ["csv-json-jsonl-parquet-hdf5-action-files", "pyarrow", "h5py", "numpy"],
"sensor_timestamp_sync_validate": ["timestamp-csv-jsonl"],
"segmentation_mask_validate": ["segmentation-mask-images", "opencv-python-headless", "numpy"],
"semantic_prelabel": ["json-artifacts", "annotation-schema"],
"shell_command": ["container-shell"],
"sim3d_asset_generate": ["json-artifacts", "obj-mtl-gltf-files"],
"template_render": ["json-artifacts"],
"tfrecord_structural_validate": ["tfrecord-files", "crc32c-structural-validation"],
"tf_transform_tree_validate": ["json-jsonl-csv-yaml-files", "optional-pyyaml"],
"trajectory_integrity_validate": ["trajectory-json", "local-media-files"],
"umi_zarr_dataset_validate": ["umi-zarr-dataset", "zarr", "numpy"],
"upload": ["filesystem", "optional-baidu-bos"],
"video_annotate": ["video-files", "json-artifacts"],
"video_face_blur": ["ffmpeg", "opencv-python-headless", "opencv-yunet-onnx", "verified-local-model-manifest", "video-files"],
"video_filter": ["json-artifacts"],
"video_quality": ["ffprobe", "video-files"],
"video_slice": ["ffmpeg", "video-files"],
"video_target_clip": ["opencv-python-headless", "numpy", "ffmpeg", "video-files", "optional-opencv-dnn-model"],
"vla_model_evaluate": ["vla-inference-result-json", "vla-sample-action-manifest"],
"vla_model_inference": ["vla-policy-model-json", "vla-sample-action-manifest"],
"vla_model_train": ["vla-sample-action-manifest", "json-jsonl-csv-files"],
"vla_sample_manifest_validate": ["json-jsonl-csv-parquet-files", "image-files", "pyarrow"],
"wilor_hand_pose": ["wilor-inference-service", "image-video-files", "http-api", "gpu-l20-service"],
"wilor_valid_event_detection": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files"],
"webdataset_shard_summary": ["webdataset-tar-shards", "python-tarfile"],
"xplatform_dataset_index": ["xplatform-dataset-files"],
"xplatform_dataset_validate": ["xplatform-dataset-index"],
"qwen_temporal_localization": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files"],
"qwen_behavior_rough_cut": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "bce-python-sdk", "real-video-files", "orbbec-pts-csv"],
"qwen_label_crop_classification": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "ms-swift", "peft", "torch-cuda", "accepted-annotation-dataset"],
"yolo_detection_validate": ["yolo-label-files", "image-files", "pyyaml"],
"yolo_embodied_detection_train": ["real-yolo-training-command", "yolo-or-coco-dataset", "gpu-a800"],
"yolo_video_target_clip": ["yolo-onnx-model", "opencv-python-headless", "numpy", "ffmpeg", "video-files", "gpu-a800"],
"zarr_trajectory_summary": ["zarr-v2-v3-metadata", "filesystem"],
}
STRICT_EXTERNALS: dict[str, list[str]] = {
"bos_incremental_scan": ["bos"],
"mongodb_write": ["mongo"],
"upload": ["bos"],
}
def load_json(path: pathlib.Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def dump_json(payload: Any) -> str:
return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
def operator_slug(operator_id: str) -> str:
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
def legacy_manifest(
*,
slug: str,
metadata: dict[str, Any],
catalog_item: dict[str, Any],
image: str,
image_id: str,
dependencies: list[str],
) -> dict[str, Any]:
acceptance = dict(metadata["acceptance"])
manifest: dict[str, Any] = {
"schemaVersion": "droboticflow.operator.repository/v1",
"repositoryId": f"builtin/{slug}",
"operatorId": metadata["id"],
"operatorSlug": slug,
"preferredSlug": operator_slug(metadata["id"]),
"name": metadata["name"],
"category": metadata["category"],
"type": metadata["type"],
"sourceRoot": f"backend/operators/builtin/{slug}",
"entrypoint": "operator.py",
"metadata": "operator.json",
"runtimeSource": metadata["runtimeSource"],
"runtimeHandler": metadata["runtimeHandler"],
"containerEntrypoint": "backend/operator-runner/main.py",
"image": image,
"imageId": image_id,
"containerCommand": catalog_item.get("containerCommand", []),
"containerArgs": catalog_item.get("containerArgs", []),
"runtimeContract": catalog_item.get("runtimeContract", {}),
"pythonImport": f"backend.operators.builtin.{slug}.operator:run",
"runtimeDependencies": dependencies,
"strictPocAcceptanceExternals": STRICT_EXTERNALS.get(slug, []),
"acceptance": {
**acceptance,
"pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh",
"requiresPersistentWorkspaceEvidence": True,
},
}
if "externalApi" in metadata:
manifest["externalApi"] = metadata["externalApi"]
return manifest
def expected_manifests(slugs: set[str] | None = None) -> dict[pathlib.Path, dict[str, Any]]:
catalog_items = {
item["id"]: item for item in load_json(CATALOG_PATH) if item.get("type") == "builtin"
}
manifests: dict[pathlib.Path, dict[str, Any]] = {}
split_operator_ids: set[str] = set()
metadata_paths = sorted(OPERATOR_REPO_ROOT.glob("*/operator.json"))
if slugs is not None:
available = {path.parent.name for path in metadata_paths} - UNMIGRATED_HISTORICAL_REPOSITORIES
if not slugs or slugs - available:
raise RuntimeError("unknown or empty operator repository selection")
metadata_paths = [path for path in metadata_paths if path.parent.name in slugs]
for metadata_path in metadata_paths:
if metadata_path.parent.name in UNMIGRATED_HISTORICAL_REPOSITORIES:
continue
metadata = load_json(metadata_path)
operator_id = metadata["id"]
if operator_id not in catalog_items:
raise RuntimeError(f"{metadata_path} operator id is missing from catalog: {operator_id}")
catalog_item = catalog_items[operator_id]
slug = metadata_path.parent.name
runtime_source = f"operator-repos/builtin/{slug}/src/operator.py"
if metadata.get("runtimeSource") != runtime_source:
raise RuntimeError(f"{metadata_path} runtimeSource must be {runtime_source}")
dependencies = RUNTIME_DEPENDENCIES.get(slug)
if dependencies is None:
raise RuntimeError(f"{slug} is missing runtime dependency metadata")
acceptance = dict(metadata["acceptance"])
is_draft = str(catalog_item.get("publicationStatus", "")).strip().lower() == "draft"
try:
release = load_release(slug)
image, image_id = release_image_reference(slug)
except RuntimeError:
if not is_draft:
raise
release = {}
image = str(catalog_item.get("image", "")).strip()
image_id = str(catalog_item.get("imageId", "")).strip()
manifest: dict[str, Any] = {
"schemaVersion": "droboticflow.operator.repository/v1",
"repositoryId": f"builtin/{slug}",
"operatorId": operator_id,
"operatorSlug": slug,
"preferredSlug": operator_slug(operator_id),
"name": metadata["name"],
"category": metadata["category"],
"type": metadata["type"],
"sourceRoot": f"operator-repos/builtin/{slug}",
"entrypoint": "src/operator.py",
"metadata": "operator.json",
"runtimeSource": runtime_source,
"runtimeHandler": metadata["runtimeHandler"],
"containerEntrypoint": "backend/operator-runner/main.py",
"releaseManifest": "release.json",
"releaseSchemaVersion": release.get("schemaVersion", RELEASE_SCHEMA_VERSION),
"sourceTreeSha256": source_tree_sha256(slug),
"image": image,
"imageId": image_id,
"containerCommand": catalog_item.get("containerCommand", []),
"containerArgs": catalog_item.get("containerArgs", []),
"runtimeContract": catalog_item.get("runtimeContract", {}),
"pythonImport": f"operator-repos/builtin/{slug}/src/operator.py:run",
"runtimeDependencies": dependencies,
"strictPocAcceptanceExternals": STRICT_EXTERNALS.get(slug, []),
"acceptance": {
**acceptance,
"pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh",
"requiresPersistentWorkspaceEvidence": True,
"requiresChineseWorkflowAcceptance": True,
"requiresChainedWorkflow": True,
"fixtureContract": "tests/acceptance/fixture_contract.json",
},
}
if "externalApi" in metadata:
manifest["externalApi"] = metadata["externalApi"]
manifests[metadata_path.with_name("repository.json")] = manifest
split_operator_ids.add(operator_id)
legacy_metadata_path = OPERATOR_ROOT / slug / "operator.json"
if not legacy_metadata_path.exists():
raise RuntimeError(f"{slug} is missing legacy runtime metadata")
legacy_metadata = load_json(legacy_metadata_path)
if legacy_metadata.get("id") != operator_id:
raise RuntimeError(
f"{legacy_metadata_path} operator id {legacy_metadata.get('id')!r} does not match {operator_id!r}"
)
manifests[legacy_metadata_path.with_name("repository.json")] = legacy_manifest(
slug=slug,
metadata=legacy_metadata,
catalog_item=catalog_item,
image=image,
image_id=image_id,
dependencies=dependencies,
)
missing_catalog = sorted(set(catalog_items) - split_operator_ids) if slugs is None else []
if missing_catalog:
raise RuntimeError(f"catalog operators missing repository manifests: {', '.join(missing_catalog)}")
return manifests
def write_manifests(manifests: dict[pathlib.Path, dict[str, Any]]) -> None:
for path, manifest in manifests.items():
path.write_text(dump_json(manifest), encoding="utf-8")
def check_manifests(manifests: dict[pathlib.Path, dict[str, Any]]) -> int:
mismatches: list[str] = []
for path, manifest in manifests.items():
expected = dump_json(manifest)
if not path.exists():
mismatches.append(f"missing {path.relative_to(ROOT)}")
continue
actual = path.read_text(encoding="utf-8")
if actual != expected:
mismatches.append(f"outdated {path.relative_to(ROOT)}")
if mismatches:
for mismatch in mismatches:
print(mismatch, file=sys.stderr)
return 1
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="Generate or check builtin operator repository manifests.")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--write", action="store_true", help="write repository.json files")
mode.add_argument("--check", action="store_true", help="check repository.json files without writing")
parser.add_argument("--slug", action="append", help="publish/check only these independent repositories; default checks every operator")
args = parser.parse_args()
manifests = expected_manifests(set(args.slug) if args.slug is not None else None)
if args.check:
return check_manifests(manifests)
write_manifests(manifests)
return 0
if __name__ == "__main__":
raise SystemExit(main())