forked from longtao.wu/droboticflow
340 lines
19 KiB
Python
340 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
from typing import Any
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from operator_release_manifest import RELEASE_SCHEMA_VERSION, load_release, release_image_reference, source_tree_sha256
|
|
|
|
CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
|
|
OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin"
|
|
OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin"
|
|
UNMIGRATED_HISTORICAL_REPOSITORIES = {"temporal_causal_eval"}
|
|
|
|
RUNTIME_DEPENDENCIES: dict[str, list[str]] = {
|
|
"action_state_quality_validate": ["json-jsonl-parquet-files", "pyarrow"],
|
|
"act_policy_train": ["huggingface-lerobot-act", "lerobot-dataset-or-local-root", "gpu-a800"],
|
|
"agibot_world2026_validate": ["agibotworld2026-meta-data-video-files", "pyarrow", "parquet-files"],
|
|
"aloha_hdf5_validate": ["aloha-act-hdf5-files", "h5py", "numpy"],
|
|
"annotation_gate": ["video-artifacts", "annotation-revisions"],
|
|
"audit_sample": ["json-artifacts"],
|
|
"bop_dataset_validate": ["bop-scenewise-dataset-files", "json-files", "rgb-depth-mask-image-files", "ply-model-files"],
|
|
"bos_incremental_scan": ["baidu-bos"],
|
|
"bridgedata_tfds_manifest_validate": ["bridgedata-tfds-rlds-directory", "tfrecord-files", "crc32c-structural-validation"],
|
|
"camera_calibration_validate": ["camera-calibration-files", "json-files", "optional-yaml-files"],
|
|
"orbbec_left_undistort": ["orbbec-calibration-files", "pyyaml", "numpy", "opencv-python-headless", "ffmpeg", "video-artifacts"],
|
|
"calvin_episode_validate": ["calvin-episode-npz-files", "numpy", "language-annotation-files"],
|
|
"coco_annotation_validate": ["coco-json", "image-files"],
|
|
"conditional_route": ["json-artifacts"],
|
|
"csv_to_json": ["csv-files"],
|
|
"dataset_schema_contract_validate": ["csv-json-jsonl-parquet-hdf5-npz-files", "pyarrow", "h5py", "numpy"],
|
|
"dataset_split_leakage_validate": ["split-manifest-csv-json-jsonl-parquet-files", "pyarrow"],
|
|
"dataset_split_manifest": ["json-artifacts", "csv-files"],
|
|
"dataset_provenance_audit": ["dataset-card-files", "license-files", "pyyaml"],
|
|
"delivery_options": ["json-artifacts"],
|
|
"dexycb_dataset_validate": ["dexycb-dataset-files", "numpy", "npz-label-files", "rgb-depth-image-files"],
|
|
"diffusion_policy_train": ["real-stanford-diffusion-policy", "zarr-or-lerobot-trajectory-files", "gpu-a800"],
|
|
"directory_scan": ["filesystem"],
|
|
"droid_raw_episode_validate": ["droid-raw-episode-hdf5-metadata-video-files", "h5py", "numpy"],
|
|
"embodiment_metadata_validate": ["csv-json-jsonl-parquet-metadata-files", "pyarrow"],
|
|
"episode_boundary_validate": ["json-jsonl-parquet-files", "pyarrow"],
|
|
"force_torque_sensor_validate": ["csv-json-jsonl-parquet-hdf5-force-torque-files", "pyarrow", "h5py", "numpy"],
|
|
"furniturebench_pkl_demonstration_validate": ["furniturebench-pkl-files", "pickle", "numpy"],
|
|
"graspnet_dataset_validate": ["graspnet-1billion-dataset-files", "numpy", "npz-npy-files", "rgb-depth-mask-png-files", "xml-annotations"],
|
|
"hdf5_demonstration_summary": ["hdf5-files", "h5py"],
|
|
"hf_dataset_probe": ["huggingface-dataset-viewer-api"],
|
|
"hand_object_pose_validate": ["json-jsonl-csv-parquet-files", "image-files", "mask-files", "pyarrow"],
|
|
"hand_removal": ["propainter-inference-service", "sam3-mask-artifacts", "image-video-files", "http-api", "gpu-a800-service"],
|
|
"head_pose": ["6drepnet-service", "video-artifacts", "http-api"],
|
|
"ho3d_dataset_validate": ["ho3d-h2o3d-dataset-files", "pickle", "numpy", "rgb-depth-image-files"],
|
|
"holobrain_data_prepare": ["official-robotwin2-dataset", "huggingface-hub", "real-robotwin-files"],
|
|
"holobrain_export": ["roboorchardlab-holobrain-export", "holobrain-checkpoint", "gpu-a800"],
|
|
"holobrain_inference_test": ["roboorchardlab-holobrain-pipeline", "real-sample-path", "gpu-a800"],
|
|
"holobrain_robotwin_eval": ["roboorchardlab-robotwin-eval", "robotwin-benchmark", "gpu-a800"],
|
|
"holobrain_train": ["roboorchardlab-holobrain-train", "official-robotwin2-dataset", "qwen-vl-checkpoint", "gpu-a800"],
|
|
"human_review": ["json-artifacts"],
|
|
"json_field_extract": ["json-artifacts"],
|
|
"json_filter": ["json-artifacts"],
|
|
"json_map_rename": ["json-artifacts"],
|
|
"json_to_csv": ["json-artifacts"],
|
|
"joint_limit_validate": ["urdf-files", "json-jsonl-csv-parquet-files", "pyarrow"],
|
|
"language_instruction_validate": ["json-jsonl-csv-parquet-files", "pyarrow"],
|
|
"lerobot_dataset_validate": ["lerobot-dataset-files", "pyarrow", "parquet-files"],
|
|
"lerobot_metadata_consistency_validate": ["lerobot-v3-meta-files", "pyarrow", "parquet-files"],
|
|
"lerobot_v3_episode_index_validate": ["lerobot-v3-meta-data-files", "pyarrow", "parquet-files"],
|
|
"lerobot_video_alignment_validate": ["lerobot-v3-data-files", "pyarrow", "parquet-files", "video-files", "opencv-python-headless"],
|
|
"lerobot_summary": ["lerobot-dataset-files"],
|
|
"libero_hdf5_task_validate": ["libero-hdf5-demo-files", "libero-bddl-files", "h5py", "numpy"],
|
|
"maniskill_trajectory_validate": ["maniskill-hdf5-json-files", "h5py", "numpy"],
|
|
"media_reference_validate": ["csv-json-jsonl-parquet-manifests", "image-video-depth-pointcloud-files", "opencv-python-headless", "numpy", "pyarrow", "ffprobe"],
|
|
"mcap_topic_summary": ["mcap-files", "mcap-python"],
|
|
"merge": ["json-artifacts", "bounded-upstream-manifests"],
|
|
"mjcf_actuator_validate": ["mjcf-files", "xml-parser"],
|
|
"model_inference": ["json-model-or-command"],
|
|
"mongodb_write": ["mongodb"],
|
|
"ms_swift_dataset_export": ["accepted-annotation-revisions", "real-video-files", "ffmpeg", "jsonl"],
|
|
"package": ["ffmpeg", "video-files", "accepted-annotation-partitions", "headpose-manifests", "optional-handpose-manifests"],
|
|
"openvla_finetune": ["openvla-lora-finetune", "rlds-vla-dataset", "peft", "gpu-a800"],
|
|
"openvla_inference": ["openvla-transformers-inference", "openvla-model-or-upstream-adapter", "gpu-a800"],
|
|
"orbbec_data_normalize": ["json-artifacts", "orbbec-ego-layout"],
|
|
"partnet_mobility_asset_validate": ["partnet-mobility-sapien-assets", "urdf-files", "json-files", "semantics-txt", "mesh-obj-files"],
|
|
"parquet_table_summary": ["parquet-files", "pyarrow"],
|
|
"pointcloud_summary": ["pcd-files", "ply-files"],
|
|
"real_source": ["filesystem"],
|
|
"rgbd_frame_pair_validate": ["rgb-images", "depth-images", "opencv-python-headless", "numpy", "pyyaml"],
|
|
"rlds_episode_summary": ["rlds-json-or-tfds-files"],
|
|
"rh20t_scene_validate": ["rh20t-scene-directory", "numpy", "npy-files", "image-video-files"],
|
|
"robomind_hdf5_quality_validate": ["robomind-trajectory-hdf5-files", "h5py", "numpy"],
|
|
"robomimic_hdf5_validate": ["robomimic-hdf5-files", "h5py", "numpy"],
|
|
"robocasa_hdf5_demonstration_validate": ["robocasa-hdf5-demo-files", "h5py", "numpy"],
|
|
"roboturk_real_hdf5_validate": ["roboturk-real-aligned-hdf5-files", "h5py", "numpy"],
|
|
"roboset_hdf5_validate": ["roboset-robohive-hdf5-trajectory-files", "h5py", "numpy"],
|
|
"robot_model_asset_validate": ["urdf-files", "mjcf-files", "mesh-assets"],
|
|
"rlbench_dataset_validate": ["rlbench-episode-files", "pickle-files", "image-files"],
|
|
"ros_joint_state_validate": ["ros-joint-state-files", "json-jsonl-csv-parquet-files", "pyarrow"],
|
|
"rosbag_summary": ["rosbag2-files", "sqlite"],
|
|
"rtx_action_schema_validate": ["csv-json-jsonl-parquet-hdf5-action-files", "pyarrow", "h5py", "numpy"],
|
|
"sensor_timestamp_sync_validate": ["timestamp-csv-jsonl"],
|
|
"segmentation_mask_validate": ["segmentation-mask-images", "opencv-python-headless", "numpy"],
|
|
"semantic_prelabel": ["json-artifacts", "annotation-schema"],
|
|
"shell_command": ["container-shell"],
|
|
"sim3d_asset_generate": ["json-artifacts", "obj-mtl-gltf-files"],
|
|
"template_render": ["json-artifacts"],
|
|
"tfrecord_structural_validate": ["tfrecord-files", "crc32c-structural-validation"],
|
|
"tf_transform_tree_validate": ["json-jsonl-csv-yaml-files", "optional-pyyaml"],
|
|
"trajectory_integrity_validate": ["trajectory-json", "local-media-files"],
|
|
"umi_zarr_dataset_validate": ["umi-zarr-dataset", "zarr", "numpy"],
|
|
"upload": ["filesystem", "optional-baidu-bos"],
|
|
"video_annotate": ["video-files", "json-artifacts"],
|
|
"video_face_blur": ["ffmpeg", "opencv-python-headless", "opencv-yunet-onnx", "verified-local-model-manifest", "video-files"],
|
|
"video_filter": ["json-artifacts"],
|
|
"video_quality": ["ffprobe", "video-files"],
|
|
"video_slice": ["ffmpeg", "video-files"],
|
|
"video_target_clip": ["opencv-python-headless", "numpy", "ffmpeg", "video-files", "optional-opencv-dnn-model"],
|
|
"vla_model_evaluate": ["vla-inference-result-json", "vla-sample-action-manifest"],
|
|
"vla_model_inference": ["vla-policy-model-json", "vla-sample-action-manifest"],
|
|
"vla_model_train": ["vla-sample-action-manifest", "json-jsonl-csv-files"],
|
|
"vla_sample_manifest_validate": ["json-jsonl-csv-parquet-files", "image-files", "pyarrow"],
|
|
"wilor_hand_pose": ["wilor-inference-service", "image-video-files", "http-api", "gpu-l20-service"],
|
|
"wilor_valid_event_detection": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files"],
|
|
"webdataset_shard_summary": ["webdataset-tar-shards", "python-tarfile"],
|
|
"xplatform_dataset_index": ["xplatform-dataset-files"],
|
|
"xplatform_dataset_validate": ["xplatform-dataset-index"],
|
|
"qwen_temporal_localization": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files"],
|
|
"qwen_behavior_rough_cut": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "bce-python-sdk", "real-video-files", "orbbec-pts-csv"],
|
|
"qwen_label_crop_classification": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "ms-swift", "peft", "torch-cuda", "accepted-annotation-dataset"],
|
|
"yolo_detection_validate": ["yolo-label-files", "image-files", "pyyaml"],
|
|
"yolo_embodied_detection_train": ["real-yolo-training-command", "yolo-or-coco-dataset", "gpu-a800"],
|
|
"yolo_video_target_clip": ["yolo-onnx-model", "opencv-python-headless", "numpy", "ffmpeg", "video-files", "gpu-a800"],
|
|
"zarr_trajectory_summary": ["zarr-v2-v3-metadata", "filesystem"],
|
|
}
|
|
|
|
STRICT_EXTERNALS: dict[str, list[str]] = {
|
|
"bos_incremental_scan": ["bos"],
|
|
"mongodb_write": ["mongo"],
|
|
"upload": ["bos"],
|
|
}
|
|
|
|
|
|
def load_json(path: pathlib.Path) -> Any:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def dump_json(payload: Any) -> str:
|
|
return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
|
|
|
|
|
def operator_slug(operator_id: str) -> str:
|
|
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
|
|
|
|
|
|
def legacy_manifest(
|
|
*,
|
|
slug: str,
|
|
metadata: dict[str, Any],
|
|
catalog_item: dict[str, Any],
|
|
image: str,
|
|
image_id: str,
|
|
dependencies: list[str],
|
|
) -> dict[str, Any]:
|
|
acceptance = dict(metadata["acceptance"])
|
|
manifest: dict[str, Any] = {
|
|
"schemaVersion": "droboticflow.operator.repository/v1",
|
|
"repositoryId": f"builtin/{slug}",
|
|
"operatorId": metadata["id"],
|
|
"operatorSlug": slug,
|
|
"preferredSlug": operator_slug(metadata["id"]),
|
|
"name": metadata["name"],
|
|
"category": metadata["category"],
|
|
"type": metadata["type"],
|
|
"sourceRoot": f"backend/operators/builtin/{slug}",
|
|
"entrypoint": "operator.py",
|
|
"metadata": "operator.json",
|
|
"runtimeSource": metadata["runtimeSource"],
|
|
"runtimeHandler": metadata["runtimeHandler"],
|
|
"containerEntrypoint": "backend/operator-runner/main.py",
|
|
"image": image,
|
|
"imageId": image_id,
|
|
"containerCommand": catalog_item.get("containerCommand", []),
|
|
"containerArgs": catalog_item.get("containerArgs", []),
|
|
"runtimeContract": catalog_item.get("runtimeContract", {}),
|
|
"pythonImport": f"backend.operators.builtin.{slug}.operator:run",
|
|
"runtimeDependencies": dependencies,
|
|
"strictPocAcceptanceExternals": STRICT_EXTERNALS.get(slug, []),
|
|
"acceptance": {
|
|
**acceptance,
|
|
"pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh",
|
|
"requiresPersistentWorkspaceEvidence": True,
|
|
},
|
|
}
|
|
if "externalApi" in metadata:
|
|
manifest["externalApi"] = metadata["externalApi"]
|
|
return manifest
|
|
|
|
|
|
def expected_manifests(slugs: set[str] | None = None) -> dict[pathlib.Path, dict[str, Any]]:
|
|
catalog_items = {
|
|
item["id"]: item for item in load_json(CATALOG_PATH) if item.get("type") == "builtin"
|
|
}
|
|
manifests: dict[pathlib.Path, dict[str, Any]] = {}
|
|
split_operator_ids: set[str] = set()
|
|
metadata_paths = sorted(OPERATOR_REPO_ROOT.glob("*/operator.json"))
|
|
if slugs is not None:
|
|
available = {path.parent.name for path in metadata_paths} - UNMIGRATED_HISTORICAL_REPOSITORIES
|
|
if not slugs or slugs - available:
|
|
raise RuntimeError("unknown or empty operator repository selection")
|
|
metadata_paths = [path for path in metadata_paths if path.parent.name in slugs]
|
|
for metadata_path in metadata_paths:
|
|
if metadata_path.parent.name in UNMIGRATED_HISTORICAL_REPOSITORIES:
|
|
continue
|
|
metadata = load_json(metadata_path)
|
|
operator_id = metadata["id"]
|
|
if operator_id not in catalog_items:
|
|
raise RuntimeError(f"{metadata_path} operator id is missing from catalog: {operator_id}")
|
|
catalog_item = catalog_items[operator_id]
|
|
slug = metadata_path.parent.name
|
|
runtime_source = f"operator-repos/builtin/{slug}/src/operator.py"
|
|
if metadata.get("runtimeSource") != runtime_source:
|
|
raise RuntimeError(f"{metadata_path} runtimeSource must be {runtime_source}")
|
|
dependencies = RUNTIME_DEPENDENCIES.get(slug)
|
|
if dependencies is None:
|
|
raise RuntimeError(f"{slug} is missing runtime dependency metadata")
|
|
acceptance = dict(metadata["acceptance"])
|
|
is_draft = str(catalog_item.get("publicationStatus", "")).strip().lower() == "draft"
|
|
try:
|
|
release = load_release(slug)
|
|
image, image_id = release_image_reference(slug)
|
|
except RuntimeError:
|
|
if not is_draft:
|
|
raise
|
|
release = {}
|
|
image = str(catalog_item.get("image", "")).strip()
|
|
image_id = str(catalog_item.get("imageId", "")).strip()
|
|
manifest: dict[str, Any] = {
|
|
"schemaVersion": "droboticflow.operator.repository/v1",
|
|
"repositoryId": f"builtin/{slug}",
|
|
"operatorId": operator_id,
|
|
"operatorSlug": slug,
|
|
"preferredSlug": operator_slug(operator_id),
|
|
"name": metadata["name"],
|
|
"category": metadata["category"],
|
|
"type": metadata["type"],
|
|
"sourceRoot": f"operator-repos/builtin/{slug}",
|
|
"entrypoint": "src/operator.py",
|
|
"metadata": "operator.json",
|
|
"runtimeSource": runtime_source,
|
|
"runtimeHandler": metadata["runtimeHandler"],
|
|
"containerEntrypoint": "backend/operator-runner/main.py",
|
|
"releaseManifest": "release.json",
|
|
"releaseSchemaVersion": release.get("schemaVersion", RELEASE_SCHEMA_VERSION),
|
|
"sourceTreeSha256": source_tree_sha256(slug),
|
|
"image": image,
|
|
"imageId": image_id,
|
|
"containerCommand": catalog_item.get("containerCommand", []),
|
|
"containerArgs": catalog_item.get("containerArgs", []),
|
|
"runtimeContract": catalog_item.get("runtimeContract", {}),
|
|
"pythonImport": f"operator-repos/builtin/{slug}/src/operator.py:run",
|
|
"runtimeDependencies": dependencies,
|
|
"strictPocAcceptanceExternals": STRICT_EXTERNALS.get(slug, []),
|
|
"acceptance": {
|
|
**acceptance,
|
|
"pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh",
|
|
"requiresPersistentWorkspaceEvidence": True,
|
|
"requiresChineseWorkflowAcceptance": True,
|
|
"requiresChainedWorkflow": True,
|
|
"fixtureContract": "tests/acceptance/fixture_contract.json",
|
|
},
|
|
}
|
|
if "externalApi" in metadata:
|
|
manifest["externalApi"] = metadata["externalApi"]
|
|
manifests[metadata_path.with_name("repository.json")] = manifest
|
|
split_operator_ids.add(operator_id)
|
|
|
|
legacy_metadata_path = OPERATOR_ROOT / slug / "operator.json"
|
|
if not legacy_metadata_path.exists():
|
|
raise RuntimeError(f"{slug} is missing legacy runtime metadata")
|
|
legacy_metadata = load_json(legacy_metadata_path)
|
|
if legacy_metadata.get("id") != operator_id:
|
|
raise RuntimeError(
|
|
f"{legacy_metadata_path} operator id {legacy_metadata.get('id')!r} does not match {operator_id!r}"
|
|
)
|
|
manifests[legacy_metadata_path.with_name("repository.json")] = legacy_manifest(
|
|
slug=slug,
|
|
metadata=legacy_metadata,
|
|
catalog_item=catalog_item,
|
|
image=image,
|
|
image_id=image_id,
|
|
dependencies=dependencies,
|
|
)
|
|
missing_catalog = sorted(set(catalog_items) - split_operator_ids) if slugs is None else []
|
|
if missing_catalog:
|
|
raise RuntimeError(f"catalog operators missing repository manifests: {', '.join(missing_catalog)}")
|
|
return manifests
|
|
|
|
|
|
def write_manifests(manifests: dict[pathlib.Path, dict[str, Any]]) -> None:
|
|
for path, manifest in manifests.items():
|
|
path.write_text(dump_json(manifest), encoding="utf-8")
|
|
|
|
|
|
def check_manifests(manifests: dict[pathlib.Path, dict[str, Any]]) -> int:
|
|
mismatches: list[str] = []
|
|
for path, manifest in manifests.items():
|
|
expected = dump_json(manifest)
|
|
if not path.exists():
|
|
mismatches.append(f"missing {path.relative_to(ROOT)}")
|
|
continue
|
|
actual = path.read_text(encoding="utf-8")
|
|
if actual != expected:
|
|
mismatches.append(f"outdated {path.relative_to(ROOT)}")
|
|
if mismatches:
|
|
for mismatch in mismatches:
|
|
print(mismatch, file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Generate or check builtin operator repository manifests.")
|
|
mode = parser.add_mutually_exclusive_group()
|
|
mode.add_argument("--write", action="store_true", help="write repository.json files")
|
|
mode.add_argument("--check", action="store_true", help="check repository.json files without writing")
|
|
parser.add_argument("--slug", action="append", help="publish/check only these independent repositories; default checks every operator")
|
|
args = parser.parse_args()
|
|
|
|
manifests = expected_manifests(set(args.slug) if args.slug is not None else None)
|
|
if args.check:
|
|
return check_manifests(manifests)
|
|
write_manifests(manifests)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|