#!/usr/bin/env python3 from __future__ import annotations import argparse import importlib.util import json import pathlib import sys from typing import Any ROOT = pathlib.Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) from operator_release_manifest import release_image_reference CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json" OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin" OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin" IMAGE_ROOT = ROOT / "operator-images" / "builtin" MANIFEST_PATH = IMAGE_ROOT / "operator-images.json" DEFAULT_REGISTRY = "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc" IMAGE_TAG = "builtin" DEFAULT_OPERATOR_BASE_IMAGE = f"{DEFAULT_REGISTRY}/droboticflow-operator-runner:{IMAGE_TAG}" DEFAULT_GPU_BASE_IMAGE = f"{DEFAULT_REGISTRY}/droboticflow-operator-gpu-base:{IMAGE_TAG}" DEFAULT_HOLOBRAIN_BASE_IMAGE = f"{DEFAULT_REGISTRY}/droboticflow-operator-holobrain-base:{IMAGE_TAG}" DEFAULT_YOLO_BASE_IMAGE = f"{DEFAULT_REGISTRY}/droboticflow-operator-yolo-base:{IMAGE_TAG}" COMMAND_ENTRYPOINT = ["python", "/app/backend/operator-runner/main.py"] GPU_SLUGS = { "act_policy_train", "diffusion_policy_train", "holobrain_export", "holobrain_inference_test", "holobrain_robotwin_eval", "holobrain_train", "openvla_finetune", "openvla_inference", "yolo_embodied_detection_train", "yolo_video_target_clip", # Fanyu v5 always executes Qwen3-VL on an allocated CUDA GPU. These # operators intentionally have no CPU, heuristic, or mock fallback. "qwen_behavior_rough_cut", "qwen_temporal_localization", "wilor_valid_event_detection", "qwen_label_crop_classification", } QWEN3VL_SLUGS = { "qwen_behavior_rough_cut", "qwen_temporal_localization", "wilor_valid_event_detection", "qwen_label_crop_classification", } BACKEND_OVERLAY_SLUGS = { # The worker intercepts this operator in normal workflow execution, but # the independent image must still be runnable for diagnostics and # acceptance. The shared CPU base predates the native annotation-gate # handler, so overlay the current runner/backend in this image. "annotation_gate", } FANYU_V5_ALLOW_EMPTY_SEMANTIC_RESULT_SLUGS = { # A real inference can legitimately find no action/event, and an accepted # annotation can be an explicit negative sample. Their manifests and # provenance must still be non-empty and cryptographically verifiable. "qwen_behavior_rough_cut", "qwen_temporal_localization", "wilor_valid_event_detection", "ms_swift_dataset_export", "qwen_label_crop_classification", } HOLOBRAIN_SLUGS = { "holobrain_data_prepare", "holobrain_export", "holobrain_inference_test", "holobrain_robotwin_eval", "holobrain_train", } HOLOBRAIN_RUNTIME_SLUGS = { "holobrain_export", "holobrain_inference_test", "holobrain_robotwin_eval", "holobrain_train", } HOLOBRAIN_ROBOTWIN_EVAL_RUNTIME_SLUGS = { "holobrain_inference_test", "holobrain_robotwin_eval", } HOLOBRAIN_ROBOTWIN_VENV = "/opt/holobrain-robotwin-venv" LONG_RUNNING_SLUGS = { "holobrain_data_prepare", "holobrain_train", } | QWEN3VL_SLUGS REMOTE_GPU_SERVICE_SLUGS = { "hand_removal", "wilor_hand_pose", } GPU_TYPE_OVERRIDES = { "wilor_hand_pose": "NVIDIA L20 48GB", } COMMAND_WRAPPER_ALLOWED: set[str] = set() CONTAINER_HANDLER_ALIASES = { "builtin_source": "source", "builtin_sim3d_asset_generate": "sim3d_asset_generate", "builtin_vla_model_evaluate": "vla_model_evaluate", "builtin_vla_model_inference": "vla_model_inference", "builtin_vla_model_train": "vla_model_train", "builtin_vla_sample_manifest_validate": "vla_sample_manifest_validate", "builtin_lerobot_summary": "lerobot", "builtin_rosbag_summary": "rosbag", } STRUCTURED_OPERATOR_OVERRIDES: dict[str, dict[str, Any]] = { "operator-builtin-model-inference": { "skillDefinition": "执行结构化模型推理节点;读取 MODEL_JSON 或 MODEL_PATH,支持 rule、linear、nearest_centroid 三类可解释模型;自定义命令请使用 Shell 命令算子,constant 等硬编码输出模型会被拒绝。", }, "operator-builtin-act-policy-train": { "skillDefinition": "ACT 策略训练算子;生产默认使用 Hugging Face LeRobot ACT 训练入口,读取 DATASET_REPO_ID 或 DATASET_PATH 指向的真实 LeRobot 数据集并输出 checkpoint 与 metrics。推荐 1 张 A800 80GB;仅本地单元测试可显式设置 TRAINING_BACKEND=torch_mlp。", "hyperparams": [ {"key": "TRAINING_BACKEND", "value": "lerobot", "description": "生产训练后端;默认 lerobot,测试 fallback 才允许 torch_mlp", "type": "string"}, {"key": "DATASET_REPO_ID", "value": "", "description": "Hugging Face LeRobot 数据集 repo_id,例如 org/robot_dataset", "type": "string"}, {"key": "DATASET_PATH", "value": "", "description": "POC/PVC 中的真实 LeRobot 数据集根目录;与 DATASET_REPO_ID 至少提供一个", "type": "string"}, {"key": "POLICY_REPO_ID", "value": "", "description": "可选的 LeRobot policy repo_id,用于后续发布或复现实验", "type": "string"}, {"key": "MAX_STEPS", "value": "1000", "description": "短训最大 step 数,生产长训按数据规模调大", "type": "number"}, {"key": "BATCH_SIZE", "value": "8", "description": "训练 batch size", "type": "number"}, {"key": "LEARNING_RATE", "value": "0.0001", "description": "LeRobot optimizer.lr", "type": "number"}, {"key": "CHUNK_SIZE", "value": "100", "description": "ACT action chunk size", "type": "number"}, {"key": "LEROBOT_OVERRIDES_JSON", "value": "[]", "description": "额外 Hydra override 字符串数组,不允许 shell 命令", "type": "json"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-diffusion-policy-train": { "skillDefinition": "Diffusion Policy 训练算子;生产默认使用 real-stanford/diffusion_policy 的 Hydra train.py,读取真实 Zarr/轨迹数据并输出 .ckpt、logs 和 metrics。推荐 1 张 A800 80GB;仅本地单元测试可显式设置 TRAINING_BACKEND=torch_mlp。", "hyperparams": [ {"key": "TRAINING_BACKEND", "value": "diffusion_policy", "description": "生产训练后端;默认 diffusion_policy,测试 fallback 才允许 torch_mlp", "type": "string"}, {"key": "DIFFUSION_POLICY_ROOT", "value": "/opt/diffusion_policy", "description": "real-stanford/diffusion_policy 仓库路径", "type": "string"}, {"key": "DATASET_PATH", "value": "", "description": "真实 Zarr/轨迹数据路径,会作为 task.dataset.zarr_path 传入", "type": "string"}, {"key": "CONFIG_NAME", "value": "image_pusht_diffusion_policy_cnn", "description": "Hydra config name 或 config yaml 名称", "type": "string"}, {"key": "TASK_NAME", "value": "", "description": "可选 task override", "type": "string"}, {"key": "EPOCHS", "value": "1", "description": "短训 epoch 数,生产长训按数据规模调大", "type": "number"}, {"key": "BATCH_SIZE", "value": "8", "description": "训练 batch size", "type": "number"}, {"key": "LEARNING_RATE", "value": "0.0001", "description": "optimizer.lr", "type": "number"}, {"key": "HORIZON", "value": "16", "description": "预测 horizon", "type": "number"}, {"key": "DIFFUSION_STEPS", "value": "32", "description": "扩散步数配置,用于训练记录和下游复现", "type": "number"}, {"key": "HYDRA_OVERRIDES_JSON", "value": "[]", "description": "额外 Hydra override 字符串数组,不允许 shell 命令", "type": "json"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-openvla-finetune": { "skillDefinition": "OpenVLA 微调算子;生产默认使用 OpenVLA 官方 vla-scripts/finetune.py 的 PEFT/LoRA 路径,读取真实 RLDS 数据集并输出 LoRA adapter artifact 与 metrics。推荐 1 张 A800 80GB。", "hyperparams": [ {"key": "TRAINING_BACKEND", "value": "openvla_lora", "description": "生产训练后端;默认 openvla_lora,兼容测试可显式 transformers_text", "type": "string"}, {"key": "OPENVLA_ROOT", "value": "/opt/openvla", "description": "OpenVLA 仓库路径", "type": "string"}, {"key": "BASE_MODEL", "value": "openvla/openvla-7b", "description": "OpenVLA 基础模型路径或 HF 模型名", "type": "string"}, {"key": "DATA_ROOT_DIR", "value": "", "description": "真实 RLDS 数据集根目录", "type": "string"}, {"key": "DATASET_NAME", "value": "", "description": "RLDS 数据集名称,例如 droid_wipe", "type": "string"}, {"key": "MAX_STEPS", "value": "1000", "description": "短训最大 step 数,生产长训按数据规模调大", "type": "number"}, {"key": "SAVE_STEPS", "value": "500", "description": "checkpoint 保存间隔", "type": "number"}, {"key": "BATCH_SIZE", "value": "8", "description": "LoRA fine-tune batch size", "type": "number"}, {"key": "LEARNING_RATE", "value": "0.0005", "description": "LoRA 学习率", "type": "number"}, {"key": "LORA_RANK", "value": "32", "description": "LoRA rank", "type": "number"}, {"key": "USE_QUANTIZATION", "value": "false", "description": "是否启用 4-bit quantization", "type": "boolean"}, {"key": "PRECISION", "value": "bf16", "description": "推荐精度配置", "type": "string"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-openvla-inference": { "skillDefinition": "OpenVLA 推理算子;使用 MODEL_PATH 或 BASE_MODEL 加载真实 Transformers 权重,读取 PROMPT 或上游样本 instruction,并输出 inference-result.json。推荐 1 张 A800,量化模型可降低显存。", "hyperparams": [ {"key": "INFERENCE_BACKEND", "value": "openvla_transformers", "description": "推理后端;默认从 MODEL_PATH 或上游 fine-tune artifact 加载 OpenVLA/Transformers 权重", "type": "string"}, {"key": "MODEL_PATH", "value": "", "description": "本地模型目录或 Hugging Face 模型名;为空时自动消费上游模型 artifact", "type": "string"}, {"key": "BASE_MODEL", "value": "", "description": "未提供 MODEL_PATH 时使用的基础模型", "type": "string"}, {"key": "PROMPT", "value": "", "description": "推理 prompt;为空则从上游样本读取 TEXT_FIELD", "type": "string"}, {"key": "TEXT_FIELD", "value": "instruction", "description": "上游样本指令字段名", "type": "string"}, {"key": "MAX_NEW_TOKENS", "value": "64", "description": "生成 token 数", "type": "number"}, {"key": "PRECISION", "value": "bf16", "description": "推荐精度配置", "type": "string"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-holobrain-data-prepare": { "description": "校验真实 RoboTwin 2.0 数据并生成 HoloBrain manifest;支持 smoke 真实小数据和 full 完整数据模式。", "skillDefinition": "HoloBrain 数据准备算子;TRAINING_MODE=smoke 使用真实小数据契约(Arrow 默认最多 1 个 episode),full 保持完整数据要求;两种模式都拒绝合成或空输入。", "hyperparams": [ {"key": "TRAINING_MODE", "value": "full", "description": "full 保持完整训练数据行为;smoke 使用真实小数据完成端到端验证", "type": "string"}, {"key": "DATA_SOURCE", "value": "local", "description": "数据来源;仅支持 local,不支持运行时下载", "type": "string"}, {"key": "ROBOTWIN_SOURCE_PATH", "value": "", "description": "已预置的 RoboTwin/HoloBrain 真实数据根目录;必填", "type": "string"}, {"key": "EXPECTED_LMDB_DIRS_JSON", "value": "", "description": "可选覆盖 LMDB 目录;空值时 smoke 默认使用三相机 UR5/WSG 真实小数据,full 要求三个官方目录", "type": "json"}, {"key": "SMOKE_MAX_EPISODES", "value": "1", "description": "smoke 从真实 Arrow 数据转换的最大 episode 数", "type": "number"}, {"key": "ARROW_MAX_EPISODES", "value": "", "description": "显式 Arrow 转换 episode 上限;空值时由模式决定", "type": "number"}, ], }, "operator-builtin-holobrain-train": { "description": "执行 HoloBrain 训练;full 保留官方长训默认,smoke 用真实小数据短训并输出可恢复 checkpoint 与结构化进度。", "skillDefinition": "HoloBrain 训练算子;TRAINING_MODE=full 保留官方 100000 step 默认,smoke 默认在真实 RoboTwin 小数据上执行 2 step 并生成 checkpoint。失败重试默认恢复最新有效 Accelerate checkpoint;进度写入 workspace 并以 DROBOTICFLOW_PROGRESS {json} 输出;dtype 在启动前检查。", "hyperparams": [ {"key": "TRAINING_MODE", "value": "full", "description": "full 或 smoke;full 不覆盖官方长训默认,smoke 使用短步数真实训练", "type": "string"}, {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "镜像内预置的 RoboOrchardLab 仓库路径;缺失时直接失败", "type": "string"}, {"key": "CONFIG_PATH", "value": "projects/holobrain/configs/config_holobrain_qwen_common.py", "description": "HoloBrain Qwen 训练配置", "type": "string"}, {"key": "ROBOTWIN_DATA_ROOT", "value": "", "description": "RoboTwin 数据根目录;为空时消费上游数据准备 manifest", "type": "string"}, {"key": "WORKSPACE", "value": "", "description": "训练 workspace;为空时使用节点输出目录下 holobrain-workspace", "type": "string"}, {"key": "LAUNCH_MODE", "value": "single", "description": "single 或 accelerate", "type": "string"}, {"key": "GPU_COUNT", "value": "1", "description": "accelerate 模式使用的进程/GPU 数", "type": "number"}, {"key": "CONFIG_KWARGS_JSON", "value": "{}", "description": "传给 train.py --kwargs 的 JSON;默认不覆盖 max_step=100000", "type": "json"}, {"key": "SMOKE_MAX_STEP", "value": "2", "description": "smoke 真实训练最大 step;默认 2", "type": "number"}, {"key": "SMOKE_BATCH_SIZE", "value": "1", "description": "smoke batch size", "type": "number"}, {"key": "SMOKE_NUM_WORKERS", "value": "0", "description": "smoke 数据加载进程数", "type": "number"}, {"key": "RESUME_POLICY", "value": "auto", "description": "auto 自动选择 workspace/checkpoints 下最新有效训练状态;explicit 要求 RESUME_FROM;never 禁用恢复", "type": "string"}, {"key": "SKIP_INITIAL_CHECKPOINT_ON_RESUME", "value": "true", "description": "恢复完整 Accelerate 状态时跳过冗余的初始模型权重加载", "type": "boolean"}, {"key": "RESUME_FROM", "value": "", "description": "显式 Accelerate 训练状态 checkpoint 目录", "type": "string"}, {"key": "INITIAL_CHECKPOINT_PATH", "value": "", "description": "可选初始模型权重文件;与训练状态恢复分开", "type": "string"}, {"key": "CHECKPOINT_ARTIFACT", "value": "latest_valid", "description": "latest_valid、explicit 或 none;控制训练结果清单暴露的模型产物", "type": "string"}, {"key": "OUTPUT_CHECKPOINT_PATH", "value": "", "description": "CHECKPOINT_ARTIFACT=explicit 时使用的模型 checkpoint", "type": "string"}, {"key": "PROGRESS_INTERVAL_SECONDS", "value": "15", "description": "结构化进度写入 droboticflow-progress.json、holobrain-progress.json 并输出 DROBOTICFLOW_PROGRESS {json};该值为最长刷新间隔", "type": "number"}, {"key": "MODEL_DTYPE", "value": "bfloat16", "description": "官方 HoloBrain Qwen 模型 dtype;不兼容值在启动前失败", "type": "string"}, {"key": "INPUT_DTYPE", "value": "float32", "description": "官方 RoboTwin processor 输入 tensor dtype;不兼容值在启动前失败", "type": "string"}, {"key": "WANDB_MODE", "value": "offline", "description": "训练日志模式", "type": "string"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-holobrain-export": { "description": "从显式、上游或训练 workspace 的最新有效 checkpoint 导出可部署 HoloBrain 模型。", "skillDefinition": "HoloBrain 模型导出算子;按 CHECKPOINT_SELECTION 消费显式、上游或训练 workspace 中最新有效 checkpoint。smoke 模式没有真实训练 checkpoint 时直接失败。", "hyperparams": [ {"key": "TRAINING_MODE", "value": "full", "description": "full 或 smoke;smoke 强制要求上游真实 checkpoint", "type": "string"}, {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "RoboOrchardLab 仓库路径", "type": "string"}, {"key": "CONFIG_PATH", "value": "projects/holobrain/configs/config_holobrain_qwen_common.py", "description": "HoloBrain Qwen 导出配置", "type": "string"}, {"key": "TRAINING_WORKSPACE", "value": "", "description": "训练 workspace;为空时消费上游训练结果清单", "type": "string"}, {"key": "CHECKPOINT_PATH", "value": "", "description": "显式 checkpoint 路径;为空时使用最新训练 checkpoint", "type": "string"}, {"key": "CHECKPOINT_SELECTION", "value": "latest_valid", "description": "latest_valid、upstream 或 explicit", "type": "string"}, {"key": "EXPORT_WORKSPACE", "value": "", "description": "导出目录;为空时使用节点输出目录下 holobrain-export", "type": "string"}, {"key": "CONFIG_KWARGS_JSON", "value": "{}", "description": "额外 export.py --kwargs JSON", "type": "json"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "export reload-test 使用 CUDA", "type": "string"}, ], }, "operator-builtin-holobrain-robotwin-eval": { "description": "调用官方 robotwin_eval.py 执行真实 HoloBrain RoboTwin 评测;smoke 为单任务单 episode。", "skillDefinition": "HoloBrain RoboTwin 评测算子;full 保留官方多任务评测默认,smoke 固定执行 1 个真实 RoboTwin episode,必须由官方 evaluator 产生指标。", "hyperparams": [ {"key": "TRAINING_MODE", "value": "full", "description": "full 或 smoke;smoke 使用专用单任务单 episode 参数", "type": "string"}, {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "RoboOrchardLab 仓库路径", "type": "string"}, {"key": "ROBOTWIN_ROOT", "value": "/opt/RoboTwin", "description": "镜像内固定版本的 RoboTwin 源码根目录", "type": "string"}, {"key": "ROBOTWIN_ASSET_ROOT", "value": "", "description": "可选只读仿真资产根目录;为空时使用 ROBOTWIN_ROOT/assets", "type": "string"}, {"key": "MODEL_DIR", "value": "", "description": "导出模型目录;为空时消费上游导出 manifest", "type": "string"}, {"key": "INFERENCE_PREFIX", "value": "", "description": "导出 pipeline config prefix;空值时从上游 export manifest 自动选择", "type": "string"}, {"key": "UR5_WSG_ARM_DISTANCE", "value": "0.8", "description": "UR5-WSG 左右基座间距,须与训练用合并双臂 URDF 一致", "type": "number"}, {"key": "ROBOTWIN_CAMERA_SHADER", "value": "default", "description": "RoboTwin 相机 shader;POC 默认使用非光追 default 以兼容容器 Vulkan", "type": "string"}, {"key": "VULKAN_ICD_FILENAMES", "value": "", "description": "可选 Vulkan ICD JSON;非光追模式为空时自动使用镜像内 Lavapipe", "type": "string"}, {"key": "EVAL_MODE", "value": "ray", "description": "official eval mode:ray 或 local", "type": "string"}, {"key": "TASK_NAMES_JSON", "value": "[]", "description": "评测任务名数组;空数组使用官方默认 SEM_TASKS_16", "type": "json"}, {"key": "EPISODE_NUM", "value": "100", "description": "每任务 episode 数", "type": "number"}, {"key": "GPU_IDS_JSON", "value": "[0]", "description": "Ray 评测 GPU id 数组", "type": "json"}, {"key": "WORKERS_PER_GPU", "value": "1", "description": "每张 GPU 的并发 worker 数", "type": "number"}, {"key": "DEVICE", "value": "cuda", "description": "full 模式模型执行设备", "type": "string"}, {"key": "SAVE_VIDEO", "value": "false", "description": "是否保存评测视频", "type": "boolean"}, {"key": "SMOKE_TASK_NAME", "value": "beat_block_hammer", "description": "smoke 实际执行的 RoboTwin 任务", "type": "string"}, {"key": "SMOKE_EVAL_EPISODE_NUM", "value": "1", "description": "smoke episode 数", "type": "number"}, {"key": "SMOKE_EVAL_MODE", "value": "local", "description": "smoke evaluator 模式", "type": "string"}, {"key": "SMOKE_DEVICE", "value": "cuda", "description": "smoke 模型执行设备", "type": "string"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-holobrain-inference-test": { "description": "加载导出模型并通过官方 RoboTwin evaluator 执行真实模型推理,不接受文件存在性或硬编码结果冒充成功。", "skillDefinition": "HoloBrain 模型测试算子;加载导出的真实模型并通过官方 RoboTwin evaluator 执行至少 1 个 episode,不再用文件存在性或硬编码结果冒充推理。", "hyperparams": [ {"key": "TRAINING_MODE", "value": "full", "description": "记录上游 full/smoke 模式", "type": "string"}, {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "RoboOrchardLab 仓库路径", "type": "string"}, {"key": "ROBOTWIN_ROOT", "value": "/opt/RoboTwin", "description": "镜像内固定版本的 RoboTwin 源码根目录", "type": "string"}, {"key": "ROBOTWIN_ASSET_ROOT", "value": "", "description": "可选只读仿真资产根目录;为空时使用 ROBOTWIN_ROOT/assets", "type": "string"}, {"key": "MODEL_DIR", "value": "", "description": "导出模型目录;为空时消费上游导出 manifest", "type": "string"}, {"key": "INFERENCE_PREFIX", "value": "", "description": "导出 pipeline config prefix;空值时从上游 export manifest 自动选择", "type": "string"}, {"key": "UR5_WSG_ARM_DISTANCE", "value": "0.8", "description": "UR5-WSG 左右基座间距,须与训练用合并双臂 URDF 一致", "type": "number"}, {"key": "ROBOTWIN_CAMERA_SHADER", "value": "default", "description": "RoboTwin 相机 shader;POC 默认使用非光追 default 以兼容容器 Vulkan", "type": "string"}, {"key": "VULKAN_ICD_FILENAMES", "value": "", "description": "可选 Vulkan ICD JSON;非光追模式为空时自动使用镜像内 Lavapipe", "type": "string"}, {"key": "TASK_NAME", "value": "beat_block_hammer", "description": "实际执行的 RoboTwin 推理任务", "type": "string"}, {"key": "EPISODE_NUM", "value": "1", "description": "真实推理 episode 数,至少为 1", "type": "number"}, {"key": "INFERENCE_MODE", "value": "local", "description": "local 或 ray", "type": "string"}, {"key": "DEVICE", "value": "cuda", "description": "模型推理设备", "type": "string"}, {"key": "GPU_IDS_JSON", "value": "[0]", "description": "ray 模式 GPU id 数组", "type": "json"}, {"key": "WORKERS_PER_GPU", "value": "1", "description": "ray 模式每卡 worker 数", "type": "number"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "完整推理测试推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-yolo-embodied-detection-train": { "skillDefinition": "YOLO 具身目标检测训练算子;使用 Ultralytics API 读取真实 YOLO/COCO data.yaml,输出 best/last checkpoint 与 metrics.json。A800 对 YOLOv8/YOLOv10/YOLO11 中大模型训练充足,默认请求 1 卡。", "hyperparams": [ {"key": "DATA_YAML", "value": "", "description": "真实 YOLO/COCO data.yaml 路径", "type": "string"}, {"key": "MODEL_SIZE", "value": "yolo11n.pt", "description": "Ultralytics 模型名或权重路径", "type": "string"}, {"key": "EPOCHS", "value": "10", "description": "训练轮数", "type": "number"}, {"key": "IMAGE_SIZE", "value": "640", "description": "输入图片尺寸", "type": "number"}, {"key": "BATCH_SIZE", "value": "8", "description": "训练 batch size", "type": "number"}, {"key": "WORKERS", "value": "0", "description": "数据加载 worker 数;K8s 默认共享内存较小时建议为 0", "type": "number"}, {"key": "CONF", "value": "0.25", "description": "后续检测推荐置信度", "type": "number"}, {"key": "TARGET_LABELS", "value": "", "description": "关注类别,逗号分隔", "type": "string"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-shell-command": { "skillDefinition": "受控脚本算子;只允许执行算子目录 approved_scripts 中登记的生产验收脚本,不接受自由 COMMAND/SCRIPT。", "hyperparams": [ {"key": "SCRIPT_ID", "value": "fixture_report", "description": "白名单脚本 ID", "type": "string"}, {"key": "SCRIPT_ARGS_JSON", "value": "{}", "description": "传给脚本的结构化 JSON 参数", "type": "json"}, {"key": "INPUT_PATHS_JSON", "value": "[]", "description": "允许脚本读取的输入路径 JSON 数组", "type": "json"}, {"key": "OUTPUT_CONTRACT_JSON", "value": "{}", "description": "脚本输出 artifact 契约", "type": "json"}, ], }, "operator-builtin-qwen-temporal-localization": { "description": "对真实左相机视频按最多 200 秒窗口执行 Qwen3-VL 零样本时序定位;没有 CUDA、不可变模型 revision、模型哈希或真实帧时直接失败。真实推理可返回空 segments,但仍必须输出带完整 provenance 的非空 manifest。", "skillDefinition": "真实 Qwen3-VL 200 秒窗口定位,严格拒绝默认标签、模拟预测和无 provenance 产物;正常视频允许模型真实返回空 segments,空结果不是伪造失败。", }, "operator-builtin-wilor-valid-event-detection": { "description": "保留 WiLoR 需求链中的有效行为检测节点名称以兼容工作流;实际使用真实 Qwen3-VL 零样本视频推理定位有效店内行为,不调用或冒充 WiLoR hand-pose inference。真实推理可返回空 events,但仍必须输出带完整 provenance 的非空 manifest。", "skillDefinition": "WiLoR 需求链的真实有效行为判定由 Qwen3-VL 完成;该节点不把 WiLoR 手姿服务冒充为行为模型,允许经过真实推理后的空 event 列表。", }, "operator-builtin-ms-swift-dataset-export": { "description": "将已通过最终验收的便利店切片、22 类语义真值和媒体引用导出为可供 ms-swift 真实 LoRA 训练的 JSONL;支持显式 accepted empty annotation 作为负样本,拒绝缺失标注语义或不可访问媒体。", "skillDefinition": "从最终验收真值导出真实 ms-swift 训练样本和输入哈希;显式空 actions 是合法负样本,不从伪造 package、默认标签或隐式空标注生成训练集。", }, } def load_repository_generator() -> Any: path = ROOT / "scripts" / "generate_operator_repository_manifests.py" spec = importlib.util.spec_from_file_location("operator_repository_generator", path) if spec is None or spec.loader is None: raise RuntimeError(f"cannot load {path}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module REPOSITORY_GENERATOR = load_repository_generator() RUNTIME_DEPENDENCIES: dict[str, list[str]] = REPOSITORY_GENERATOR.RUNTIME_DEPENDENCIES def load_json(path: pathlib.Path) -> Any: return json.loads(path.read_text(encoding="utf-8")) def dump_json(payload: Any) -> str: return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" def operator_slug(operator_id: str) -> str: return operator_id.removeprefix("operator-builtin-").replace("-", "_") def container_handler(item: dict[str, Any], slug: str) -> str: handler = item.get("runtimeHandler") or slug return str(handler) def image_name(slug: str, registry: str = DEFAULT_REGISTRY) -> str: return f"{registry}/droboticflow-operator-{slug.replace('_', '-')}:{IMAGE_TAG}" def docker_base(slug: str) -> str: if slug in HOLOBRAIN_SLUGS: return "${OPERATOR_HOLOBRAIN_BASE_IMAGE}" if slug.startswith("yolo_"): return "${OPERATOR_YOLO_BASE_IMAGE}" if slug in GPU_SLUGS: return "${OPERATOR_GPU_BASE_IMAGE}" return "${OPERATOR_BASE_IMAGE}" def dockerfile(slug: str, operator_id: str, handler: str) -> str: if slug not in HOLOBRAIN_SLUGS and slug not in GPU_SLUGS and not slug.startswith("yolo_"): dependency_install = ( "USER root\n" "RUN python -m pip install --no-cache-dir --force-reinstall --no-deps " "opencv-python-headless==4.11.0.86\n" "USER 10001:10001\n\n" if slug in {"video_face_blur", "orbbec_left_undistort"} else "" ) model_environment = ( "ENV DROBOTICFLOW_FACE_DETECTOR_MODEL=/app/operator-repos/builtin/video_face_blur/assets/face_detection_yunet_2023mar.onnx \\\n" " DROBOTICFLOW_FACE_DETECTOR_MANIFEST=/app/operator-repos/builtin/video_face_blur/assets/model_manifest.json\n\n" if slug == "video_face_blur" else "" ) model_verification = ( "RUN python -c \"import hashlib,json,pathlib; " "p=pathlib.Path('/app/operator-repos/builtin/video_face_blur/assets/face_detection_yunet_2023mar.onnx'); " "m=json.loads(p.with_name('model_manifest.json').read_text()); " "assert p.stat().st_size == 232589 == int(m['sizeBytes']); " "assert hashlib.sha256(p.read_bytes()).hexdigest() == m['sha256'] == '8f2383e4dd3cfbb4553ea8718107fc0423210dc964f9f4280604804ed2552fa4'; " "assert m['licenseSpdx'] == 'MIT' and p.with_name('LICENSE').is_file() and m['runtimeDownloadsAllowed'] is False\"\n" if slug == "video_face_blur" else "" ) model_verification_block = model_verification + "\n" if model_verification else "" backend_copy = "COPY backend ./backend\n" if slug in BACKEND_OVERLAY_SLUGS else "" return f"""# Generated by scripts/generate_operator_images.py. Do not edit by hand. ARG OPERATOR_BASE_IMAGE={DEFAULT_OPERATOR_BASE_IMAGE} FROM ${{OPERATOR_BASE_IMAGE}} ENV DROBOTICFLOW_OPERATOR_ID={operator_id} \\ DROBOTICFLOW_OPERATOR_SLUG={slug} \\ DROBOTICFLOW_OPERATOR_SOURCE=/app/operator-repos/builtin/{slug}/src/operator.py \\ DROBOTICFLOW_OPERATOR_REPOSITORY=/app/operator-repos/builtin/{slug} \\ DROBOTICFLOW_OPERATOR_HANDLER={handler} {dependency_install}{model_environment}{backend_copy}COPY operator-repos/builtin/{slug} ./operator-repos/builtin/{slug} {model_verification_block}ENTRYPOINT ["python", "/app/backend/operator-runner/main.py"] """ if slug in HOLOBRAIN_SLUGS: base_arg = f"ARG OPERATOR_HOLOBRAIN_BASE_IMAGE={DEFAULT_HOLOBRAIN_BASE_IMAGE}" elif slug.startswith("yolo_"): base_arg = f"ARG OPERATOR_YOLO_BASE_IMAGE={DEFAULT_YOLO_BASE_IMAGE}" else: base_arg = f"ARG OPERATOR_GPU_BASE_IMAGE={DEFAULT_GPU_BASE_IMAGE}" qwen_bos_dependency = " bce-python-sdk" if slug == "qwen_behavior_rough_cut" else "" qwen3vl_dependency_install = ( "RUN python -m pip install --no-cache-dir \\\n" " transformers==4.57.6 accelerate==1.14.0 safetensors==0.8.0 peft==0.19.0 \\\n" f" ms-swift==4.5.2 qwen-vl-utils==0.0.14 av==16.0.1 decord==0.6.0 opencv-python-headless==4.11.0.86{qwen_bos_dependency}\n" if slug in QWEN3VL_SLUGS else "" ) holobrain_shim_copy = ( "COPY operator-images/builtin/holobrain-shims/pytorch3d /app/pytorch3d\n" if slug in HOLOBRAIN_RUNTIME_SLUGS else "" ) holobrain_asset_copy = ( "COPY operator-images/builtin/holobrain-assets/urdf /opt/RoboOrchardLab/projects/holobrain/urdf\n" "COPY operator-images/builtin/holobrain-vendor/RoboOrchardLab/robo_orchard_lab/models/holobrain/processor.py /opt/RoboOrchardLab/robo_orchard_lab/models/holobrain/processor.py\n" if slug in HOLOBRAIN_RUNTIME_SLUGS else "" ) holobrain_robotwin_env_copy = ( "COPY operator-images/builtin/holobrain-vendor/RoboOrchardLab/robo_orchard_lab/envs/robotwin/env.py /opt/RoboOrchardLab/robo_orchard_lab/envs/robotwin/env.py\n" "COPY operator-images/builtin/holobrain-vendor/RoboOrchardLab/robo_orchard_lab/policy/evaluator/benchmark/robotwin.py /opt/RoboOrchardLab/robo_orchard_lab/policy/evaluator/benchmark/robotwin.py\n" "COPY operator-images/builtin/holobrain-vendor/RoboOrchardLab/robo_orchard_lab/envs/robotwin/kinematics.py /opt/RoboOrchardLab/robo_orchard_lab/envs/robotwin/kinematics.py\n" "COPY operator-images/builtin/holobrain-vendor/RoboTwin/envs/robot/robot.py /opt/RoboTwin/envs/robot/robot.py\n" "COPY operator-images/builtin/holobrain-vendor/RoboTwin/envs/_base_task.py /opt/RoboTwin/envs/_base_task.py\n" if slug in HOLOBRAIN_ROBOTWIN_EVAL_RUNTIME_SLUGS else "" ) holobrain_train_dependency_fix = "" if slug in HOLOBRAIN_RUNTIME_SLUGS: if slug in HOLOBRAIN_ROBOTWIN_EVAL_RUNTIME_SLUGS: holobrain_train_dependency_fix = ( f'RUN "$(command -v python)" -m venv --system-site-packages {HOLOBRAIN_ROBOTWIN_VENV} && \\\n' " printf 'numpy==1.26.4\\n' > /tmp/holobrain-robotwin-constraints.txt && \\\n" f" {HOLOBRAIN_ROBOTWIN_VENV}/bin/python -m pip install --no-cache-dir --force-reinstall " '-c /tmp/holobrain-robotwin-constraints.txt "numpy==1.26.4" ' '"scipy>=1.14,<1.16" "einops>=0.8,<0.9" && \\\n' f" {HOLOBRAIN_ROBOTWIN_VENV}/bin/python -m pip install --no-cache-dir --force-reinstall --no-deps \\\n" ' "transformers==4.57.6" "accelerate==1.14.0" "tokenizers==0.22.2" \\\n' ' "huggingface_hub==0.36.2" "safetensors==0.8.0" && \\\n' f" {HOLOBRAIN_ROBOTWIN_VENV}/bin/python -m pip install --no-cache-dir --no-deps " '"https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/' 'flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl" && \\\n' f" {HOLOBRAIN_ROBOTWIN_VENV}/bin/python -m pip install --no-cache-dir " "-c /tmp/holobrain-robotwin-constraints.txt \\\n" ' "mcap-protobuf-support>=0.5.3" "mcap-ros2-support" "mcap>=1.2.2" \\\n' ' "foxglove-schemas-protobuf>=0.3.0" "robo_orchard_schemas==0.2.0" "foxglove-sdk" \\\n' ' "sapien==3.0.3" "mplib" "transforms3d" "trimesh" "open3d" "toppra" "warp-lang==1.12.0" && \\\n' " hash -r && \\\n" " python -c 'import importlib.metadata as metadata; import sys; " "from pathlib import Path; import numpy as np; " f'venv = Path("{HOLOBRAIN_ROBOTWIN_VENV}").resolve(); ' 'package_path = Path(np.__file__).resolve(); distribution = metadata.distribution("numpy"); ' 'distribution_path = Path(distribution.locate_file("")).resolve(); ' 'assert np.__version__ == metadata.version("numpy") == "1.26.4"; ' 'assert Path(sys.prefix).resolve() == venv, sys.prefix; ' 'assert Path(sys.executable).is_relative_to(venv), sys.executable; ' 'assert package_path.is_relative_to(venv), package_path; ' 'assert distribution_path.is_relative_to(venv), distribution_path; import accelerate\'\n' ) else: holobrain_train_dependency_fix = ( 'RUN python -m pip install --no-cache-dir --force-reinstall "numpy>=2.0,<2.3" ' '"scipy>=1.14,<1.16" "einops>=0.8,<0.9" && \\\n' ' python -m pip install --no-cache-dir --force-reinstall --no-deps \\\n' ' "transformers==4.57.6" "accelerate==1.14.0" "tokenizers==0.22.2" \\\n' ' "huggingface_hub==0.36.2" "safetensors==0.8.0" && \\\n' ' python -m pip install --no-cache-dir --no-deps "https://github.com/Dao-AILab/' 'flash-attention/releases/download/v2.7.4.post1/' 'flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp311-cp311-linux_x86_64.whl"\n' ) holobrain_runtime_path_env = ( f" PATH={HOLOBRAIN_ROBOTWIN_VENV}/bin:${{PATH}} \\\n" if slug in HOLOBRAIN_ROBOTWIN_EVAL_RUNTIME_SLUGS else "" ) return f"""# Generated by scripts/generate_operator_images.py. Do not edit by hand. {base_arg} FROM {docker_base(slug)} WORKDIR /app ENV PYTHONUNBUFFERED=1 \\ PYTHONPATH=/app \\ {holobrain_runtime_path_env} DROBOTICFLOW_OPERATOR_ID={operator_id} \\ DROBOTICFLOW_OPERATOR_SLUG={slug} \\ DROBOTICFLOW_OPERATOR_SOURCE=/app/operator-repos/builtin/{slug}/src/operator.py \\ DROBOTICFLOW_OPERATOR_REPOSITORY=/app/operator-repos/builtin/{slug} \\ DROBOTICFLOW_OPERATOR_HANDLER={handler} \\ USER=droboticflow \\ LOGNAME=droboticflow \\ HOME=/tmp/droboticflow-home RUN mkdir -p /tmp/droboticflow-home /workspace /outputs COPY backend ./backend {holobrain_train_dependency_fix}{qwen3vl_dependency_install}{holobrain_shim_copy}{holobrain_asset_copy}{holobrain_robotwin_env_copy}COPY operator-repos/builtin/{slug} ./operator-repos/builtin/{slug} ENTRYPOINT ["python", "/app/backend/operator-runner/main.py"] """ def gpu_base_dockerfile() -> str: return """# Generated by scripts/generate_operator_images.py. Do not edit by hand. FROM pytorch/pytorch:2.6.0-cuda12.4-cudnn9-runtime WORKDIR /app ENV PYTHONUNBUFFERED=1 \\ PYTHONPATH=/app \\ USER=droboticflow \\ LOGNAME=droboticflow \\ HOME=/tmp/droboticflow-home RUN apt-get update && apt-get install -y --no-install-recommends \\ build-essential \\ ffmpeg \\ git \\ libglib2.0-0 \\ libgl1 \\ linux-libc-dev \\ python3-dev \\ && rm -rf /var/lib/apt/lists/* RUN printf 'torch==2.6.0\\ntorchvision==0.21.0\\n' > /tmp/gpu-constraints.txt RUN python -m pip install --no-cache-dir -c /tmp/gpu-constraints.txt transformers accelerate safetensors nvidia-ml-py RUN python -m pip install --no-cache-dir -c /tmp/gpu-constraints.txt lerobot peft draccus wandb hydra-core omegaconf zarr numcodecs dill einops diffusers RUN git clone --depth=1 https://github.com/openvla/openvla.git /opt/openvla && \ git clone --depth=1 https://github.com/real-stanford/diffusion_policy.git /opt/diffusion_policy RUN mkdir -p /tmp/droboticflow-home /workspace /outputs """ def holobrain_base_dockerfile() -> str: return """# Generated by scripts/generate_operator_images.py. Do not edit by hand. ARG HOLOBRAIN_OFFLINE_PARENT_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-base@sha256:cc23ad2eb5d7515f2135ef9380ca69e9b5578be3b74044e1e6d708c47f08d127 FROM $HOLOBRAIN_OFFLINE_PARENT_IMAGE ARG UBUNTU_MIRROR=https://mirrors.aliyun.com/ubuntu WORKDIR /app ENV PYTHONUNBUFFERED=1 \\ PYTHONPATH=/app:/opt/RoboOrchardLab:/opt/RoboTwin \\ PIP_DEFAULT_TIMEOUT=120 \\ PIP_RETRIES=10 \\ USER=droboticflow \\ LOGNAME=droboticflow \\ HOME=/tmp/droboticflow-home \\ ROBOORCHARDLAB_ROOT=/opt/RoboOrchardLab \\ RoboTwin_PATH=/opt/RoboTwin \\ NVIDIA_DRIVER_CAPABILITIES=compute,utility,graphics \\ HF_HOME=/opt/hf-cache \\ HF_HUB_CACHE=/opt/hf-cache/hub \\ TRANSFORMERS_CACHE=/opt/hf-cache/transformers \\ HOLOBRAIN_MODEL_DIR=/opt/models/HoloBrain_v0.0_Qwen RUN sed -i \\ -e "s|http://archive.ubuntu.com/ubuntu|$UBUNTU_MIRROR|g" \\ -e "s|http://security.ubuntu.com/ubuntu|$UBUNTU_MIRROR|g" \\ /etc/apt/sources.list && \\ apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update && \\ apt-get -o Acquire::Retries=5 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 install -y --no-install-recommends \\ build-essential \\ ffmpeg \\ git \\ libglib2.0-0 \\ libgl1 \\ libvulkan1 \\ linux-libc-dev \\ mesa-vulkan-drivers \\ python3-dev \\ && rm -rf /var/lib/apt/lists/* RUN printf 'torch==2.6.0\\ntorchvision==0.21.0\\n' > /tmp/holobrain-constraints.txt RUN python -m pip install --no-cache-dir -c /tmp/holobrain-constraints.txt \\ accelerate==0.34.2 \\ cloudpickle \\ datasets \\ deprecated \\ diffusers \\ "duckdb<=1.3.2" \\ duckdb-engine \\ flask \\ gevent \\ gymnasium \\ h5py \\ huggingface_hub \\ imageio[ffmpeg] \\ lmdb \\ "numpy>=2.0,<2.3" \\ numpydantic \\ nvidia-ml-py \\ opencv-python \\ ordered-set \\ pandas \\ peft \\ pydantic-settings \\ pytorch-kinematics \\ pyzstd \\ ray[default] \\ rtoml \\ safetensors \\ "scipy>=1.14,<1.16" \\ sortedcontainers \\ tensorboard \\ terminaltables \\ timeout-decorator \\ torchmetrics \\ "transformers<=5.8" \\ "websockets>=10.0" RUN rm -rf /opt/robo_orchard_core /opt/RoboOrchardLab /opt/RoboTwin COPY operator-images/builtin/holobrain-vendor/robo_orchard_core /opt/robo_orchard_core COPY operator-images/builtin/holobrain-vendor/RoboOrchardLab /opt/RoboOrchardLab COPY operator-images/builtin/holobrain-vendor/RoboTwin /opt/RoboTwin RUN cd /opt/robo_orchard_core && \\ python -m pip install --no-cache-dir -e . --no-deps && \\ cd /opt/RoboOrchardLab && \\ python -m pip install --no-cache-dir -e . --no-deps RUN python -m pip install --no-cache-dir accelerate==0.34.2 RUN test -d /opt/models/HoloBrain_v0.0_Qwen && \\ find /opt/models/HoloBrain_v0.0_Qwen -name '*.safetensors' -print -quit | grep -q . && \\ test -d /opt/hf-cache/hub/models--Qwen--Qwen2.5-VL-3B-Instruct && \\ find /opt/hf-cache/hub/models--Qwen--Qwen2.5-VL-3B-Instruct -name '*.safetensors' -print -quit | grep -q . ENV HF_HUB_OFFLINE=1 \\ TRANSFORMERS_OFFLINE=1 \\ HF_DATASETS_OFFLINE=1 RUN mkdir -p /opt/RoboOrchardLab/projects/holobrain/data /tmp/droboticflow-home /workspace /outputs && \\ chown -R 10001:10001 /opt/RoboOrchardLab/projects/holobrain/data /tmp/droboticflow-home && \\ chmod -R a+rX /opt/RoboOrchardLab /opt/RoboTwin /opt/hf-cache /opt/models """ def yolo_base_dockerfile() -> str: return """# Generated by scripts/generate_operator_images.py. Do not edit by hand. FROM ultralytics/ultralytics:latest WORKDIR /app ENV PYTHONUNBUFFERED=1 \\ PYTHONPATH=/app \\ USER=droboticflow \\ LOGNAME=droboticflow \\ HOME=/tmp/droboticflow-home RUN python -m pip install --no-cache-dir nvidia-ml-py RUN mkdir -p /tmp/droboticflow-home /workspace /outputs """ def contract_for(item: dict[str, Any], slug: str) -> dict[str, Any]: dependencies = RUNTIME_DEPENDENCIES.get(slug, []) gpu = slug in GPU_SLUGS remote_gpu_service = slug in REMOTE_GPU_SERVICE_SLUGS handler = container_handler(item, slug) resource_contract: dict[str, Any] = { "requiresGpu": gpu, "recommendedGpuType": GPU_TYPE_OVERRIDES.get(slug, "NVIDIA A800 80GB") if gpu or remote_gpu_service else "", "minGpuCount": 1 if gpu else 0, } if remote_gpu_service: resource_contract["externalGpuService"] = True contract = { "schemaVersion": "droboticflow.operator.runtime/v1", "operatorId": item["id"], "operatorSlug": slug, "handler": handler, "entrypoint": "/app/backend/operator-runner/main.py", "imagePolicy": { "independentFinalImage": True, "allowsSharedBaseLayer": True, "forbidsSharedOperatorRunnerFinalImage": True, }, "runtimeDependencies": dependencies, "resources": resource_contract, "parameters": { "source": "hyperparams", "structured": True, "commandWrapperAllowed": False, "controlledScriptAllowed": item["id"] == "operator-builtin-shell-command", }, "artifacts": { "requiresManifest": True, "requiresChecksum": True, "requiresLineage": True, }, "acceptance": { "requiresRealFixture": True, "requiresFixtureChecksum": True, "requiresArtifactChecksum": True, "requiresWorkflowTransfer": True, "requiresGpuEvidence": gpu or remote_gpu_service, }, } if slug in FANYU_V5_ALLOW_EMPTY_SEMANTIC_RESULT_SLUGS: contract["acceptance"].update( { "requiresNonEmptyOutput": False, "allowsEmptyResultList": True, "requiresNonEmptyManifest": True, } ) if slug in LONG_RUNNING_SLUGS: contract["longRunning"] = True return contract def strip_disallowed_command_params(item: dict[str, Any]) -> dict[str, Any]: if item["id"] in COMMAND_WRAPPER_ALLOWED: return item hyperparams = [ param for param in item.get("hyperparams", []) if str(param.get("key", "")).strip().upper() not in {"COMMAND", "SCRIPT"} ] if len(hyperparams) != len(item.get("hyperparams", [])): item = {**item, "hyperparams": hyperparams} return item def expected_catalog() -> list[dict[str, Any]]: items = load_json(CATALOG_PATH) next_items: list[dict[str, Any]] = [] for item in items: if item.get("type") != "builtin": next_items.append(item) continue slug = operator_slug(item["id"]) handler = item.get("runtimeHandler") or slug next_item = strip_disallowed_command_params(dict(item)) overrides = STRUCTURED_OPERATOR_OVERRIDES.get(next_item["id"], {}) if "description" in overrides: next_item["description"] = overrides["description"] if "skillDefinition" in overrides: next_item["skillDefinition"] = overrides["skillDefinition"] if "hyperparams" in overrides: next_item["hyperparams"] = overrides["hyperparams"] # Draft source may intentionally differ from the last published image. # Keep the catalog's previous image reference until a new digest-pinned # release is built; published/built entries still require release truth. is_draft = str(next_item.get("publicationStatus", "")).strip().lower() == "draft" try: next_item["image"], next_item["imageId"] = release_image_reference(slug) except RuntimeError: if not is_draft: raise next_item["containerCommand"] = COMMAND_ENTRYPOINT next_item["containerArgs"] = [] next_item["runtimeSource"] = f"operator-repos/builtin/{slug}/src/operator.py" next_item["runtimeContract"] = contract_for(next_item, slug) next_items.append(next_item) return next_items def expected_files(catalog_items: list[dict[str, Any]]) -> dict[pathlib.Path, str]: files: dict[pathlib.Path, str] = {} manifest_items: list[dict[str, Any]] = [] files[IMAGE_ROOT / "gpu-base.Dockerfile"] = gpu_base_dockerfile() files[IMAGE_ROOT / "holobrain-base.Dockerfile"] = holobrain_base_dockerfile() files[IMAGE_ROOT / "yolo-base.Dockerfile"] = yolo_base_dockerfile() for item in catalog_items: if item.get("type") != "builtin": continue slug = operator_slug(item["id"]) metadata_path = OPERATOR_REPO_ROOT / slug / "operator.json" if not metadata_path.exists(): raise RuntimeError(f"missing operator metadata for {item['id']}: {metadata_path.relative_to(ROOT)}") handler = container_handler(item, slug) files[OPERATOR_REPO_ROOT / slug / "Dockerfile"] = dockerfile(slug, item["id"], handler) manifest_items.append( { "operatorId": item["id"], "operatorSlug": slug, "image": item["image"], "imageId": item["imageId"], "dockerfile": f"operator-repos/builtin/{slug}/Dockerfile", "sourceRoot": f"operator-repos/builtin/{slug}", "runtimeSource": f"operator-repos/builtin/{slug}/src/operator.py", "runtimeHandler": handler, "baseImage": docker_base(slug), "dependencyProfile": RUNTIME_DEPENDENCIES.get(slug, []), "requiresGpu": slug in GPU_SLUGS, "containerCommand": COMMAND_ENTRYPOINT, "containerArgs": [], } ) files[MANIFEST_PATH] = dump_json( { "schemaVersion": "droboticflow.operator-images/v1", "registry": DEFAULT_REGISTRY, "tag": IMAGE_TAG, "items": sorted(manifest_items, key=lambda item: item["operatorId"]), } ) return files def write_files(catalog_items: list[dict[str, Any]], files: dict[pathlib.Path, str]) -> None: CATALOG_PATH.write_text(dump_json(catalog_items), encoding="utf-8") for path, content in files.items(): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") def check_files(catalog_items: list[dict[str, Any]], files: dict[pathlib.Path, str]) -> int: mismatches: list[str] = [] expected_catalog_text = dump_json(catalog_items) if CATALOG_PATH.read_text(encoding="utf-8") != expected_catalog_text: mismatches.append(f"outdated {CATALOG_PATH.relative_to(ROOT)}") for path, content in files.items(): if not path.exists(): mismatches.append(f"missing {path.relative_to(ROOT)}") continue if path.read_text(encoding="utf-8") != content: mismatches.append(f"outdated {path.relative_to(ROOT)}") if mismatches: for mismatch in mismatches: print(mismatch, file=sys.stderr) return 1 return 0 def main() -> int: parser = argparse.ArgumentParser(description="Generate or check independent builtin operator images.") mode = parser.add_mutually_exclusive_group() mode.add_argument("--write", action="store_true", help="write catalog image contracts and Dockerfiles") mode.add_argument("--check", action="store_true", help="check generated catalog image contracts and Dockerfiles") args = parser.parse_args() catalog_items = expected_catalog() files = expected_files(catalog_items) if args.check: return check_files(catalog_items, files) write_files(catalog_items, files) return 0 if __name__ == "__main__": raise SystemExit(main())