droboticflow/scripts/generate_operator_repositories.py

200 lines
7.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import pathlib
import sys
from typing import Any
ROOT = pathlib.Path(__file__).resolve().parents[1]
CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
LEGACY_OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin"
OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin"
IMAGE_ROOT = ROOT / "operator-images" / "builtin"
sys.path.insert(0, str(ROOT / "scripts"))
from generate_operator_images import container_handler as image_container_handler
from generate_operator_images import dockerfile as image_dockerfile
CONTAINER_HANDLER_ALIASES = {
"builtin_lerobot_summary": "lerobot",
"builtin_rosbag_summary": "rosbag",
}
# This directory arrived from the upstream merge with an operator id whose
# slug does not match its legacy source directory. It remains preserved for
# historical runs but is deliberately not reconciled as a current catalog
# builtin until its digest-pinned release migration is completed.
UNMIGRATED_HISTORICAL_REPOSITORIES = {"temporal_causal_eval"}
def load_json(path: pathlib.Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def dump_json(payload: Any) -> str:
return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
def operator_slug(operator_id: str) -> str:
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
def container_handler(handler: str) -> str:
return str(handler)
def read_runtime_contract(catalog_item: dict[str, Any]) -> dict[str, Any]:
contract = dict(catalog_item.get("runtimeContract") or {})
handler = contract.get("handler") or catalog_item.get("runtimeHandler") or operator_slug(catalog_item["id"])
contract["handler"] = container_handler(str(handler))
contract["sourceRoot"] = f"operator-repos/builtin/{operator_slug(catalog_item['id'])}"
contract["runtimeSource"] = "src/operator.py"
return contract
def dockerfile_text(slug: str, catalog_item: dict[str, Any]) -> str:
return image_dockerfile(slug, catalog_item["id"], image_container_handler(catalog_item, slug))
def readme_text(slug: str, catalog_item: dict[str, Any]) -> str:
role_note = ""
if slug == "wilor_valid_event_detection":
role_note = (
"## 运行语义\n\n"
"该节点保留 WiLoR 需求链中的历史名称以保持工作流兼容;有效行为判定实际由本地、固定 revision 的 **Qwen3-VL** 完成。"
"它不会调用或把 WiLoR hand-pose inference 冒充为动作分类。真实推理可以得到空 event 列表,但必须产生可校验的非空 manifest、模型 provenance 与真实视频证据。\n"
)
base = f"""# {catalog_item.get('name', slug)}
独立 builtin 算子目录。该目录是 `{catalog_item['id']}` 的生产运行权威来源,包含镜像构建、运行契约、源码和验收输入定义。
- 发布镜像:以 `release.json` 中的 digest-pinned CCR imageId 为准
- Repository manifest:`repository.json`
- Runtime handler:`{catalog_item.get('runtimeHandler', slug)}`
- Source:`src/operator.py`
生产验收要求真实 fixture、artifact checksum、lineage 证据;不接受 mock、仿真、smoke 或空文件验收。
"""
return base + ("\n" + role_note if role_note else "")
def requirements_text(slug: str, catalog_item: dict[str, Any]) -> str:
deps = ((catalog_item.get("runtimeContract") or {}).get("runtimeDependencies") or [])
package_hints = {
"pyarrow": "pyarrow",
"h5py": "h5py",
"numpy": "numpy",
"pyyaml": "pyyaml",
"zarr": "zarr",
"mcap-python": "mcap",
"opencv-python-headless": "opencv-python-headless",
"mongodb": "pymongo",
"huggingface-dataset-viewer-api": "requests",
}
packages = sorted({package_hints[item] for item in deps if item in package_hints})
return "\n".join(packages) + ("\n" if packages else "")
def fixture_contract(catalog_item: dict[str, Any]) -> dict[str, Any]:
return {
"schemaVersion": "droboticflow.operator.fixture/v1",
"operatorId": catalog_item["id"],
"operatorSlug": operator_slug(catalog_item["id"]),
"fixtures": [],
"rules": {
"requiresPublicSourceUrl": True,
"requiresLicense": True,
"requiresSha256": True,
"forbidsMock": True,
"forbidsSynthetic": True,
"forbidsSimulationOnly": True,
"forbidsSmokeOnly": True,
},
"minimumEvidence": [
"input checksum",
"artifact checksum",
"lineage",
"schema validation",
],
}
def expected_files() -> dict[pathlib.Path, str]:
catalog_items = [item for item in load_json(CATALOG_PATH) if item.get("type") == "builtin"]
files: dict[pathlib.Path, str] = {}
for item in catalog_items:
slug = operator_slug(item["id"])
legacy_dir = LEGACY_OPERATOR_ROOT / slug
metadata_path = legacy_dir / "operator.json"
source_path = legacy_dir / "operator.py"
if not metadata_path.exists() or not source_path.exists():
raise RuntimeError(f"missing legacy operator source for {item['id']}")
metadata = load_json(metadata_path)
target = OPERATOR_REPO_ROOT / slug
files[target / "Dockerfile"] = dockerfile_text(slug, item)
files[target / "operator.json"] = dump_json(
{
**metadata,
"runtimeHandler": item.get("runtimeHandler", metadata.get("runtimeHandler")),
"runtimeSource": f"operator-repos/builtin/{slug}/src/operator.py",
}
)
files[target / "runtime_contract.json"] = dump_json(read_runtime_contract(item))
files[target / "README.md"] = readme_text(slug, item)
files[target / "requirements.txt"] = requirements_text(slug, item)
files[target / "src" / "__init__.py"] = ""
files[target / "src" / "operator.py"] = source_path.read_text(encoding="utf-8")
if slug == "shell_command":
approved = LEGACY_OPERATOR_ROOT / slug / "approved_scripts" / "fixture_report.py"
if approved.exists():
files[target / "approved_scripts" / "fixture_report.py"] = approved.read_text(encoding="utf-8")
files[target / "tests" / "acceptance" / "fixture_contract.json"] = dump_json(fixture_contract(item))
return files
def write_files(files: dict[pathlib.Path, str]) -> None:
for path, content in files.items():
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
def check_files(files: dict[pathlib.Path, str]) -> int:
mismatches: list[str] = []
for path, expected in files.items():
if not path.exists():
mismatches.append(f"missing {path.relative_to(ROOT)}")
continue
if path.read_text(encoding="utf-8") != expected:
mismatches.append(f"outdated {path.relative_to(ROOT)}")
expected_dirs = {path.parent for path in files if path.name == "operator.json"}
actual_dirs = {
path.parent
for path in OPERATOR_REPO_ROOT.glob("*/operator.json")
if path.parent.name not in UNMIGRATED_HISTORICAL_REPOSITORIES
}
for extra in sorted(actual_dirs - expected_dirs):
mismatches.append(f"unexpected {extra.relative_to(ROOT)}")
if mismatches:
for mismatch in mismatches:
print(mismatch, file=sys.stderr)
return 1
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="Generate or check split builtin operator repositories.")
mode = parser.add_mutually_exclusive_group()
mode.add_argument("--write", action="store_true")
mode.add_argument("--check", action="store_true")
args = parser.parse_args()
files = expected_files()
if args.check:
return check_files(files)
write_files(files)
return 0
if __name__ == "__main__":
raise SystemExit(main())