forked from longtao.wu/droboticflow
200 lines
7.9 KiB
Python
200 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import pathlib
|
||
import sys
|
||
from typing import Any
|
||
|
||
|
||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||
CATALOG_PATH = ROOT / "internal" / "api" / "builtin_operators_catalog.json"
|
||
LEGACY_OPERATOR_ROOT = ROOT / "backend" / "operators" / "builtin"
|
||
OPERATOR_REPO_ROOT = ROOT / "operator-repos" / "builtin"
|
||
IMAGE_ROOT = ROOT / "operator-images" / "builtin"
|
||
sys.path.insert(0, str(ROOT / "scripts"))
|
||
from generate_operator_images import container_handler as image_container_handler
|
||
from generate_operator_images import dockerfile as image_dockerfile
|
||
|
||
CONTAINER_HANDLER_ALIASES = {
|
||
"builtin_lerobot_summary": "lerobot",
|
||
"builtin_rosbag_summary": "rosbag",
|
||
}
|
||
# This directory arrived from the upstream merge with an operator id whose
|
||
# slug does not match its legacy source directory. It remains preserved for
|
||
# historical runs but is deliberately not reconciled as a current catalog
|
||
# builtin until its digest-pinned release migration is completed.
|
||
UNMIGRATED_HISTORICAL_REPOSITORIES = {"temporal_causal_eval"}
|
||
|
||
|
||
def load_json(path: pathlib.Path) -> Any:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def dump_json(payload: Any) -> str:
|
||
return json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||
|
||
|
||
def operator_slug(operator_id: str) -> str:
|
||
return operator_id.removeprefix("operator-builtin-").replace("-", "_")
|
||
|
||
|
||
def container_handler(handler: str) -> str:
|
||
return str(handler)
|
||
|
||
|
||
def read_runtime_contract(catalog_item: dict[str, Any]) -> dict[str, Any]:
|
||
contract = dict(catalog_item.get("runtimeContract") or {})
|
||
handler = contract.get("handler") or catalog_item.get("runtimeHandler") or operator_slug(catalog_item["id"])
|
||
contract["handler"] = container_handler(str(handler))
|
||
contract["sourceRoot"] = f"operator-repos/builtin/{operator_slug(catalog_item['id'])}"
|
||
contract["runtimeSource"] = "src/operator.py"
|
||
return contract
|
||
|
||
|
||
def dockerfile_text(slug: str, catalog_item: dict[str, Any]) -> str:
|
||
return image_dockerfile(slug, catalog_item["id"], image_container_handler(catalog_item, slug))
|
||
|
||
|
||
def readme_text(slug: str, catalog_item: dict[str, Any]) -> str:
|
||
role_note = ""
|
||
if slug == "wilor_valid_event_detection":
|
||
role_note = (
|
||
"## 运行语义\n\n"
|
||
"该节点保留 WiLoR 需求链中的历史名称以保持工作流兼容;有效行为判定实际由本地、固定 revision 的 **Qwen3-VL** 完成。"
|
||
"它不会调用或把 WiLoR hand-pose inference 冒充为动作分类。真实推理可以得到空 event 列表,但必须产生可校验的非空 manifest、模型 provenance 与真实视频证据。\n"
|
||
)
|
||
base = f"""# {catalog_item.get('name', slug)}
|
||
|
||
独立 builtin 算子目录。该目录是 `{catalog_item['id']}` 的生产运行权威来源,包含镜像构建、运行契约、源码和验收输入定义。
|
||
|
||
- 发布镜像:以 `release.json` 中的 digest-pinned CCR imageId 为准
|
||
- Repository manifest:`repository.json`
|
||
- Runtime handler:`{catalog_item.get('runtimeHandler', slug)}`
|
||
- Source:`src/operator.py`
|
||
|
||
生产验收要求真实 fixture、artifact checksum、lineage 证据;不接受 mock、仿真、smoke 或空文件验收。
|
||
"""
|
||
return base + ("\n" + role_note if role_note else "")
|
||
|
||
|
||
def requirements_text(slug: str, catalog_item: dict[str, Any]) -> str:
|
||
deps = ((catalog_item.get("runtimeContract") or {}).get("runtimeDependencies") or [])
|
||
package_hints = {
|
||
"pyarrow": "pyarrow",
|
||
"h5py": "h5py",
|
||
"numpy": "numpy",
|
||
"pyyaml": "pyyaml",
|
||
"zarr": "zarr",
|
||
"mcap-python": "mcap",
|
||
"opencv-python-headless": "opencv-python-headless",
|
||
"mongodb": "pymongo",
|
||
"huggingface-dataset-viewer-api": "requests",
|
||
}
|
||
packages = sorted({package_hints[item] for item in deps if item in package_hints})
|
||
return "\n".join(packages) + ("\n" if packages else "")
|
||
|
||
|
||
def fixture_contract(catalog_item: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"schemaVersion": "droboticflow.operator.fixture/v1",
|
||
"operatorId": catalog_item["id"],
|
||
"operatorSlug": operator_slug(catalog_item["id"]),
|
||
"fixtures": [],
|
||
"rules": {
|
||
"requiresPublicSourceUrl": True,
|
||
"requiresLicense": True,
|
||
"requiresSha256": True,
|
||
"forbidsMock": True,
|
||
"forbidsSynthetic": True,
|
||
"forbidsSimulationOnly": True,
|
||
"forbidsSmokeOnly": True,
|
||
},
|
||
"minimumEvidence": [
|
||
"input checksum",
|
||
"artifact checksum",
|
||
"lineage",
|
||
"schema validation",
|
||
],
|
||
}
|
||
|
||
|
||
def expected_files() -> dict[pathlib.Path, str]:
|
||
catalog_items = [item for item in load_json(CATALOG_PATH) if item.get("type") == "builtin"]
|
||
files: dict[pathlib.Path, str] = {}
|
||
for item in catalog_items:
|
||
slug = operator_slug(item["id"])
|
||
legacy_dir = LEGACY_OPERATOR_ROOT / slug
|
||
metadata_path = legacy_dir / "operator.json"
|
||
source_path = legacy_dir / "operator.py"
|
||
if not metadata_path.exists() or not source_path.exists():
|
||
raise RuntimeError(f"missing legacy operator source for {item['id']}")
|
||
metadata = load_json(metadata_path)
|
||
target = OPERATOR_REPO_ROOT / slug
|
||
files[target / "Dockerfile"] = dockerfile_text(slug, item)
|
||
files[target / "operator.json"] = dump_json(
|
||
{
|
||
**metadata,
|
||
"runtimeHandler": item.get("runtimeHandler", metadata.get("runtimeHandler")),
|
||
"runtimeSource": f"operator-repos/builtin/{slug}/src/operator.py",
|
||
}
|
||
)
|
||
files[target / "runtime_contract.json"] = dump_json(read_runtime_contract(item))
|
||
files[target / "README.md"] = readme_text(slug, item)
|
||
files[target / "requirements.txt"] = requirements_text(slug, item)
|
||
files[target / "src" / "__init__.py"] = ""
|
||
files[target / "src" / "operator.py"] = source_path.read_text(encoding="utf-8")
|
||
if slug == "shell_command":
|
||
approved = LEGACY_OPERATOR_ROOT / slug / "approved_scripts" / "fixture_report.py"
|
||
if approved.exists():
|
||
files[target / "approved_scripts" / "fixture_report.py"] = approved.read_text(encoding="utf-8")
|
||
files[target / "tests" / "acceptance" / "fixture_contract.json"] = dump_json(fixture_contract(item))
|
||
return files
|
||
|
||
|
||
def write_files(files: dict[pathlib.Path, str]) -> None:
|
||
for path, content in files.items():
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(content, encoding="utf-8")
|
||
|
||
|
||
def check_files(files: dict[pathlib.Path, str]) -> int:
|
||
mismatches: list[str] = []
|
||
for path, expected in files.items():
|
||
if not path.exists():
|
||
mismatches.append(f"missing {path.relative_to(ROOT)}")
|
||
continue
|
||
if path.read_text(encoding="utf-8") != expected:
|
||
mismatches.append(f"outdated {path.relative_to(ROOT)}")
|
||
expected_dirs = {path.parent for path in files if path.name == "operator.json"}
|
||
actual_dirs = {
|
||
path.parent
|
||
for path in OPERATOR_REPO_ROOT.glob("*/operator.json")
|
||
if path.parent.name not in UNMIGRATED_HISTORICAL_REPOSITORIES
|
||
}
|
||
for extra in sorted(actual_dirs - expected_dirs):
|
||
mismatches.append(f"unexpected {extra.relative_to(ROOT)}")
|
||
if mismatches:
|
||
for mismatch in mismatches:
|
||
print(mismatch, file=sys.stderr)
|
||
return 1
|
||
return 0
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description="Generate or check split builtin operator repositories.")
|
||
mode = parser.add_mutually_exclusive_group()
|
||
mode.add_argument("--write", action="store_true")
|
||
mode.add_argument("--check", action="store_true")
|
||
args = parser.parse_args()
|
||
files = expected_files()
|
||
if args.check:
|
||
return check_files(files)
|
||
write_files(files)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|