droboticflow/scripts/generate_operator_docs.py
eust-w 934527d746
Some checks are pending
CI / verify (push) Waiting to run
CI / docker-build (push) Waiting to run
🧾 Make operator doc source explicit
2026-06-16 22:26:42 +08:00

623 lines
32 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import re
import urllib.request
from collections import defaultdict
from pathlib import Path
from typing import Any
DEFAULT_BUILTIN_OUTPUT_DIR = "docs/operators"
DEFAULT_CUSTOM_OUTPUT_DIR = "docs/custom-operators/instances"
DEFAULT_BUILTIN_CATALOG_PATH = "internal/api/builtin_operators_catalog.json"
def load_builtin_handler_hints(path: str = DEFAULT_BUILTIN_CATALOG_PATH) -> dict[str, tuple[str, str]]:
catalog_path = Path(path)
if not catalog_path.exists():
return {}
payload = json.loads(catalog_path.read_text(encoding="utf-8"))
if not isinstance(payload, list):
raise SystemExit(f"{path} must contain a JSON array")
hints: dict[str, tuple[str, str]] = {}
for item in payload:
if not isinstance(item, dict):
continue
operator_id = str(item.get("id", "")).strip()
runtime_source = str(item.get("runtimeSource", "")).strip()
runtime_handler = str(item.get("runtimeHandler", "")).strip()
if operator_id and runtime_source and runtime_handler:
hints[operator_id] = (runtime_source, runtime_handler)
return hints
BUILTIN_HANDLER_HINTS = load_builtin_handler_hints()
PORT_TYPE_HINTS = {
"json": "适合结构化清单、规则结果、摘要对象等可直接序列化的数据。",
"image": "适合单帧图片、截图、标注图或图片引用。",
"video": "适合视频文件或 clip 引用,通常不会以内联字节返回。",
"tensor": "适合模型中间结果或高维数组,通常需要外置存储。",
"file": "适合 CSV、压缩包、文档、配置文件等任意文件产物。",
}
TRANSMISSION_HINTS = {
"bus": "通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。",
"oss": "更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。",
"shm": "偏向节点内共享内存/本地工作目录语义,要求上下游执行环境紧耦合。",
"stream": "偏向流式消费场景,需要上下游在时序和消费方式上兼容。",
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Generate detailed markdown manuals for builtin and custom operators")
parser.add_argument("--api-base-url", default="", help="Fetch operator payload from a specific live API /api/operators endpoint")
parser.add_argument("--input-json", default="", help="Read operator payload from a local JSON file instead of calling /api/operators")
parser.add_argument("--builtin-output-dir", default=DEFAULT_BUILTIN_OUTPUT_DIR)
parser.add_argument("--custom-output-dir", default=DEFAULT_CUSTOM_OUTPUT_DIR)
return parser.parse_args()
def request_json(url: str) -> Any:
with urllib.request.urlopen(url, timeout=60) as resp:
return json.loads(resp.read().decode("utf-8"))
def load_operator_payload(args: argparse.Namespace) -> tuple[Any, str]:
if args.input_json and args.api_base_url:
raise SystemExit("set only one of --input-json or --api-base-url")
if args.input_json:
input_path = Path(args.input_json)
return json.loads(input_path.read_text(encoding="utf-8")), str(input_path)
if args.api_base_url:
api_base_url = args.api_base_url.rstrip("/")
return request_json(f"{api_base_url}/api/operators"), f"{api_base_url}/api/operators"
catalog_path = Path(DEFAULT_BUILTIN_CATALOG_PATH)
return json.loads(catalog_path.read_text(encoding="utf-8")), str(catalog_path)
def load_operator_items(args: argparse.Namespace) -> list[dict[str, Any]]:
payload, _ = load_operator_payload(args)
return operator_items_from_payload(payload)
def operator_items_from_payload(payload: Any) -> list[dict[str, Any]]:
if isinstance(payload, dict) and isinstance(payload.get("items"), list):
items = payload.get("items", [])
elif isinstance(payload, list):
items = payload
elif isinstance(payload, dict) and str(payload.get("name", "")).strip():
items = [payload]
else:
raise SystemExit("operator payload must be one of: {items:[...]}, [...], or a single operator object")
return [item for item in items if isinstance(item, dict)]
def slugify(value: str) -> str:
cleaned = re.sub(r"[\s/]+", "-", value.strip())
cleaned = re.sub(r"[^0-9A-Za-z\-_.\u4e00-\u9fff]+", "-", cleaned)
cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-")
return cleaned or "operator"
def choose_builtin_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
for item in items:
if str(item.get("type", "")).strip() != "builtin":
continue
name = str(item.get("name", "")).strip()
if name:
grouped[name].append(item)
chosen: list[dict[str, Any]] = []
for name in sorted(grouped):
records = grouped[name]
records.sort(key=lambda item: (0 if str(item.get("type")) == "builtin" else 1, str(item.get("id", ""))))
chosen.append(records[0])
return chosen
def choose_custom_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
chosen = [item for item in items if str(item.get("type", "")).strip() == "custom" and str(item.get("name", "")).strip()]
chosen.sort(key=lambda item: (str(item.get("category", "")).strip(), str(item.get("name", "")).strip(), str(item.get("id", "")).strip()))
return chosen
def clean_markdown_dir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
for item in path.glob("*.md"):
item.unlink()
def stringify(value: Any) -> str:
if value is None:
return ""
if isinstance(value, (dict, list)):
return json.dumps(value, ensure_ascii=False)
return str(value)
def render_port_detail(port: dict[str, Any], direction: str) -> str:
port_id = str(port.get("id", "")).strip() or "-"
name = str(port.get("name", "")).strip() or "-"
port_type = str(port.get("type", "")).strip() or "-"
transmission = str(port.get("transmission", "")).strip() or "-"
description = str(port.get("description", "")).strip() or "未额外说明。"
type_hint = PORT_TYPE_HINTS.get(port_type, "表示该端口承载的是平台允许的对应数据形态。")
transmission_hint = TRANSMISSION_HINTS.get(transmission, "表示该端口采用平台约定的传输语义。")
return (
f"- `{port_id}` / **{name}** / `{port_type}` / `{transmission}`:{description} "
f"作为{direction}端口时,{type_hint} {transmission_hint}"
)
def render_hyperparam_detail(param: dict[str, Any]) -> str:
key = str(param.get("key", "")).strip() or "-"
default_value = stringify(param.get("value", "")).strip()
default_text = f"`{default_value}`" if default_value else "空字符串或由工作流显式填写"
description = str(param.get("description", "")).strip() or "未提供描述。"
ptype = str(param.get("type", "")).strip() or "string"
required_hint = "通常建议在工作流中显式配置。" if not default_value else "可直接使用默认值,也可在节点级覆盖。"
return f"- `{key}`:类型 `{ptype}`,默认值 {default_text}。{description} {required_hint}"
def render_skill_definition_block(operator: dict[str, Any]) -> list[str]:
skill_definition = str(operator.get("skillDefinition", "")).strip()
if not skill_definition:
return []
return ["## 平台语义", "", skill_definition, ""]
def operator_positioning(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
category = str(operator.get("category", "")).strip() or "OTHER"
operator_type = str(operator.get("type", "")).strip() or "unknown"
lines = [f"- 该算子属于 `{category}` 类,当前登记类型是 `{operator_type}`。"]
if operator_type == "builtin":
lines.append("- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。")
else:
lines.append("- 该算子属于自定义算子,平台只负责登记元数据与调度,实际业务逻辑由镜像、脚本或命令实现。")
if name in {"真实数据源", "BOS 增量扫描", "目录扫描"}:
lines.append("- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。")
if name in {"结果上传", "MongoDB 写入"}:
lines.append("- 它更偏向工作流出口,职责是把平台内部产物同步到外部对象存储或业务数据库。")
if name in {"视频质检", "视频切片", "视频标注", "成品打包"}:
lines.append("- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。")
if name in {"JSON 字段提取", "JSON 过滤", "JSON 映射重命名", "JSON 转 CSV", "CSV 转 JSON", "条件分流", "数据合并", "文本模板渲染"}:
lines.append("- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。")
return lines
def operator_runtime_contract(operator: dict[str, Any]) -> list[str]:
operator_id = str(operator.get("id", "")).strip()
operator_type = str(operator.get("type", "")).strip()
lines = [
"- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。",
"- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。",
"- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。",
]
if operator_type == "builtin":
source_file, handler_name = BUILTIN_HANDLER_HINTS.get(operator_id, ("backend/operator-runner/main.py", "builtin_generic"))
lines.append(f"- 当前实现入口通常可追到 `{source_file}` 中的 `{handler_name}(...)`。")
else:
lines.append("- 当前类型是自定义算子,建议在元数据中通过 `skillDefinition` 明确输入要求、超参数含义、产物结构和失败语义。")
if operator_id in {"operator-builtin-real-source", "operator-builtin-bos-incremental-scan"}:
lines.append("- 该类源算子没有显式上游输入,主要依赖节点超参数和运行时注入的存储连接信息。")
if operator_id == "operator-builtin-upload":
lines.append("- 该类末端算子通常没有输出端口,验收重点是外部对象存储中的真实写入结果。")
return lines
def operator_boundaries(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
lines = [
"- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。",
"- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。",
]
if name == "结果上传":
lines.append("- 上传节点只应负责收集并落地结果,不应顺便做复杂转换;转换应在上传前完成。")
if name == "人工审核":
lines.append("- 审核节点的职责是等待外部决策并路由分支,不应继续承担后置业务处理。")
if name in {"Shell 命令执行", "模型推理"}:
lines.append("- 这类通用执行算子适合快速验证,但当逻辑稳定后应考虑沉淀为专用内建算子或受控自定义算子。")
return lines
def operator_preparation(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
lines = [
"- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。",
"- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。",
"- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。",
]
if name in {"BOS 增量扫描", "结果上传", "MongoDB 写入"}:
lines.append("- 需要预先准备连接配置、凭据和目标路径,优先通过平台连接配置而不是把密钥散落到节点超参数里。")
if name in {"视频质检", "视频切片", "视频标注", "成品打包", "Shell 命令执行"}:
lines.append("- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。")
if name == "人工审核":
lines.append("- 需要提前确认审核 API、审核页面或人工流程是否可用,否则工作流会停在等待状态。")
return lines
def operator_data_path(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
lines: list[str] = []
if name in {"BOS 增量扫描", "真实数据源"}:
lines.extend(
[
"- 源算子通常先输出轻量级 manifest,记录 `entries`、`catalogEntries`、`bos://` 路径或摘要信息,而不是直接下载整个数据集。",
"- 下游第一次真正读取文件内容时,才会通过 runner 的物化逻辑把远程对象缓存到共享 PVC 或本地工作目录。",
]
)
elif name == "结果上传":
lines.extend(
[
"- 该节点读取上游的 `artifact.path`、`dataRef.payloadPath` 或本地文件目录,然后把真实文件上传到对象存储。",
"- 平台运行记录里通常只保留上传摘要、目标路径和状态,不重复存放完整文件内容。",
]
)
elif name in {"视频质检", "视频切片", "视频标注", "成品打包"}:
lines.extend(
[
"- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。",
"- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。",
]
)
elif name == "MongoDB 写入":
lines.extend(
[
"- 该节点主要消费结构化 JSON payload,并把整理后的文档写入目标 Mongo 集合。",
"- 平台主库只保留写入摘要、错误信息和执行状态,避免把已写入业务库的整批文档再次冗余保存。",
]
)
else:
lines.extend(
[
"- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。",
"- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。",
]
)
return lines
def operator_combinations(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
lines: list[str] = []
if name in {"BOS 增量扫描", "真实数据源"}:
lines.extend(
[
"- 常接 `X 平台数据集索引`、`目录扫描`、`模型推理` 或任意自定义 JSON 处理节点,作为工作流入口。",
"- 当需要持续监听新增目录或新增对象时,应把它放在整条 DAG 的最上游,并让后续节点只消费本轮增量。 ",
]
)
elif name == "X 平台数据集索引":
lines.extend(
[
"- 上游通常是 `BOS 增量扫描` 或 `真实数据源`,下游再接 `采集目录校验`、`数据过滤`、`视频质检` 等节点。",
"- 如果后续要做抽检或上传,建议把索引结果先标准化为结构化 manifest,再进入视频主链路。",
]
)
elif name == "采集目录校验":
lines.append("- 适合放在索引之后,用来把不完整目录尽早筛掉,减少后续视频处理成本。 ")
elif name == "视频质检":
lines.append("- 常位于 `视频过滤` 或 `数据过滤` 之前,为后续筛选提供清晰度、抖动度、遮挡率等量化依据。")
elif name == "视频切片":
lines.append("- 常接在视频筛选之后,并把 clip 结果交给 `视频标注`、`成品打包` 或单独的 `结果上传`。")
elif name == "视频标注":
lines.append("- 上游一般是 `视频切片`,下游则是 `成品打包`、`结果上传` 或人工质检支路。")
elif name == "成品打包":
lines.append("- 建议在打包后单独接一个上传节点;如果同时上传 clip、标注和 package,应该拆成多条上传支路。")
elif name == "MongoDB 写入":
lines.append("- 通常接在 JSON 提取、映射、模板渲染或清洗节点之后,确保入库文档已经满足目标库 schema。")
elif name == "人工审核":
lines.append("- 审核节点之后应明确分成 `approved` 和 `rejected` 两条支路,避免后续节点混收不同语义的数据。")
else:
lines.append("- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。")
return lines
def operator_examples(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
if name == "BOS 增量扫描":
return [
"- 示例配置:`SOURCE_PATH=bos://<bucket>/<prefix>`,`SCAN_MODE=INCREMENTAL`,`INCREMENTAL_UNIT=DIRECTORY`。",
"- 典型输出:`source-manifest.json`,其中 `entries` 表示本轮增量对象,`catalogEntries` 表示本轮完整列举结果。",
]
if name == "结果上传":
return [
"- 典型输入:来自上游的 `packages-manifest.json`、`annotation.json`、`clip.mp4` 或任意 `dataRef.payloadPath`。",
"- 典型输出:`uploaded-results.json`,记录每个对象的目标路径、对象大小、状态和错误信息。",
]
if name == "视频切片":
return [
"- 典型输入:带有视频路径的 manifest 或上游过滤结果。",
"- 典型输出:`clips-manifest.json` 与多个 `*_clip001.mp4` 文件,clip 实体位于当前节点 `OUTPUT_DIR/clips/` 下。",
]
if name == "视频标注":
return [
"- 典型输入:来自切片节点的 clip 视频与 `clips-manifest.json`。",
"- 典型输出:`annotations-manifest.json` 与逐 clip 的标注 JSON 文件。",
]
if name == "成品打包":
return [
"- 典型输入:clip、annotation、quality、配置文件或传感器补充数据。",
"- 典型输出:`packages-manifest.json` 和每个成品包目录。 ",
]
if name == "MongoDB 写入":
return [
"- 典型输入:结构化 JSON 行列表、单文档对象或经过模板渲染后的入库 payload。",
"- 典型输出:写入摘要、命中集合名、成功数量、失败数量和错误原因。",
]
return [
"- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。",
"- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。",
]
def operator_success_signals(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
lines = [
"- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。",
"- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。",
]
if name in {"结果上传", "MongoDB 写入"}:
lines.append("- 外部系统中存在真实副作用,例如对象已上传、文档已写入,而不只是平台内显示成功。")
if name == "人工审核":
lines.append("- 审核节点收到明确的 approve/reject 决策,并将正确的分支结果送达下游。")
if name in {"BOS 增量扫描", "真实数据源"}:
lines.append("- 输出 manifest 与真实源路径内容一致,没有漏扫、重扫或错误分类。")
return lines
def operator_failures(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
lines = [
"- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。",
"- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。",
"- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。",
]
if name in {"BOS 增量扫描", "真实数据源", "结果上传"}:
lines.append("- 对象存储异常时,先核对 `SOURCE_PATH`、bucket/prefix、endpoint、AK/SK 和目标对象是否真实存在。")
if name in {"视频质检", "视频切片", "视频标注", "成品打包"}:
lines.append("- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。")
if name == "MongoDB 写入":
lines.append("- 若写库失败,先确认目标集合不在保留名单中,再核对连接串、认证信息和文档结构是否可序列化。")
if name == "人工审核":
lines.append("- 节点停在等待状态通常不是故障,而是还没有收到人工决策。")
return lines
def operator_acceptance(operator: dict[str, Any]) -> list[str]:
name = str(operator.get("name", "")).strip()
lines = [
"- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。",
"- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。",
"- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。",
]
if name in {"BOS 增量扫描", "真实数据源"}:
lines.append("- 对照真实源目录核验 `entries`、目录数、对象数和增量语义是否正确。")
if name == "结果上传":
lines.append("- 到目标对象存储前缀核对真实对象、对象大小和命名规则,确认并非只在平台侧生成了摘要。")
if name == "MongoDB 写入":
lines.append("- 到目标 Mongo 集合查询最近写入数据,确认记录数量、关键字段和值类型都符合预期。")
return lines
def render_config_example(operator: dict[str, Any]) -> str:
params = {}
for item in operator.get("hyperparams") or []:
key = str(item.get("key", "")).strip()
if not key:
continue
raw_value = item.get("value", "")
params[key] = raw_value if raw_value != "" else "<按节点填写>"
example = {
"label": str(operator.get("name", "")).strip(),
"operatorId": str(operator.get("id", "")).strip(),
"type": str(operator.get("type", "")).strip(),
"category": str(operator.get("category", "")).strip(),
"hyperparams": params,
}
return json.dumps(example, ensure_ascii=False, indent=2)
def render_operator_doc(operator: dict[str, Any], related_index: str, related_docs: list[tuple[str, str]]) -> str:
name = str(operator.get("name", "")).strip()
description = str(operator.get("description", "")).strip() or "无描述"
operator_id = str(operator.get("id", "")).strip()
operator_type = str(operator.get("type", "")).strip()
category = str(operator.get("category", "")).strip() or "OTHER"
inputs = operator.get("inputs") or []
outputs = operator.get("outputs") or []
hyperparams = operator.get("hyperparams") or []
lines = [
f"# {name}",
"",
f"- 算子 ID:`{operator_id}`",
f"- 类别:`{category}`",
f"- 类型:`{operator_type}`",
"",
"## 作用",
"",
description,
"",
"## 定位与职责边界",
"",
]
lines.extend(operator_positioning(operator))
lines.extend([""])
lines.extend(render_skill_definition_block(operator))
lines.extend(["## 运行时契约", ""])
lines.extend(operator_runtime_contract(operator))
lines.extend(["", "## 设计边界", ""])
lines.extend(operator_boundaries(operator))
lines.extend(["", "## 数据流示意", "", "```mermaid", "flowchart LR", f" A[上游输入或运行时注入] --> B[{name}]", " B --> C[artifact / stage manifest / 外部副作用]", "```", ""])
lines.extend(["## 输入端口详解", ""])
if inputs:
lines.extend([render_port_detail(port, "输入") for port in inputs])
else:
lines.append("- 无显式输入端口。通常作为源算子使用,主要依赖节点超参数和运行时环境。")
lines.extend(["", "## 输出端口详解", ""])
if outputs:
lines.extend([render_port_detail(port, "输出") for port in outputs])
else:
lines.append("- 无显式输出端口。通常意味着该节点以落库、上传或触发外部副作用为主。")
lines.extend(["", "## 关键超参数", ""])
if hyperparams:
lines.extend([render_hyperparam_detail(param) for param in hyperparams])
else:
lines.append("- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。")
lines.extend(["", "## 执行前准备", ""])
lines.extend(operator_preparation(operator))
lines.extend(["", "## 数据通信与存储路径", ""])
lines.extend(operator_data_path(operator))
lines.extend(["", "## 典型上下游编排", ""])
lines.extend(operator_combinations(operator))
lines.extend(["", "## 节点配置示例", "", "```json", render_config_example(operator), "```", ""])
lines.extend(["## 输入输出示例", ""])
lines.extend(operator_examples(operator))
lines.extend(["", "## 成功判定", ""])
lines.extend(operator_success_signals(operator))
lines.extend(["", "## 常见失败与排查", ""])
lines.extend(operator_failures(operator))
lines.extend(["", "## 验收检查清单", ""])
lines.extend(operator_acceptance(operator))
lines.extend(["", "## 相关文档", "", f"- [{category} 类索引]({related_index})"])
lines.extend([f"- [{label}]({path})" for label, path in related_docs])
lines.append("")
return "\n".join(lines)
def write_builtin_docs(items: list[dict[str, Any]], output_dir: Path) -> int:
clean_markdown_dir(output_dir)
grouped: dict[str, list[tuple[str, str]]] = defaultdict(list)
for operator in items:
name = str(operator.get("name", "")).strip()
file_name = f"{slugify(name)}.md"
(output_dir / file_name).write_text(
render_operator_doc(
operator,
"README.md",
[
("算子开发文档", "../operator-authoring.md"),
("内建算子总手册", "../builtin-operators-manual.zh-CN.md"),
],
),
encoding="utf-8",
)
grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, file_name))
readme_lines = [
"# 内建算子分册",
"",
"当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。",
"",
f"- 当前收录数量:`{len(items)}`",
"- 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。",
"- 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md)",
"",
"## 使用建议",
"",
"- 先阅读总手册理解平台级规则,再进入本目录阅读单算子分册。",
"- 设计 DAG 时优先确认端口类型、artifact 语义、外部副作用和验收口径,而不是只看算子名称。",
"",
"## 分类索引",
"",
]
for category in sorted(grouped):
readme_lines.append(f"### {category}")
readme_lines.append("")
for name, file_name in sorted(grouped[category], key=lambda item: item[0]):
readme_lines.append(f"- [{name}]({file_name})")
readme_lines.append("")
(output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8")
return len(items)
def write_custom_docs(items: list[dict[str, Any]], output_dir: Path) -> int:
clean_markdown_dir(output_dir)
grouped: dict[str, list[tuple[str, str, str]]] = defaultdict(list)
for operator in items:
name = str(operator.get("name", "")).strip()
operator_id = str(operator.get("id", "")).strip()
suffix = slugify(operator_id[-8:] if len(operator_id) >= 8 else operator_id or name)
file_name = f"{slugify(name)}-{suffix}.md"
(output_dir / file_name).write_text(
render_operator_doc(
operator,
"README.md",
[
("自定义算子总手册", "../../custom-operator-manual.zh-CN.md"),
("自定义算子模板与示例", "../README.md"),
],
),
encoding="utf-8",
)
grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, operator_id, file_name))
readme_lines = [
"# 自定义算子实例分册",
"",
"当前目录专门存放从算子注册表导出的“实例级自定义算子 Markdown”。",
"",
f"- 当前实例数量:`{len(items)}`",
"- 配套总览:[`../README.md`](../README.md)",
"- 配套手册:[`../../custom-operator-manual.zh-CN.md`](../../custom-operator-manual.zh-CN.md)",
"",
]
if not items:
readme_lines.extend(
[
"## 当前状态",
"",
"当前导出的算子目录中没有 `custom` 类型实例,因此本目录暂时只有索引页。",
"",
"建议做法:",
"",
"- 先参考上一级目录中的模板与示例撰写自定义算子说明。",
"- 当环境中出现真实自定义算子后,再重新运行生成脚本为每个实例导出单独 Markdown。",
"",
]
)
else:
readme_lines.extend(["## 分类索引", ""])
for category in sorted(grouped):
readme_lines.append(f"### {category}")
readme_lines.append("")
for name, operator_id, file_name in sorted(grouped[category], key=lambda item: item[0]):
readme_lines.append(f"- [{name}]({file_name}),实例 ID:`{operator_id}`")
readme_lines.append("")
(output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8")
return len(items)
def main() -> int:
args = parse_args()
payload, source = load_operator_payload(args)
items = operator_items_from_payload(payload)
builtin_items = choose_builtin_records(items)
custom_items = choose_custom_records(items)
builtin_count = write_builtin_docs(builtin_items, Path(args.builtin_output_dir))
custom_count = write_custom_docs(custom_items, Path(args.custom_output_dir))
print(
json.dumps(
{
"status": "ok",
"builtinCount": builtin_count,
"customCount": custom_count,
"builtinOutputDir": args.builtin_output_dir,
"customOutputDir": args.custom_output_dir,
"source": source,
},
ensure_ascii=False,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())