#!/usr/bin/env python3 from __future__ import annotations import argparse import json import re import urllib.request from collections import defaultdict from pathlib import Path from typing import Any DEFAULT_API_BASE_URL = "http://127.0.0.1:3001" DEFAULT_BUILTIN_OUTPUT_DIR = "docs/operators" DEFAULT_CUSTOM_OUTPUT_DIR = "docs/custom-operators/instances" BUILTIN_HANDLER_HINTS: dict[str, tuple[str, str]] = { "operator-builtin-real-source": ("backend/operator-runner/main.py", "builtin_source"), "operator-builtin-bos-incremental-scan": ("backend/operator-runner/main.py", "builtin_source"), "operator-builtin-model-inference": ("backend/operator-runner/main.py", "builtin_generic"), "operator-builtin-merge": ("backend/operator-runner/main.py", "builtin_merge"), "operator-builtin-upload": ("backend/operator-runner/main.py", "builtin_upload"), "operator-builtin-human-review": ("backend/operator-runner/main.py", "builtin_condition_router"), "operator-builtin-directory-scan": ("backend/operator-runner/main.py", "builtin_directory_scan"), "operator-builtin-csv-to-json": ("backend/operator-runner/main.py", "builtin_csv_to_json"), "operator-builtin-json-to-csv": ("backend/operator-runner/main.py", "builtin_json_to_csv"), "operator-builtin-json-field-extract": ("backend/operator-runner/main.py", "builtin_json_extract"), "operator-builtin-json-filter": ("backend/operator-runner/main.py", "builtin_json_filter"), "operator-builtin-json-map-rename": ("backend/operator-runner/main.py", "builtin_json_map"), "operator-builtin-template-render": ("backend/operator-runner/main.py", "builtin_text_template"), "operator-builtin-conditional-route": ("backend/operator-runner/main.py", "builtin_condition_router"), "operator-builtin-xplatform-dataset-index": ("backend/operator-runner/main.py", "builtin_xplatform_index"), "operator-builtin-xplatform-dataset-validate": ("backend/operator-runner/main.py", "builtin_collection_validate"), "operator-builtin-video-quality": ("backend/operator-runner/main.py", "builtin_video_quality"), "operator-builtin-video-filter": ("backend/operator-runner/main.py", "builtin_dataset_filter"), "operator-builtin-video-slice": ("backend/operator-runner/main.py", "builtin_video_slicing"), "operator-builtin-video-annotate": ("backend/operator-runner/main.py", "builtin_video_annotation"), "operator-builtin-package": ("backend/operator-runner/main.py", "builtin_result_package"), "operator-builtin-audit-sample": ("backend/operator-runner/main.py", "builtin_audit_sampling"), "operator-builtin-mongodb-write": ("backend/operator-runner/main.py", "builtin_mongodb_write"), "operator-builtin-lerobot-summary": ("backend/operator-runner/main.py", "builtin_lerobot_summary"), "operator-builtin-rosbag-summary": ("backend/operator-runner/main.py", "builtin_rosbag_summary"), "operator-builtin-shell-command": ("backend/operator-runner/main.py", "builtin_generic"), } PORT_TYPE_HINTS = { "json": "适合结构化清单、规则结果、摘要对象等可直接序列化的数据。", "image": "适合单帧图片、截图、标注图或图片引用。", "video": "适合视频文件或 clip 引用,通常不会以内联字节返回。", "tensor": "适合模型中间结果或高维数组,通常需要外置存储。", "file": "适合 CSV、压缩包、文档、配置文件等任意文件产物。", } TRANSMISSION_HINTS = { "bus": "通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。", "oss": "更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。", "shm": "偏向节点内共享内存/本地工作目录语义,要求上下游执行环境紧耦合。", "stream": "偏向流式消费场景,需要上下游在时序和消费方式上兼容。", } def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Generate detailed markdown manuals for builtin and custom operators") parser.add_argument("--api-base-url", default=DEFAULT_API_BASE_URL) parser.add_argument("--input-json", default="", help="Read operator payload from a local JSON file instead of calling /api/operators") parser.add_argument("--builtin-output-dir", default=DEFAULT_BUILTIN_OUTPUT_DIR) parser.add_argument("--custom-output-dir", default=DEFAULT_CUSTOM_OUTPUT_DIR) return parser.parse_args() def request_json(url: str) -> Any: with urllib.request.urlopen(url, timeout=60) as resp: return json.loads(resp.read().decode("utf-8")) def load_operator_items(args: argparse.Namespace) -> list[dict[str, Any]]: if args.input_json: payload = json.loads(Path(args.input_json).read_text(encoding="utf-8")) else: api_base_url = args.api_base_url.rstrip("/") payload = request_json(f"{api_base_url}/api/operators") if isinstance(payload, dict) and isinstance(payload.get("items"), list): items = payload.get("items", []) elif isinstance(payload, list): items = payload elif isinstance(payload, dict) and str(payload.get("name", "")).strip(): items = [payload] else: raise SystemExit("operator payload must be one of: {items:[...]}, [...], or a single operator object") return [item for item in items if isinstance(item, dict)] def slugify(value: str) -> str: cleaned = re.sub(r"[\s/]+", "-", value.strip()) cleaned = re.sub(r"[^0-9A-Za-z\-_.\u4e00-\u9fff]+", "-", cleaned) cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-") return cleaned or "operator" def choose_builtin_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]: grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) for item in items: if str(item.get("type", "")).strip() != "builtin": continue name = str(item.get("name", "")).strip() if name: grouped[name].append(item) chosen: list[dict[str, Any]] = [] for name in sorted(grouped): records = grouped[name] records.sort(key=lambda item: (0 if str(item.get("type")) == "builtin" else 1, str(item.get("id", "")))) chosen.append(records[0]) return chosen def choose_custom_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]: chosen = [item for item in items if str(item.get("type", "")).strip() == "custom" and str(item.get("name", "")).strip()] chosen.sort(key=lambda item: (str(item.get("category", "")).strip(), str(item.get("name", "")).strip(), str(item.get("id", "")).strip())) return chosen def clean_markdown_dir(path: Path) -> None: path.mkdir(parents=True, exist_ok=True) for item in path.glob("*.md"): item.unlink() def stringify(value: Any) -> str: if value is None: return "" if isinstance(value, (dict, list)): return json.dumps(value, ensure_ascii=False) return str(value) def render_port_detail(port: dict[str, Any], direction: str) -> str: port_id = str(port.get("id", "")).strip() or "-" name = str(port.get("name", "")).strip() or "-" port_type = str(port.get("type", "")).strip() or "-" transmission = str(port.get("transmission", "")).strip() or "-" description = str(port.get("description", "")).strip() or "未额外说明。" type_hint = PORT_TYPE_HINTS.get(port_type, "表示该端口承载的是平台允许的对应数据形态。") transmission_hint = TRANSMISSION_HINTS.get(transmission, "表示该端口采用平台约定的传输语义。") return ( f"- `{port_id}` / **{name}** / `{port_type}` / `{transmission}`:{description} " f"作为{direction}端口时,{type_hint} {transmission_hint}" ) def render_hyperparam_detail(param: dict[str, Any]) -> str: key = str(param.get("key", "")).strip() or "-" default_value = stringify(param.get("value", "")).strip() default_text = f"`{default_value}`" if default_value else "空字符串或由工作流显式填写" description = str(param.get("description", "")).strip() or "未提供描述。" ptype = str(param.get("type", "")).strip() or "string" required_hint = "通常建议在工作流中显式配置。" if not default_value else "可直接使用默认值,也可在节点级覆盖。" return f"- `{key}`:类型 `{ptype}`,默认值 {default_text}。{description} {required_hint}" def render_skill_definition_block(operator: dict[str, Any]) -> list[str]: skill_definition = str(operator.get("skillDefinition", "")).strip() if not skill_definition: return [] return ["## 平台语义", "", skill_definition, ""] def operator_positioning(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() category = str(operator.get("category", "")).strip() or "OTHER" operator_type = str(operator.get("type", "")).strip() or "unknown" lines = [f"- 该算子属于 `{category}` 类,当前登记类型是 `{operator_type}`。"] if operator_type == "builtin": lines.append("- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 的既有 handler 接管。") else: lines.append("- 该算子属于自定义算子,平台只负责登记元数据与调度,实际业务逻辑由镜像、脚本或命令实现。") if name in {"真实数据源", "BOS 增量扫描", "目录扫描"}: lines.append("- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。") if name in {"结果上传", "MongoDB 写入"}: lines.append("- 它更偏向工作流出口,职责是把平台内部产物同步到外部对象存储或业务数据库。") if name in {"视频质检", "视频切片", "视频标注", "成品打包"}: lines.append("- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。") if name in {"JSON 字段提取", "JSON 过滤", "JSON 映射重命名", "JSON 转 CSV", "CSV 转 JSON", "条件分流", "数据合并", "文本模板渲染"}: lines.append("- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。") return lines def operator_runtime_contract(operator: dict[str, Any]) -> list[str]: operator_id = str(operator.get("id", "")).strip() operator_type = str(operator.get("type", "")).strip() lines = [ "- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。", "- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。", "- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。", ] if operator_type == "builtin": source_file, handler_name = BUILTIN_HANDLER_HINTS.get(operator_id, ("backend/operator-runner/main.py", "builtin_generic")) lines.append(f"- 当前实现入口通常可追到 `{source_file}` 中的 `{handler_name}(...)`。") else: lines.append("- 当前类型是自定义算子,建议在元数据中通过 `skillDefinition` 明确输入要求、超参数含义、产物结构和失败语义。") if operator_id in {"operator-builtin-real-source", "operator-builtin-bos-incremental-scan"}: lines.append("- 该类源算子没有显式上游输入,主要依赖节点超参数和运行时注入的存储连接信息。") if operator_id == "operator-builtin-upload": lines.append("- 该类末端算子通常没有输出端口,验收重点是外部对象存储中的真实写入结果。") return lines def operator_boundaries(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() lines = [ "- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。", "- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。", ] if name == "结果上传": lines.append("- 上传节点只应负责收集并落地结果,不应顺便做复杂转换;转换应在上传前完成。") if name == "人工审核": lines.append("- 审核节点的职责是等待外部决策并路由分支,不应继续承担后置业务处理。") if name in {"Shell 命令执行", "模型推理"}: lines.append("- 这类通用执行算子适合快速验证,但当逻辑稳定后应考虑沉淀为专用内建算子或受控自定义算子。") return lines def operator_preparation(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() lines = [ "- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。", "- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。", "- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。", ] if name in {"BOS 增量扫描", "结果上传", "MongoDB 写入"}: lines.append("- 需要预先准备连接配置、凭据和目标路径,优先通过平台连接配置而不是把密钥散落到节点超参数里。") if name in {"视频质检", "视频切片", "视频标注", "成品打包", "Shell 命令执行"}: lines.append("- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。") if name == "人工审核": lines.append("- 需要提前确认审核 API、审核页面或人工流程是否可用,否则工作流会停在等待状态。") return lines def operator_data_path(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() lines: list[str] = [] if name in {"BOS 增量扫描", "真实数据源"}: lines.extend( [ "- 源算子通常先输出轻量级 manifest,记录 `entries`、`catalogEntries`、`bos://` 路径或摘要信息,而不是直接下载整个数据集。", "- 下游第一次真正读取文件内容时,才会通过 runner 的物化逻辑把远程对象缓存到共享 PVC 或本地工作目录。", ] ) elif name == "结果上传": lines.extend( [ "- 该节点读取上游的 `artifact.path`、`dataRef.payloadPath` 或本地文件目录,然后把真实文件上传到对象存储。", "- 平台运行记录里通常只保留上传摘要、目标路径和状态,不重复存放完整文件内容。", ] ) elif name in {"视频质检", "视频切片", "视频标注", "成品打包"}: lines.extend( [ "- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。", "- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。", ] ) elif name == "MongoDB 写入": lines.extend( [ "- 该节点主要消费结构化 JSON payload,并把整理后的文档写入目标 Mongo 集合。", "- 平台主库只保留写入摘要、错误信息和执行状态,避免把已写入业务库的整批文档再次冗余保存。", ] ) else: lines.extend( [ "- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。", "- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。", ] ) return lines def operator_combinations(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() lines: list[str] = [] if name in {"BOS 增量扫描", "真实数据源"}: lines.extend( [ "- 常接 `X 平台数据集索引`、`目录扫描`、`模型推理` 或任意自定义 JSON 处理节点,作为工作流入口。", "- 当需要持续监听新增目录或新增对象时,应把它放在整条 DAG 的最上游,并让后续节点只消费本轮增量。 ", ] ) elif name == "X 平台数据集索引": lines.extend( [ "- 上游通常是 `BOS 增量扫描` 或 `真实数据源`,下游再接 `采集目录校验`、`数据过滤`、`视频质检` 等节点。", "- 如果后续要做抽检或上传,建议把索引结果先标准化为结构化 manifest,再进入视频主链路。", ] ) elif name == "采集目录校验": lines.append("- 适合放在索引之后,用来把不完整目录尽早筛掉,减少后续视频处理成本。 ") elif name == "视频质检": lines.append("- 常位于 `视频过滤` 或 `数据过滤` 之前,为后续筛选提供清晰度、抖动度、遮挡率等量化依据。") elif name == "视频切片": lines.append("- 常接在视频筛选之后,并把 clip 结果交给 `视频标注`、`成品打包` 或单独的 `结果上传`。") elif name == "视频标注": lines.append("- 上游一般是 `视频切片`,下游则是 `成品打包`、`结果上传` 或人工质检支路。") elif name == "成品打包": lines.append("- 建议在打包后单独接一个上传节点;如果同时上传 clip、标注和 package,应该拆成多条上传支路。") elif name == "MongoDB 写入": lines.append("- 通常接在 JSON 提取、映射、模板渲染或清洗节点之后,确保入库文档已经满足目标库 schema。") elif name == "人工审核": lines.append("- 审核节点之后应明确分成 `approved` 和 `rejected` 两条支路,避免后续节点混收不同语义的数据。") else: lines.append("- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。") return lines def operator_examples(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() if name == "BOS 增量扫描": return [ "- 示例配置:`SOURCE_PATH=bos:///`,`SCAN_MODE=INCREMENTAL`,`INCREMENTAL_UNIT=DIRECTORY`。", "- 典型输出:`source-manifest.json`,其中 `entries` 表示本轮增量对象,`catalogEntries` 表示本轮完整列举结果。", ] if name == "结果上传": return [ "- 典型输入:来自上游的 `packages-manifest.json`、`annotation.json`、`clip.mp4` 或任意 `dataRef.payloadPath`。", "- 典型输出:`uploaded-results.json`,记录每个对象的目标路径、对象大小、状态和错误信息。", ] if name == "视频切片": return [ "- 典型输入:带有视频路径的 manifest 或上游过滤结果。", "- 典型输出:`clips-manifest.json` 与多个 `*_clip001.mp4` 文件,clip 实体位于当前节点 `OUTPUT_DIR/clips/` 下。", ] if name == "视频标注": return [ "- 典型输入:来自切片节点的 clip 视频与 `clips-manifest.json`。", "- 典型输出:`annotations-manifest.json` 与逐 clip 的标注 JSON 文件。", ] if name == "成品打包": return [ "- 典型输入:clip、annotation、quality、配置文件或传感器补充数据。", "- 典型输出:`packages-manifest.json` 和每个成品包目录。 ", ] if name == "MongoDB 写入": return [ "- 典型输入:结构化 JSON 行列表、单文档对象或经过模板渲染后的入库 payload。", "- 典型输出:写入摘要、命中集合名、成功数量、失败数量和错误原因。", ] return [ "- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。", "- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。", ] def operator_success_signals(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() lines = [ "- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。", "- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。", ] if name in {"结果上传", "MongoDB 写入"}: lines.append("- 外部系统中存在真实副作用,例如对象已上传、文档已写入,而不只是平台内显示成功。") if name == "人工审核": lines.append("- 审核节点收到明确的 approve/reject 决策,并将正确的分支结果送达下游。") if name in {"BOS 增量扫描", "真实数据源"}: lines.append("- 输出 manifest 与真实源路径内容一致,没有漏扫、重扫或错误分类。") return lines def operator_failures(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() lines = [ "- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。", "- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。", "- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。", ] if name in {"BOS 增量扫描", "真实数据源", "结果上传"}: lines.append("- 对象存储异常时,先核对 `SOURCE_PATH`、bucket/prefix、endpoint、AK/SK 和目标对象是否真实存在。") if name in {"视频质检", "视频切片", "视频标注", "成品打包"}: lines.append("- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。") if name == "MongoDB 写入": lines.append("- 若写库失败,先确认目标集合不在保留名单中,再核对连接串、认证信息和文档结构是否可序列化。") if name == "人工审核": lines.append("- 节点停在等待状态通常不是故障,而是还没有收到人工决策。") return lines def operator_acceptance(operator: dict[str, Any]) -> list[str]: name = str(operator.get("name", "")).strip() lines = [ "- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。", "- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。", "- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。", ] if name in {"BOS 增量扫描", "真实数据源"}: lines.append("- 对照真实源目录核验 `entries`、目录数、对象数和增量语义是否正确。") if name == "结果上传": lines.append("- 到目标对象存储前缀核对真实对象、对象大小和命名规则,确认并非只在平台侧生成了摘要。") if name == "MongoDB 写入": lines.append("- 到目标 Mongo 集合查询最近写入数据,确认记录数量、关键字段和值类型都符合预期。") return lines def render_config_example(operator: dict[str, Any]) -> str: params = {} for item in operator.get("hyperparams") or []: key = str(item.get("key", "")).strip() if not key: continue raw_value = item.get("value", "") params[key] = raw_value if raw_value != "" else "<按节点填写>" example = { "label": str(operator.get("name", "")).strip(), "operatorId": str(operator.get("id", "")).strip(), "type": str(operator.get("type", "")).strip(), "category": str(operator.get("category", "")).strip(), "hyperparams": params, } return json.dumps(example, ensure_ascii=False, indent=2) def render_operator_doc(operator: dict[str, Any], related_index: str, related_docs: list[tuple[str, str]]) -> str: name = str(operator.get("name", "")).strip() description = str(operator.get("description", "")).strip() or "无描述" operator_id = str(operator.get("id", "")).strip() operator_type = str(operator.get("type", "")).strip() category = str(operator.get("category", "")).strip() or "OTHER" inputs = operator.get("inputs") or [] outputs = operator.get("outputs") or [] hyperparams = operator.get("hyperparams") or [] lines = [ f"# {name}", "", f"- 算子 ID:`{operator_id}`", f"- 类别:`{category}`", f"- 类型:`{operator_type}`", "", "## 作用", "", description, "", "## 定位与职责边界", "", ] lines.extend(operator_positioning(operator)) lines.extend([""]) lines.extend(render_skill_definition_block(operator)) lines.extend(["## 运行时契约", ""]) lines.extend(operator_runtime_contract(operator)) lines.extend(["", "## 设计边界", ""]) lines.extend(operator_boundaries(operator)) lines.extend(["", "## 数据流示意", "", "```mermaid", "flowchart LR", f" A[上游输入或运行时注入] --> B[{name}]", " B --> C[artifact / stage manifest / 外部副作用]", "```", ""]) lines.extend(["## 输入端口详解", ""]) if inputs: lines.extend([render_port_detail(port, "输入") for port in inputs]) else: lines.append("- 无显式输入端口。通常作为源算子使用,主要依赖节点超参数和运行时环境。") lines.extend(["", "## 输出端口详解", ""]) if outputs: lines.extend([render_port_detail(port, "输出") for port in outputs]) else: lines.append("- 无显式输出端口。通常意味着该节点以落库、上传或触发外部副作用为主。") lines.extend(["", "## 关键超参数", ""]) if hyperparams: lines.extend([render_hyperparam_detail(param) for param in hyperparams]) else: lines.append("- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。") lines.extend(["", "## 执行前准备", ""]) lines.extend(operator_preparation(operator)) lines.extend(["", "## 数据通信与存储路径", ""]) lines.extend(operator_data_path(operator)) lines.extend(["", "## 典型上下游编排", ""]) lines.extend(operator_combinations(operator)) lines.extend(["", "## 节点配置示例", "", "```json", render_config_example(operator), "```", ""]) lines.extend(["## 输入输出示例", ""]) lines.extend(operator_examples(operator)) lines.extend(["", "## 成功判定", ""]) lines.extend(operator_success_signals(operator)) lines.extend(["", "## 常见失败与排查", ""]) lines.extend(operator_failures(operator)) lines.extend(["", "## 验收检查清单", ""]) lines.extend(operator_acceptance(operator)) lines.extend(["", "## 相关文档", "", f"- [{category} 类索引]({related_index})"]) lines.extend([f"- [{label}]({path})" for label, path in related_docs]) lines.append("") return "\n".join(lines) def write_builtin_docs(items: list[dict[str, Any]], output_dir: Path) -> int: clean_markdown_dir(output_dir) grouped: dict[str, list[tuple[str, str]]] = defaultdict(list) for operator in items: name = str(operator.get("name", "")).strip() file_name = f"{slugify(name)}.md" (output_dir / file_name).write_text( render_operator_doc( operator, "README.md", [ ("算子开发文档", "../operator-authoring.md"), ("内建算子总手册", "../builtin-operators-manual.zh-CN.md"), ], ), encoding="utf-8", ) grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, file_name)) readme_lines = [ "# 内建算子分册", "", "当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。", "", f"- 当前收录数量:`{len(items)}`", "- 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。", "- 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md)", "", "## 使用建议", "", "- 先阅读总手册理解平台级规则,再进入本目录阅读单算子分册。", "- 设计 DAG 时优先确认端口类型、artifact 语义、外部副作用和验收口径,而不是只看算子名称。", "", "## 分类索引", "", ] for category in sorted(grouped): readme_lines.append(f"### {category}") readme_lines.append("") for name, file_name in sorted(grouped[category], key=lambda item: item[0]): readme_lines.append(f"- [{name}]({file_name})") readme_lines.append("") (output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8") return len(items) def write_custom_docs(items: list[dict[str, Any]], output_dir: Path) -> int: clean_markdown_dir(output_dir) grouped: dict[str, list[tuple[str, str, str]]] = defaultdict(list) for operator in items: name = str(operator.get("name", "")).strip() operator_id = str(operator.get("id", "")).strip() suffix = slugify(operator_id[-8:] if len(operator_id) >= 8 else operator_id or name) file_name = f"{slugify(name)}-{suffix}.md" (output_dir / file_name).write_text( render_operator_doc( operator, "README.md", [ ("自定义算子总手册", "../../custom-operator-manual.zh-CN.md"), ("自定义算子模板与示例", "../README.md"), ], ), encoding="utf-8", ) grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, operator_id, file_name)) readme_lines = [ "# 自定义算子实例分册", "", "当前目录专门存放从算子注册表导出的“实例级自定义算子 Markdown”。", "", f"- 当前实例数量:`{len(items)}`", "- 配套总览:[`../README.md`](../README.md)", "- 配套手册:[`../../custom-operator-manual.zh-CN.md`](../../custom-operator-manual.zh-CN.md)", "", ] if not items: readme_lines.extend( [ "## 当前状态", "", "当前导出的算子目录中没有 `custom` 类型实例,因此本目录暂时只有索引页。", "", "建议做法:", "", "- 先参考上一级目录中的模板与示例撰写自定义算子说明。", "- 当环境中出现真实自定义算子后,再重新运行生成脚本为每个实例导出单独 Markdown。", "", ] ) else: readme_lines.extend(["## 分类索引", ""]) for category in sorted(grouped): readme_lines.append(f"### {category}") readme_lines.append("") for name, operator_id, file_name in sorted(grouped[category], key=lambda item: item[0]): readme_lines.append(f"- [{name}]({file_name}),实例 ID:`{operator_id}`") readme_lines.append("") (output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8") return len(items) def main() -> int: args = parse_args() items = load_operator_items(args) builtin_items = choose_builtin_records(items) custom_items = choose_custom_records(items) builtin_count = write_builtin_docs(builtin_items, Path(args.builtin_output_dir)) custom_count = write_custom_docs(custom_items, Path(args.custom_output_dir)) print( json.dumps( { "status": "ok", "builtinCount": builtin_count, "customCount": custom_count, "builtinOutputDir": args.builtin_output_dir, "customOutputDir": args.custom_output_dir, }, ensure_ascii=False, ) ) return 0 if __name__ == "__main__": raise SystemExit(main())