661 lines
34 KiB
Python
661 lines
34 KiB
Python
#!/usr/bin/env python3
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import urllib.request
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
DEFAULT_BUILTIN_OUTPUT_DIR = "docs/operators"
|
||
DEFAULT_CUSTOM_OUTPUT_DIR = "docs/custom-operators/instances"
|
||
DEFAULT_BUILTIN_CATALOG_PATH = "internal/api/builtin_operators_catalog.json"
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
|
||
|
||
def load_builtin_handler_hints(path: str = DEFAULT_BUILTIN_CATALOG_PATH) -> dict[str, tuple[str, str]]:
|
||
catalog_path = Path(path)
|
||
if not catalog_path.exists():
|
||
return {}
|
||
payload = json.loads(catalog_path.read_text(encoding="utf-8"))
|
||
if not isinstance(payload, list):
|
||
raise SystemExit(f"{path} must contain a JSON array")
|
||
hints: dict[str, tuple[str, str]] = {}
|
||
for item in payload:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
operator_id = str(item.get("id", "")).strip()
|
||
runtime_source = str(item.get("runtimeSource", "")).strip()
|
||
runtime_handler = str(item.get("runtimeHandler", "")).strip()
|
||
if operator_id and runtime_source and runtime_handler:
|
||
hints[operator_id] = (runtime_source, runtime_handler)
|
||
return hints
|
||
|
||
|
||
BUILTIN_HANDLER_HINTS = load_builtin_handler_hints()
|
||
|
||
PORT_TYPE_HINTS = {
|
||
"json": "适合结构化清单、规则结果、摘要对象等可直接序列化的数据。",
|
||
"image": "适合单帧图片、截图、标注图或图片引用。",
|
||
"video": "适合视频文件或 clip 引用,通常不会以内联字节返回。",
|
||
"tensor": "适合模型中间结果或高维数组,通常需要外置存储。",
|
||
"file": "适合 CSV、压缩包、文档、配置文件等任意文件产物。",
|
||
}
|
||
|
||
TRANSMISSION_HINTS = {
|
||
"bus": "通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。",
|
||
"oss": "更偏向远程对象存储传递,适合需要跨 Job 生命周期保留的大文件。",
|
||
"shm": "偏向节点内共享内存/本地工作目录语义,要求上下游执行环境紧耦合。",
|
||
"stream": "偏向流式消费场景,需要上下游在时序和消费方式上兼容。",
|
||
}
|
||
|
||
|
||
def parse_args() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(description="Generate detailed markdown manuals for builtin and custom operators")
|
||
parser.add_argument("--api-base-url", default="", help="Fetch operator payload from a specific live API /api/operators endpoint")
|
||
parser.add_argument("--input-json", default="", help="Read operator payload from a local JSON file instead of calling /api/operators")
|
||
parser.add_argument("--builtin-output-dir", default=DEFAULT_BUILTIN_OUTPUT_DIR)
|
||
parser.add_argument("--custom-output-dir", default=DEFAULT_CUSTOM_OUTPUT_DIR)
|
||
return parser.parse_args()
|
||
|
||
|
||
def request_json(url: str) -> Any:
|
||
with urllib.request.urlopen(url, timeout=60) as resp:
|
||
return json.loads(resp.read().decode("utf-8"))
|
||
|
||
|
||
def load_operator_payload(args: argparse.Namespace) -> tuple[Any, str]:
|
||
if args.input_json and args.api_base_url:
|
||
raise SystemExit("set only one of --input-json or --api-base-url")
|
||
if args.input_json:
|
||
input_path = Path(args.input_json)
|
||
return json.loads(input_path.read_text(encoding="utf-8")), str(input_path)
|
||
if args.api_base_url:
|
||
api_base_url = args.api_base_url.rstrip("/")
|
||
return request_json(f"{api_base_url}/api/operators"), f"{api_base_url}/api/operators"
|
||
catalog_path = Path(DEFAULT_BUILTIN_CATALOG_PATH)
|
||
return json.loads(catalog_path.read_text(encoding="utf-8")), str(catalog_path)
|
||
|
||
|
||
def load_operator_items(args: argparse.Namespace) -> list[dict[str, Any]]:
|
||
payload, _ = load_operator_payload(args)
|
||
return operator_items_from_payload(payload)
|
||
|
||
|
||
def operator_items_from_payload(payload: Any) -> list[dict[str, Any]]:
|
||
if isinstance(payload, dict) and isinstance(payload.get("items"), list):
|
||
items = payload.get("items", [])
|
||
elif isinstance(payload, list):
|
||
items = payload
|
||
elif isinstance(payload, dict) and str(payload.get("name", "")).strip():
|
||
items = [payload]
|
||
else:
|
||
raise SystemExit("operator payload must be one of: {items:[...]}, [...], or a single operator object")
|
||
return [item for item in items if isinstance(item, dict)]
|
||
|
||
|
||
def slugify(value: str) -> str:
|
||
cleaned = re.sub(r"[\s/]+", "-", value.strip())
|
||
cleaned = re.sub(r"[^0-9A-Za-z\-_.\u4e00-\u9fff]+", "-", cleaned)
|
||
cleaned = re.sub(r"-{2,}", "-", cleaned).strip("-")
|
||
return cleaned or "operator"
|
||
|
||
|
||
def choose_builtin_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||
for item in items:
|
||
if str(item.get("type", "")).strip() != "builtin":
|
||
continue
|
||
name = str(item.get("name", "")).strip()
|
||
if name:
|
||
grouped[name].append(item)
|
||
chosen: list[dict[str, Any]] = []
|
||
for name in sorted(grouped):
|
||
records = grouped[name]
|
||
records.sort(key=lambda item: (0 if str(item.get("type")) == "builtin" else 1, str(item.get("id", ""))))
|
||
chosen.append(records[0])
|
||
return chosen
|
||
|
||
|
||
def choose_custom_records(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
chosen = [item for item in items if str(item.get("type", "")).strip() == "custom" and str(item.get("name", "")).strip()]
|
||
chosen.sort(key=lambda item: (str(item.get("category", "")).strip(), str(item.get("name", "")).strip(), str(item.get("id", "")).strip()))
|
||
return chosen
|
||
|
||
|
||
def clean_markdown_dir(path: Path) -> None:
|
||
path.mkdir(parents=True, exist_ok=True)
|
||
for item in path.glob("*.md"):
|
||
item.unlink()
|
||
|
||
|
||
def stringify(value: Any) -> str:
|
||
if value is None:
|
||
return ""
|
||
if isinstance(value, (dict, list)):
|
||
return json.dumps(value, ensure_ascii=False)
|
||
return str(value)
|
||
|
||
|
||
def render_port_detail(port: dict[str, Any], direction: str) -> str:
|
||
port_id = str(port.get("id", "")).strip() or "-"
|
||
name = str(port.get("name", "")).strip() or "-"
|
||
port_type = str(port.get("type", "")).strip() or "-"
|
||
transmission = str(port.get("transmission", "")).strip() or "-"
|
||
description = str(port.get("description", "")).strip() or "未额外说明。"
|
||
type_hint = PORT_TYPE_HINTS.get(port_type, "表示该端口承载的是平台允许的对应数据形态。")
|
||
transmission_hint = TRANSMISSION_HINTS.get(transmission, "表示该端口采用平台约定的传输语义。")
|
||
return (
|
||
f"- `{port_id}` / **{name}** / `{port_type}` / `{transmission}`:{description} "
|
||
f"作为{direction}端口时,{type_hint} {transmission_hint}"
|
||
)
|
||
|
||
|
||
def render_hyperparam_detail(param: dict[str, Any]) -> str:
|
||
key = str(param.get("key", "")).strip() or "-"
|
||
default_value = stringify(param.get("value", "")).strip()
|
||
default_text = f"`{default_value}`" if default_value else "空字符串或由工作流显式填写"
|
||
description = str(param.get("description", "")).strip() or "未提供描述。"
|
||
ptype = str(param.get("type", "")).strip() or "string"
|
||
required_hint = "通常建议在工作流中显式配置。" if not default_value else "可直接使用默认值,也可在节点级覆盖。"
|
||
return f"- `{key}`:类型 `{ptype}`,默认值 {default_text}。{description} {required_hint}"
|
||
|
||
|
||
def render_skill_definition_block(operator: dict[str, Any]) -> list[str]:
|
||
skill_definition = str(operator.get("skillDefinition", "")).strip()
|
||
if not skill_definition:
|
||
return []
|
||
return ["## 平台语义", "", skill_definition, ""]
|
||
|
||
|
||
def local_operator_metadata(operator: dict[str, Any]) -> dict[str, Any]:
|
||
runtime_source = str(operator.get("runtimeSource", "")).strip()
|
||
if not runtime_source.startswith("backend/operators/builtin/") or not runtime_source.endswith("/operator.py"):
|
||
return {}
|
||
metadata_path = ROOT / Path(runtime_source).with_name("operator.json")
|
||
if not metadata_path.exists():
|
||
return {}
|
||
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||
return metadata if isinstance(metadata, dict) else {}
|
||
|
||
|
||
def render_external_api_block(operator: dict[str, Any]) -> list[str]:
|
||
external_api = operator.get("externalApi")
|
||
if not isinstance(external_api, dict) or not external_api:
|
||
external_api = local_operator_metadata(operator).get("externalApi")
|
||
if not isinstance(external_api, dict) or not external_api:
|
||
return []
|
||
source_repo = str(external_api.get("sourceRepo", "")).strip()
|
||
fat_route = str(external_api.get("fatRoute", "")).strip()
|
||
probe = external_api.get("acceptanceProbe") if isinstance(external_api.get("acceptanceProbe"), dict) else {}
|
||
probe_method = str(probe.get("method", "")).strip()
|
||
probe_endpoint = str(probe.get("endpoint", "")).strip()
|
||
lines = ["## 外部 API 契约", ""]
|
||
if source_repo:
|
||
lines.append(f"- 源码仓库:`{source_repo}`。")
|
||
if fat_route:
|
||
lines.append(f"- FAT 路由根:`{fat_route}`。")
|
||
if probe_method or probe_endpoint:
|
||
lines.append(f"- 验收探针:`{probe_method or 'GET'} {probe_endpoint}`。")
|
||
for key, label in (("requestJson", "请求体"), ("queryJson", "查询参数")):
|
||
if key in probe:
|
||
lines.append(f"- 探针{label}:`{json.dumps(probe[key], ensure_ascii=False, sort_keys=True)}`。")
|
||
lines.append("- 外部服务地址、认证和租户上下文必须由节点参数或运行时 Secret 注入,不允许在算子源码或 metadata 中硬编码。")
|
||
return lines + [""]
|
||
|
||
|
||
def operator_positioning(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
category = str(operator.get("category", "")).strip() or "OTHER"
|
||
operator_type = str(operator.get("type", "")).strip() or "unknown"
|
||
lines = [f"- 该算子属于 `{category}` 类,当前登记类型是 `{operator_type}`。"]
|
||
if operator_type == "builtin":
|
||
lines.append("- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 委派到独立算子入口。")
|
||
else:
|
||
lines.append("- 该算子属于自定义算子,平台只负责登记元数据与调度,实际业务逻辑由镜像、脚本或命令实现。")
|
||
if name in {"真实数据源", "BOS 增量扫描", "目录扫描"}:
|
||
lines.append("- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。")
|
||
if name in {"结果上传", "MongoDB 写入"}:
|
||
lines.append("- 它更偏向工作流出口,职责是把平台内部产物同步到外部对象存储或业务数据库。")
|
||
if name in {"视频质检", "视频切片", "视频标注", "成品打包"}:
|
||
lines.append("- 它位于视频主处理链路中,对共享 PVC、本地文件路径和 manifest 契约依赖较强。")
|
||
if name in {"JSON 字段提取", "JSON 过滤", "JSON 映射重命名", "JSON 转 CSV", "CSV 转 JSON", "条件分流", "数据合并", "文本模板渲染"}:
|
||
lines.append("- 它更偏向结构化中间层,用于整理、筛选、转换、路由或补充数据。")
|
||
return lines
|
||
|
||
|
||
def operator_runtime_contract(operator: dict[str, Any]) -> list[str]:
|
||
operator_id = str(operator.get("id", "")).strip()
|
||
operator_type = str(operator.get("type", "")).strip()
|
||
lines = [
|
||
"- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。",
|
||
"- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。",
|
||
"- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。",
|
||
]
|
||
if operator_type == "builtin":
|
||
source_file, handler_name = BUILTIN_HANDLER_HINTS.get(operator_id, ("backend/operator-runner/main.py", "builtin_generic"))
|
||
lines.append(f"- 当前独立实现入口是 `{source_file}` 中的 `run(...)`,平台 runtimeHandler 为 `{handler_name}`。")
|
||
else:
|
||
lines.append("- 当前类型是自定义算子,建议在元数据中通过 `skillDefinition` 明确输入要求、超参数含义、产物结构和失败语义。")
|
||
if operator_id in {"operator-builtin-real-source", "operator-builtin-bos-incremental-scan"}:
|
||
lines.append("- 该类源算子没有显式上游输入,主要依赖节点超参数和运行时注入的存储连接信息。")
|
||
if operator_id == "operator-builtin-upload":
|
||
lines.append("- 该类末端算子通常没有输出端口,验收重点是外部对象存储中的真实写入结果。")
|
||
return lines
|
||
|
||
|
||
def operator_boundaries(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
lines = [
|
||
"- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。",
|
||
"- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。",
|
||
]
|
||
if name == "结果上传":
|
||
lines.append("- 上传节点只应负责收集并落地结果,不应顺便做复杂转换;转换应在上传前完成。")
|
||
if name == "人工审核":
|
||
lines.append("- 审核节点的职责是等待外部决策并路由分支,不应继续承担后置业务处理。")
|
||
if name in {"Shell 命令执行", "模型推理"}:
|
||
lines.append("- 这类通用执行算子适合快速验证,但当逻辑稳定后应考虑沉淀为专用内建算子或受控自定义算子。")
|
||
return lines
|
||
|
||
|
||
def operator_preparation(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
lines = [
|
||
"- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。",
|
||
"- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。",
|
||
"- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。",
|
||
]
|
||
if name in {"BOS 增量扫描", "结果上传", "MongoDB 写入"}:
|
||
lines.append("- 需要预先准备连接配置、凭据和目标路径,优先通过平台连接配置而不是把密钥散落到节点超参数里。")
|
||
if name in {"视频质检", "视频切片", "视频标注", "成品打包", "Shell 命令执行"}:
|
||
lines.append("- 需要确认运行镜像内已经包含依赖工具,例如 `ffmpeg`、`ffprobe`、OpenCV、业务脚本或系统命令。")
|
||
if name == "人工审核":
|
||
lines.append("- 需要提前确认审核 API、审核页面或人工流程是否可用,否则工作流会停在等待状态。")
|
||
return lines
|
||
|
||
|
||
def operator_data_path(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
lines: list[str] = []
|
||
if name in {"BOS 增量扫描", "真实数据源"}:
|
||
lines.extend(
|
||
[
|
||
"- 源算子通常先输出轻量级 manifest,记录 `entries`、`catalogEntries`、`bos://` 路径或摘要信息,而不是直接下载整个数据集。",
|
||
"- 下游第一次真正读取文件内容时,才会通过 runner 的物化逻辑把远程对象缓存到共享 PVC 或本地工作目录。",
|
||
]
|
||
)
|
||
elif name == "结果上传":
|
||
lines.extend(
|
||
[
|
||
"- 该节点读取上游的 `artifact.path`、`dataRef.payloadPath` 或本地文件目录,然后把真实文件上传到对象存储。",
|
||
"- 平台运行记录里通常只保留上传摘要、目标路径和状态,不重复存放完整文件内容。",
|
||
]
|
||
)
|
||
elif name in {"视频质检", "视频切片", "视频标注", "成品打包"}:
|
||
lines.extend(
|
||
[
|
||
"- 视频类处理主要依赖共享 PVC 的本地路径执行,必要时会把 `bos://...` 输入缓存到 `_bos_cache` 或当前节点工作目录。",
|
||
"- 上下游之间传递的重点是 clip 路径、manifest、annotation JSON、package 目录等引用,而不是媒体字节本身。",
|
||
]
|
||
)
|
||
elif name == "MongoDB 写入":
|
||
lines.extend(
|
||
[
|
||
"- 该节点主要消费结构化 JSON payload,并把整理后的文档写入目标 Mongo 集合。",
|
||
"- 平台主库只保留写入摘要、错误信息和执行状态,避免把已写入业务库的整批文档再次冗余保存。",
|
||
]
|
||
)
|
||
else:
|
||
lines.extend(
|
||
[
|
||
"- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。",
|
||
"- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。",
|
||
]
|
||
)
|
||
return lines
|
||
|
||
|
||
def operator_combinations(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
lines: list[str] = []
|
||
if name in {"BOS 增量扫描", "真实数据源"}:
|
||
lines.extend(
|
||
[
|
||
"- 常接 `X 平台数据集索引`、`目录扫描`、`模型推理` 或任意自定义 JSON 处理节点,作为工作流入口。",
|
||
"- 当需要持续监听新增目录或新增对象时,应把它放在整条 DAG 的最上游,并让后续节点只消费本轮增量。 ",
|
||
]
|
||
)
|
||
elif name == "X 平台数据集索引":
|
||
lines.extend(
|
||
[
|
||
"- 上游通常是 `BOS 增量扫描` 或 `真实数据源`,下游再接 `采集目录校验`、`数据过滤`、`视频质检` 等节点。",
|
||
"- 如果后续要做抽检或上传,建议把索引结果先标准化为结构化 manifest,再进入视频主链路。",
|
||
]
|
||
)
|
||
elif name == "采集目录校验":
|
||
lines.append("- 适合放在索引之后,用来把不完整目录尽早筛掉,减少后续视频处理成本。 ")
|
||
elif name == "视频质检":
|
||
lines.append("- 常位于 `视频过滤` 或 `数据过滤` 之前,为后续筛选提供清晰度、抖动度、遮挡率等量化依据。")
|
||
elif name == "视频切片":
|
||
lines.append("- 常接在视频筛选之后,并把 clip 结果交给 `视频标注`、`成品打包` 或单独的 `结果上传`。")
|
||
elif name == "视频标注":
|
||
lines.append("- 上游一般是 `视频切片`,下游则是 `成品打包`、`结果上传` 或人工质检支路。")
|
||
elif name == "成品打包":
|
||
lines.append("- 建议在打包后单独接一个上传节点;如果同时上传 clip、标注和 package,应该拆成多条上传支路。")
|
||
elif name == "MongoDB 写入":
|
||
lines.append("- 通常接在 JSON 提取、映射、模板渲染或清洗节点之后,确保入库文档已经满足目标库 schema。")
|
||
elif name == "人工审核":
|
||
lines.append("- 审核节点之后应明确分成 `approved` 和 `rejected` 两条支路,避免后续节点混收不同语义的数据。")
|
||
else:
|
||
lines.append("- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。")
|
||
return lines
|
||
|
||
|
||
def operator_examples(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
if name == "BOS 增量扫描":
|
||
return [
|
||
"- 示例配置:`SOURCE_PATH=bos://<bucket>/<prefix>`,`SCAN_MODE=INCREMENTAL`,`INCREMENTAL_UNIT=DIRECTORY`。",
|
||
"- 典型输出:`source-manifest.json`,其中 `entries` 表示本轮增量对象,`catalogEntries` 表示本轮完整列举结果。",
|
||
]
|
||
if name == "结果上传":
|
||
return [
|
||
"- 典型输入:来自上游的 `packages-manifest.json`、`annotation.json`、`clip.mp4` 或任意 `dataRef.payloadPath`。",
|
||
"- 典型输出:`uploaded-results.json`,记录每个对象的目标路径、对象大小、状态和错误信息。",
|
||
]
|
||
if name == "视频切片":
|
||
return [
|
||
"- 典型输入:带有视频路径的 manifest 或上游过滤结果。",
|
||
"- 典型输出:`clips-manifest.json` 与多个 `*_clip001.mp4` 文件,clip 实体位于当前节点 `OUTPUT_DIR/clips/` 下。",
|
||
]
|
||
if name == "视频标注":
|
||
return [
|
||
"- 典型输入:来自切片节点的 clip 视频与 `clips-manifest.json`。",
|
||
"- 典型输出:`annotations-manifest.json` 与逐 clip 的标注 JSON 文件。",
|
||
]
|
||
if name == "成品打包":
|
||
return [
|
||
"- 典型输入:clip、annotation、quality、配置文件或传感器补充数据。",
|
||
"- 典型输出:`packages-manifest.json` 和每个成品包目录。 ",
|
||
]
|
||
if name == "MongoDB 写入":
|
||
return [
|
||
"- 典型输入:结构化 JSON 行列表、单文档对象或经过模板渲染后的入库 payload。",
|
||
"- 典型输出:写入摘要、命中集合名、成功数量、失败数量和错误原因。",
|
||
]
|
||
return [
|
||
"- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。",
|
||
"- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。",
|
||
]
|
||
|
||
|
||
def operator_success_signals(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
lines = [
|
||
"- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。",
|
||
"- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。",
|
||
]
|
||
if name in {"结果上传", "MongoDB 写入"}:
|
||
lines.append("- 外部系统中存在真实副作用,例如对象已上传、文档已写入,而不只是平台内显示成功。")
|
||
if name == "人工审核":
|
||
lines.append("- 审核节点收到明确的 approve/reject 决策,并将正确的分支结果送达下游。")
|
||
if name in {"BOS 增量扫描", "真实数据源"}:
|
||
lines.append("- 输出 manifest 与真实源路径内容一致,没有漏扫、重扫或错误分类。")
|
||
return lines
|
||
|
||
|
||
def operator_failures(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
lines = [
|
||
"- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。",
|
||
"- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。",
|
||
"- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。",
|
||
]
|
||
if name in {"BOS 增量扫描", "真实数据源", "结果上传"}:
|
||
lines.append("- 对象存储异常时,先核对 `SOURCE_PATH`、bucket/prefix、endpoint、AK/SK 和目标对象是否真实存在。")
|
||
if name in {"视频质检", "视频切片", "视频标注", "成品打包"}:
|
||
lines.append("- 视频链路异常时,先在 runner 容器内用 `ffprobe`/`ffmpeg` 验证输入文件是否可读,再检查缓存路径和磁盘空间。")
|
||
if name == "MongoDB 写入":
|
||
lines.append("- 若写库失败,先确认目标集合不在保留名单中,再核对连接串、认证信息和文档结构是否可序列化。")
|
||
if name == "人工审核":
|
||
lines.append("- 节点停在等待状态通常不是故障,而是还没有收到人工决策。")
|
||
return lines
|
||
|
||
|
||
def operator_acceptance(operator: dict[str, Any]) -> list[str]:
|
||
name = str(operator.get("name", "")).strip()
|
||
lines = [
|
||
"- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。",
|
||
"- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。",
|
||
"- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。",
|
||
]
|
||
if name in {"BOS 增量扫描", "真实数据源"}:
|
||
lines.append("- 对照真实源目录核验 `entries`、目录数、对象数和增量语义是否正确。")
|
||
if name == "结果上传":
|
||
lines.append("- 到目标对象存储前缀核对真实对象、对象大小和命名规则,确认并非只在平台侧生成了摘要。")
|
||
if name == "MongoDB 写入":
|
||
lines.append("- 到目标 Mongo 集合查询最近写入数据,确认记录数量、关键字段和值类型都符合预期。")
|
||
return lines
|
||
|
||
|
||
def render_config_example(operator: dict[str, Any]) -> str:
|
||
params = {}
|
||
for item in operator.get("hyperparams") or []:
|
||
key = str(item.get("key", "")).strip()
|
||
if not key:
|
||
continue
|
||
raw_value = item.get("value", "")
|
||
params[key] = raw_value if raw_value != "" else "<按节点填写>"
|
||
example = {
|
||
"label": str(operator.get("name", "")).strip(),
|
||
"operatorId": str(operator.get("id", "")).strip(),
|
||
"type": str(operator.get("type", "")).strip(),
|
||
"category": str(operator.get("category", "")).strip(),
|
||
"hyperparams": params,
|
||
}
|
||
return json.dumps(example, ensure_ascii=False, indent=2)
|
||
|
||
|
||
def render_operator_doc(operator: dict[str, Any], related_index: str, related_docs: list[tuple[str, str]]) -> str:
|
||
name = str(operator.get("name", "")).strip()
|
||
description = str(operator.get("description", "")).strip() or "无描述"
|
||
operator_id = str(operator.get("id", "")).strip()
|
||
operator_type = str(operator.get("type", "")).strip()
|
||
category = str(operator.get("category", "")).strip() or "OTHER"
|
||
inputs = operator.get("inputs") or []
|
||
outputs = operator.get("outputs") or []
|
||
hyperparams = operator.get("hyperparams") or []
|
||
|
||
lines = [
|
||
f"# {name}",
|
||
"",
|
||
f"- 算子 ID:`{operator_id}`",
|
||
f"- 类别:`{category}`",
|
||
f"- 类型:`{operator_type}`",
|
||
"",
|
||
"## 作用",
|
||
"",
|
||
description,
|
||
"",
|
||
"## 定位与职责边界",
|
||
"",
|
||
]
|
||
lines.extend(operator_positioning(operator))
|
||
lines.extend([""])
|
||
lines.extend(render_skill_definition_block(operator))
|
||
lines.extend(render_external_api_block(operator))
|
||
lines.extend(["## 运行时契约", ""])
|
||
lines.extend(operator_runtime_contract(operator))
|
||
lines.extend(["", "## 设计边界", ""])
|
||
lines.extend(operator_boundaries(operator))
|
||
lines.extend(["", "## 数据流示意", "", "```mermaid", "flowchart LR", f" A[上游输入或运行时注入] --> B[{name}]", " B --> C[artifact / stage manifest / 外部副作用]", "```", ""])
|
||
lines.extend(["## 输入端口详解", ""])
|
||
if inputs:
|
||
lines.extend([render_port_detail(port, "输入") for port in inputs])
|
||
else:
|
||
lines.append("- 无显式输入端口。通常作为源算子使用,主要依赖节点超参数和运行时环境。")
|
||
lines.extend(["", "## 输出端口详解", ""])
|
||
if outputs:
|
||
lines.extend([render_port_detail(port, "输出") for port in outputs])
|
||
else:
|
||
lines.append("- 无显式输出端口。通常意味着该节点以落库、上传或触发外部副作用为主。")
|
||
lines.extend(["", "## 关键超参数", ""])
|
||
if hyperparams:
|
||
lines.extend([render_hyperparam_detail(param) for param in hyperparams])
|
||
else:
|
||
lines.append("- 当前元数据未声明额外超参数。若为自定义算子,建议在注册阶段补齐。")
|
||
lines.extend(["", "## 执行前准备", ""])
|
||
lines.extend(operator_preparation(operator))
|
||
lines.extend(["", "## 数据通信与存储路径", ""])
|
||
lines.extend(operator_data_path(operator))
|
||
lines.extend(["", "## 典型上下游编排", ""])
|
||
lines.extend(operator_combinations(operator))
|
||
lines.extend(["", "## 节点配置示例", "", "```json", render_config_example(operator), "```", ""])
|
||
lines.extend(["## 输入输出示例", ""])
|
||
lines.extend(operator_examples(operator))
|
||
lines.extend(["", "## 成功判定", ""])
|
||
lines.extend(operator_success_signals(operator))
|
||
lines.extend(["", "## 常见失败与排查", ""])
|
||
lines.extend(operator_failures(operator))
|
||
lines.extend(["", "## 验收检查清单", ""])
|
||
lines.extend(operator_acceptance(operator))
|
||
lines.extend(["", "## 相关文档", "", f"- [{category} 类索引]({related_index})"])
|
||
lines.extend([f"- [{label}]({path})" for label, path in related_docs])
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def write_builtin_docs(items: list[dict[str, Any]], output_dir: Path) -> int:
|
||
clean_markdown_dir(output_dir)
|
||
grouped: dict[str, list[tuple[str, str]]] = defaultdict(list)
|
||
for operator in items:
|
||
name = str(operator.get("name", "")).strip()
|
||
file_name = f"{slugify(name)}.md"
|
||
(output_dir / file_name).write_text(
|
||
render_operator_doc(
|
||
operator,
|
||
"README.md",
|
||
[
|
||
("算子开发文档", "../operator-authoring.md"),
|
||
("内建算子总手册", "../builtin-operators-manual.zh-CN.md"),
|
||
],
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, file_name))
|
||
|
||
readme_lines = [
|
||
"# 内建算子分册",
|
||
"",
|
||
"当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。",
|
||
"",
|
||
f"- 当前收录数量:`{len(items)}`",
|
||
"- 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。",
|
||
"- 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md)",
|
||
"",
|
||
"## 使用建议",
|
||
"",
|
||
"- 先阅读总手册理解平台级规则,再进入本目录阅读单算子分册。",
|
||
"- 设计 DAG 时优先确认端口类型、artifact 语义、外部副作用和验收口径,而不是只看算子名称。",
|
||
"",
|
||
"## 分类索引",
|
||
"",
|
||
]
|
||
for category in sorted(grouped):
|
||
readme_lines.append(f"### {category}")
|
||
readme_lines.append("")
|
||
for name, file_name in sorted(grouped[category], key=lambda item: item[0]):
|
||
readme_lines.append(f"- [{name}]({file_name})")
|
||
readme_lines.append("")
|
||
(output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8")
|
||
return len(items)
|
||
|
||
|
||
def write_custom_docs(items: list[dict[str, Any]], output_dir: Path) -> int:
|
||
clean_markdown_dir(output_dir)
|
||
grouped: dict[str, list[tuple[str, str, str]]] = defaultdict(list)
|
||
for operator in items:
|
||
name = str(operator.get("name", "")).strip()
|
||
operator_id = str(operator.get("id", "")).strip()
|
||
suffix = slugify(operator_id[-8:] if len(operator_id) >= 8 else operator_id or name)
|
||
file_name = f"{slugify(name)}-{suffix}.md"
|
||
(output_dir / file_name).write_text(
|
||
render_operator_doc(
|
||
operator,
|
||
"README.md",
|
||
[
|
||
("自定义算子总手册", "../../custom-operator-manual.zh-CN.md"),
|
||
("自定义算子模板与示例", "../README.md"),
|
||
],
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
grouped[str(operator.get("category", "")).strip() or "OTHER"].append((name, operator_id, file_name))
|
||
|
||
readme_lines = [
|
||
"# 自定义算子实例分册",
|
||
"",
|
||
"当前目录专门存放从算子注册表导出的“实例级自定义算子 Markdown”。",
|
||
"",
|
||
f"- 当前实例数量:`{len(items)}`",
|
||
"- 配套总览:[`../README.md`](../README.md)",
|
||
"- 配套手册:[`../../custom-operator-manual.zh-CN.md`](../../custom-operator-manual.zh-CN.md)",
|
||
"",
|
||
]
|
||
if not items:
|
||
readme_lines.extend(
|
||
[
|
||
"## 当前状态",
|
||
"",
|
||
"当前导出的算子目录中没有 `custom` 类型实例,因此本目录暂时只有索引页。",
|
||
"",
|
||
"建议做法:",
|
||
"",
|
||
"- 先参考上一级目录中的模板与示例撰写自定义算子说明。",
|
||
"- 当环境中出现真实自定义算子后,再重新运行生成脚本为每个实例导出单独 Markdown。",
|
||
"",
|
||
]
|
||
)
|
||
else:
|
||
readme_lines.extend(["## 分类索引", ""])
|
||
for category in sorted(grouped):
|
||
readme_lines.append(f"### {category}")
|
||
readme_lines.append("")
|
||
for name, operator_id, file_name in sorted(grouped[category], key=lambda item: item[0]):
|
||
readme_lines.append(f"- [{name}]({file_name}),实例 ID:`{operator_id}`")
|
||
readme_lines.append("")
|
||
(output_dir / "README.md").write_text("\n".join(readme_lines), encoding="utf-8")
|
||
return len(items)
|
||
|
||
|
||
def main() -> int:
|
||
args = parse_args()
|
||
payload, source = load_operator_payload(args)
|
||
items = operator_items_from_payload(payload)
|
||
builtin_items = choose_builtin_records(items)
|
||
custom_items = choose_custom_records(items)
|
||
builtin_count = write_builtin_docs(builtin_items, Path(args.builtin_output_dir))
|
||
custom_count = write_custom_docs(custom_items, Path(args.custom_output_dir))
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"status": "ok",
|
||
"builtinCount": builtin_count,
|
||
"customCount": custom_count,
|
||
"builtinOutputDir": args.builtin_output_dir,
|
||
"customOutputDir": args.custom_output_dir,
|
||
"source": source,
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|