forked from longtao.wu/droboticflow
✨ Add native annotation gate operator
This commit is contained in:
parent
8aaa50fe3f
commit
cf17bf46df
1
backend/operators/builtin/annotation_gate/__init__.py
Normal file
1
backend/operators/builtin/annotation_gate/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
|
||||
16
backend/operators/builtin/annotation_gate/operator.json
Normal file
16
backend/operators/builtin/annotation_gate/operator.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"hardcodedOutputAccepted": false,
|
||||
"mockOnlyAccepted": false,
|
||||
"requiresRealInputs": true,
|
||||
"shallowOnlyAccepted": false,
|
||||
"simulationOnlyAccepted": false
|
||||
},
|
||||
"category": "HUMAN",
|
||||
"id": "operator-builtin-annotation-gate",
|
||||
"migrationStatus": "migrated",
|
||||
"name": "原生视频标注 Gate",
|
||||
"runtimeHandler": "builtin_annotation_gate",
|
||||
"runtimeSource": "backend/operators/builtin/annotation_gate/operator.py",
|
||||
"type": "builtin"
|
||||
}
|
||||
44
backend/operators/builtin/annotation_gate/operator.py
Normal file
44
backend/operators/builtin/annotation_gate/operator.py
Normal file
@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
HelperMap = dict[str, Callable[..., Any]]
|
||||
|
||||
|
||||
def run(
|
||||
run_id: str,
|
||||
node_id: str,
|
||||
output_dir: Any,
|
||||
node_data: dict[str, Any],
|
||||
inputs: list[dict[str, Any]],
|
||||
helpers: HelperMap,
|
||||
) -> list[dict[str, Any]]:
|
||||
params = helpers["hyperparams"](node_data)
|
||||
template_id = params.get("ANNOTATION_TEMPLATE_ID", "").strip()
|
||||
workgroup_id = params.get("ANNOTATION_WORKGROUP_ID", "").strip()
|
||||
if not template_id or not workgroup_id:
|
||||
raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID 和 ANNOTATION_WORKGROUP_ID")
|
||||
payloads = helpers["payloads_from_inputs"](inputs)
|
||||
items = helpers["collect_items"](payloads)
|
||||
if not items:
|
||||
raise RuntimeError("原生视频标注 Gate 需要上游真实视频切片或预标输入")
|
||||
return [
|
||||
helpers["write_json_artifact"](
|
||||
run_id,
|
||||
node_id,
|
||||
output_dir,
|
||||
"annotation-gate-diagnostic.json",
|
||||
{
|
||||
"datasetType": "annotation-gate-diagnostic",
|
||||
"runId": run_id,
|
||||
"nodeId": node_id,
|
||||
"templateId": template_id,
|
||||
"workgroupId": workgroup_id,
|
||||
"itemCount": len(items),
|
||||
"items": items,
|
||||
"interceptedByWorker": False,
|
||||
},
|
||||
output=helpers["output_binding"](node_data),
|
||||
)
|
||||
]
|
||||
77
backend/operators/builtin/annotation_gate/repository.json
Normal file
77
backend/operators/builtin/annotation_gate/repository.json
Normal file
@ -0,0 +1,77 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"hardcodedOutputAccepted": false,
|
||||
"mockOnlyAccepted": false,
|
||||
"pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh",
|
||||
"requiresPersistentWorkspaceEvidence": true,
|
||||
"requiresRealInputs": true,
|
||||
"shallowOnlyAccepted": false,
|
||||
"simulationOnlyAccepted": false
|
||||
},
|
||||
"category": "HUMAN",
|
||||
"containerArgs": [],
|
||||
"containerCommand": [
|
||||
"python",
|
||||
"/app/backend/operator-runner/main.py"
|
||||
],
|
||||
"containerEntrypoint": "backend/operator-runner/main.py",
|
||||
"entrypoint": "operator.py",
|
||||
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate:fanyu-annotation-gate-bootstrap",
|
||||
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"metadata": "operator.json",
|
||||
"name": "原生视频标注 Gate",
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"preferredSlug": "annotation_gate",
|
||||
"pythonImport": "backend.operators.builtin.annotation_gate.operator:run",
|
||||
"repositoryId": "builtin/annotation_gate",
|
||||
"runtimeContract": {
|
||||
"acceptance": {
|
||||
"requiresArtifactChecksum": true,
|
||||
"requiresFixtureChecksum": true,
|
||||
"requiresGpuEvidence": false,
|
||||
"requiresRealFixture": true,
|
||||
"requiresWorkflowTransfer": true
|
||||
},
|
||||
"artifacts": {
|
||||
"requiresChecksum": true,
|
||||
"requiresLineage": true,
|
||||
"requiresManifest": true
|
||||
},
|
||||
"entrypoint": "/app/backend/operator-runner/main.py",
|
||||
"handler": "builtin_annotation_gate",
|
||||
"imagePolicy": {
|
||||
"allowsSharedBaseLayer": true,
|
||||
"forbidsSharedOperatorRunnerFinalImage": true,
|
||||
"independentFinalImage": true
|
||||
},
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"parameters": {
|
||||
"commandWrapperAllowed": false,
|
||||
"controlledScriptAllowed": false,
|
||||
"source": "hyperparams",
|
||||
"structured": true
|
||||
},
|
||||
"resources": {
|
||||
"minGpuCount": 0,
|
||||
"recommendedGpuType": "",
|
||||
"requiresGpu": false
|
||||
},
|
||||
"runtimeDependencies": [
|
||||
"video-artifacts",
|
||||
"annotation-revisions"
|
||||
],
|
||||
"schemaVersion": "droboticflow.operator.runtime/v1"
|
||||
},
|
||||
"runtimeDependencies": [
|
||||
"video-artifacts",
|
||||
"annotation-revisions"
|
||||
],
|
||||
"runtimeHandler": "builtin_annotation_gate",
|
||||
"runtimeSource": "backend/operators/builtin/annotation_gate/operator.py",
|
||||
"schemaVersion": "droboticflow.operator.repository/v1",
|
||||
"sourceRoot": "backend/operators/builtin/annotation_gate",
|
||||
"strictPocAcceptanceExternals": [],
|
||||
"type": "builtin"
|
||||
}
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。
|
||||
|
||||
- 当前收录数量:`103`
|
||||
- 当前收录数量:`104`
|
||||
- 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。
|
||||
- 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md)
|
||||
|
||||
@ -59,6 +59,7 @@
|
||||
### HUMAN
|
||||
|
||||
- [人工审核](人工审核.md)
|
||||
- [原生视频标注 Gate](原生视频标注-Gate.md)
|
||||
|
||||
### IO
|
||||
|
||||
|
||||
113
docs/operators/原生视频标注-Gate.md
Normal file
113
docs/operators/原生视频标注-Gate.md
Normal file
@ -0,0 +1,113 @@
|
||||
# 原生视频标注 Gate
|
||||
|
||||
- 算子 ID:`operator-builtin-annotation-gate`
|
||||
- 类别:`HUMAN`
|
||||
- 类型:`builtin`
|
||||
|
||||
## 作用
|
||||
|
||||
将视频切片和语义预标转换为原生逐段标注任务;质检与验收通过后恢复同一 Workflow Run。
|
||||
|
||||
## 定位与职责边界
|
||||
|
||||
- 该算子属于 `HUMAN` 类,当前登记类型是 `builtin`。
|
||||
- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 委派到独立算子入口。
|
||||
|
||||
## 平台语义
|
||||
|
||||
原生标注门禁:从 Workflow Run 自动创建逐段任务,等待 annotation quality/acceptance 决定,并把 accepted revision manifest 回传到同一 Run。
|
||||
|
||||
## 运行时契约
|
||||
|
||||
- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。
|
||||
- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。
|
||||
- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。
|
||||
- 当前独立实现入口是 `operator-repos/builtin/annotation_gate/src/operator.py` 中的 `run(...)`,平台 runtimeHandler 为 `builtin_annotation_gate`。
|
||||
|
||||
## 设计边界
|
||||
|
||||
- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。
|
||||
- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。
|
||||
|
||||
## 数据流示意
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
A[上游输入或运行时注入] --> B[原生视频标注 Gate]
|
||||
B --> C[artifact / stage manifest / 外部副作用]
|
||||
```
|
||||
|
||||
## 输入端口详解
|
||||
|
||||
- `clips` / **clips** / `json` / `bus`:视频切片 manifest 与真实 mp4 artifacts 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。
|
||||
- `prelabels` / **prelabels** / `json` / `bus`:视频语义预标 JSON artifacts 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。
|
||||
|
||||
## 输出端口详解
|
||||
|
||||
- `approved` / **approved** / `json` / `bus`:已通过质检与验收的 revision artifacts 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。
|
||||
- `rejected` / **rejected** / `json` / `bus`:拒绝或返工分支 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。
|
||||
|
||||
## 关键超参数
|
||||
|
||||
- `ANNOTATION_TEMPLATE_ID`:类型 `string`,默认值 空字符串或由工作流显式填写。当前项目中已发布的语义模板 ID 通常建议在工作流中显式配置。
|
||||
- `ANNOTATION_WORKGROUP_ID`:类型 `string`,默认值 空字符串或由工作流显式填写。目标供应商工作组 ID 通常建议在工作流中显式配置。
|
||||
- `ANNOTATION_ASSIGNED_USER_ID`:类型 `string`,默认值 空字符串或由工作流显式填写。可选:首轮标注员用户 ID 通常建议在工作流中显式配置。
|
||||
|
||||
## 执行前准备
|
||||
|
||||
- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。
|
||||
- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。
|
||||
- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。
|
||||
|
||||
## 数据通信与存储路径
|
||||
|
||||
- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。
|
||||
- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。
|
||||
|
||||
## 典型上下游编排
|
||||
|
||||
- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。
|
||||
|
||||
## 节点配置示例
|
||||
|
||||
```json
|
||||
{
|
||||
"label": "原生视频标注 Gate",
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"type": "builtin",
|
||||
"category": "HUMAN",
|
||||
"hyperparams": {
|
||||
"ANNOTATION_TEMPLATE_ID": "<按节点填写>",
|
||||
"ANNOTATION_WORKGROUP_ID": "<按节点填写>",
|
||||
"ANNOTATION_ASSIGNED_USER_ID": "<按节点填写>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 输入输出示例
|
||||
|
||||
- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。
|
||||
- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。
|
||||
|
||||
## 成功判定
|
||||
|
||||
- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。
|
||||
- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。
|
||||
|
||||
## 常见失败与排查
|
||||
|
||||
- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。
|
||||
- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。
|
||||
- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。
|
||||
|
||||
## 验收检查清单
|
||||
|
||||
- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。
|
||||
- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。
|
||||
- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。
|
||||
|
||||
## 相关文档
|
||||
|
||||
- [HUMAN 类索引](README.md)
|
||||
- [算子开发文档](../operator-authoring.md)
|
||||
- [内建算子总手册](../builtin-operators-manual.zh-CN.md)
|
||||
@ -521,6 +521,114 @@
|
||||
"skillDefinition": "基于真实输入生成审核任务并等待人工处理。",
|
||||
"type": "builtin"
|
||||
},
|
||||
{
|
||||
"category": "HUMAN",
|
||||
"containerArgs": [],
|
||||
"containerCommand": [
|
||||
"python",
|
||||
"/app/backend/operator-runner/main.py"
|
||||
],
|
||||
"description": "将视频切片和语义预标转换为原生逐段标注任务;质检与验收通过后恢复同一 Workflow Run。",
|
||||
"hyperparams": [
|
||||
{
|
||||
"description": "当前项目中已发布的语义模板 ID",
|
||||
"key": "ANNOTATION_TEMPLATE_ID",
|
||||
"type": "string",
|
||||
"value": ""
|
||||
},
|
||||
{
|
||||
"description": "目标供应商工作组 ID",
|
||||
"key": "ANNOTATION_WORKGROUP_ID",
|
||||
"type": "string",
|
||||
"value": ""
|
||||
},
|
||||
{
|
||||
"description": "可选:首轮标注员用户 ID",
|
||||
"key": "ANNOTATION_ASSIGNED_USER_ID",
|
||||
"type": "string",
|
||||
"value": ""
|
||||
}
|
||||
],
|
||||
"id": "operator-builtin-annotation-gate",
|
||||
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate:fanyu-annotation-gate-bootstrap",
|
||||
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"inputs": [
|
||||
{
|
||||
"description": "视频切片 manifest 与真实 mp4 artifacts",
|
||||
"id": "clips",
|
||||
"name": "clips",
|
||||
"transmission": "bus",
|
||||
"type": "json"
|
||||
},
|
||||
{
|
||||
"description": "视频语义预标 JSON artifacts",
|
||||
"id": "prelabels",
|
||||
"name": "prelabels",
|
||||
"transmission": "bus",
|
||||
"type": "json"
|
||||
}
|
||||
],
|
||||
"name": "原生视频标注 Gate",
|
||||
"outputs": [
|
||||
{
|
||||
"description": "已通过质检与验收的 revision artifacts",
|
||||
"id": "approved",
|
||||
"name": "approved",
|
||||
"transmission": "bus",
|
||||
"type": "json"
|
||||
},
|
||||
{
|
||||
"description": "拒绝或返工分支",
|
||||
"id": "rejected",
|
||||
"name": "rejected",
|
||||
"transmission": "bus",
|
||||
"type": "json"
|
||||
}
|
||||
],
|
||||
"runtimeContract": {
|
||||
"acceptance": {
|
||||
"requiresArtifactChecksum": true,
|
||||
"requiresFixtureChecksum": true,
|
||||
"requiresGpuEvidence": false,
|
||||
"requiresRealFixture": true,
|
||||
"requiresWorkflowTransfer": true
|
||||
},
|
||||
"artifacts": {
|
||||
"requiresChecksum": true,
|
||||
"requiresLineage": true,
|
||||
"requiresManifest": true
|
||||
},
|
||||
"entrypoint": "/app/backend/operator-runner/main.py",
|
||||
"handler": "builtin_annotation_gate",
|
||||
"imagePolicy": {
|
||||
"allowsSharedBaseLayer": true,
|
||||
"forbidsSharedOperatorRunnerFinalImage": true,
|
||||
"independentFinalImage": true
|
||||
},
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"parameters": {
|
||||
"commandWrapperAllowed": false,
|
||||
"controlledScriptAllowed": false,
|
||||
"source": "hyperparams",
|
||||
"structured": true
|
||||
},
|
||||
"resources": {
|
||||
"minGpuCount": 0,
|
||||
"recommendedGpuType": "",
|
||||
"requiresGpu": false
|
||||
},
|
||||
"runtimeDependencies": [
|
||||
"video-artifacts",
|
||||
"annotation-revisions"
|
||||
],
|
||||
"schemaVersion": "droboticflow.operator.runtime/v1"
|
||||
},
|
||||
"runtimeHandler": "builtin_annotation_gate",
|
||||
"runtimeSource": "operator-repos/builtin/annotation_gate/src/operator.py",
|
||||
"skillDefinition": "原生标注门禁:从 Workflow Run 自动创建逐段任务,等待 annotation quality/acceptance 决定,并把 accepted revision manifest 回传到同一 Run。",
|
||||
"type": "builtin"
|
||||
},
|
||||
{
|
||||
"category": "IO",
|
||||
"containerArgs": [],
|
||||
|
||||
@ -34,3 +34,26 @@ func TestUpsertBuiltinOperatorPreservesSharedMemoryOverride(t *testing.T) {
|
||||
t.Fatalf("expected catalog CPU and user SHM override, got %#v", items[0].Resources)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltinCatalogIncludesNativeAnnotationGateContract(t *testing.T) {
|
||||
items, err := builtinOperatorRecords("2026-08-18T00:00:00Z", "registry.example/droboticflow-operator-runner:test")
|
||||
if err != nil {
|
||||
t.Fatalf("load builtin operator catalog: %v", err)
|
||||
}
|
||||
item := findBuiltinOperator(items, "operator-builtin-annotation-gate", "")
|
||||
if item == nil {
|
||||
t.Fatal("expected native annotation gate in builtin catalog")
|
||||
}
|
||||
if item.Category != "HUMAN" || len(item.Inputs) != 2 || len(item.Outputs) != 2 {
|
||||
t.Fatalf("unexpected annotation gate ports: %#v", item)
|
||||
}
|
||||
keys := map[string]bool{}
|
||||
for _, hyperparam := range item.Hyperparams {
|
||||
keys[hyperparam.Key] = true
|
||||
}
|
||||
for _, key := range []string{"ANNOTATION_TEMPLATE_ID", "ANNOTATION_WORKGROUP_ID", "ANNOTATION_ASSIGNED_USER_ID"} {
|
||||
if !keys[key] {
|
||||
t.Fatalf("annotation gate is missing hyperparameter %s", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -87,6 +87,27 @@
|
||||
"runtimeSource": "operator-repos/builtin/aloha_hdf5_validate/src/operator.py",
|
||||
"sourceRoot": "operator-repos/builtin/aloha_hdf5_validate"
|
||||
},
|
||||
{
|
||||
"baseImage": "${OPERATOR_BASE_IMAGE}",
|
||||
"containerArgs": [],
|
||||
"containerCommand": [
|
||||
"python",
|
||||
"/app/backend/operator-runner/main.py"
|
||||
],
|
||||
"dependencyProfile": [
|
||||
"video-artifacts",
|
||||
"annotation-revisions"
|
||||
],
|
||||
"dockerfile": "operator-repos/builtin/annotation_gate/Dockerfile",
|
||||
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate:fanyu-annotation-gate-bootstrap",
|
||||
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"requiresGpu": false,
|
||||
"runtimeHandler": "builtin_annotation_gate",
|
||||
"runtimeSource": "operator-repos/builtin/annotation_gate/src/operator.py",
|
||||
"sourceRoot": "operator-repos/builtin/annotation_gate"
|
||||
},
|
||||
{
|
||||
"baseImage": "${OPERATOR_BASE_IMAGE}",
|
||||
"containerArgs": [],
|
||||
|
||||
12
operator-repos/builtin/annotation_gate/Dockerfile
Normal file
12
operator-repos/builtin/annotation_gate/Dockerfile
Normal file
@ -0,0 +1,12 @@
|
||||
# Generated by scripts/generate_operator_images.py. Do not edit by hand.
|
||||
ARG OPERATOR_BASE_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-runner:builtin
|
||||
FROM ${OPERATOR_BASE_IMAGE}
|
||||
|
||||
ENV DROBOTICFLOW_OPERATOR_ID=operator-builtin-annotation-gate \
|
||||
DROBOTICFLOW_OPERATOR_SLUG=annotation_gate \
|
||||
DROBOTICFLOW_OPERATOR_SOURCE=/app/operator-repos/builtin/annotation_gate/src/operator.py \
|
||||
DROBOTICFLOW_OPERATOR_REPOSITORY=/app/operator-repos/builtin/annotation_gate \
|
||||
DROBOTICFLOW_OPERATOR_HANDLER=builtin_annotation_gate
|
||||
|
||||
COPY operator-repos/builtin/annotation_gate ./operator-repos/builtin/annotation_gate
|
||||
ENTRYPOINT ["python", "/app/backend/operator-runner/main.py"]
|
||||
10
operator-repos/builtin/annotation_gate/README.md
Normal file
10
operator-repos/builtin/annotation_gate/README.md
Normal file
@ -0,0 +1,10 @@
|
||||
# 原生视频标注 Gate
|
||||
|
||||
独立 builtin 算子目录。`operator-builtin-annotation-gate` 由 Worker 原生拦截:它不会启动 Kubernetes Job,而是从上游真实视频切片与预标 artifacts 创建逐段标注任务,并在质检、验收完成后恢复同一个 Workflow Run。
|
||||
|
||||
- 发布镜像:以 `release.json` 中的 digest-pinned CCR imageId 为准
|
||||
- Repository manifest:`repository.json`
|
||||
- Runtime handler:`builtin_annotation_gate`
|
||||
- Source:`src/operator.py`
|
||||
|
||||
独立镜像保留为契约与故障诊断入口;正常运行必须经过 Worker annotation gate runtime。
|
||||
16
operator-repos/builtin/annotation_gate/operator.json
Normal file
16
operator-repos/builtin/annotation_gate/operator.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"hardcodedOutputAccepted": false,
|
||||
"mockOnlyAccepted": false,
|
||||
"requiresRealInputs": true,
|
||||
"shallowOnlyAccepted": false,
|
||||
"simulationOnlyAccepted": false
|
||||
},
|
||||
"category": "HUMAN",
|
||||
"id": "operator-builtin-annotation-gate",
|
||||
"migrationStatus": "migrated",
|
||||
"name": "原生视频标注 Gate",
|
||||
"runtimeHandler": "builtin_annotation_gate",
|
||||
"runtimeSource": "operator-repos/builtin/annotation_gate/src/operator.py",
|
||||
"type": "builtin"
|
||||
}
|
||||
30
operator-repos/builtin/annotation_gate/release.json
Normal file
30
operator-repos/builtin/annotation_gate/release.json
Normal file
@ -0,0 +1,30 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"artifactId": "",
|
||||
"projectId": "",
|
||||
"runId": "",
|
||||
"verifiedAt": "",
|
||||
"workflowId": ""
|
||||
},
|
||||
"builtAt": "",
|
||||
"digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"environmentReleases": {
|
||||
"poc": {
|
||||
"digest": "sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate:fanyu-annotation-gate-bootstrap",
|
||||
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"registry": "ccr-29eug8s3-pub.cnc.bj.baidubce.com",
|
||||
"tag": "fanyu-annotation-gate-bootstrap"
|
||||
}
|
||||
},
|
||||
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate:fanyu-annotation-gate-bootstrap",
|
||||
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"kanikoJob": "",
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"registry": "ccr-29eug8s3-pub.cnc.bj.baidubce.com",
|
||||
"schemaVersion": "droboticflow.operator.release/v1",
|
||||
"slug": "annotation_gate",
|
||||
"sourceCommit": "8aaa50fe3fca259b23b81bf3fed4edc19de918cf",
|
||||
"sourceTreeSha256": "4e045f7d7c0b650dcad679a457a62c8560a073c6c031daa5da49e9143f9ee1a0",
|
||||
"tag": "fanyu-annotation-gate-bootstrap"
|
||||
}
|
||||
83
operator-repos/builtin/annotation_gate/repository.json
Normal file
83
operator-repos/builtin/annotation_gate/repository.json
Normal file
@ -0,0 +1,83 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"fixtureContract": "tests/acceptance/fixture_contract.json",
|
||||
"hardcodedOutputAccepted": false,
|
||||
"mockOnlyAccepted": false,
|
||||
"pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh",
|
||||
"requiresChainedWorkflow": true,
|
||||
"requiresChineseWorkflowAcceptance": true,
|
||||
"requiresPersistentWorkspaceEvidence": true,
|
||||
"requiresRealInputs": true,
|
||||
"shallowOnlyAccepted": false,
|
||||
"simulationOnlyAccepted": false
|
||||
},
|
||||
"category": "HUMAN",
|
||||
"containerArgs": [],
|
||||
"containerCommand": [
|
||||
"python",
|
||||
"/app/backend/operator-runner/main.py"
|
||||
],
|
||||
"containerEntrypoint": "backend/operator-runner/main.py",
|
||||
"entrypoint": "src/operator.py",
|
||||
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate:fanyu-annotation-gate-bootstrap",
|
||||
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-annotation-gate@sha256:0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"metadata": "operator.json",
|
||||
"name": "原生视频标注 Gate",
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"preferredSlug": "annotation_gate",
|
||||
"pythonImport": "operator-repos/builtin/annotation_gate/src/operator.py:run",
|
||||
"releaseManifest": "release.json",
|
||||
"releaseSchemaVersion": "droboticflow.operator.release/v1",
|
||||
"repositoryId": "builtin/annotation_gate",
|
||||
"runtimeContract": {
|
||||
"acceptance": {
|
||||
"requiresArtifactChecksum": true,
|
||||
"requiresFixtureChecksum": true,
|
||||
"requiresGpuEvidence": false,
|
||||
"requiresRealFixture": true,
|
||||
"requiresWorkflowTransfer": true
|
||||
},
|
||||
"artifacts": {
|
||||
"requiresChecksum": true,
|
||||
"requiresLineage": true,
|
||||
"requiresManifest": true
|
||||
},
|
||||
"entrypoint": "/app/backend/operator-runner/main.py",
|
||||
"handler": "builtin_annotation_gate",
|
||||
"imagePolicy": {
|
||||
"allowsSharedBaseLayer": true,
|
||||
"forbidsSharedOperatorRunnerFinalImage": true,
|
||||
"independentFinalImage": true
|
||||
},
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"parameters": {
|
||||
"commandWrapperAllowed": false,
|
||||
"controlledScriptAllowed": false,
|
||||
"source": "hyperparams",
|
||||
"structured": true
|
||||
},
|
||||
"resources": {
|
||||
"minGpuCount": 0,
|
||||
"recommendedGpuType": "",
|
||||
"requiresGpu": false
|
||||
},
|
||||
"runtimeDependencies": [
|
||||
"video-artifacts",
|
||||
"annotation-revisions"
|
||||
],
|
||||
"schemaVersion": "droboticflow.operator.runtime/v1"
|
||||
},
|
||||
"runtimeDependencies": [
|
||||
"video-artifacts",
|
||||
"annotation-revisions"
|
||||
],
|
||||
"runtimeHandler": "builtin_annotation_gate",
|
||||
"runtimeSource": "operator-repos/builtin/annotation_gate/src/operator.py",
|
||||
"schemaVersion": "droboticflow.operator.repository/v1",
|
||||
"sourceRoot": "operator-repos/builtin/annotation_gate",
|
||||
"sourceTreeSha256": "4e045f7d7c0b650dcad679a457a62c8560a073c6c031daa5da49e9143f9ee1a0",
|
||||
"strictPocAcceptanceExternals": [],
|
||||
"type": "builtin"
|
||||
}
|
||||
1
operator-repos/builtin/annotation_gate/requirements.txt
Normal file
1
operator-repos/builtin/annotation_gate/requirements.txt
Normal file
@ -0,0 +1 @@
|
||||
|
||||
41
operator-repos/builtin/annotation_gate/runtime_contract.json
Normal file
41
operator-repos/builtin/annotation_gate/runtime_contract.json
Normal file
@ -0,0 +1,41 @@
|
||||
{
|
||||
"acceptance": {
|
||||
"requiresArtifactChecksum": true,
|
||||
"requiresFixtureChecksum": true,
|
||||
"requiresGpuEvidence": false,
|
||||
"requiresRealFixture": true,
|
||||
"requiresWorkflowTransfer": true
|
||||
},
|
||||
"artifacts": {
|
||||
"requiresChecksum": true,
|
||||
"requiresLineage": true,
|
||||
"requiresManifest": true
|
||||
},
|
||||
"entrypoint": "/app/backend/operator-runner/main.py",
|
||||
"handler": "builtin_annotation_gate",
|
||||
"imagePolicy": {
|
||||
"allowsSharedBaseLayer": true,
|
||||
"forbidsSharedOperatorRunnerFinalImage": true,
|
||||
"independentFinalImage": true
|
||||
},
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"parameters": {
|
||||
"commandWrapperAllowed": false,
|
||||
"controlledScriptAllowed": false,
|
||||
"source": "hyperparams",
|
||||
"structured": true
|
||||
},
|
||||
"resources": {
|
||||
"minGpuCount": 0,
|
||||
"recommendedGpuType": "",
|
||||
"requiresGpu": false
|
||||
},
|
||||
"runtimeDependencies": [
|
||||
"video-artifacts",
|
||||
"annotation-revisions"
|
||||
],
|
||||
"runtimeSource": "src/operator.py",
|
||||
"schemaVersion": "droboticflow.operator.runtime/v1",
|
||||
"sourceRoot": "operator-repos/builtin/annotation_gate"
|
||||
}
|
||||
1
operator-repos/builtin/annotation_gate/src/__init__.py
Normal file
1
operator-repos/builtin/annotation_gate/src/__init__.py
Normal file
@ -0,0 +1 @@
|
||||
|
||||
48
operator-repos/builtin/annotation_gate/src/operator.py
Normal file
48
operator-repos/builtin/annotation_gate/src/operator.py
Normal file
@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
HelperMap = dict[str, Callable[..., Any]]
|
||||
|
||||
|
||||
def run(
|
||||
run_id: str,
|
||||
node_id: str,
|
||||
output_dir: Any,
|
||||
node_data: dict[str, Any],
|
||||
inputs: list[dict[str, Any]],
|
||||
helpers: HelperMap,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Diagnostic fallback; normal execution is intercepted by the Worker runtime."""
|
||||
params = helpers["hyperparams"](node_data)
|
||||
template_id = params.get("ANNOTATION_TEMPLATE_ID", "").strip()
|
||||
workgroup_id = params.get("ANNOTATION_WORKGROUP_ID", "").strip()
|
||||
if not template_id or not workgroup_id:
|
||||
raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID 和 ANNOTATION_WORKGROUP_ID")
|
||||
|
||||
payloads = helpers["payloads_from_inputs"](inputs)
|
||||
items = helpers["collect_items"](payloads)
|
||||
if not items:
|
||||
raise RuntimeError("原生视频标注 Gate 需要上游真实视频切片或预标输入")
|
||||
|
||||
manifest = {
|
||||
"datasetType": "annotation-gate-diagnostic",
|
||||
"runId": run_id,
|
||||
"nodeId": node_id,
|
||||
"templateId": template_id,
|
||||
"workgroupId": workgroup_id,
|
||||
"itemCount": len(items),
|
||||
"items": items,
|
||||
"interceptedByWorker": False,
|
||||
}
|
||||
return [
|
||||
helpers["write_json_artifact"](
|
||||
run_id,
|
||||
node_id,
|
||||
output_dir,
|
||||
"annotation-gate-diagnostic.json",
|
||||
manifest,
|
||||
output=helpers["output_binding"](node_data),
|
||||
)
|
||||
]
|
||||
@ -0,0 +1,21 @@
|
||||
{
|
||||
"fixtures": [],
|
||||
"minimumEvidence": [
|
||||
"real clip video checksum",
|
||||
"accepted revision checksum",
|
||||
"artifact lineage",
|
||||
"same-run resume"
|
||||
],
|
||||
"operatorId": "operator-builtin-annotation-gate",
|
||||
"operatorSlug": "annotation_gate",
|
||||
"rules": {
|
||||
"forbidsMock": true,
|
||||
"forbidsSimulationOnly": true,
|
||||
"forbidsSmokeOnly": true,
|
||||
"forbidsSynthetic": true,
|
||||
"requiresLicense": true,
|
||||
"requiresPublicSourceUrl": true,
|
||||
"requiresSha256": true
|
||||
},
|
||||
"schemaVersion": "droboticflow.operator.fixture/v1"
|
||||
}
|
||||
@ -21,6 +21,7 @@ RUNTIME_DEPENDENCIES: dict[str, list[str]] = {
|
||||
"act_policy_train": ["huggingface-lerobot-act", "lerobot-dataset-or-local-root", "gpu-a800"],
|
||||
"agibot_world2026_validate": ["agibotworld2026-meta-data-video-files", "pyarrow", "parquet-files"],
|
||||
"aloha_hdf5_validate": ["aloha-act-hdf5-files", "h5py", "numpy"],
|
||||
"annotation_gate": ["video-artifacts", "annotation-revisions"],
|
||||
"audit_sample": ["json-artifacts"],
|
||||
"bop_dataset_validate": ["bop-scenewise-dataset-files", "json-files", "rgb-depth-mask-image-files", "ply-model-files"],
|
||||
"bos_incremental_scan": ["baidu-bos"],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user