From 6aa4058e364e16a1c514faeb3bd7ea64efd3f9bc Mon Sep 17 00:00:00 2001 From: eust-w Date: Fri, 21 Aug 2026 12:29:07 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20Build=20PTS-grounded=20Fanyu=20h?= =?UTF-8?q?uman=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/operator-runner/main.py | 3 +- backend/operator-runner/test_main.py | 44 + .../builtin/directory_scan/operator.json | 2 +- .../builtin/directory_scan/operator.py | 94 +- .../builtin/directory_scan/repository.json | 8 +- backend/operators/builtin/package/operator.py | 14 +- .../operators/builtin/package/repository.json | 6 +- .../qwen_behavior_rough_cut/__init__.py | 1 + .../qwen_behavior_rough_cut/operator.json | 16 + .../qwen_behavior_rough_cut/operator.py | 1007 +++++++++++++++++ .../qwen_behavior_rough_cut/repository.json | 89 ++ docs/fanyu-v5-workflow.md | 26 +- docs/operators/Qwen3-VL-语义行为粗切-PTS.md | 124 ++ docs/operators/README.md | 5 +- docs/operators/成品打包.md | 1 - docs/operators/{目录扫描.md => 目录读取.md} | 17 +- internal/api/annotation_operations.go | 287 ++++- internal/api/annotation_operations_test.go | 13 + internal/api/builtin_operators_catalog.json | 190 +++- internal/api/fanyu_template.go | 60 +- internal/api/fanyu_template_test.go | 28 +- .../fanyu_v5_four_stage_human_loop_test.go | 280 +++++ internal/api/human_loop.go | 141 +-- internal/api/human_loop_test.go | 6 +- internal/api/plugins.go | 2 +- internal/api/server.go | 4 +- internal/api/server_test.go | 2 +- internal/domain/annotation_operations.go | 38 +- internal/domain/human_loop.go | 45 +- internal/worker/annotation_gate.go | 61 +- internal/worker/human_release_test.go | 11 + internal/worker/runtime.go | 44 +- internal/worker/runtime_test.go | 15 + .../builtin/directory_scan/README.md | 2 +- .../builtin/directory_scan/operator.json | 2 +- .../directory_scan/runtime_contract.json | 3 +- .../builtin/directory_scan/src/operator.py | 94 +- .../builtin/package/runtime_contract.json | 3 +- .../builtin/package/src/operator.py | 14 +- .../qwen_behavior_rough_cut/Dockerfile | 23 + .../builtin/qwen_behavior_rough_cut/README.md | 10 + .../qwen_behavior_rough_cut/operator.json | 16 + .../qwen_behavior_rough_cut/requirements.txt | 0 .../runtime_contract.json | 49 + .../qwen_behavior_rough_cut/src/__init__.py | 0 .../qwen_behavior_rough_cut/src/operator.py | 1007 +++++++++++++++++ .../tests/acceptance/fixture_contract.json | 21 + scripts/generate_operator_images.py | 3 + .../generate_operator_repository_manifests.py | 1 + src/App.tsx | 2 +- src/components/AnnotationTimelineEditor.tsx | 33 +- src/components/Sidebar.tsx | 4 +- src/pages/AnnotationOperations.tsx | 60 +- src/types.ts | 4 + tests/backend/test_delivery_pose_contracts.py | 66 +- tests/backend/test_qwen_behavior_rough_cut.py | 242 ++++ tests/frontend/annotation-operations.test.mjs | 2 + tests/frontend/fanyu-v5-four-stage.test.mjs | 25 + 58 files changed, 4097 insertions(+), 273 deletions(-) create mode 100644 backend/operators/builtin/qwen_behavior_rough_cut/__init__.py create mode 100644 backend/operators/builtin/qwen_behavior_rough_cut/operator.json create mode 100644 backend/operators/builtin/qwen_behavior_rough_cut/operator.py create mode 100644 backend/operators/builtin/qwen_behavior_rough_cut/repository.json create mode 100644 docs/operators/Qwen3-VL-语义行为粗切-PTS.md rename docs/operators/{目录扫描.md => 目录读取.md} (78%) create mode 100644 internal/api/fanyu_v5_four_stage_human_loop_test.go create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/Dockerfile create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/README.md create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/operator.json create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/requirements.txt create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/runtime_contract.json create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/src/__init__.py create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py create mode 100644 operator-repos/builtin/qwen_behavior_rough_cut/tests/acceptance/fixture_contract.json create mode 100644 tests/backend/test_qwen_behavior_rough_cut.py create mode 100644 tests/frontend/fanyu-v5-four-stage.test.mjs diff --git a/backend/operator-runner/main.py b/backend/operator-runner/main.py index 40654d9b..f64ab47f 100644 --- a/backend/operator-runner/main.py +++ b/backend/operator-runner/main.py @@ -5338,7 +5338,7 @@ BUILTIN_HANDLER_RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("json_map", ("json映射", "json 映射", "rename", "映射重命名")), ("text_template", ("模板渲染", "template")), ("condition_router", ("条件分流", "conditional", "router", "route")), - ("directory_scan", ("目录扫描", "directory", "browse", "browse dir")), + ("directory_scan", ("目录读取", "目录扫描", "directory", "browse", "browse dir")), ("merge", ("merge", "合并")), ("intersection", ("intersect", "交集")), ("distribution", ("distribution", "分配")), @@ -5376,6 +5376,7 @@ BUILTIN_HANDLERS: dict[str, BuiltinHandler] = { "dataset_filter": operator_handler('backend.operators.builtin.video_filter.operator'), "video_slicing": operator_handler('backend.operators.builtin.video_slice.operator'), "video_face_blur": operator_handler('backend.operators.builtin.video_face_blur.operator'), + "qwen_behavior_rough_cut": operator_handler('backend.operators.builtin.qwen_behavior_rough_cut.operator'), "qwen_temporal_localization": operator_handler('backend.operators.builtin.qwen_temporal_localization.operator'), "wilor_valid_event_detection": operator_handler('backend.operators.builtin.wilor_valid_event_detection.operator'), "ms_swift_dataset_export": operator_handler('backend.operators.builtin.ms_swift_dataset_export.operator'), diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py index 1c742d66..6b3678f5 100644 --- a/backend/operator-runner/test_main.py +++ b/backend/operator-runner/test_main.py @@ -383,6 +383,50 @@ class RuntimeFileContractTests(unittest.TestCase): self.assertEqual(MODULE.select_handler("数据集切分泄漏校验"), "dataset_split_leakage_validate") self.assertEqual(MODULE.select_handler("媒体引用可读性校验"), "media_reference_validate") self.assertEqual(MODULE.select_handler("数据集Schema合约校验"), "dataset_schema_contract_validate") + self.assertEqual(MODULE.select_handler("目录读取(必填)"), "directory_scan") + + def test_directory_read_consumes_required_bos_directory_ref_without_incremental_state(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + entries = [ + { + "path": "capture/camera_left_part0001.mp4", + "storagePath": "bos://testxyz/fanyu/capture/camera_left_part0001.mp4", + "sizeBytes": 123, + "source": {"path": "bos://testxyz/fanyu/capture/camera_left_part0001.mp4"}, + }, + { + "path": "capture/camera_left_part0001_pts.csv", + "storagePath": "bos://testxyz/fanyu/capture/camera_left_part0001_pts.csv", + "sizeBytes": 45, + "source": {"path": "bos://testxyz/fanyu/capture/camera_left_part0001_pts.csv"}, + }, + ] + node_data = { + "label": "目录读取(必填)", + "hyperparams": [ + {"key": "BOS_ENDPOINT", "value": "https://bj.bcebos.com"}, + {"key": "BOS_ACCESS_KEY_ID", "value": "ak"}, + {"key": "BOS_SECRET_ACCESS_KEY", "value": "sk"}, + ], + "outputs": [{"id": "out-1", "name": "manifest", "type": "json"}], + } + inputs = [{ + "name": "sourceDirectory", + "type": "file", + "path": "bos://testxyz/fanyu/capture", + "payload": {"mode": "ref", "type": "file", "ref": {"provider": "BOS", "connectionId": "storage-1", "bucket": "testxyz", "prefix": "fanyu/capture", "manifestKey": "fanyu/capture/_droboticflow/manifests/run.json", "kind": "directory"}}, + }] + with mock.patch.dict(MODULE.OPERATOR_HELPERS, {"list_bos_entries": mock.Mock(return_value=entries)}): + artifacts = MODULE.BUILTIN_HANDLERS["directory_scan"]("run-directory", "directory", output_dir, node_data, inputs) + + payload = json.loads(pathlib.Path(artifacts[0]["path"]).read_text(encoding="utf-8")) + self.assertEqual(payload["schemaVersion"], "droboticflow.directory-read/v1") + self.assertEqual(payload["sourcePath"], "bos://testxyz/fanyu/capture") + self.assertEqual(payload["fileCount"], 2) + self.assertEqual(payload["entries"], entries) + self.assertNotIn("scanMode", payload) + self.assertNotIn("checkpoint", payload) def test_builtin_handler_aliases_match_runtime_contract_names(self): self.assertIs(MODULE.BUILTIN_HANDLERS["builtin_lerobot"], MODULE.BUILTIN_HANDLERS["lerobot"]) diff --git a/backend/operators/builtin/directory_scan/operator.json b/backend/operators/builtin/directory_scan/operator.json index b21738d3..2ad5f8a5 100644 --- a/backend/operators/builtin/directory_scan/operator.json +++ b/backend/operators/builtin/directory_scan/operator.json @@ -1,6 +1,6 @@ { "id": "operator-builtin-directory-scan", - "name": "目录扫描", + "name": "目录读取", "category": "IO", "type": "builtin", "runtimeHandler": "builtin_directory_scan", diff --git a/backend/operators/builtin/directory_scan/operator.py b/backend/operators/builtin/directory_scan/operator.py index 7173b6d8..04f9b60c 100644 --- a/backend/operators/builtin/directory_scan/operator.py +++ b/backend/operators/builtin/directory_scan/operator.py @@ -1,11 +1,57 @@ from __future__ import annotations +import os +import pathlib from typing import Any, Callable HelperMap = dict[str, Callable[..., Any]] +def _directory_ref(inputs: list[dict[str, Any]]) -> tuple[str, str] | None: + for artifact in inputs: + payload = artifact.get("payload") if isinstance(artifact.get("payload"), dict) else {} + ref = payload.get("ref") if isinstance(payload.get("ref"), dict) else {} + if str(ref.get("kind", "")).strip().lower() != "directory": + continue + bucket = str(ref.get("bucket", "")).strip() + prefix = str(ref.get("prefix", "")).strip().strip("/") + if bucket: + return bucket, prefix + raw_path = str(artifact.get("path", "")).strip() + if raw_path.startswith("bos://"): + without_scheme = raw_path[len("bos://") :] + parsed_bucket, _, parsed_prefix = without_scheme.partition("/") + if parsed_bucket: + return parsed_bucket, parsed_prefix.strip("/") + return None + + +def _bos_entries( + bucket: str, + prefix: str, + pattern: str, + params: dict[str, str], + helpers: HelperMap, +) -> list[dict[str, Any]]: + endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip() + access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip() + secret_access_key = str(params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "")).strip() + if not endpoint or not access_key_id or not secret_access_key: + raise RuntimeError("目录读取需要输入目录对应的 BOS 连接凭证") + entries = helpers["list_bos_entries"]( + bucket, + prefix, + pattern, + endpoint, + access_key_id, + secret_access_key, + ) + if not entries: + raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件") + return entries + + def run( run_id: str, node_id: str, @@ -15,12 +61,56 @@ def run( helpers: HelperMap, ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) + pattern = str(params.get("PATTERN", "")).strip() + directory_ref = _directory_ref(inputs) + if directory_ref is not None: + bucket, prefix = directory_ref + entries = _bos_entries(bucket, prefix, pattern, params, helpers) + source_path = helpers["canonical_bos_path"](bucket, prefix) + payload = { + "schemaVersion": "droboticflow.directory-read/v1", + "datasetType": "directory", + "dataSource": "BOS", + "sourcePath": source_path, + "bucket": bucket, + "prefix": prefix, + "pattern": pattern, + "fileCount": len(entries), + "totalBytes": sum(int(item.get("sizeBytes", 0) or 0) for item in entries), + "entries": entries, + } + return [ + helpers["write_json_artifact"]( + run_id, + node_id, + output_dir, + "directory-manifest.json", + payload, + output=helpers["output_binding"](node_data), + ) + ] + source_path = helpers["resolve_source_path"](params, inputs) if source_path is not None: root = source_path if source_path.is_dir() else source_path.parent + entries = helpers["list_source_entries"](root, pattern) payload = { + "schemaVersion": "droboticflow.directory-read/v1", + "datasetType": "directory", + "dataSource": "FILESYSTEM", "sourcePath": str(source_path), "scanRoot": str(root), + "pattern": pattern, + "fileCount": len(entries), + "totalBytes": sum(int(item.get("sizeBytes", 0) or 0) for item in entries), + "entries": [ + { + **item, + "storagePath": str(root / pathlib.PurePosixPath(str(item.get("path", "")))), + "source": {"path": str(root / pathlib.PurePosixPath(str(item.get("path", ""))))}, + } + for item in entries + ], "summary": helpers["summarize_directory_tree"](root), } elif inputs: @@ -30,13 +120,13 @@ def run( "summary": helpers["summarize_input_artifacts"](inputs), } else: - raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录") + raise RuntimeError("目录读取需要必填目录输入或 SOURCE_PATH 指向真实目录") return [ helpers["write_json_artifact"]( run_id, node_id, output_dir, - "directory-scan.json", + "directory-manifest.json", payload, output=helpers["output_binding"](node_data), ) diff --git a/backend/operators/builtin/directory_scan/repository.json b/backend/operators/builtin/directory_scan/repository.json index e76bb08a..06aef054 100644 --- a/backend/operators/builtin/directory_scan/repository.json +++ b/backend/operators/builtin/directory_scan/repository.json @@ -19,7 +19,7 @@ "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-directory-scan:builtin", "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-directory-scan@sha256:f591c68e5a9f63ed1203347dedfddfe7f9f8786266d69af1fbdab74bc4eece9f", "metadata": "operator.json", - "name": "目录扫描", + "name": "目录读取", "operatorId": "operator-builtin-directory-scan", "operatorSlug": "directory_scan", "preferredSlug": "directory_scan", @@ -59,12 +59,14 @@ "requiresGpu": false }, "runtimeDependencies": [ - "filesystem" + "filesystem", + "bos-directory-ref" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, "runtimeDependencies": [ - "filesystem" + "filesystem", + "bos-directory-ref" ], "runtimeHandler": "builtin_directory_scan", "runtimeSource": "backend/operators/builtin/directory_scan/operator.py", diff --git a/backend/operators/builtin/package/operator.py b/backend/operators/builtin/package/operator.py index a4c3ebe0..442aabd1 100644 --- a/backend/operators/builtin/package/operator.py +++ b/backend/operators/builtin/package/operator.py @@ -124,7 +124,14 @@ def _materialize_final_mp4(item: dict[str, Any], package_dir: pathlib.Path, help destination = package_dir / name source_path = pathlib.Path(str(item.get("clipPath", "")).strip()) if not source_path.is_file() or source_path.stat().st_size <= 0: - raise RuntimeError(f"final segment source clip is missing or empty: {source_path}") + source = item.get("source") if isinstance(item.get("source"), dict) else {} + source_path = helpers["source_path_from_item"]( + {"source": source or {"path": str(item.get("clipPath", "")).strip()}}, + str(item.get("sourceVideo") or pathlib.PurePosixPath(str(item.get("clipPath", ""))).name or "source.mp4"), + package_dir, + ) + if not source_path.is_file() or source_path.stat().st_size <= 0: + raise RuntimeError(f"final segment source video is missing or empty: {source_path}") start = float(item.get("startSecond", 0.0)) end = float(item.get("endSecond", 0.0)) if not (0 <= start < end): @@ -153,11 +160,8 @@ def _run_fanyu_v5(run_id: str, node_id: str, output_dir: Any, node_data: dict[st payloads = _payload_tree(helpers["payloads_from_inputs"](inputs)) headpose = _first_payload(payloads, "droboticflow.headpose/v1") handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1") - training = _first_payload(payloads, "droboticflow.qwen3vl.continuous-training/v2") if require_headpose and (headpose is None or not headpose.get("enabled") or not isinstance(headpose.get("frames"), list)): raise RuntimeError("package requires a real Headpose manifest with frames") - if require_final_acceptance and (training is None or training.get("status") != "succeeded" or not isinstance(training.get("adapter"), dict) or not isinstance(training.get("provenance"), dict)): - raise RuntimeError("v5 package requires successful real Qwen3-VL training evidence before packaging") accepted = next((payload for payload in payloads if isinstance(payload.get("items"), list) and ("annotationCount" in payload or payload.get("schemaVersion") == "droboticflow.annotation.accepted-partition/v1")), None) if accepted is None: raise RuntimeError("package requires an accepted annotation partition") @@ -201,7 +205,7 @@ def _run_fanyu_v5(run_id: str, node_id: str, output_dir: Any, node_data: dict[st package_items.append({"packageName": package_dir.name, "clipId": clip_id, "clipName": video_path.name, "storagePath": str(package_dir), "startSecond": start, "endSecond": end, "duration": end - start, "fileCount": len([path for path in package_dir.iterdir() if path.is_file()]), "quality": quality, "annotation": annotation, "segmentManifest": segment_manifest}) if not package_items: raise RuntimeError("package produced zero final segment packages") - manifest = {"schemaVersion": "droboticflow.delivery.manifest/v2", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "items": package_items, "trainingProvenance": training.get("provenance") if training else None} + manifest = {"schemaVersion": "droboticflow.delivery.manifest/v2", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "items": package_items, "trainingEnabled": False} artifacts = [helpers["write_json_artifact"](run_id, node_id, output, "packages-manifest.json", manifest, output=binding)] artifacts.extend(helpers["collect_output_artifacts"](run_id, node_id, output / "packages", node_data=node_data, output=binding)) return artifacts diff --git a/backend/operators/builtin/package/repository.json b/backend/operators/builtin/package/repository.json index 8b34455a..f3ebbde5 100644 --- a/backend/operators/builtin/package/repository.json +++ b/backend/operators/builtin/package/repository.json @@ -63,8 +63,7 @@ "video-files", "accepted-annotation-partitions", "headpose-manifests", - "optional-handpose-manifests", - "qwen-training-provenance" + "optional-handpose-manifests" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, @@ -73,8 +72,7 @@ "video-files", "accepted-annotation-partitions", "headpose-manifests", - "optional-handpose-manifests", - "qwen-training-provenance" + "optional-handpose-manifests" ], "runtimeHandler": "builtin_result_package", "runtimeSource": "backend/operators/builtin/package/operator.py", diff --git a/backend/operators/builtin/qwen_behavior_rough_cut/__init__.py b/backend/operators/builtin/qwen_behavior_rough_cut/__init__.py new file mode 100644 index 00000000..763311b8 --- /dev/null +++ b/backend/operators/builtin/qwen_behavior_rough_cut/__init__.py @@ -0,0 +1 @@ +"""Qwen3-VL PTS-grounded behaviour rough-cut builtin.""" diff --git a/backend/operators/builtin/qwen_behavior_rough_cut/operator.json b/backend/operators/builtin/qwen_behavior_rough_cut/operator.json new file mode 100644 index 00000000..8bac7efe --- /dev/null +++ b/backend/operators/builtin/qwen_behavior_rough_cut/operator.json @@ -0,0 +1,16 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "id": "operator-builtin-qwen-behavior-rough-cut", + "migrationStatus": "migrated", + "name": "Qwen3-VL 语义行为粗切(PTS)", + "runtimeHandler": "builtin_qwen_behavior_rough_cut", + "runtimeSource": "operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py", + "type": "builtin" +} diff --git a/backend/operators/builtin/qwen_behavior_rough_cut/operator.py b/backend/operators/builtin/qwen_behavior_rough_cut/operator.py new file mode 100644 index 00000000..45df192d --- /dev/null +++ b/backend/operators/builtin/qwen_behavior_rough_cut/operator.py @@ -0,0 +1,1007 @@ +"""PTS-grounded Qwen3-VL behaviour rough-cut for the Fanyu workflow. + +This is deliberately *not* a duration slicer. It asks a verified local +Qwen3-VL checkpoint to describe hand-action changes (subject-verb-object) and +uses only a PTS value visibly burned into every model input frame as timing +evidence. The bounded technical blocks exist solely to cap a single inference +request; they are never emitted as clip boundaries. + +Candidates are virtual clips: each one retains the original full, face-blurred +video artifact reference. The human cutter therefore works against the full +sanitized source and can correct Qwen's proposed boundaries before any physical +MP4 is materialized by the downstream package stage. +""" +from __future__ import annotations + +import copy +import csv +import hashlib +import json +import math +import pathlib +import re +import uuid +from datetime import datetime, timezone +from typing import Any, Callable + +from backend.operators.builtin.qwen3vl_model_manifest import ( + REVISION as QWEN3VL_8B_INSTRUCT_REVISION, + stage_preverified_model_to_local, + verify as verify_qwen3vl_model_manifest, +) + + +HelperMap = dict[str, Callable[..., Any]] +VIDEO_SUFFIXES = {".mp4", ".mov", ".m4v", ".avi", ".mkv", ".webm"} +PTS_COLUMN = "timestamp_us" +DEFAULT_TECHNICAL_BLOCK_SECONDS = 10.0 +DEFAULT_FRAMES_PER_TECHNICAL_BLOCK = 18 +REFERENCE_GROUP_SCRIPT_SHA256 = "2cf5fdb34fc11911dbcdbd4c51965d9cfe17239f862ebd9cad2e18078f90afc2" +# The operator itself writes exactly ``PTS_US=`` into the top-right +# overlay. Accepting a bare number or another clock format would make it +# impossible to prove that a model response was copied from that overlay. +_PTS_TEXT = re.compile(r"^PTS_US=(\d+)$") + + +def _sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _hyperparams(node_data: dict[str, Any], helpers: HelperMap) -> dict[str, str]: + fn = helpers.get("hyperparams") + if callable(fn): + return {str(key): str(value) for key, value in fn(node_data).items()} + return { + str(item.get("key")): str(item.get("value", "")) + for item in node_data.get("hyperparams", []) + if isinstance(item, dict) and item.get("key") + } + + +def _boolean(params: dict[str, str], key: str, default: bool) -> bool: + raw = str(params.get(key, str(default))).strip().lower() + if raw in {"1", "true", "yes", "on"}: + return True + if raw in {"0", "false", "no", "off"}: + return False + raise RuntimeError(f"{key} must be a boolean") + + +def _positive_float(params: dict[str, str], key: str, default: float) -> float: + try: + value = float(params.get(key, default) or default) + except (TypeError, ValueError) as exc: + raise RuntimeError(f"{key} must be a finite positive number") from exc + if not math.isfinite(value) or value <= 0: + raise RuntimeError(f"{key} must be a finite positive number") + return value + + +def _positive_int(params: dict[str, str], key: str, default: int) -> int: + value = _positive_float(params, key, float(default)) + if not value.is_integer(): + raise RuntimeError(f"{key} must be a positive integer") + return int(value) + + +def _payloads(inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]: + fn = helpers.get("payloads_from_inputs") + if callable(fn): + values = fn(inputs) + if isinstance(values, list): + return [value for value in values if isinstance(value, dict)] + values: list[dict[str, Any]] = [] + for item in inputs: + data = item.get("data") if isinstance(item, dict) else None + if isinstance(data, dict): + values.append(data) + return values + + +def _face_blur_manifest(payloads: list[dict[str, Any]]) -> dict[str, Any]: + for payload in payloads: + schema = str(payload.get("schemaVersion", "")).strip() + items = payload.get("items") + if schema.startswith("droboticflow.video_face_blur.") and isinstance(items, list): + return payload + if isinstance(items, list) and any(isinstance(item, dict) and item.get("sanitizedVideoPath") for item in items): + return payload + raise RuntimeError("Qwen 行为粗切需要视频人脸脱敏输出 manifest 与真实脱敏视频 artifact") + + +def _orbbec_manifest(payloads: list[dict[str, Any]]) -> dict[str, Any]: + for payload in payloads: + schema = str(payload.get("schemaVersion", "")).strip() + if schema == "droboticflow.orbbec.normalized/v1" and isinstance(payload.get("entries"), list): + return payload + if payload.get("datasetType") == "orbbec-ego" and isinstance(payload.get("entries"), list): + return payload + raise RuntimeError("Qwen 行为粗切需要 Orbbec 标准化 manifest,以定位每帧 PTS CSV") + + +def _basename(raw: Any) -> str: + return pathlib.PurePosixPath(str(raw or "").replace("\\", "/")).name + + +def _original_video_name(item: dict[str, Any]) -> str: + for key in ("relativePath", "originalFilename", "sourceVideo"): + candidate = _basename(item.get(key)) + if pathlib.PurePosixPath(candidate).suffix.lower() in VIDEO_SUFFIXES: + return candidate + filename = _basename(item.get("filename") or item.get("sanitizedFilename")) + if filename.startswith("blurred_"): + filename = filename[len("blurred_") :] + if pathlib.PurePosixPath(filename).suffix.lower() in VIDEO_SUFFIXES: + return filename + raise RuntimeError("视频人脸脱敏 manifest 缺少原始视频文件名") + + +def _sanitized_video_name(item: dict[str, Any]) -> str: + for key in ("sanitizedFilename", "filename", "sanitizedVideoPath"): + candidate = _basename(item.get(key)) + if pathlib.PurePosixPath(candidate).suffix.lower() in VIDEO_SUFFIXES: + return candidate + raise RuntimeError("视频人脸脱敏 manifest 缺少已脱敏视频文件名") + + +def _artifact_by_names(inputs: list[dict[str, Any]], names: list[str], suffixes: set[str]) -> dict[str, Any] | None: + expected = {name.lower() for name in names if name} + for artifact in inputs: + if not isinstance(artifact, dict): + continue + name = _basename(artifact.get("name") or artifact.get("path")) + if name.lower() in expected and pathlib.PurePosixPath(name).suffix.lower() in suffixes: + return artifact + return None + + +def _entry_by_names(entries: list[dict[str, Any]], names: list[str]) -> dict[str, Any] | None: + expected = {name.lower() for name in names if name} + for entry in entries: + if not isinstance(entry, dict): + continue + name = _basename(entry.get("relativePath") or entry.get("path") or entry.get("name")) + if name.lower() in expected: + return entry + return None + + +def _artifact_source_item(artifact: dict[str, Any], helpers: HelperMap) -> dict[str, Any]: + source_ref = helpers.get("source_ref_from_artifact") + source = source_ref(artifact) if callable(source_ref) else {} + if not isinstance(source, dict): + source = {} + return {"path": str(artifact.get("path", "")).strip(), "source": source} + + +def _materialize_item(item: dict[str, Any], default_name: str, output: pathlib.Path, helpers: HelperMap) -> pathlib.Path: + raw_path = str(item.get("path", "")).strip() + if raw_path: + direct = pathlib.Path(raw_path) + if direct.is_file() and direct.stat().st_size > 0: + return direct + source = item.get("source") if isinstance(item.get("source"), dict) else {} + direct_source = str(source.get("path", "")).strip() + if direct_source: + direct = pathlib.Path(direct_source) + if direct.is_file() and direct.stat().st_size > 0: + return direct + materialize = helpers.get("source_path_from_item") + if not callable(materialize): + raise RuntimeError(f"Qwen 行为粗切无法物化真实输入文件: {default_name}") + path = materialize(item, default_name, output, {}) + if not isinstance(path, pathlib.Path): + path = pathlib.Path(path) + if not path.is_file() or path.stat().st_size <= 0: + raise RuntimeError(f"Qwen 行为粗切物化的输入文件为空: {default_name}") + return path + + +def _pts_candidates( + inputs: list[dict[str, Any]], + normalized_entries: list[dict[str, Any]], + blur_item: dict[str, Any], + original_video_name: str, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + original_pts = pathlib.PurePosixPath(original_video_name).stem + "_pts.csv" + blurred_pts = "blurred_" + original_pts + paired = blur_item.get("pairedFiles") if isinstance(blur_item.get("pairedFiles"), dict) else {} + paired_pts = _basename(paired.get("pts")) + names = [paired_pts, blurred_pts, original_pts] + artifacts: list[dict[str, Any]] = [] + entries: list[dict[str, Any]] = [] + for name in names: + artifact = _artifact_by_names(inputs, [name], {".csv"}) + if artifact is not None and artifact not in artifacts: + artifacts.append(artifact) + entry = _entry_by_names(normalized_entries, [name]) + if entry is not None and entry not in entries: + entries.append(entry) + # A copied PTS artifact is preferred because it travelled with the blur + # output. The standardised source entry is retained as a durable fallback. + return artifacts, entries + + +def _read_pts_csv(path: pathlib.Path) -> list[int]: + try: + handle = path.open("r", encoding="utf-8-sig", newline="") + except OSError as exc: + raise RuntimeError(f"cannot open PTS CSV {path}: {exc}") from exc + with handle: + reader = csv.DictReader(handle) + fields = {str(field).strip(): field for field in (reader.fieldnames or []) if field is not None} + source_field = fields.get(PTS_COLUMN) + if source_field is None: + raise RuntimeError(f"PTS CSV {path.name} must contain exactly-named {PTS_COLUMN} column") + values: list[int] = [] + for line_number, row in enumerate(reader, start=2): + raw = str(row.get(source_field, "")).strip() + if not re.fullmatch(r"\d+", raw): + raise RuntimeError(f"PTS CSV {path.name} line {line_number} has invalid {PTS_COLUMN}") + value = int(raw) + if values and value <= values[-1]: + raise RuntimeError(f"PTS CSV {path.name} {PTS_COLUMN} must be strictly increasing") + values.append(value) + if not values: + raise RuntimeError(f"PTS CSV {path.name} has no timestamp rows") + return values + + +def _decode_frame_count(video: pathlib.Path) -> tuple[int, float, int, int]: + try: + import cv2 + except Exception as exc: # pragma: no cover - image dependency boundary + raise RuntimeError("Qwen 行为粗切需要 OpenCV 解码真实视频") from exc + capture = cv2.VideoCapture(str(video)) + if not capture.isOpened(): + raise RuntimeError(f"cannot decode sanitized video {video}") + try: + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + if not math.isfinite(fps) or fps <= 0 or width <= 0 or height <= 0: + raise RuntimeError("sanitized video has invalid FPS or resolution") + count = 0 + while True: + ok, frame = capture.read() + if not ok or frame is None: + break + count += 1 + if count <= 0: + raise RuntimeError("sanitized video decoded zero frames") + return count, fps, width, height + finally: + capture.release() + + +def _technical_blocks(frame_count: int, fps: float, technical_block_seconds: float, frames_per_block: int) -> list[dict[str, Any]]: + if frame_count <= 0 or fps <= 0: + raise RuntimeError("technical inference planning requires positive frame count and FPS") + frames_per_duration = max(1, int(math.ceil(technical_block_seconds * fps))) + blocks: list[dict[str, Any]] = [] + for offset in range(0, frame_count, frames_per_duration): + end = min(frame_count, offset + frames_per_duration) + available = end - offset + sample_count = min(frames_per_block, available) + if sample_count <= 0: + continue + if sample_count == 1: + sample_indices = [offset] + else: + sample_indices = [offset + int(round((available - 1) * index / (sample_count - 1))) for index in range(sample_count)] + if len(set(sample_indices)) != len(sample_indices): + raise RuntimeError("technical inference frame plan produced duplicate samples") + blocks.append( + { + "technicalOnly": True, + "technicalBlockIndex": len(blocks) + 1, + "startFrameIndex": offset, + "endFrameIndexExclusive": end, + "sampleFrameIndices": sample_indices, + } + ) + if not blocks: + raise RuntimeError("technical inference plan contains no blocks") + return blocks + + +def _overlay_pts(frame: Any, timestamp_us: int) -> Any: + import cv2 + + image = frame.copy() + height, width = image.shape[:2] + text = f"PTS_US={timestamp_us}" + font = cv2.FONT_HERSHEY_SIMPLEX + scale = 0.8 + thickness = 2 + while scale > 0.28: + (text_width, text_height), baseline = cv2.getTextSize(text, font, scale, thickness) + if text_width + 20 <= width: + break + scale -= 0.08 + (text_width, text_height), baseline = cv2.getTextSize(text, font, scale, thickness) + x = max(5, width - text_width - 10) + y = max(text_height + baseline + 8, min(height - 8, text_height + baseline + 8)) + top_left = (max(0, x - 5), max(0, y - text_height - baseline - 5)) + bottom_right = (min(width - 1, x + text_width + 5), min(height - 1, y + baseline + 5)) + cv2.rectangle(image, top_left, bottom_right, (0, 0, 0), -1) + cv2.putText(image, text, (x, y), font, scale, (255, 255, 255), thickness, cv2.LINE_AA) + return image + + +def _write_overlaid_samples( + video: pathlib.Path, + timestamps_us: list[int], + blocks: list[dict[str, Any]], + output: pathlib.Path, +) -> list[dict[str, Any]]: + try: + import cv2 + except Exception as exc: # pragma: no cover - image dependency boundary + raise RuntimeError("Qwen 行为粗切需要 OpenCV 写入带 PTS 叠加的真实帧") from exc + requested: dict[int, tuple[int, int]] = {} + for block_index, block in enumerate(blocks): + for sample_position, frame_index in enumerate(block["sampleFrameIndices"]): + if frame_index in requested: + raise RuntimeError("technical inference blocks must not reuse sampled frames") + requested[frame_index] = (block_index, sample_position) + capture = cv2.VideoCapture(str(video)) + if not capture.isOpened(): + raise RuntimeError(f"cannot decode sanitized video {video}") + sample_root = output / "_qwen_behavior_rough_cut_frames" + sample_root.mkdir(parents=True, exist_ok=True) + materialized: list[list[dict[str, Any] | None]] = [ + [None for _ in block["sampleFrameIndices"]] for block in blocks + ] + try: + index = 0 + while True: + ok, frame = capture.read() + if not ok or frame is None: + break + target = requested.get(index) + if target is not None: + block_index, sample_position = target + timestamp_us = timestamps_us[index] + rendered = _overlay_pts(frame, timestamp_us) + path = sample_root / f"block-{block_index + 1:05d}-frame-{index:08d}.jpg" + if not cv2.imwrite(str(path), rendered) or not path.is_file() or path.stat().st_size <= 0: + raise RuntimeError(f"failed to write PTS-overlaid model frame {index}") + materialized[block_index][sample_position] = { + "path": str(path), + "frameIndex": index, + "timestampUs": timestamp_us, + "overlayText": f"PTS_US={timestamp_us}", + } + index += 1 + finally: + capture.release() + results: list[dict[str, Any]] = [] + for block_index, items in enumerate(materialized): + if any(item is None for item in items): + missing = [blocks[block_index]["sampleFrameIndices"][position] for position, item in enumerate(items) if item is None] + raise RuntimeError(f"unable to decode planned model frame(s): {missing}") + completed = [item for item in items if isinstance(item, dict)] + blocks[block_index]["sampledFrames"] = completed + blocks[block_index]["sampledPTSUs"] = [int(item["timestampUs"]) for item in completed] + results.append(blocks[block_index]) + return results + + +def _load_qwen(model_path: pathlib.Path) -> tuple[Any, Any, Any]: + try: + import torch + from transformers import AutoProcessor, Qwen3VLForConditionalGeneration + except Exception as exc: # pragma: no cover - image dependency boundary + raise RuntimeError("Qwen3-VL runtime requires torch and transformers in the operator image") from exc + if not torch.cuda.is_available(): + raise RuntimeError("Qwen3-VL 行为粗切需要已分配 CUDA GPU;没有 CPU、mock 或 heuristic fallback") + try: + processor = AutoProcessor.from_pretrained(str(model_path), local_files_only=True) + model = Qwen3VLForConditionalGeneration.from_pretrained( + str(model_path), local_files_only=True, torch_dtype="auto", device_map="auto" + ) + model.eval() + except Exception as exc: + raise RuntimeError(f"cannot load verified local Qwen3-VL checkpoint {model_path}: {exc}") from exc + return torch, processor, model + + +def _json_behaviors(text: str) -> dict[str, Any]: + candidate = re.sub(r"```(?:json)?", "", text, flags=re.IGNORECASE).strip() + decoder = json.JSONDecoder() + matches: list[dict[str, Any]] = [] + for start, character in enumerate(candidate): + if character != "{": + continue + try: + payload, _ = decoder.raw_decode(candidate[start:]) + except json.JSONDecodeError: + continue + if isinstance(payload, dict) and isinstance(payload.get("behaviors"), list): + matches.append(payload) + if not matches: + response_sha = hashlib.sha256(text.encode("utf-8")).hexdigest() + raise RuntimeError( + "Qwen3-VL behaviour response must be a JSON object containing behaviors " + f"(sha256={response_sha}, chars={len(text)})" + ) + return matches[-1] + + +def _previous_context(events: list[dict[str, Any]]) -> list[dict[str, str]]: + return [ + { + "behavior": str(event["behavior"]), + "object_change": str(event.get("object_change", "")), + "start_timestamp": str(event["start_timestamp"]), + "end_timestamp": str(event["end_timestamp"]), + } + for event in events + ] + + +def _infer_technical_block( + torch: Any, + processor: Any, + model: Any, + frames: list[dict[str, Any]], + previous_events: list[dict[str, Any]], + max_new_tokens: int, +) -> tuple[dict[str, Any], str]: + prompt = ( + "Analyze the chronological images from a convenience-store egocentric video. " + "Identify every change in hand action as subject-verb-object and describe what happens to the manipulated object. " + "The images are a bounded technical inference request only; they are NOT delivery clips or proposed fixed-duration cuts.\n\n" + "IMPORTANT: every image contains a high-contrast PTS_US timestamp overlay in the top-right corner. " + "That top-right PTS_US overlay is the ONLY source of time. Read it directly. " + "Do NOT derive time from frame ordering, FPS, video duration, request boundaries, or any offset. " + "For every action return exact PTS_US strings visibly printed in the images.\n\n" + "Return JSON only: {\"behaviors\":[{\"behavior\":string,\"object_change\":string," + "\"start_timestamp\":string,\"end_timestamp\":string}]}. " + "Use an empty behaviors array only when no hand-action change is visible. " + "Do not include Markdown, explanation, confidence, invented timestamps, or extra fields." + ) + if previous_events: + prompt += ( + "\n\nPrevious technical-block analysis:\n" + + json.dumps(_previous_context(previous_events), ensure_ascii=False, separators=(",", ":")) + + "\nContinue behavior naming consistently. Timestamps must still come only from the visible PTS_US overlay." + ) + try: + from qwen_vl_utils import process_vision_info + + conversation = [ + { + "role": "user", + "content": [ + *[{"type": "image", "image": str(frame["path"])} for frame in frames], + {"type": "text", "text": prompt}, + ], + } + ] + text = processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True) + image_inputs, video_inputs = process_vision_info(conversation) + model_inputs = processor( + text=[text], images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True + ).to(model.device) + with torch.inference_mode(): + generated = model.generate(**model_inputs, max_new_tokens=max_new_tokens, do_sample=False) + trimmed = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated)] + response = processor.batch_decode(trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + except Exception as exc: + raise RuntimeError(f"Qwen3-VL behaviour inference failed on real PTS-overlaid frames: {exc}") from exc + return _json_behaviors(response), response + + +def _parse_overlay_timestamp(raw: Any) -> tuple[str, int]: + if not isinstance(raw, str): + raise RuntimeError("Qwen3-VL behavior timestamp must be a string read from the PTS_US overlay") + value = raw.strip() + matched = _PTS_TEXT.fullmatch(value) + if matched is None: + raise RuntimeError("Qwen3-VL behavior timestamp is not an exact PTS_US overlay value") + return value, int(matched.group(1)) + + +def _normalize_behavior( + raw: dict[str, Any], + sampled_timestamps_us: set[int], + timestamp_to_frame: dict[int, int], + pts_origin_us: int, +) -> dict[str, Any]: + required = {"behavior", "object_change", "start_timestamp", "end_timestamp"} + if set(raw) != required: + raise RuntimeError("Qwen3-VL behavior event must contain exactly behavior, object_change, start_timestamp and end_timestamp") + behavior = raw.get("behavior") + object_change = raw.get("object_change") + if not isinstance(behavior, str) or not behavior.strip(): + raise RuntimeError("Qwen3-VL behavior event requires non-empty behavior") + if not isinstance(object_change, str): + raise RuntimeError("Qwen3-VL behavior event object_change must be a string") + start_raw, start_us = _parse_overlay_timestamp(raw.get("start_timestamp")) + end_raw, end_us = _parse_overlay_timestamp(raw.get("end_timestamp")) + if start_us not in sampled_timestamps_us or end_us not in sampled_timestamps_us: + raise RuntimeError("Qwen3-VL behavior timestamps must exactly match PTS_US values in this request's displayed frames") + if start_us >= end_us: + raise RuntimeError("Qwen3-VL behavior event end timestamp must be greater than start timestamp") + if start_us not in timestamp_to_frame or end_us not in timestamp_to_frame: + raise RuntimeError("Qwen3-VL behavior timestamp is not present in the verified PTS CSV") + return { + "behavior": behavior.strip(), + "object_change": object_change.strip(), + "start_timestamp": start_raw, + "end_timestamp": end_raw, + "start_us": start_us, + "end_us": end_us, + "start_frame_index": timestamp_to_frame[start_us], + "end_frame_index": timestamp_to_frame[end_us], + "start_second": round((start_us - pts_origin_us) / 1_000_000.0, 6), + "end_second": round((end_us - pts_origin_us) / 1_000_000.0, 6), + } + + +def _merge_adjacent_behaviors(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Mirror the group reference: adjacent events with exact same behavior merge. + + The original raw events remain in the manifest. We intentionally do not + merge by duration, similarity, time gap, taxonomy, or any heuristic. + """ + merged: list[dict[str, Any]] = [] + for event in events: + if merged and event["behavior"] == merged[-1]["behavior"]: + prior = merged[-1] + prior["end_timestamp"] = event["end_timestamp"] + prior["end_us"] = event["end_us"] + prior["end_frame_index"] = event["end_frame_index"] + prior["end_second"] = event["end_second"] + prior["technical_block_indexes"].extend(event["technical_block_indexes"]) + prior["raw_event_indexes"].extend(event["raw_event_indexes"]) + prior.setdefault("raw_response_sha256s", [prior["raw_response_sha256"]]) + prior["raw_response_sha256s"].extend( + event.get("raw_response_sha256s", [event["raw_response_sha256"]]) + ) + continue + merged.append(copy.deepcopy(event)) + return merged + + +def _artifact_size_bytes(artifact: dict[str, Any], source_path: pathlib.Path) -> int: + raw = artifact.get("sizeBytes") + if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw >= 0: + return int(raw) + payload = artifact.get("payload") if isinstance(artifact.get("payload"), dict) else {} + ref = payload.get("ref") if isinstance(payload.get("ref"), dict) else {} + ref_size = ref.get("sizeBytes") + if isinstance(ref_size, (int, float)) and not isinstance(ref_size, bool) and ref_size >= 0: + return int(ref_size) + return source_path.stat().st_size + + +def _source_ref(artifact: dict[str, Any], helpers: HelperMap) -> dict[str, Any]: + fn = helpers.get("source_ref_from_artifact") + value = fn(artifact) if callable(fn) else {} + if isinstance(value, dict) and value: + return copy.deepcopy(value) + raw_path = str(artifact.get("path", "")).strip() + return {"path": raw_path} if raw_path else {} + + +def _virtual_clip_artifact( + run_id: str, + node_id: str, + binding: dict[str, Any] | None, + source_artifact: dict[str, Any], + source_path: pathlib.Path, + clip: dict[str, Any], + helpers: HelperMap, +) -> dict[str, Any]: + build = helpers.get("build_virtual_artifact") + if not callable(build): + raise RuntimeError("Qwen 行为粗切运行时缺少 virtual artifact builder") + payload = source_artifact.get("payload") if isinstance(source_artifact.get("payload"), dict) else None + payload_ref = payload.get("ref") if isinstance(payload, dict) and isinstance(payload.get("ref"), dict) else None + metadata = source_artifact.get("metadata") if isinstance(source_artifact.get("metadata"), dict) else {} + data: dict[str, Any] = { + "clipId": clip["clipId"], + "clipName": clip["clipName"], + "source": _source_ref(source_artifact, helpers), + "sourceIsFullVideo": True, + "virtualClip": True, + "startSecond": clip["startSecond"], + "endSecond": clip["endSecond"], + "sourceDurationSeconds": clip["sourceDurationSeconds"], + "behavior": clip["behavior"], + "objectChange": clip["objectChange"], + } + if isinstance(payload_ref, dict): + data["ref"] = copy.deepcopy(payload_ref) + result = build( + run_id, + node_id, + str(clip["clipName"]), + str(source_artifact.get("path", "")).strip() or str(source_path), + "video", + _artifact_size_bytes(source_artifact, source_path), + data=data, + metadata=copy.deepcopy(metadata), + output=binding, + ) + if isinstance(payload, dict): + # This is the important virtual-clip invariant: retain the exact + # credential-free ref / payload of the complete face-blurred source, + # rather than uploading or slicing a second media object. + result["payload"] = copy.deepcopy(payload) + return result + + +def _source_contexts( + inputs: list[dict[str, Any]], output: pathlib.Path, helpers: HelperMap +) -> list[dict[str, Any]]: + payloads = _payloads(inputs, helpers) + blur_manifest = _face_blur_manifest(payloads) + normalized = _orbbec_manifest(payloads) + normalized_entries = [entry for entry in normalized.get("entries", []) if isinstance(entry, dict)] + contexts: list[dict[str, Any]] = [] + seen_video_artifacts: set[str] = set() + for blur_item in blur_manifest.get("items", []): + if not isinstance(blur_item, dict): + continue + sanitized_name = _sanitized_video_name(blur_item) + original_name = _original_video_name(blur_item) + video_artifact = _artifact_by_names(inputs, [sanitized_name, _basename(blur_item.get("sanitizedVideoPath"))], VIDEO_SUFFIXES) + if video_artifact is None: + raise RuntimeError(f"Qwen 行为粗切未收到人脸脱敏视频 artifact: {sanitized_name}") + artifact_key = str(video_artifact.get("id") or video_artifact.get("path") or sanitized_name) + if artifact_key in seen_video_artifacts: + continue + seen_video_artifacts.add(artifact_key) + source_path = _materialize_item(_artifact_source_item(video_artifact, helpers), sanitized_name, output, helpers) + pts_artifacts, pts_entries = _pts_candidates(inputs, normalized_entries, blur_item, original_name) + pts_path: pathlib.Path | None = None + pts_origin = "" + for artifact in pts_artifacts: + try: + pts_path = _materialize_item(_artifact_source_item(artifact, helpers), _basename(artifact.get("name")), output, helpers) + except RuntimeError: + continue + pts_origin = "face-blur-output-artifact" + break + if pts_path is None: + for entry in pts_entries: + try: + pts_path = _materialize_item(entry, _basename(entry.get("relativePath") or entry.get("path")), output, helpers) + except RuntimeError: + continue + pts_origin = "orbbec-normalized-manifest" + break + if pts_path is None: + expected = pathlib.PurePosixPath(original_name).stem + "_pts.csv" + raise RuntimeError(f"Qwen 行为粗切无法从 Orbbec manifest 物化对应 PTS CSV: {expected}") + contexts.append( + { + "blurItem": blur_item, + "originalVideoName": original_name, + "sanitizedVideoName": sanitized_name, + "sourceArtifact": video_artifact, + "sourcePath": source_path, + "ptsPath": pts_path, + "ptsOrigin": pts_origin, + } + ) + if not contexts: + raise RuntimeError("视频人脸脱敏 manifest 不包含可供 Qwen 行为粗切的真实视频") + return contexts + + +def run( + run_id: str, + node_id: str, + output_dir: Any, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], + helpers: HelperMap, +) -> list[dict[str, Any]]: + output = pathlib.Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + params = _hyperparams(node_data, helpers) + technical_block_seconds = _positive_float(params, "TECHNICAL_BLOCK_SECONDS", DEFAULT_TECHNICAL_BLOCK_SECONDS) + frames_per_block = _positive_int(params, "FRAMES_PER_TECHNICAL_BLOCK", DEFAULT_FRAMES_PER_TECHNICAL_BLOCK) + if frames_per_block > 96: + raise RuntimeError("FRAMES_PER_TECHNICAL_BLOCK must not exceed 96") + max_new_tokens = _positive_int(params, "MAX_NEW_TOKENS", 2048) + require_non_empty = _boolean(params, "REQUIRE_NON_EMPTY_RESULT", False) + model_path = pathlib.Path(str(params.get("MODEL_PATH", "/workspace/models/Qwen3-VL-8B-Instruct")).strip()) + revision = str(params.get("MODEL_REVISION", QWEN3VL_8B_INSTRUCT_REVISION)).strip() + manifest_raw = str(params.get("MODEL_MANIFEST_PATH", "")).strip() + manifest_path = pathlib.Path(manifest_raw) if manifest_raw else None + trust_preverified = _boolean(params, "TRUST_PREVERIFIED_MODEL", False) + + contexts = _source_contexts(inputs, output, helpers) + model_manifest, verified_model_sha = verify_qwen3vl_model_manifest( + model_path, + revision, + manifest_path, + str(params.get("MODEL_MANIFEST_SHA256", "")).strip(), + trust_preverified, + ) + runtime_model_path, staged_model_bytes = ( + stage_preverified_model_to_local(model_path, model_manifest, verified_model_sha) + if trust_preverified + else (model_path, 0) + ) + torch, processor, model = _load_qwen(runtime_model_path) + + binding_fn = helpers.get("output_binding") + binding = binding_fn(node_data) if callable(binding_fn) else None + raw_behavior_events: list[dict[str, Any]] = [] + merged_behavior_events: list[dict[str, Any]] = [] + technical_blocks: list[dict[str, Any]] = [] + source_provenance: list[dict[str, Any]] = [] + virtual_artifacts: list[dict[str, Any]] = [] + candidate_count = 0 + + for context_index, context in enumerate(contexts, start=1): + source_path = context["sourcePath"] + timestamps_us = _read_pts_csv(context["ptsPath"]) + frame_count, fps, width, height = _decode_frame_count(source_path) + if frame_count != len(timestamps_us): + raise RuntimeError( + "PTS CSV row count must exactly equal decoded sanitized-video frame count " + f"for {context['sanitizedVideoName']}: ptsRows={len(timestamps_us)}, decodedFrames={frame_count}" + ) + timestamp_to_frame = {timestamp: index for index, timestamp in enumerate(timestamps_us)} + decoded_duration_seconds = round(frame_count / fps, 6) + pts_duration_seconds = round((timestamps_us[-1] - timestamps_us[0]) / 1_000_000.0, 6) + if pts_duration_seconds > decoded_duration_seconds + max(1.0 / fps, 0.001): + raise RuntimeError( + "PTS CSV duration is inconsistent with decoded sanitized-video duration " + f"for {context['sanitizedVideoName']}: ptsDuration={pts_duration_seconds}, " + f"decodedDuration={decoded_duration_seconds}" + ) + blocks = _technical_blocks(frame_count, fps, technical_block_seconds, frames_per_block) + blocks = _write_overlaid_samples(source_path, timestamps_us, blocks, output) + previous_events: list[dict[str, Any]] = [] + source_events: list[dict[str, Any]] = [] + for block in blocks: + result, raw_response = _infer_technical_block( + torch, + processor, + model, + block["sampledFrames"], + previous_events, + max_new_tokens, + ) + raw_values = result.get("behaviors") + if not isinstance(raw_values, list): + raise RuntimeError("Qwen3-VL behavior response is missing behaviors array") + accepted: list[dict[str, Any]] = [] + prior_start_us: int | None = None + for model_index, raw in enumerate(raw_values): + if not isinstance(raw, dict): + raise RuntimeError("Qwen3-VL behavior response behaviors must contain objects only") + event = _normalize_behavior( + raw, + set(block["sampledPTSUs"]), + timestamp_to_frame, + timestamps_us[0], + ) + if prior_start_us is not None and event["start_us"] < prior_start_us: + raise RuntimeError("Qwen3-VL behavior events must be ordered by visible PTS_US timestamp") + prior_start_us = event["start_us"] + event.update( + { + "technical_block_indexes": [block["technicalBlockIndex"]], + "raw_event_indexes": [len(raw_behavior_events) + 1], + "model_event_index": model_index, + "source_video_index": context_index, + "raw_response_sha256": hashlib.sha256(raw_response.encode("utf-8")).hexdigest(), + "raw_response_sha256s": [hashlib.sha256(raw_response.encode("utf-8")).hexdigest()], + } + ) + accepted.append(event) + raw_behavior_events.append( + { + "rawEventIndex": len(raw_behavior_events) + 1, + "sourceVideoIndex": context_index, + "technicalBlockIndex": block["technicalBlockIndex"], + "technicalOnly": True, + "modelEventIndex": model_index, + "behavior": event["behavior"], + "objectChange": event["object_change"], + "rawPTS": { + "startTimestamp": event["start_timestamp"], + "endTimestamp": event["end_timestamp"], + "startTimestampUs": event["start_us"], + "endTimestampUs": event["end_us"], + }, + "startSecond": event["start_second"], + "endSecond": event["end_second"], + "rawResponseSHA256": event["raw_response_sha256"], + } + ) + source_events.append(event) + block["previousTechnicalBlockContextCount"] = len(previous_events) + block["rawResponseSHA256"] = hashlib.sha256(raw_response.encode("utf-8")).hexdigest() + block["acceptedBehaviorCount"] = len(accepted) + # Same as the group reference script: only the immediately previous + # technical block is context for the next request. + previous_events = accepted + for merged in _merge_adjacent_behaviors(source_events): + candidate_count += 1 + clip_stem = pathlib.PurePosixPath(context["sanitizedVideoName"]).stem + event_key = f"{context['sanitizedVideoName']}|{merged['start_us']}|{merged['end_us']}|{merged['behavior']}" + suffix = hashlib.sha256(event_key.encode("utf-8")).hexdigest()[:12] + clip_id = f"{clip_stem}_rough_{candidate_count:04d}_{suffix}" + clip_name = clip_id + ".mp4" + source_payload = ( + context["sourceArtifact"].get("payload") + if isinstance(context["sourceArtifact"].get("payload"), dict) + else {} + ) + source_payload_ref = source_payload.get("ref") if isinstance(source_payload.get("ref"), dict) else None + clip = { + "clipId": clip_id, + "clipName": clip_name, + "sourceVideo": context["sanitizedVideoName"], + "sourceIsFullVideo": True, + "virtualClip": True, + "source": _source_ref(context["sourceArtifact"], helpers), + "startSecond": merged["start_second"], + "endSecond": merged["end_second"], + "sourceDurationSeconds": decoded_duration_seconds, + "duration": round(merged["end_second"] - merged["start_second"], 6), + "behavior": merged["behavior"], + "objectChange": merged["object_change"], + "rawPTS": { + "startTimestamp": merged["start_timestamp"], + "endTimestamp": merged["end_timestamp"], + "startTimestampUs": merged["start_us"], + "endTimestampUs": merged["end_us"], + "startFrameIndex": merged["start_frame_index"], + "endFrameIndex": merged["end_frame_index"], + "originTimestampUs": timestamps_us[0], + }, + "provenance": { + "model": "Qwen/Qwen3-VL-8B-Instruct", + "timeSource": "visible upper-right PTS_US overlay only", + "sourceVideoArtifactId": str(context["sourceArtifact"].get("id", "")), + "technicalOnly": True, + "technicalBlockIndexes": merged["technical_block_indexes"], + "rawBehaviorEventIndexes": merged["raw_event_indexes"], + "rawResponseSHA256s": merged["raw_response_sha256s"], + }, + "annotation": { + "schemaVersion": "droboticflow.qwen3vl.behavior-rough-prelabel/v1", + "clipId": clip_id, + "sourceVideo": context["sanitizedVideoName"], + "actions": [ + { + "id": clip_id + "-action-001", + # Qwen's free-form rough-cut behavior is not a + # member of the later 22-class semantic taxonomy. + # Keep this prelabel deliberately unlabeled; the + # semantic annotator makes the taxonomy decision. + "label": "", + "start": merged["start_second"], + "end": merged["end_second"], + "phrases": [ + value + for value in (merged["behavior"], merged["object_change"]) + if value + ], + "source": "qwen3-vl-behavior-rough-cut", + } + ], + "objects": [merged["object_change"]] if merged["object_change"] else [], + }, + } + if isinstance(source_payload_ref, dict): + # The manifest mirrors the same durable reference as the + # matching virtual artifact; no candidate has its own MP4. + clip["ref"] = copy.deepcopy(source_payload_ref) + for raw_event_index in merged["raw_event_indexes"]: + raw_behavior_events[raw_event_index - 1]["mergedCandidateClipId"] = clip_id + merged_behavior_events.append(clip) + virtual_artifacts.append( + _virtual_clip_artifact( + run_id, + node_id, + binding if isinstance(binding, dict) else None, + context["sourceArtifact"], + source_path, + clip, + helpers, + ) + ) + source_provenance.append( + { + "sourceVideoIndex": context_index, + "originalVideoName": context["originalVideoName"], + "sanitizedVideoName": context["sanitizedVideoName"], + "sanitizedVideoSHA256": _sha256(source_path), + "sourceVideoArtifactId": str(context["sourceArtifact"].get("id", "")), + "sourceVideoArtifactPath": str(context["sourceArtifact"].get("path", "")), + "ptsCSVName": context["ptsPath"].name, + "ptsSHA256": _sha256(context["ptsPath"]), + "ptsSource": context["ptsOrigin"], + "ptsRows": len(timestamps_us), + "decodedFrameCount": frame_count, + "ptsOriginTimestampUs": timestamps_us[0], + "decodedDurationSeconds": decoded_duration_seconds, + "ptsDurationSeconds": pts_duration_seconds, + "fps": fps, + "width": width, + "height": height, + } + ) + technical_blocks.extend( + [ + { + "sourceVideoIndex": context_index, + "sourceVideo": context["sanitizedVideoName"], + "technicalOnly": True, + "technicalBlockIndex": block["technicalBlockIndex"], + "startFrameIndex": block["startFrameIndex"], + "endFrameIndexExclusive": block["endFrameIndexExclusive"], + "sampleFrameIndices": block["sampleFrameIndices"], + "sampledPTSUs": block["sampledPTSUs"], + "previousTechnicalBlockContextCount": block["previousTechnicalBlockContextCount"], + "acceptedBehaviorCount": block["acceptedBehaviorCount"], + "rawResponseSHA256": block["rawResponseSHA256"], + } + for block in blocks + ] + ) + if require_non_empty and not merged_behavior_events: + raise RuntimeError("Qwen3-VL returned no accepted behavior rough-cut candidates while REQUIRE_NON_EMPTY_RESULT=true") + + manifest = { + "schemaVersion": "droboticflow.qwen3vl.behavior-rough-cut/v1", + "datasetType": "qwen3vl-behavior-rough-cut", + "status": "succeeded", + "clipCount": len(merged_behavior_events), + "itemCount": len(merged_behavior_events), + "sourceIsFullVideo": True, + "physicalMP4Slicing": False, + "items": merged_behavior_events, + "rawBehaviorEvents": raw_behavior_events, + "technicalBlocks": technical_blocks, + "parameters": { + "technicalBlockSeconds": technical_block_seconds, + "framesPerTechnicalBlock": frames_per_block, + "technicalOnly": True, + "technicalBlocksAreClipBoundaries": False, + "modelTimeSource": "visible upper-right PTS_US overlay only", + "requireNonEmptyResult": require_non_empty, + }, + "sourceVideos": source_provenance, + "provenance": { + "inferenceId": f"{run_id}:{node_id}:{uuid.uuid4().hex}", + "semanticReference": { + "source": "Feishu 云平台-泛宇交流群 video_segmentation.py", + "sha256": REFERENCE_GROUP_SCRIPT_SHA256, + }, + "modelId": model_manifest["modelId"], + "modelRevision": model_manifest["revision"], + "modelSha256": verified_model_sha, + "modelManifestPath": str(manifest_path or model_path / "MODEL_MANIFEST.json"), + "modelRuntimePath": str(runtime_model_path), + "modelStagedBytes": staged_model_bytes, + "device": torch.cuda.get_device_name(0), + "executedAt": datetime.now(timezone.utc).isoformat(), + }, + } + write = helpers.get("write_json_artifact") + if not callable(write): + path = output / "clips-manifest.json" + path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + return [{"name": path.name, "path": str(path), "type": "json", "data": manifest}, *virtual_artifacts] + return [ + write(run_id, node_id, output, "clips-manifest.json", manifest, output=binding if isinstance(binding, dict) else None), + *virtual_artifacts, + ] diff --git a/backend/operators/builtin/qwen_behavior_rough_cut/repository.json b/backend/operators/builtin/qwen_behavior_rough_cut/repository.json new file mode 100644 index 00000000..216c9d43 --- /dev/null +++ b/backend/operators/builtin/qwen_behavior_rough_cut/repository.json @@ -0,0 +1,89 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "containerEntrypoint": "backend/operator-runner/main.py", + "entrypoint": "operator.py", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:pending-fanyu-qwen-behavior-rough-cut", + "metadata": "operator.json", + "name": "Qwen3-VL 语义行为粗切(PTS)", + "operatorId": "operator-builtin-qwen-behavior-rough-cut", + "operatorSlug": "qwen_behavior_rough_cut", + "preferredSlug": "qwen_behavior_rough_cut", + "pythonImport": "backend.operators.builtin.qwen_behavior_rough_cut.operator:run", + "repositoryId": "builtin/qwen_behavior_rough_cut", + "runtimeContract": { + "acceptance": { + "allowsEmptyResultList": true, + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresNonEmptyManifest": true, + "requiresNonEmptyOutput": false, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_qwen_behavior_rough_cut", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "operatorId": "operator-builtin-qwen-behavior-rough-cut", + "operatorSlug": "qwen_behavior_rough_cut", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 1, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": true + }, + "runtimeDependencies": [ + "qwen3-vl-8b-instruct", + "verified-local-model-manifest", + "torch-cuda", + "opencv", + "qwen-vl-utils", + "real-video-files", + "orbbec-pts-csv" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeDependencies": [ + "qwen3-vl-8b-instruct", + "verified-local-model-manifest", + "torch-cuda", + "opencv", + "qwen-vl-utils", + "real-video-files", + "orbbec-pts-csv" + ], + "runtimeHandler": "builtin_qwen_behavior_rough_cut", + "runtimeSource": "operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py", + "schemaVersion": "droboticflow.operator.repository/v1", + "sourceRoot": "backend/operators/builtin/qwen_behavior_rough_cut", + "strictPocAcceptanceExternals": [], + "type": "builtin" +} diff --git a/docs/fanyu-v5-workflow.md b/docs/fanyu-v5-workflow.md index 7819e68b..f3ac3245 100644 --- a/docs/fanyu-v5-workflow.md +++ b/docs/fanyu-v5-workflow.md @@ -4,11 +4,13 @@ v5 是新建的左相机工作流,历史 v1–v4 不会被修改。它以真 流程如下: -`BOS → 左相机标准化 → 人脸脱敏 → 有效行为(Qwen3-VL, 5s/5s) → 时间定位(Qwen3-VL, 200s/180s/20s overlap) → 同任务切片校正+22类标注 → 统一质检 → 一个聚合包前验收任务(仅确认) → ms-swift 数据导出 → Qwen3-VL LoRA → 最终切片 MP4/语义 JSON/manifest 包` +`必填 BOS 目录引用 → 目录读取 → Orbbec 左相机标准化 → 人脸高斯模糊 → Qwen3-VL 手部行为语义粗切 → 人工调切 → 剪切审核(驳回回人工调切)→ 语义标注 → 标注审核(驳回回语义标注)→ 一个人工聚合确认任务(仅确认)→ 最终切片 MP4/语义 JSON/manifest 打包` -工作流中历史命名为 “WiLoR 有效行为检测” 的节点,仅表示它处于 WiLoR 需求链:实际有效行为判定由 Qwen3-VL 运行,**不会**调用或把 WiLoR hand-pose inference 冒充为行为分类。两类 Qwen 真实推理均可得到空结果列表;空列表仍要伴随真实视频、模型和 checksum 的非空 manifest。显式空 `actions` 则是合法的已验收负样本。 +目录读取没有 checkpoint、增量游标或历史扫描状态;每次 Run 都以用户明确选择的目录引用为完整输入。粗切不会按照 200 秒或任意固定交付时长生成片段,也不再运行历史的有效行为/时间定位组合。它参考泛宇实验脚本,把真实逐帧 `timestamp_us` PTS 叠加到送入 Qwen3-VL 的采样画面,识别每次手部主谓宾动作变化、物体变化和起止 PTS,再把模型时间严格映射回源视频秒数。 -当前网页验收范围暂不执行 Headpose 或 Handpose,也不暴露 `deliveryOptions` 姿态开关。两个 Qwen 节点仍有数据依赖,因此一次只申请一张 GPU:有效行为先完成,时间定位再开始。LoRA 训练位于人工确认之后,并同样申请一张 GPU。姿态能力保留为未来独立版本,不在本次成功条件中。 +Qwen 请求内部可按 10 秒、18 帧拆成技术块以控制单次显存和上下文,并携带上一技术块结果保持连续性;这些块在 manifest 中标为 `technicalOnly`,绝不作为粗切或最终交付边界。相同行为只有在模型输出相邻时才合并。没有真实 PTS、PTS 与视频帧数不一致、时间无法解析、模型未加载或模型没有返回非空合法行为时,节点失败关闭。 + +当前流程不执行 HeadPose、HandPose、ms-swift 导出或 LoRA 微调,也不暴露姿态或训练输入。姿态与训练能力留给未来独立版本,不在本次成功条件中。 ## 模型完整性 @@ -28,16 +30,18 @@ python3 scripts/verify_qwen3vl_model_manifest.py --model-root /workspace/models/ ## 人在环与交付 -- 标注员领取公共供应商隔离池中的一个任务,在同一工作台任意顺序编辑切片和 22 类语义;一次原子提交。 -- 质检员统一查看最终切片和语义,能够打回到同一标注阶段。 -- 所有任务质检通过后生成一个、而非每片一个的包前验收任务。验收员可以查看所有切片、标签与受限 artifact 预览;API 和页面都只允许“确认”。 -- 确认后才释放 `ms-swift`/LoRA/打包 child run;v5 不会生成历史的 post-package reject-capable 验收任务。 -- 每个最终切片都重新生成独立 MP4,并写入 `semantic.json` 与 `segment.json`。当前版本不生成 Headpose/Handpose 文件。 +- 标注员先从供应商隔离的公共任务池领取“人工剪切调整”,在完整脱敏源视频上校正 Qwen 粗切边界。 +- 剪切质检员审核;驳回时系统自动回到该任务的“人工剪切调整”,不能跳到语义标注。 +- 剪切通过后,标注员领取“语义标注”,在已审核边界上完成 22 类语义与对象短语。 +- 标注质检员审核;驳回时系统自动回到“语义标注”,不会推翻已通过的剪切审核。 +- 所有任务标注审核通过后生成一个、而非每片一个的包前聚合确认任务。验收员查看全部最终切片、标签和统计;API 与页面只允许“确认”。 +- 未确认时打包节点保持等待;确认后直接释放打包 child run,不经过导出或训练。 +- 打包阶段才根据人工最终边界物化独立 MP4,并写入 `semantic.json` 与 `segment.json`;当前版本不生成 HeadPose/HandPose 文件。 ## POC/S100 运行前置条件 -1. 独立构建并发布 `video-face-blur`、三个 Qwen 相关算子及 ms-swift 导出镜像,写入 digest-pinned release manifest。 +1. 独立构建并发布 `directory-scan`(目录读取语义)、`video-face-blur`、`qwen-behavior-rough-cut` 与 `package` 镜像,写入 digest-pinned release manifest。 2. 预置并校验 Qwen3-VL 本地快照;当前无模型或未校验 manifest 即为预期 fail-closed 状态。 -3. 为有效行为、时间定位、LoRA 顺序腾出 1 张 GPU;不需要也不应触碰 GSStudio/HoloBrain 的 GPU。 -4. 提供 `FANYU_V5_ANNOTATION_WORKGROUP_ID`,并保持 `FANYU_HEADPOSE_*` 的真实服务/许可证/模型 digest 配置。 +3. 为 Qwen3-VL 语义粗切腾出 1 张 GPU;目录读取、人工阶段与打包不申请 GPU。 +4. 提供 `FANYU_V5_ANNOTATION_WORKGROUP_ID`;本版本不读取 `FANYU_HEADPOSE_*`。 5. 仅在 `dev/poc` 验收环境中部署 `dev`,S100 使用同一镜像 digest 和以上 PVC/模型配置,不把草稿算子直接标为已发布。 diff --git a/docs/operators/Qwen3-VL-语义行为粗切-PTS.md b/docs/operators/Qwen3-VL-语义行为粗切-PTS.md new file mode 100644 index 00000000..7f5678e8 --- /dev/null +++ b/docs/operators/Qwen3-VL-语义行为粗切-PTS.md @@ -0,0 +1,124 @@ +# Qwen3-VL 语义行为粗切(PTS) + +- 算子 ID:`operator-builtin-qwen-behavior-rough-cut` +- 类别:`AI` +- 类型:`builtin` + +## 作用 + +读取真实人脸脱敏视频和同目录逐帧 timestamp_us CSV;在送入 Qwen3-VL 的每个采样帧右上角叠加真实 PTS_US,以手部主谓宾动作/物体变化生成自动粗切候选。技术块仅限制单次模型负载,不是最终切片边界;输出引用完整脱敏源视频的 virtual clips,供人工调切。 + +## 定位与职责边界 + +- 该算子属于 `AI` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 委派到独立算子入口。 + +## 平台语义 + +真实 Qwen3-VL 基于右上角真实 PTS_US 识别手部主谓宾行为变化并生成自动粗切建议;不按固定时长输出、没有 200 秒定位。每个候选复用完整脱敏视频,人工调切后才物理切片。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前独立实现入口是 `operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py` 中的 `run(...)`,平台 runtimeHandler 为 `builtin_qwen_behavior_rough_cut`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[Qwen3-VL 语义行为粗切(PTS)] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `video` / **video** / `json` / `bus`:视频人脸脱敏输出:blurred-manifest 与完整脱敏视频 artifact 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 +- `metadata` / **metadata** / `json` / `bus`:Orbbec 标准化 manifest:用于精确定位匹配视频的逐帧 timestamp_us CSV 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **roughCuts** / `json` / `bus`:行为驱动的 clips-manifest 与复用完整脱敏源视频引用的 virtual clip artifacts 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `MODEL_PATH`:类型 `string`,默认值 `/workspace/models/Qwen3-VL-8B-Instruct`。只读的本地 Qwen3-VL-8B-Instruct 快照目录 可直接使用默认值,也可在节点级覆盖。 +- `MODEL_REVISION`:类型 `string`,默认值 `0c351dd01ed87e9c1b53cbc748cba10e6187ff3b`。锁定的 Qwen3-VL-8B-Instruct revision 可直接使用默认值,也可在节点级覆盖。 +- `MODEL_MANIFEST_PATH`:类型 `string`,默认值 `/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json`。逐 shard SHA/大小经过校验的模型 manifest 可直接使用默认值,也可在节点级覆盖。 +- `MODEL_MANIFEST_SHA256`:类型 `string`,默认值 `3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a`。预载 Qwen3-VL manifest 的固定 SHA-256 可直接使用默认值,也可在节点级覆盖。 +- `TRUST_PREVERIFIED_MODEL`:类型 `boolean`,默认值 `true`。复用已校验预载收据,仍校验固定 manifest 与布局 可直接使用默认值,也可在节点级覆盖。 +- `TECHNICAL_BLOCK_SECONDS`:类型 `number`,默认值 `10`。单次 Qwen 请求的技术负载上限(仅技术块,不产生切片边界) 可直接使用默认值,也可在节点级覆盖。 +- `FRAMES_PER_TECHNICAL_BLOCK`:类型 `number`,默认值 `18`。每个技术块送入 Qwen3-VL 的真实 PTS 叠加帧数(仅技术负载) 可直接使用默认值,也可在节点级覆盖。 +- `MAX_NEW_TOKENS`:类型 `number`,默认值 `2048`。Qwen JSON 输出的最大 token 数 可直接使用默认值,也可在节点级覆盖。 +- `REQUIRE_NON_EMPTY_RESULT`:类型 `boolean`,默认值 `false`。验收时可要求真实 Qwen 至少输出一个行为;正常视频允许真实空结果 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "Qwen3-VL 语义行为粗切(PTS)", + "operatorId": "operator-builtin-qwen-behavior-rough-cut", + "type": "builtin", + "category": "AI", + "hyperparams": { + "MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", + "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", + "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", + "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", + "TRUST_PREVERIFIED_MODEL": "true", + "TECHNICAL_BLOCK_SECONDS": "10", + "FRAMES_PER_TECHNICAL_BLOCK": "18", + "MAX_NEW_TOKENS": "2048", + "REQUIRE_NON_EMPTY_RESULT": "false" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [AI 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/docs/operators/README.md b/docs/operators/README.md index 5694e1f7..f468b843 100644 --- a/docs/operators/README.md +++ b/docs/operators/README.md @@ -2,7 +2,7 @@ 当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。 -- 当前收录数量:`113` +- 当前收录数量:`114` - 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。 - 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md) @@ -27,6 +27,7 @@ - [OpenVLA 推理](OpenVLA-推理.md) - [Qwen3-VL 200秒时间区间定位(真实)](Qwen3-VL-200秒时间区间定位-真实.md) - [Qwen3-VL 22类持续训练(真实 LoRA)](Qwen3-VL-22类持续训练-真实-LoRA.md) +- [Qwen3-VL 语义行为粗切(PTS)](Qwen3-VL-语义行为粗切-PTS.md) - [WiLoR 3D手部姿态估计](WiLoR-3D手部姿态估计.md) - [YOLO 具身目标检测训练](YOLO-具身目标检测训练.md) - [有效行为检测(真实 Qwen3-VL)](有效行为检测-真实-Qwen3-VL.md) @@ -70,7 +71,7 @@ ### IO - [BOS 增量扫描](BOS-增量扫描.md) -- [目录扫描](目录扫描.md) +- [目录读取](目录读取.md) - [真实数据源](真实数据源.md) - [结果上传](结果上传.md) diff --git a/docs/operators/成品打包.md b/docs/operators/成品打包.md index 1bcd59c6..be5bffd3 100644 --- a/docs/operators/成品打包.md +++ b/docs/operators/成品打包.md @@ -43,7 +43,6 @@ flowchart LR - `in-1` / **annotations** / `json` / `bus`:切片与人工标注结果 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 - `in-2` / **headpose** / `json` / `bus`:必选 Headpose manifest 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 - `in-3` / **handpose** / `json` / `bus`:可选 Handpose manifest 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 -- `in-4` / **trainingEvidence** / `json` / `bus`:真实 Qwen3-VL 连续训练报告;作为包前强制依赖与 provenance 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 ## 输出端口详解 diff --git a/docs/operators/目录扫描.md b/docs/operators/目录读取.md similarity index 78% rename from docs/operators/目录扫描.md rename to docs/operators/目录读取.md index 8595f275..4e2369c6 100644 --- a/docs/operators/目录扫描.md +++ b/docs/operators/目录读取.md @@ -1,4 +1,4 @@ -# 目录扫描 +# 目录读取 - 算子 ID:`operator-builtin-directory-scan` - 类别:`IO` @@ -6,17 +6,16 @@ ## 作用 -对真实目录做文件统计、扩展名分布和样例文件扫描。 +读取必填目录输入;支持 BOS directory ref 与本地目录,输出完整文件清单供下游算子消费,不保存增量扫描状态。 ## 定位与职责边界 - 该算子属于 `IO` 类,当前登记类型是 `builtin`。 - 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 委派到独立算子入口。 -- 它更偏向工作流入口,职责是把外部世界的数据引用整理成平台统一 artifact。 ## 平台语义 -读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。 +读取必填 BOS directory ref 或本地 SOURCE_PATH,输出可直接进入数据标准化算子的完整目录 manifest;无增量状态。 ## 运行时契约 @@ -34,21 +33,21 @@ ```mermaid flowchart LR - A[上游输入或运行时注入] --> B[目录扫描] + A[上游输入或运行时注入] --> B[目录读取] B --> C[artifact / stage manifest / 外部副作用] ``` ## 输入端口详解 -- `in-1` / **input** / `json` / `bus`:目录输入 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 +- `in-1` / **input** / `file` / `bus`:目录输入 作为输入端口时,适合 CSV、压缩包、文档、配置文件等任意文件产物。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 ## 输出端口详解 -- `out-1` / **summary** / `json` / `bus`:目录概览 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 +- `out-1` / **manifest** / `json` / `bus`:目录文件清单 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 ## 关键超参数 -- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。待扫描目录 通常建议在工作流中显式配置。 +- `SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。本地目录兼容输入;Workflow 生产运行应使用必填目录引用 通常建议在工作流中显式配置。 ## 执行前准备 @@ -69,7 +68,7 @@ flowchart LR ```json { - "label": "目录扫描", + "label": "目录读取", "operatorId": "operator-builtin-directory-scan", "type": "builtin", "category": "IO", diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index 2b30cf48..5f65fd70 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -475,6 +475,14 @@ type annotationTaskInput struct { AssignedUserID string `json:"assignedUserId,omitempty"` } +// annotationTaskListRecord keeps task-pool routing server-authoritative without +// persisting UI-only state on AnnotationTaskRecord. The workbench uses it to +// show a task only in the matching human stage queue. +type annotationTaskListRecord struct { + domain.AnnotationTaskRecord + HumanStageID string `json:"humanStageId,omitempty"` +} + func (server *server) handleAnnotationTasksList(writer http.ResponseWriter, request *http.Request) { projectID := strings.TrimSpace(request.URL.Query().Get("projectId")) status := domain.AnnotationTaskStatus(strings.TrimSpace(request.URL.Query().Get("status"))) @@ -489,6 +497,17 @@ func (server *server) handleAnnotationTasksList(writer http.ResponseWriter, requ renderError(writer, http.StatusInternalServerError, err) return } + humanItems, humanItemsErr := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: projectID}) + if humanItemsErr != nil { + renderError(writer, http.StatusInternalServerError, humanItemsErr) + return + } + humanStageByTaskID := make(map[string]string, len(humanItems)) + for _, humanItem := range humanItems { + if strings.TrimSpace(humanItem.LegacyTaskID) != "" { + humanStageByTaskID[humanItem.LegacyTaskID] = humanItem.CurrentStageID + } + } principal, _ := currentPrincipal(request) var memberships []domain.ProjectAnnotationMembershipRecord if !annotationManager(principal) && principal != nil { @@ -498,7 +517,7 @@ func (server *server) handleAnnotationTasksList(writer http.ResponseWriter, requ return } } - filtered := make([]domain.AnnotationTaskRecord, 0, len(items)) + filtered := make([]annotationTaskListRecord, 0, len(items)) for _, item := range items { if status != "" && item.Status != status { continue @@ -524,7 +543,7 @@ func (server *server) handleAnnotationTasksList(writer http.ResponseWriter, requ if !annotationManager(principal) && !membershipAllowsTaskRead(memberships, item) { continue } - filtered = append(filtered, item) + filtered = append(filtered, annotationTaskListRecord{AnnotationTaskRecord: item, HumanStageID: humanStageByTaskID[item.ID]}) } renderJSON(writer, http.StatusOK, map[string]any{"items": filtered}) } @@ -947,6 +966,13 @@ func (server *server) handleAnnotationTaskReject(writer http.ResponseWriter, req renderJSON(writer, http.StatusCreated, map[string]any{"task": task, "item": item}) } +func annotationTaskEditableRange(task domain.AnnotationTaskRecord) (float64, float64) { + if task.SourceIsFullVideo && task.SourceDurationSeconds > 0 { + return 0, task.SourceDurationSeconds + } + return task.ClipStart, task.ClipEnd +} + func validateSliceSegments(segments []domain.SliceSegment, task domain.AnnotationTaskRecord) []validationIssue { issues := make([]validationIssue, 0) if len(segments) == 0 { @@ -954,11 +980,12 @@ func validateSliceSegments(segments []domain.SliceSegment, task domain.Annotatio } sorted := append([]domain.SliceSegment(nil), segments...) sort.Slice(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start }) + allowedStart, allowedEnd := annotationTaskEditableRange(task) for index, segment := range sorted { if strings.TrimSpace(segment.ID) == "" { issues = append(issues, validationIssue{Field: fmt.Sprintf("segments.%d.id", index), Message: "slice id is required"}) } - if math.IsNaN(segment.Start) || math.IsNaN(segment.End) || segment.Start < task.ClipStart || segment.End > task.ClipEnd || segment.End <= segment.Start { + if math.IsNaN(segment.Start) || math.IsNaN(segment.End) || segment.Start < allowedStart || segment.End > allowedEnd || segment.End <= segment.Start { issues = append(issues, validationIssue{Field: fmt.Sprintf("segments.%d", index), Message: "slice must be finite and inside the episode range"}) } if index > 0 && segment.Start < sorted[index-1].End { @@ -986,7 +1013,27 @@ func (server *server) handleSliceRevisionCreate(writer http.ResponseWriter, requ renderJSON(writer, http.StatusForbidden, map[string]string{"error": "annotator assignment is required"}) return } - if _, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "split_annotation"); !claimed { + humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "") + if !claimed { + return + } + batch, batchErr := server.store.GetHumanBatch(request.Context(), humanItem.BatchID) + if batchErr != nil { + renderError(writer, http.StatusInternalServerError, batchErr) + return + } + if batch == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Human batch not found"}) + return + } + cutAdjustmentStageID := "split_annotation" + cutReviewStageID := "split_quality_review" + if isFanyuV5FourStageBatch(batch) { + cutAdjustmentStageID = domain.FanyuV5StageCutAdjustment + cutReviewStageID = domain.FanyuV5StageCutReview + } + if humanItem.CurrentStageID != cutAdjustmentStageID { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item is not in the cut-adjustment stage", "code": "human_item_stage_conflict"}) return } var payload sliceRevisionInput @@ -1025,7 +1072,7 @@ func (server *server) handleSliceRevisionCreate(writer http.ResponseWriter, requ } if item != nil { previousVersion := item.Version - item.CurrentStageID = "split_quality_review" + item.CurrentStageID = cutReviewStageID item.Status = domain.HumanItemReady item.AssignedUserID = "" item.ClaimedAt = "" @@ -1040,7 +1087,7 @@ func (server *server) handleSliceRevisionCreate(writer http.ResponseWriter, requ } return } - _ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: "split_annotation", Round: item.ReviewRound, Status: domain.HumanItemSubmitted, AssigneeUserID: principal.User.ID, SubmissionRef: revision.ID}) + _ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: cutAdjustmentStageID, Round: item.ReviewRound, Status: domain.HumanItemSubmitted, AssigneeUserID: principal.User.ID, SubmissionRef: revision.ID}) _ = server.appendHumanEvent(request.Context(), *item, "submitted", principal.User.ID, map[string]any{"submissionRef": revision.ID, "sliceRevision": revision.Revision}) } taskMode, isTemp := server.resolveTaskRunMode(request.Context(), task.RunID) @@ -1052,11 +1099,12 @@ func validateAnnotationSegments(segments []domain.AnnotationSegment, task domain issues := make([]validationIssue, 0) sorted := append([]domain.AnnotationSegment(nil), segments...) sort.Slice(sorted, func(left int, right int) bool { return sorted[left].Start < sorted[right].Start }) + allowedStart, allowedEnd := annotationTaskEditableRange(task) for index, segment := range sorted { if strings.TrimSpace(segment.LabelID) == "" || strings.TrimSpace(segment.Label) == "" { issues = append(issues, validationIssue{Field: fmt.Sprintf("segments.%d", index), Message: "labelId and label are required"}) } - if segment.Start < task.ClipStart || segment.End > task.ClipEnd || segment.End <= segment.Start || math.IsNaN(segment.Start) || math.IsNaN(segment.End) { + if segment.Start < allowedStart || segment.End > allowedEnd || segment.End <= segment.Start || math.IsNaN(segment.Start) || math.IsNaN(segment.End) { issues = append(issues, validationIssue{Field: fmt.Sprintf("segments.%d", index), Message: "segment must be finite and inside the clip range"}) } if segment.Confidence < 0 || segment.Confidence > 1 || math.IsNaN(segment.Confidence) { @@ -1224,6 +1272,202 @@ func (server *server) queueAcceptedAnnotationGateRun(ctx context.Context, task d return server.store.UpsertRun(ctx, *run) } +func fanyuV5FourStageReviewRoute(stage domain.AnnotationReviewStage) (reviewStageID string, rejectTargetStageID string, ok bool) { + switch stage { + case domain.AnnotationReviewStageSplitQuality: + return domain.FanyuV5StageCutReview, domain.FanyuV5StageCutAdjustment, true + case domain.AnnotationReviewStageQuality: + return domain.FanyuV5StageSemanticReview, domain.FanyuV5StageSemanticAnnotation, true + default: + return "", "", false + } +} + +// handleFanyuV5FourStageReview implements the supplier-facing rejection loop +// as an immediate, auditable rework transition. We deliberately do not leave +// a v5 review failure in the generic reject zone: cut review must go straight +// back to cut adjustment, and semantic review must go straight back to +// semantic annotation. The HumanReject record is still append-only evidence +// of the decision and records the automatic reopen target. +func (server *server) handleFanyuV5FourStageReview(writer http.ResponseWriter, request *http.Request, task *domain.AnnotationTaskRecord, principal *authenticatedPrincipal, item *domain.HumanItemRecord, batch *domain.HumanBatchRecord, payload annotationReviewInput) bool { + if task == nil || item == nil || batch == nil || !isFanyuV5FourStageBatch(batch) { + return false + } + reviewStageID, rejectTargetStageID, routed := fanyuV5FourStageReviewRoute(payload.Stage) + if !routed { + renderValidationError(writer, validationIssue{Field: "stage", Message: "v5 requires cut review or semantic review before aggregate acceptance"}) + return true + } + if item.CurrentStageID != reviewStageID { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "v5 item is not in the requested review stage", "code": "human_item_stage_conflict"}) + return true + } + + reviewRevision := task.CurrentRevision + if payload.Stage == domain.AnnotationReviewStageSplitQuality { + sliceRevisions, sliceErr := server.store.ListSliceRevisions(request.Context(), task.OrganizationID, task.ID) + if sliceErr != nil { + renderError(writer, http.StatusInternalServerError, sliceErr) + return true + } + if len(sliceRevisions) == 0 { + renderValidationError(writer, validationIssue{Field: "revision", Message: "a cut adjustment is required before cut review"}) + return true + } + latestSlice := sliceRevisions[len(sliceRevisions)-1] + reviewRevision = latestSlice.Revision + if latestSlice.AuthorUserID == principal.User.ID { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "A submitter cannot review their own cut adjustment", "code": "separation_of_duties"}) + return true + } + } else { + if task.CurrentRevision <= 0 { + renderValidationError(writer, validationIssue{Field: "revision", Message: "a semantic annotation is required before semantic review"}) + return true + } + revisions, revisionErr := server.store.ListAnnotationRevisions(request.Context(), task.OrganizationID, task.ID) + if revisionErr != nil { + renderError(writer, http.StatusInternalServerError, revisionErr) + return true + } + for _, revision := range revisions { + if revision.Revision == task.CurrentRevision && revision.AuthorUserID == principal.User.ID { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "A submitter cannot review their own semantic annotation", "code": "separation_of_duties"}) + return true + } + } + } + if payload.Decision != domain.AnnotationReviewApproved && payload.Decision != domain.AnnotationReviewRejected { + renderValidationError(writer, validationIssue{Field: "decision", Message: "decision must be approved or rejected"}) + return true + } + if payload.Decision == domain.AnnotationReviewRejected { + if strings.TrimSpace(payload.Comment) == "" || len(payload.ReasonCodes) == 0 { + renderValidationError(writer, validationIssue{Field: "rejection", Message: "reasonCodes and comment are required for rework"}) + return true + } + if len(payload.RejectTargets) > 0 && (len(payload.RejectTargets) != 1 || payload.RejectTargets[0] != rejectTargetStageID) { + renderValidationError(writer, validationIssue{Field: "rejectTargets", Message: "v5 review can return only to its configured immediate edit stage"}) + return true + } + } + existing, existingErr := server.store.ListAnnotationReviews(request.Context(), task.OrganizationID, task.ID) + if existingErr != nil { + renderError(writer, http.StatusInternalServerError, existingErr) + return true + } + for _, review := range existing { + if review.Revision == reviewRevision && review.Stage == payload.Stage { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "review already exists for this revision and stage"}) + return true + } + } + + now := nowISO() + review := domain.AnnotationReviewRecord{ + ID: domain.NewID("annotation-review"), OrganizationID: task.OrganizationID, TaskID: task.ID, + Revision: reviewRevision, Stage: payload.Stage, Decision: payload.Decision, ReviewerUserID: principal.User.ID, + Comment: strings.TrimSpace(payload.Comment), VideoQuality: strings.TrimSpace(payload.VideoQuality), + AnnotationAccuracy: payload.AnnotationAccuracy, AnnotationCompleteness: payload.AnnotationCompleteness, + OverallGrade: strings.TrimSpace(payload.OverallGrade), ReasonCodes: payload.ReasonCodes, CreatedAt: now, + } + if payload.Decision == domain.AnnotationReviewRejected { + review.RejectTargets = []string{rejectTargetStageID} + if payload.Stage == domain.AnnotationReviewStageQuality { + review.NextRevision = task.CurrentRevision + 1 + } + task.Status = domain.AnnotationTaskStatusRework + task.AssignedUserID = "" + } else if payload.Stage == domain.AnnotationReviewStageSplitQuality { + // A cut has no semantic revision yet; the next available task is the + // semantic annotation stage, not a re-run of automatic rough cutting. + task.CurrentRevision = 0 + task.Status = domain.AnnotationTaskStatusPendingAnnotation + task.AssignedUserID = "" + } else { + task.Status = domain.AnnotationTaskStatusAccepted + task.AssignedUserID = "" + } + if err := server.store.UpsertAnnotationReview(request.Context(), review); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return true + } + if err := server.store.UpsertAnnotationTask(request.Context(), *task); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return true + } + + previousVersion := item.Version + item.AssignedUserID = "" + item.ClaimedAt = "" + item.Version++ + if payload.Decision == domain.AnnotationReviewRejected { + item.ReviewRound++ + item.CurrentStageID = rejectTargetStageID + if item.ReviewRound > item.MaxReworkRounds { + item.Status = domain.HumanItemEscalated + task.Status = domain.AnnotationTaskStatusRejected + if err := server.store.UpsertAnnotationTask(request.Context(), *task); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return true + } + } else { + item.Status = domain.HumanItemRework + } + } else if payload.Stage == domain.AnnotationReviewStageSplitQuality { + item.CurrentStageID = domain.FanyuV5StageSemanticAnnotation + item.Status = domain.HumanItemReady + } else { + item.Status = domain.HumanItemAccepted + item.AcceptedAt = now + item.AcceptedByUserID = principal.User.ID + } + updated, updateErr := server.store.UpdateHumanItemVersioned(request.Context(), *item, previousVersion) + if updateErr != nil || !updated { + if updateErr != nil { + renderError(writer, http.StatusInternalServerError, updateErr) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"}) + } + return true + } + _ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{ + OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: reviewStageID, Round: item.ReviewRound, + Status: item.Status, AssigneeUserID: principal.User.ID, ReviewedRevision: review.Revision, + Decision: domain.HumanDecision(payload.Decision), RejectTargets: review.RejectTargets, ReasonCodes: review.ReasonCodes, Comment: review.Comment, + }) + if payload.Decision == domain.AnnotationReviewRejected { + _ = server.store.UpsertHumanReject(request.Context(), domain.HumanRejectRecord{ + OrganizationID: item.OrganizationID, BatchID: item.BatchID, ItemID: item.ID, ProjectID: item.ProjectID, + WorkflowID: item.WorkflowID, RunID: item.RunID, StageID: reviewStageID, AttemptRound: item.ReviewRound, + ReviewedRevision: review.Revision, RejectedByUserID: principal.User.ID, ReasonCodes: review.ReasonCodes, + Comment: review.Comment, AllowedReopenStages: []string{rejectTargetStageID}, Status: domain.HumanRejectReopened, + ReopenCount: 1, ReopenedByUserID: principal.User.ID, ReopenTargetStageID: rejectTargetStageID, + ReopenComment: "automatic v5 rework routing", ReopenedAt: now, + }) + } + eventType := reviewStageID + "_" + string(payload.Decision) + _ = server.appendHumanEvent(request.Context(), *item, eventType, principal.User.ID, map[string]any{ + "reviewedRevision": review.Revision, "rejectTargets": review.RejectTargets, "reasonCodes": review.ReasonCodes, + "comment": review.Comment, "routedToStage": rejectTargetStageID, + }) + taskMode, isTemp := server.resolveTaskRunMode(request.Context(), task.RunID) + _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ + ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, + UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, + EventType: eventType, Mode: taskMode, Temporary: isTemp, Revision: review.Revision, TemplateVersion: task.TemplateVersion, + DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: now, + }) + if item.Status == domain.HumanItemAccepted { + if err := server.flushHumanReleasePartitions(request.Context(), batch, false); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return true + } + } + renderJSON(writer, http.StatusCreated, map[string]any{"task": task, "review": review, "humanItem": item}) + return true +} + func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWriter, request *http.Request, task *domain.AnnotationTaskRecord, principal *authenticatedPrincipal, item *domain.HumanItemRecord, batch *domain.HumanBatchRecord, payload annotationReviewInput) bool { if task == nil || item == nil || batch == nil || !batch.RequirePrePackageAcceptance { return false @@ -1367,17 +1611,40 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r renderJSON(writer, http.StatusForbidden, map[string]string{"error": string(requiredRole) + " assignment is required"}) return } + humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, "") + if !claimed { + return + } + batch, batchErr := server.store.GetHumanBatch(request.Context(), humanItem.BatchID) + if batchErr != nil { + renderError(writer, http.StatusInternalServerError, batchErr) + return + } + if batch == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Human batch not found"}) + return + } requiredStageID := "quality_review" if payload.Stage == domain.AnnotationReviewStageAcceptance { requiredStageID = "acceptance" } else if payload.Stage == domain.AnnotationReviewStageSplitQuality { requiredStageID = "split_quality_review" } - humanItem, claimed := server.requireLegacyHumanItemClaim(writer, request, *task, requiredStageID) - if !claimed { + if isFanyuV5FourStageBatch(batch) { + if stageID, _, routed := fanyuV5FourStageReviewRoute(payload.Stage); routed { + requiredStageID = stageID + } + } + if humanItem.CurrentStageID != requiredStageID { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item is not in the required review stage", "code": "human_item_stage_conflict"}) return } - if batch, batchErr := server.store.GetHumanBatch(request.Context(), humanItem.BatchID); batchErr == nil && batch != nil { + if isFanyuV5FourStageBatch(batch) { + if server.handleFanyuV5FourStageReview(writer, request, task, principal, humanItem, batch, payload) { + return + } + } + if batch != nil { if server.handleFanyuV5CombinedQualityReview(writer, request, task, principal, humanItem, batch, payload) { return } diff --git a/internal/api/annotation_operations_test.go b/internal/api/annotation_operations_test.go index e9add803..67e6aee8 100644 --- a/internal/api/annotation_operations_test.go +++ b/internal/api/annotation_operations_test.go @@ -28,6 +28,19 @@ func TestValidateAnnotationSegmentsAcceptsOrderedTimeline(t *testing.T) { } } +func TestVirtualFullVideoTaskCanExpandRoughCutWithinSourceDuration(t *testing.T) { + task := domain.AnnotationTaskRecord{ClipStart: 10, ClipEnd: 12, SourceIsFullVideo: true, SourceDurationSeconds: 30} + if issues := validateSliceSegments([]domain.SliceSegment{{ID: "expanded", Start: 8, End: 14}}, task); len(issues) != 0 { + t.Fatalf("full-video rough cut must be expandable: %#v", issues) + } + if issues := validateSliceSegments([]domain.SliceSegment{{ID: "outside", Start: 8, End: 31}}, task); len(issues) == 0 { + t.Fatal("full-video adjustment must remain within verified source duration") + } + if issues := validateAnnotationSegments([]domain.AnnotationSegment{{ID: "label", LabelID: "01", Label: "鲜食处理", Start: 8, End: 14, Confidence: 1}}, task); len(issues) != 0 { + t.Fatalf("semantic annotation must accept the reviewed expanded cut: %#v", issues) + } +} + func TestValidateFanyuV5CombinedSegmentsRequiresExactTemplateAndSliceContainment(t *testing.T) { template := &domain.AnnotationTemplateRecord{Labels: domain.ConvenienceStore22SemanticLabels()} slices := []domain.SliceSegment{{ID: "a", Start: 0, End: 5}, {ID: "b", Start: 5, End: 10}} diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index 5dfd0cd5..07856092 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -872,10 +872,10 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "对真实目录做文件统计、扩展名分布和样例文件扫描。", + "description": "读取必填目录输入;支持 BOS directory ref 与本地目录,输出完整文件清单供下游算子消费,不保存增量扫描状态。", "hyperparams": [ { - "description": "待扫描目录", + "description": "本地目录兼容输入;Workflow 生产运行应使用必填目录引用", "key": "SOURCE_PATH", "type": "string", "value": "" @@ -889,16 +889,17 @@ "description": "目录输入", "id": "in-1", "name": "input", + "required": true, "transmission": "bus", - "type": "json" + "type": "file" } ], - "name": "目录扫描", + "name": "目录读取", "outputs": [ { - "description": "目录概览", + "description": "目录文件清单", "id": "out-1", - "name": "summary", + "name": "manifest", "transmission": "bus", "type": "json" } @@ -937,13 +938,14 @@ "requiresGpu": false }, "runtimeDependencies": [ - "filesystem" + "filesystem", + "bos-directory-ref" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, "runtimeHandler": "builtin_directory_scan", "runtimeSource": "operator-repos/builtin/directory_scan/src/operator.py", - "skillDefinition": "读取 SOURCE_PATH 或上游 source manifest,输出真实目录概览。", + "skillDefinition": "读取必填 BOS directory ref 或本地 SOURCE_PATH,输出可直接进入数据标准化算子的完整目录 manifest;无增量状态。", "type": "builtin" }, { @@ -2302,13 +2304,6 @@ "name": "handpose", "transmission": "bus", "type": "json" - }, - { - "description": "真实 Qwen3-VL 连续训练报告;作为包前强制依赖与 provenance", - "id": "in-4", - "name": "trainingEvidence", - "transmission": "bus", - "type": "json" } ], "name": "成品打包", @@ -2359,8 +2354,7 @@ "video-files", "accepted-annotation-partitions", "headpose-manifests", - "optional-handpose-manifests", - "qwen-training-provenance" + "optional-handpose-manifests" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, @@ -13009,5 +13003,167 @@ "runtimeSource": "operator-repos/builtin/qwen_label_crop_classification/src/operator.py", "skillDefinition": "真实 ms-swift Qwen3-VL LoRA 连续训练;没有 adapter、trainer_state 或输入/模型 provenance 即失败。", "type": "builtin" + }, + { + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "requiresGpuEvidence": true, + "requiresNonEmptyProvenance": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "description": "读取真实人脸脱敏视频和同目录逐帧 timestamp_us CSV;在送入 Qwen3-VL 的每个采样帧右上角叠加真实 PTS_US,以手部主谓宾动作/物体变化生成自动粗切候选。技术块仅限制单次模型负载,不是最终切片边界;输出引用完整脱敏源视频的 virtual clips,供人工调切。", + "hyperparams": [ + { + "description": "只读的本地 Qwen3-VL-8B-Instruct 快照目录", + "key": "MODEL_PATH", + "type": "string", + "value": "/workspace/models/Qwen3-VL-8B-Instruct" + }, + { + "description": "锁定的 Qwen3-VL-8B-Instruct revision", + "key": "MODEL_REVISION", + "type": "string", + "value": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b" + }, + { + "description": "逐 shard SHA/大小经过校验的模型 manifest", + "key": "MODEL_MANIFEST_PATH", + "type": "string", + "value": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json" + }, + { + "description": "预载 Qwen3-VL manifest 的固定 SHA-256", + "key": "MODEL_MANIFEST_SHA256", + "type": "string", + "value": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a" + }, + { + "description": "复用已校验预载收据,仍校验固定 manifest 与布局", + "key": "TRUST_PREVERIFIED_MODEL", + "type": "boolean", + "value": "true" + }, + { + "description": "单次 Qwen 请求的技术负载上限(仅技术块,不产生切片边界)", + "key": "TECHNICAL_BLOCK_SECONDS", + "type": "number", + "value": "10" + }, + { + "description": "每个技术块送入 Qwen3-VL 的真实 PTS 叠加帧数(仅技术负载)", + "key": "FRAMES_PER_TECHNICAL_BLOCK", + "type": "number", + "value": "18" + }, + { + "description": "Qwen JSON 输出的最大 token 数", + "key": "MAX_NEW_TOKENS", + "type": "number", + "value": "2048" + }, + { + "description": "验收时可要求真实 Qwen 至少输出一个行为;正常视频允许真实空结果", + "key": "REQUIRE_NON_EMPTY_RESULT", + "type": "boolean", + "value": "false" + } + ], + "id": "operator-builtin-qwen-behavior-rough-cut", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:pending-fanyu-qwen-behavior-rough-cut", + "inputs": [ + { + "description": "视频人脸脱敏输出:blurred-manifest 与完整脱敏视频 artifact", + "id": "video", + "name": "video", + "required": true, + "transmission": "bus", + "type": "json" + }, + { + "description": "Orbbec 标准化 manifest:用于精确定位匹配视频的逐帧 timestamp_us CSV", + "id": "metadata", + "name": "metadata", + "required": true, + "transmission": "bus", + "type": "json" + } + ], + "name": "Qwen3-VL 语义行为粗切(PTS)", + "outputs": [ + { + "description": "行为驱动的 clips-manifest 与复用完整脱敏源视频引用的 virtual clip artifacts", + "id": "out-1", + "name": "roughCuts", + "transmission": "bus", + "type": "json" + } + ], + "publicationStatus": "draft", + "resources": { + "cpuLimit": "4", + "gpuLimit": "1", + "gpuType": "NVIDIA A800 80GB", + "memoryLimit": "32Gi" + }, + "runtimeContract": { + "acceptance": { + "allowsEmptyResultList": true, + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresNonEmptyManifest": true, + "requiresNonEmptyOutput": false, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_qwen_behavior_rough_cut", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "operatorId": "operator-builtin-qwen-behavior-rough-cut", + "operatorSlug": "qwen_behavior_rough_cut", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 1, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": true + }, + "runtimeDependencies": [ + "qwen3-vl-8b-instruct", + "verified-local-model-manifest", + "torch-cuda", + "opencv", + "qwen-vl-utils", + "real-video-files", + "orbbec-pts-csv" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeHandler": "builtin_qwen_behavior_rough_cut", + "runtimeSource": "operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py", + "skillDefinition": "真实 Qwen3-VL 基于右上角真实 PTS_US 识别手部主谓宾行为变化并生成自动粗切建议;不按固定时长输出、没有 200 秒定位。每个候选复用完整脱敏视频,人工调切后才物理切片。", + "type": "builtin" } ] diff --git a/internal/api/fanyu_template.go b/internal/api/fanyu_template.go index 2d2b8738..031deb66 100644 --- a/internal/api/fanyu_template.go +++ b/internal/api/fanyu_template.go @@ -2,7 +2,6 @@ package api import ( "context" - "encoding/json" "fmt" "os" "sort" @@ -120,10 +119,8 @@ func (server *server) ensureFanyuDeliveryTemplateV5(ctx context.Context, organiz return err } operatorIDs := []string{ - "operator-builtin-bos-incremental-scan", "operator-builtin-orbbec-data-normalize", "operator-builtin-video-face-blur", - "operator-builtin-video-slice", "operator-builtin-qwen-temporal-localization", "operator-builtin-wilor-valid-event-detection", - "operator-builtin-annotation-gate", "operator-builtin-merge", "operator-builtin-ms-swift-dataset-export", - "operator-builtin-qwen-label-crop-classification", "operator-builtin-package", + "operator-builtin-directory-scan", "operator-builtin-orbbec-data-normalize", "operator-builtin-video-face-blur", + "operator-builtin-qwen-behavior-rough-cut", "operator-builtin-annotation-gate", "operator-builtin-package", } operators := map[string]domain.OperatorRecord{} for _, operatorID := range operatorIDs { @@ -136,39 +133,30 @@ func (server *server) ensureFanyuDeliveryTemplateV5(ctx context.Context, organiz } operators[operatorID] = *operator } - taxonomyBytes, err := json.Marshal(domain.ConvenienceStore22SemanticLabels()) - if err != nil { - return fmt.Errorf("marshal fanyu v5 taxonomy: %w", err) - } annotationTemplateID := fmt.Sprintf("tpl-fanyu-convenience-store-v5-%s", normalizedOrganizationID(organizationID)) annotationWorkgroupID := strings.TrimSpace(os.Getenv("FANYU_V5_ANNOTATION_WORKGROUP_ID")) nodes := []domain.WorkflowNode{ - fanyuNode("fanyu-v5-scan", "BOS 增量扫描", operators[operatorIDs[0]], 0, 260), + fanyuNode("fanyu-v5-directory", "目录读取(必填)", operators[operatorIDs[0]], 0, 260), fanyuNodeWithHyperparams("fanyu-v5-normalize", "Orbbec Ego 左相机标准化", operators[operatorIDs[1]], 240, 260, map[string]string{"CAMERA_SIDE": "left", "REQUIRE_STEREO": "true"}), fanyuNodeWithHyperparams("fanyu-v5-face-blur", "人脸脱敏(真实视频)", operators[operatorIDs[2]], 500, 260, map[string]string{"PRESERVE_AUDIO": "true", "PRESERVE_PTS": "true", "REQUIRE_DETECTIONS": "true"}), - fanyuNodeWithHyperparams("fanyu-v5-slice", "左相机初始切片", operators[operatorIDs[3]], 760, 80, map[string]string{"SEGMENT_DURATION_SECONDS": "200"}), - fanyuNodeWithHyperparams("fanyu-v5-temporal", "Qwen3-VL 200秒真实时间定位", operators[operatorIDs[4]], 760, 220, map[string]string{"MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "WINDOW_SECONDS": "200", "STRIDE_SECONDS": "180", "OVERLAP_SECONDS": "20", "FPS": "1", "MAX_FRAMES": "96", "MERGE_GAP_SECONDS": "2", "TAXONOMY_22_JSON": string(taxonomyBytes)}), - fanyuNodeWithHyperparams("fanyu-v5-valid-event", "有效行为检测(真实 Qwen3-VL)", operators[operatorIDs[5]], 760, 360, map[string]string{"MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "WINDOW_SECONDS": "5", "STRIDE_SECONDS": "5", "FPS": "1", "MAX_FRAMES": "5", "MIN_EVENT_OVERLAP": "0.5"}), - fanyuNodeWithHyperparams("fanyu-v5-human", "同任务切片校正+22类标注→统一质检→最终确认", operators[operatorIDs[6]], 1040, 100, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_WORKGROUP_ID": annotationWorkgroupID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "1", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), - fanyuNodeWithHyperparams("fanyu-v5-handoff", "验收后受控交付汇合", operators[operatorIDs[7]], 1320, 100, map[string]string{"PRESERVE_UPSTREAM_PAYLOADS": "true"}), - fanyuNodeWithHyperparams("fanyu-v5-swift-export", "已验收22类 ms-swift 数据导出", operators[operatorIDs[8]], 1560, 40, map[string]string{"EXPORT_MODE": "classification", "WINDOW_SECONDS": "200", "STRIDE_SECONDS": "180", "TRAIN_RATIO": "1.0", "VAL_RATIO": "0", "TEST_RATIO": "0", "REQUIRE_ACCEPTANCE": "true"}), - fanyuNodeWithHyperparams("fanyu-v5-train", "Qwen3-VL 22类真实 LoRA 持续训练", operators[operatorIDs[9]], 1800, 40, map[string]string{"BASE_MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "BASE_MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "EPOCHS": "1", "MAX_STEPS": "1", "BATCH_SIZE": "1", "LORA_RANK": "8"}), - fanyuNodeWithHyperparams("fanyu-v5-package", "最终切片 MP4、语义 JSON 与 manifest 打包", operators[operatorIDs[10]], 2040, 180, map[string]string{"REQUIRE_HEADPOSE": "false", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}), + fanyuNodeWithHyperparams("fanyu-v5-rough-cut", "Qwen3-VL 手部行为语义粗切", operators[operatorIDs[3]], 760, 260, map[string]string{"MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "TECHNICAL_BLOCK_SECONDS": "10", "FRAMES_PER_TECHNICAL_BLOCK": "18", "REQUIRE_NON_EMPTY_RESULT": "true"}), + fanyuNodeWithHyperparams("fanyu-v5-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认", operators[operatorIDs[4]], 1040, 260, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_WORKGROUP_ID": annotationWorkgroupID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), + fanyuNodeWithHyperparams("fanyu-v5-package", "人工聚合确认后打包最终 MP4、语义 JSON 与 manifest", operators[operatorIDs[5]], 1320, 260, map[string]string{"REQUIRE_HEADPOSE": "false", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}), } for nodeIndex := range nodes { switch nodes[nodeIndex].ID { case "fanyu-v5-package": - inputs := make([]domain.PortDefinition, 0, len(nodes[nodeIndex].Data.Inputs)-2) + inputs := make([]domain.PortDefinition, 0, 1) for _, input := range nodes[nodeIndex].Data.Inputs { - if input.ID != "in-2" && input.ID != "in-3" { + if input.ID == "in-1" { inputs = append(inputs, input) } } nodes[nodeIndex].Data.Inputs = inputs case "fanyu-v5-human": - inputs := make([]domain.PortDefinition, 0, len(nodes[nodeIndex].Data.Inputs)-2) + inputs := make([]domain.PortDefinition, 0, 1) for _, input := range nodes[nodeIndex].Data.Inputs { - if input.ID != "headpose" && input.ID != "handpose" { + if input.ID == "clips" { inputs = append(inputs, input) } } @@ -176,22 +164,18 @@ func (server *server) ensureFanyuDeliveryTemplateV5(ctx context.Context, organiz } } edges := []domain.WorkflowEdge{ - fanyuEdge("fanyu-v5-e1", "fanyu-v5-scan", "out-1", "fanyu-v5-normalize", "in-1"), + fanyuEdge("fanyu-v5-e1", "fanyu-v5-directory", "out-1", "fanyu-v5-normalize", "in-1"), fanyuEdge("fanyu-v5-e2", "fanyu-v5-normalize", "out-1", "fanyu-v5-face-blur", "in-1"), - fanyuEdge("fanyu-v5-e3", "fanyu-v5-face-blur", "out-1", "fanyu-v5-slice", "in-1"), - fanyuEdge("fanyu-v5-e4", "fanyu-v5-face-blur", "out-1", "fanyu-v5-temporal", "in-1"), - fanyuEdge("fanyu-v5-e5", "fanyu-v5-face-blur", "out-1", "fanyu-v5-valid-event", "in-1"), - fanyuEdge("fanyu-v5-e5b", "fanyu-v5-valid-event", "out-1", "fanyu-v5-temporal", "validEvents"), - fanyuEdge("fanyu-v5-e9", "fanyu-v5-slice", "out-1", "fanyu-v5-human", "clips"), - fanyuEdge("fanyu-v5-e10", "fanyu-v5-temporal", "out-1", "fanyu-v5-human", "prelabels"), - fanyuEdge("fanyu-v5-e11", "fanyu-v5-valid-event", "out-1", "fanyu-v5-human", "validEvents"), - fanyuEdge("fanyu-v5-e14", "fanyu-v5-human", "approved", "fanyu-v5-handoff", "in-1"), - fanyuEdge("fanyu-v5-e15", "fanyu-v5-handoff", "out-1", "fanyu-v5-swift-export", "in-1"), - fanyuEdge("fanyu-v5-e16", "fanyu-v5-swift-export", "out-1", "fanyu-v5-train", "in-1"), - fanyuEdge("fanyu-v5-e17", "fanyu-v5-handoff", "out-1", "fanyu-v5-package", "in-1"), - fanyuEdge("fanyu-v5-e18", "fanyu-v5-train", "out-1", "fanyu-v5-package", "in-4"), + fanyuEdge("fanyu-v5-e3", "fanyu-v5-face-blur", "out-1", "fanyu-v5-rough-cut", "video"), + fanyuEdge("fanyu-v5-e4", "fanyu-v5-normalize", "out-1", "fanyu-v5-rough-cut", "metadata"), + fanyuEdge("fanyu-v5-e5", "fanyu-v5-rough-cut", "out-1", "fanyu-v5-human", "clips"), + fanyuEdge("fanyu-v5-e6", "fanyu-v5-human", "approved", "fanyu-v5-package", "in-1"), } - workflowInputs := []domain.WorkflowInputDefinition{} + workflowInputs := []domain.WorkflowInputDefinition{{ + ID: "sourceDirectory", Name: "sourceDirectory", Description: "本次运行要处理的 Orbbec Ego BOS 目录(必填,非增量扫描)", + Type: domain.PortTypeFile, Required: true, + Bindings: []domain.WorkflowInputTarget{{NodeID: "fanyu-v5-directory", TargetPortID: "in-1"}}, + }} now := nowISO() createdAt := now if existing, err := server.store.GetTemplate(ctx, templateID); err != nil { @@ -199,7 +183,7 @@ func (server *server) ensureFanyuDeliveryTemplateV5(ctx context.Context, organiz } else if existing != nil && strings.TrimSpace(existing.CreatedAt) != "" { createdAt = existing.CreatedAt } - template := domain.TemplateRecord{ID: templateID, OrganizationID: normalizedOrganizationID(organizationID), Name: "01-左相机切分标注质检-v5", Description: "左相机标准化与人脸脱敏,真实 Qwen3-VL 时间定位/有效行为;同任务切片校正和22类语义标注、统一质检、仅确认的最终验收后生成逐段 MP4、语义 JSON 与 manifest,并触发真实 LoRA 持续训练。当前版本不执行 Headpose 或 Handpose。", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, ExecutionScope: domain.WorkflowExecutionScopeProduction, MaxConcurrentRuns: 1, WorkflowInputs: workflowInputs, Nodes: nodes, Edges: edges, CreatedAt: createdAt, UpdatedAt: now} + template := domain.TemplateRecord{ID: templateID, OrganizationID: normalizedOrganizationID(organizationID), Name: "01-左相机切分标注质检-v5", Description: "必填目录读取、左相机标准化、人脸高斯模糊、基于真实逐帧 PTS 的 Qwen3-VL 手部行为语义粗切;随后按人工调切、剪切审核、语义标注、标注审核闭环处理,全部通过后由验收员完成人工聚合确认并直接打包最终 MP4、语义 JSON 与 manifest。不执行固定时长切片、200秒定位、LoRA、HeadPose 或 HandPose。", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, ExecutionScope: domain.WorkflowExecutionScopeProduction, MaxConcurrentRuns: 1, WorkflowInputs: workflowInputs, Nodes: nodes, Edges: edges, CreatedAt: createdAt, UpdatedAt: now} if issues := validateWorkflowTopology(nodes, edges); len(issues) > 0 { return fmt.Errorf("invalid fanyu v5 template: %s", strings.TrimSpace(issues[0].Message)) } @@ -223,7 +207,7 @@ func (server *server) ensureFanyuDeliveryTemplateV5(ctx context.Context, organiz if issues := server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction); len(issues) > 0 { return nil } - release := domain.WorkflowReleaseRecord{ID: releaseID, OrganizationID: normalizedOrganizationID(organizationID), TemplateID: template.ID, Version: 5, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, WorkflowInputs: workflowInputs, Nodes: nodes, Edges: edges, Capabilities: []string{"human_annotation", "human_quality_review", "human_acceptance", "qwen3vl_temporal", "qwen3vl_training"}, RequiredPluginIDs: []string{pluginregistry.HumanLoopCoreID, pluginregistry.AnnotationQAID}, ContentDigest: workflowReleaseDigest(template), ReleaseNotes: "Left camera v5; verified Qwen3-VL, confirm-only acceptance before package, per-final-segment MP4, semantic JSON and manifest; pose stages deferred", PublishedByUserID: "system", PublishedAt: now, CreatedAt: now} + release := domain.WorkflowReleaseRecord{ID: releaseID, OrganizationID: normalizedOrganizationID(organizationID), TemplateID: template.ID, Version: 5, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, WorkflowInputs: workflowInputs, Nodes: nodes, Edges: edges, Capabilities: []string{"directory_input", "qwen3vl_behavior_rough_cut", "human_cut_adjustment", "human_cut_review", "human_semantic_annotation", "human_semantic_review", "human_acceptance", "final_segment_package"}, RequiredPluginIDs: []string{pluginregistry.HumanLoopCoreID, pluginregistry.AnnotationQAID}, ContentDigest: workflowReleaseDigest(template), ReleaseNotes: "Left camera v5; directory input, PTS-grounded Qwen3-VL semantic rough cut, four-stage human rework loop, aggregate confirm-only acceptance, then direct pose-free package", PublishedByUserID: "system", PublishedAt: now, CreatedAt: now} return server.store.UpsertWorkflowRelease(ctx, release) } diff --git a/internal/api/fanyu_template_test.go b/internal/api/fanyu_template_test.go index ce3f326e..62d405e2 100644 --- a/internal/api/fanyu_template_test.go +++ b/internal/api/fanyu_template_test.go @@ -117,14 +117,22 @@ func TestEnsureFanyuDeliveryTemplateCreatesLeftCameraV5AsFailClosedDraft(t *test if got := fanyuTestHyperparam(fanyuTestNode(template.Nodes, "fanyu-v5-normalize"), "CAMERA_SIDE"); got != "left" { t.Fatalf("v5 CAMERA_SIDE=%q", got) } - if fanyuTestNode(template.Nodes, "fanyu-v5-valid-event") == nil || fanyuTestNode(template.Nodes, "fanyu-v5-temporal") == nil || fanyuTestNode(template.Nodes, "fanyu-v5-train") == nil { - t.Fatalf("v5 real AI nodes missing: %#v", template.Nodes) + if len(template.Nodes) != 6 || len(template.Edges) != 6 { + t.Fatalf("unexpected v5 topology nodes=%d edges=%d: %#v", len(template.Nodes), len(template.Edges), template) + } + if fanyuTestNode(template.Nodes, "fanyu-v5-directory") == nil || fanyuTestNode(template.Nodes, "fanyu-v5-rough-cut") == nil { + t.Fatalf("v5 directory/Qwen rough-cut nodes missing: %#v", template.Nodes) + } + for _, obsoleteNodeID := range []string{"fanyu-v5-scan", "fanyu-v5-slice", "fanyu-v5-valid-event", "fanyu-v5-temporal", "fanyu-v5-handoff", "fanyu-v5-swift-export", "fanyu-v5-train"} { + if fanyuTestNode(template.Nodes, obsoleteNodeID) != nil { + t.Fatalf("v5 obsolete fixed-slice/temporal/training node %s must be absent", obsoleteNodeID) + } } if fanyuTestNode(template.Nodes, "fanyu-v5-headpose") != nil || fanyuTestNode(template.Nodes, "fanyu-v5-handpose") != nil || fanyuTestNode(template.Nodes, "fanyu-v5-options") != nil { t.Fatalf("v5 pose stages are deferred and must not be present: %#v", template.Nodes) } - if len(template.WorkflowInputs) != 0 { - t.Fatalf("v5 must not expose the obsolete Handpose run input: %#v", template.WorkflowInputs) + if len(template.WorkflowInputs) != 1 || template.WorkflowInputs[0].ID != "sourceDirectory" || !template.WorkflowInputs[0].Required || template.WorkflowInputs[0].DefaultPayload != nil || len(template.WorkflowInputs[0].Bindings) != 1 || template.WorkflowInputs[0].Bindings[0].NodeID != "fanyu-v5-directory" { + t.Fatalf("v5 must expose one required directory ref input: %#v", template.WorkflowInputs) } packageNode := fanyuTestNode(template.Nodes, "fanyu-v5-package") if packageNode == nil || fanyuTestHyperparam(packageNode, "REQUIRE_HEADPOSE") != "false" { @@ -132,11 +140,15 @@ func TestEnsureFanyuDeliveryTemplateCreatesLeftCameraV5AsFailClosedDraft(t *test } for _, nodeID := range []string{"fanyu-v5-package", "fanyu-v5-human"} { for _, input := range fanyuTestNode(template.Nodes, nodeID).Data.Inputs { - if input.ID == "in-2" || input.ID == "in-3" || input.ID == "headpose" || input.ID == "handpose" { - t.Fatalf("v5 deferred pose input must not be exposed on %s: %#v", nodeID, input) + if (nodeID == "fanyu-v5-package" && input.ID != "in-1") || (nodeID == "fanyu-v5-human" && input.ID != "clips") { + t.Fatalf("v5 must expose only the direct aggregate-to-package contract on %s: %#v", nodeID, input) } } } + roughCut := fanyuTestNode(template.Nodes, "fanyu-v5-rough-cut") + if fanyuTestHyperparam(roughCut, "TECHNICAL_BLOCK_SECONDS") != "10" || fanyuTestHyperparam(roughCut, "FRAMES_PER_TECHNICAL_BLOCK") != "18" || fanyuTestHyperparam(roughCut, "WINDOW_SECONDS") != "" || fanyuTestHyperparam(roughCut, "STRIDE_SECONDS") != "" || fanyuTestHyperparam(roughCut, "OVERLAP_SECONDS") != "" { + t.Fatalf("v5 rough-cut must use technical-only chunks without 200-second localization: %#v", roughCut) + } human := fanyuTestNode(template.Nodes, "fanyu-v5-human") if human == nil || fanyuTestHyperparam(human, "REQUIRE_PREPACKAGE_ACCEPTANCE") != "true" { t.Fatalf("v5 aggregate acceptance gate missing: %#v", human) @@ -145,7 +157,9 @@ func TestEnsureFanyuDeliveryTemplateCreatesLeftCameraV5AsFailClosedDraft(t *test t.Fatalf("draft operators/model preconditions must not publish v5 release: %#v err=%v", release, releaseErr) } tpl, templateErr := dataStore.GetAnnotationTemplate(ctx, "tpl-fanyu-convenience-store-v5-org-v5") - if templateErr != nil || tpl == nil || len(tpl.HumanStages) != 2 || tpl.HumanStages[0].ID != "annotation_combined" || tpl.HumanStages[1].ID != "quality_review" { + if templateErr != nil || tpl == nil || !domain.IsFanyuV5FourStageHumanLoop(tpl.HumanStages) || + tpl.HumanStages[1].RejectTargets[0] != domain.FanyuV5StageCutAdjustment || + tpl.HumanStages[3].RejectTargets[0] != domain.FanyuV5StageSemanticAnnotation { t.Fatalf("unexpected v5 human stages: %#v err=%v", tpl, templateErr) } } diff --git a/internal/api/fanyu_v5_four_stage_human_loop_test.go b/internal/api/fanyu_v5_four_stage_human_loop_test.go new file mode 100644 index 00000000..8526ff40 --- /dev/null +++ b/internal/api/fanyu_v5_four_stage_human_loop_test.go @@ -0,0 +1,280 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +type fanyuV5FourStageFixture struct { + ctx context.Context + store *store.MemoryStore + handler http.Handler + annotator domain.UserRecord + quality domain.UserRecord + batch domain.HumanBatchRecord + task domain.AnnotationTaskRecord + item domain.HumanItemRecord +} + +func newFanyuV5FourStageFixture(t *testing.T, stageID string, status domain.HumanItemStatus, assignee string, currentRevision int) fanyuV5FourStageFixture { + t.Helper() + ctx := context.Background() + dataStore := store.NewMemoryStore() + now := time.Now().UTC().Format(time.RFC3339) + annotator := domain.UserRecord{ID: "annotator-v5", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive} + quality := domain.UserRecord{ID: "quality-v5", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleQualityReviewer, Status: domain.UserStatusActive} + for _, user := range []domain.UserRecord{annotator, quality} { + if err := dataStore.UpsertUser(ctx, user); err != nil { + t.Fatal(err) + } + } + for _, membership := range []domain.ProjectAnnotationMembershipRecord{ + {ID: "membership-v5-annotator", OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-v5", UserID: annotator.ID, WorkgroupID: "workgroup-v5", VendorID: "vendor-v5", AssignmentRole: domain.AnnotationRoleAnnotator, CreatedAt: now, UpdatedAt: now}, + {ID: "membership-v5-quality", OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-v5", UserID: quality.ID, WorkgroupID: "workgroup-v5", VendorID: "vendor-v5", AssignmentRole: domain.AnnotationRoleQualityReviewer, CreatedAt: now, UpdatedAt: now}, + } { + if err := dataStore.UpsertProjectAnnotationMembership(ctx, membership); err != nil { + t.Fatal(err) + } + } + batch := domain.HumanBatchRecord{ + ID: "batch-v5-four-stage", OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", + ProjectID: "project-v5", WorkflowID: "workflow-v5", RunID: "run-v5", GateNodeID: "human-v5", + Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV5HumanStages(), + ReleasePolicy: domain.HumanReleaseAllAccepted, MaxReworkRounds: 5, RequirePrePackageAcceptance: true, + CreatedAt: now, UpdatedAt: now, + } + if err := dataStore.UpsertHumanBatch(ctx, batch); err != nil { + t.Fatal(err) + } + task := domain.AnnotationTaskRecord{ + ID: "task-v5-four-stage", OrganizationID: domain.DefaultOrganizationID, ProjectID: batch.ProjectID, + WorkflowID: batch.WorkflowID, RunID: batch.RunID, NodeID: batch.GateNodeID, ArtifactID: "artifact-v5", + SourceVideo: "bos://fixture/video.mp4", ClipStart: 0, ClipEnd: 20, TemplateID: "template-v5", TemplateVersion: 5, + WorkgroupID: "workgroup-v5", VendorID: "vendor-v5", Status: domain.AnnotationTaskStatusPendingAnnotation, + CurrentRevision: currentRevision, CreatedAt: now, UpdatedAt: now, + } + if stageID == domain.FanyuV5StageCutReview || stageID == domain.FanyuV5StageSemanticReview { + task.Status = domain.AnnotationTaskStatusPendingQuality + } + if err := dataStore.UpsertAnnotationTask(ctx, task); err != nil { + t.Fatal(err) + } + item := domain.HumanItemRecord{ + ID: "item-v5-four-stage", OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", BatchID: batch.ID, + ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, GateNodeID: batch.GateNodeID, + LegacyTaskID: task.ID, Subject: domain.HumanSubjectRef{ArtifactID: task.ArtifactID, ItemKey: "clip-v5", MediaType: "video/mp4"}, + CurrentStageID: stageID, Status: status, AssignedUserID: assignee, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, + CurrentRevision: currentRevision, MaxReworkRounds: 5, Version: 1, CreatedAt: now, UpdatedAt: now, + } + if err := dataStore.UpsertHumanItem(ctx, item); err != nil { + t.Fatal(err) + } + return fanyuV5FourStageFixture{ctx: ctx, store: dataStore, handler: NewHandler(dataStore, nil), annotator: annotator, quality: quality, batch: batch, task: task, item: item} +} + +func (fixture fanyuV5FourStageFixture) request(method string, path string, user domain.UserRecord, body any) *httptest.ResponseRecorder { + encoded, err := json.Marshal(body) + if err != nil { + panic(err) + } + request := httptest.NewRequest(method, path, bytes.NewReader(encoded)) + request.Header.Set("Content-Type", "application/json") + csrfToken := "fanyu-v5-four-stage-csrf" + request.AddCookie(&http.Cookie{Name: csrfCookieName, Value: csrfToken}) + request.Header.Set(csrfHeaderName, csrfToken) + request = request.WithContext(context.WithValue(request.Context(), authContextKey{}, &authenticatedPrincipal{ + User: user, + Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}, + })) + recorder := httptest.NewRecorder() + fixture.handler.ServeHTTP(recorder, request) + return recorder +} + +func TestFanyuV5FourStageRejectTargetsImmediatelyReturnToExactEditStage(t *testing.T) { + t.Run("cut review returns only to cut adjustment", func(t *testing.T) { + fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutReview, domain.HumanItemClaimed, "quality-v5", 0) + if err := fixture.store.UpsertSliceRevision(fixture.ctx, domain.SliceRevisionRecord{ + ID: "slice-v5-1", OrganizationID: domain.DefaultOrganizationID, TaskID: fixture.task.ID, Revision: 1, + Segments: []domain.SliceSegment{{ID: "slice-1", Start: 0, End: 20}}, AuthorUserID: fixture.annotator.ID, CreatedAt: nowISO(), + }); err != nil { + t.Fatal(err) + } + recorder := fixture.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+fixture.task.ID+"/reviews", fixture.quality, map[string]any{ + "stage": "split_quality", "decision": "rejected", "comment": "边界需要调整", "reasonCodes": []string{"boundary"}, + "rejectTargets": []string{domain.FanyuV5StageCutAdjustment}, + }) + if recorder.Code != http.StatusCreated { + t.Fatalf("cut rejection status=%d body=%s", recorder.Code, recorder.Body.String()) + } + item, _ := fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.CurrentStageID != domain.FanyuV5StageCutAdjustment || item.Status != domain.HumanItemRework || item.ReviewRound != 1 { + t.Fatalf("cut rework item=%#v", item) + } + rejects, _ := fixture.store.ListHumanRejects(fixture.ctx, domain.DefaultOrganizationID, domain.HumanRejectQuery{ItemID: fixture.item.ID}) + if len(rejects) != 1 || rejects[0].StageID != domain.FanyuV5StageCutReview || rejects[0].Status != domain.HumanRejectReopened || rejects[0].ReopenTargetStageID != domain.FanyuV5StageCutAdjustment { + t.Fatalf("cut reject routing=%#v", rejects) + } + }) + + t.Run("semantic review returns only to semantic annotation", func(t *testing.T) { + fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemClaimed, "quality-v5", 1) + if err := fixture.store.UpsertAnnotationRevision(fixture.ctx, domain.AnnotationRevisionRecord{ + ID: "semantic-v5-1", OrganizationID: domain.DefaultOrganizationID, TaskID: fixture.task.ID, Revision: 1, TemplateVersion: 5, + Segments: []domain.AnnotationSegment{{ID: "semantic-1", LabelID: "01", Label: "测试动作", Start: 0, End: 20, Confidence: 1}}, AuthorUserID: fixture.annotator.ID, CreatedAt: nowISO(), + }); err != nil { + t.Fatal(err) + } + recorder := fixture.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+fixture.task.ID+"/reviews", fixture.quality, map[string]any{ + "stage": "quality", "decision": "rejected", "comment": "标签需要补全", "reasonCodes": []string{"semantic_gap"}, + "rejectTargets": []string{domain.FanyuV5StageSemanticAnnotation}, + }) + if recorder.Code != http.StatusCreated { + t.Fatalf("semantic rejection status=%d body=%s", recorder.Code, recorder.Body.String()) + } + item, _ := fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.CurrentStageID != domain.FanyuV5StageSemanticAnnotation || item.Status != domain.HumanItemRework || item.ReviewRound != 1 { + t.Fatalf("semantic rework item=%#v", item) + } + rejects, _ := fixture.store.ListHumanRejects(fixture.ctx, domain.DefaultOrganizationID, domain.HumanRejectQuery{ItemID: fixture.item.ID}) + if len(rejects) != 1 || rejects[0].StageID != domain.FanyuV5StageSemanticReview || rejects[0].Status != domain.HumanRejectReopened || rejects[0].ReopenTargetStageID != domain.FanyuV5StageSemanticAnnotation { + t.Fatalf("semantic reject routing=%#v", rejects) + } + }) +} + +func TestFanyuV5FourStageRejectTargetCannotCrossBackToAnotherStage(t *testing.T) { + fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutReview, domain.HumanItemClaimed, "quality-v5", 0) + if err := fixture.store.UpsertSliceRevision(fixture.ctx, domain.SliceRevisionRecord{ + ID: "slice-v5-invalid-target", OrganizationID: domain.DefaultOrganizationID, TaskID: fixture.task.ID, Revision: 1, + Segments: []domain.SliceSegment{{ID: "slice-1", Start: 0, End: 20}}, AuthorUserID: fixture.annotator.ID, CreatedAt: nowISO(), + }); err != nil { + t.Fatal(err) + } + recorder := fixture.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+fixture.task.ID+"/reviews", fixture.quality, map[string]any{ + "stage": "split_quality", "decision": "rejected", "comment": "边界需要调整", "reasonCodes": []string{"boundary"}, + "rejectTargets": []string{domain.FanyuV5StageSemanticAnnotation}, + }) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("cross-stage reject target status=%d body=%s", recorder.Code, recorder.Body.String()) + } + item, _ := fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.CurrentStageID != domain.FanyuV5StageCutReview || item.Status != domain.HumanItemClaimed { + t.Fatalf("invalid target must not mutate item: %#v", item) + } +} + +func TestFanyuV5GenericDecisionAlsoUsesTheConfiguredImmediateReworkTarget(t *testing.T) { + fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemClaimed, "quality-v5", 1) + recorder := fixture.request(http.MethodPost, "/api/human-items/"+fixture.item.ID+"/decisions", fixture.quality, map[string]any{ + "version": fixture.item.Version, "decision": "rejected", "reviewedRevision": 1, + "reasonCodes": []string{"semantic_gap"}, "comment": "需要补充动作标签", + }) + if recorder.Code != http.StatusOK { + t.Fatalf("generic decision status=%d body=%s", recorder.Code, recorder.Body.String()) + } + item, _ := fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.CurrentStageID != domain.FanyuV5StageSemanticAnnotation || item.Status != domain.HumanItemRework { + t.Fatalf("generic decision must route semantic review to semantic annotation: %#v", item) + } + rejects, _ := fixture.store.ListHumanRejects(fixture.ctx, domain.DefaultOrganizationID, domain.HumanRejectQuery{ItemID: fixture.item.ID}) + if len(rejects) != 1 || rejects[0].Status != domain.HumanRejectReopened || rejects[0].ReopenTargetStageID != domain.FanyuV5StageSemanticAnnotation { + t.Fatalf("generic decision must retain automatic rework evidence: %#v", rejects) + } +} + +func TestFanyuV5TaskListCarriesCurrentHumanStageForWorkbenchRouting(t *testing.T) { + fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1) + recorder := fixture.request(http.MethodGet, "/api/plugins/annotation-qa/tasks?projectId="+fixture.task.ProjectID, fixture.annotator, nil) + if recorder.Code != http.StatusOK { + t.Fatalf("task list status=%d body=%s", recorder.Code, recorder.Body.String()) + } + var response struct { + Items []struct { + ID string `json:"id"` + HumanStageID string `json:"humanStageId"` + } `json:"items"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if len(response.Items) != 1 || response.Items[0].ID != fixture.task.ID || response.Items[0].HumanStageID != domain.FanyuV5StageSemanticReview { + t.Fatalf("task-list human stage=%#v", response.Items) + } +} + +func TestFanyuV5FourStageProgressesToConfirmOnlyAggregateAcceptance(t *testing.T) { + fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemClaimed, "annotator-v5", 0) + + sliceSubmission := fixture.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+fixture.task.ID+"/slice-revisions", fixture.annotator, map[string]any{ + "segments": []map[string]any{{"id": "rough-cut-adjusted", "start": 0.0, "end": 20.0}}, + }) + if sliceSubmission.Code != http.StatusCreated { + t.Fatalf("cut adjustment status=%d body=%s", sliceSubmission.Code, sliceSubmission.Body.String()) + } + item, _ := fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.CurrentStageID != domain.FanyuV5StageCutReview || item.Status != domain.HumanItemReady { + t.Fatalf("cut adjustment must progress to cut review: %#v", item) + } + + claimCutReview := fixture.request(http.MethodPost, "/api/human-items/"+fixture.item.ID+"/claim", fixture.quality, map[string]any{"version": item.Version}) + if claimCutReview.Code != http.StatusOK { + t.Fatalf("claim cut review status=%d body=%s", claimCutReview.Code, claimCutReview.Body.String()) + } + approveCutReview := fixture.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+fixture.task.ID+"/reviews", fixture.quality, map[string]any{ + "stage": "split_quality", "decision": "approved", "comment": "剪切审核通过", + }) + if approveCutReview.Code != http.StatusCreated { + t.Fatalf("approve cut review status=%d body=%s", approveCutReview.Code, approveCutReview.Body.String()) + } + item, _ = fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.CurrentStageID != domain.FanyuV5StageSemanticAnnotation || item.Status != domain.HumanItemReady { + t.Fatalf("cut review approval must progress to semantic annotation: %#v", item) + } + + claimSemantic := fixture.request(http.MethodPost, "/api/human-items/"+fixture.item.ID+"/claim", fixture.annotator, map[string]any{"version": item.Version}) + if claimSemantic.Code != http.StatusOK { + t.Fatalf("claim semantic annotation status=%d body=%s", claimSemantic.Code, claimSemantic.Body.String()) + } + semanticSubmission := fixture.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+fixture.task.ID+"/revisions", fixture.annotator, map[string]any{ + "segments": []map[string]any{{"id": "semantic-1", "labelId": "01", "label": "测试动作", "start": 0.0, "end": 20.0, "confidence": 1.0}}, + }) + if semanticSubmission.Code != http.StatusCreated { + t.Fatalf("semantic annotation status=%d body=%s", semanticSubmission.Code, semanticSubmission.Body.String()) + } + item, _ = fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.CurrentStageID != domain.FanyuV5StageSemanticReview || item.Status != domain.HumanItemSubmitted { + t.Fatalf("semantic submission must progress to semantic review: %#v", item) + } + + claimSemanticReview := fixture.request(http.MethodPost, "/api/human-items/"+fixture.item.ID+"/claim", fixture.quality, map[string]any{"version": item.Version}) + if claimSemanticReview.Code != http.StatusOK { + t.Fatalf("claim semantic review status=%d body=%s", claimSemanticReview.Code, claimSemanticReview.Body.String()) + } + approveSemanticReview := fixture.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+fixture.task.ID+"/reviews", fixture.quality, map[string]any{ + "stage": "quality", "decision": "approved", "comment": "标注审核通过", + }) + if approveSemanticReview.Code != http.StatusCreated { + t.Fatalf("approve semantic review status=%d body=%s", approveSemanticReview.Code, approveSemanticReview.Body.String()) + } + item, _ = fixture.store.GetHumanItem(fixture.ctx, fixture.item.ID) + if item == nil || item.Status != domain.HumanItemAccepted { + t.Fatalf("semantic review approval must accept the source item: %#v", item) + } + aggregate, err := fixture.store.GetHumanItem(fixture.ctx, fanyuV5PrePackageAcceptanceItemID(fixture.batch.ID)) + if err != nil || aggregate == nil || aggregate.CurrentStageID != fanyuV5PrePackageAcceptanceStageID || aggregate.Status != domain.HumanItemReady { + t.Fatalf("semantic acceptance must create the aggregate confirm-only item: aggregate=%#v err=%v", aggregate, err) + } + aggregateBatch, err := fixture.store.GetHumanBatch(fixture.ctx, aggregate.BatchID) + if err != nil || aggregateBatch == nil || len(aggregateBatch.Stages) != 1 || !aggregateBatch.Stages[0].ConfirmOnly || aggregateBatch.Stages[0].RequiredRole != "acceptance_reviewer" { + t.Fatalf("aggregate acceptance must remain confirm-only for acceptance_reviewer: batch=%#v err=%v", aggregateBatch, err) + } +} diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index 7008f2f3..4d1ba88d 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -277,6 +277,10 @@ func humanStage(batch *domain.HumanBatchRecord, stageID string) (domain.HumanSta return domain.HumanStageDefinition{}, -1, false } +func isFanyuV5FourStageBatch(batch *domain.HumanBatchRecord) bool { + return batch != nil && domain.IsFanyuV5FourStageHumanLoop(batch.Stages) +} + func annotationRoleForHumanStage(stage domain.HumanStageDefinition) domain.AnnotationAssignmentRole { switch stage.RequiredRole { case "quality_reviewer": @@ -382,7 +386,8 @@ func (server *server) syncLegacyTaskFromHumanItem(ctx context.Context, item doma task.Status = domain.AnnotationTaskStatusRejected case item.Status == domain.HumanItemRework: task.Status = domain.AnnotationTaskStatusRework - case item.CurrentStageID == "split_quality_review" || item.CurrentStageID == "quality_review": + case item.CurrentStageID == "split_quality_review" || item.CurrentStageID == "quality_review" || + item.CurrentStageID == domain.FanyuV5StageCutReview || item.CurrentStageID == domain.FanyuV5StageSemanticReview: task.Status = domain.AnnotationTaskStatusPendingQuality case item.CurrentStageID == "acceptance": task.Status = domain.AnnotationTaskStatusPendingAcceptance @@ -456,10 +461,18 @@ func (server *server) syncHumanItemFromLegacyRevision(ctx context.Context, task if err != nil || item == nil { return err } + semanticReviewStageID := "quality_review" + batch, batchErr := server.store.GetHumanBatch(ctx, item.BatchID) + if batchErr != nil { + return batchErr + } + if isFanyuV5FourStageBatch(batch) { + semanticReviewStageID = domain.FanyuV5StageSemanticReview + } previousVersion := item.Version item.CurrentRevision = revision.Revision item.LatestAuthorUserID = revision.AuthorUserID - item.CurrentStageID = "quality_review" + item.CurrentStageID = semanticReviewStageID item.Status = domain.HumanItemSubmitted item.SubmittedAt = nowISO() item.AssignedUserID = "" @@ -470,11 +483,11 @@ func (server *server) syncHumanItemFromLegacyRevision(ctx context.Context, task return err } _ = server.store.UpsertHumanStageAttempt(ctx, domain.HumanStageAttemptRecord{ - OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: "semantic_annotation", Round: item.ReviewRound, + OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: domain.FanyuV5StageSemanticAnnotation, Round: item.ReviewRound, Status: domain.HumanItemSubmitted, AssigneeUserID: revision.AuthorUserID, SubmissionRef: revision.ID, }) return server.appendHumanEvent(ctx, *item, "submitted", revision.AuthorUserID, map[string]any{ - "submissionRef": revision.ID, "revision": revision.Revision, "stage": "semantic_annotation", + "submissionRef": revision.ID, "revision": revision.Revision, "stage": domain.FanyuV5StageSemanticAnnotation, }) } @@ -838,6 +851,8 @@ func (server *server) handleHumanItemDecision(writer http.ResponseWriter, reques item.Subject.Payload["statistics"] = statistics } eventType := string(payload.Decision) + automaticReworkTarget := "" + rejectAllowedStages := []string(nil) if payload.Decision == domain.HumanDecisionApproved { if stageIndex+1 >= len(batch.Stages) { item.Status = domain.HumanItemAccepted @@ -855,15 +870,39 @@ func (server *server) handleHumanItemDecision(writer http.ResponseWriter, reques ) return } - allowedReopenStages := allowedHumanReopenStages(batch, previousStage) - for _, target := range payload.RejectTargets { - if !containsHumanStage(allowedReopenStages, target) { - renderValidationError(writer, validationIssue{Field: "rejectTargets", Message: "rejectTargets must contain only current or earlier reopen stages"}) + rejectAllowedStages = allowedHumanReopenStages(batch, previousStage) + if isFanyuV5FourStageBatch(batch) { + if len(stage.RejectTargets) != 1 { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "v5 review stage has no single configured rework target", "code": "human_item_stage_contract"}) return } + automaticReworkTarget = stage.RejectTargets[0] + if len(payload.RejectTargets) == 0 { + payload.RejectTargets = []string{automaticReworkTarget} + } + if len(payload.RejectTargets) != 1 || payload.RejectTargets[0] != automaticReworkTarget { + renderValidationError(writer, validationIssue{Field: "rejectTargets", Message: "v5 review can return only to its configured immediate edit stage"}) + return + } + rejectAllowedStages = []string{automaticReworkTarget} + item.CurrentStageID = automaticReworkTarget + item.ReviewRound++ + if item.ReviewRound > item.MaxReworkRounds { + item.Status = domain.HumanItemEscalated + batch.Status = domain.HumanBatchBlocked + } else { + item.Status = domain.HumanItemRework + } + } else { + for _, target := range payload.RejectTargets { + if !containsHumanStage(rejectAllowedStages, target) { + renderValidationError(writer, validationIssue{Field: "rejectTargets", Message: "rejectTargets must contain only current or earlier reopen stages"}) + return + } + } + item.Status = domain.HumanItemRejected + item.RejectedAt = nowISO() } - item.Status = domain.HumanItemRejected - item.RejectedAt = nowISO() } item.AssignedUserID = "" item.ClaimedAt = "" @@ -884,20 +923,32 @@ func (server *server) handleHumanItemDecision(writer http.ResponseWriter, reques Decision: payload.Decision, RejectTargets: payload.RejectTargets, ReasonCodes: payload.ReasonCodes, Comment: strings.TrimSpace(payload.Comment), }) if payload.Decision == domain.HumanDecisionRejected { - allowedReopenStages := allowedHumanReopenStages(batch, previousStage) - if err := server.store.UpsertHumanReject(request.Context(), domain.HumanRejectRecord{ + reject := domain.HumanRejectRecord{ OrganizationID: item.OrganizationID, BatchID: item.BatchID, ItemID: item.ID, ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.RunID, StageID: previousStage, AttemptRound: item.ReviewRound, ReviewedRevision: payload.ReviewedRevision, RejectedByUserID: principal.User.ID, ReasonCodes: payload.ReasonCodes, - Comment: strings.TrimSpace(payload.Comment), AllowedReopenStages: allowedReopenStages, Status: domain.HumanRejectOpen, - }); err != nil { + Comment: strings.TrimSpace(payload.Comment), AllowedReopenStages: rejectAllowedStages, Status: domain.HumanRejectOpen, + } + if automaticReworkTarget != "" { + reject.ReopenCount = 1 + reject.ReopenedByUserID = principal.User.ID + reject.ReopenTargetStageID = automaticReworkTarget + reject.ReopenComment = "automatic v5 rework routing" + reject.ReopenedAt = nowISO() + if item.Status == domain.HumanItemEscalated { + reject.Status = domain.HumanRejectEscalated + } else { + reject.Status = domain.HumanRejectReopened + } + } + if err := server.store.UpsertHumanReject(request.Context(), reject); err != nil { renderError(writer, http.StatusInternalServerError, err) return } } _ = server.appendHumanEvent(request.Context(), *item, eventType, principal.User.ID, map[string]any{ "decision": payload.Decision, "reviewedRevision": payload.ReviewedRevision, "rejectTargets": payload.RejectTargets, - "reasonCodes": payload.ReasonCodes, "comment": strings.TrimSpace(payload.Comment), "videoQuality": payload.VideoQuality, "annotationAccuracy": payload.AnnotationAccuracy, "annotationCompleteness": payload.AnnotationCompleteness, "overallGrade": payload.OverallGrade, + "reasonCodes": payload.ReasonCodes, "comment": strings.TrimSpace(payload.Comment), "routedToStage": automaticReworkTarget, "videoQuality": payload.VideoQuality, "annotationAccuracy": payload.AnnotationAccuracy, "annotationCompleteness": payload.AnnotationCompleteness, "overallGrade": payload.OverallGrade, }) if err := server.syncLegacyTaskFromHumanItem(request.Context(), *item); err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -1237,9 +1288,9 @@ func acceptanceArtifactRefs(value any) []map[string]any { } // ensureFanyuV5PrePackageAcceptance creates exactly one vendor-scoped, -// partition-level acceptance item after every source item has passed combined -// QA. It deliberately aggregates all slice/annotation/prefix pose evidence; -// no per-clip acceptance stage is introduced. +// partition-level acceptance item after every source item has passed the cut +// and semantic-review loop. Pose stages are intentionally out of scope for +// this workflow; aggregate acceptance covers only final cuts and annotations. func (server *server) ensureFanyuV5PrePackageAcceptance(ctx context.Context, source *domain.HumanBatchRecord) (bool, error) { if source == nil || !source.RequirePrePackageAcceptance { return true, nil @@ -1281,9 +1332,12 @@ func (server *server) ensureFanyuV5PrePackageAcceptance(ctx context.Context, sou return acceptanceItem.Status == domain.HumanItemAccepted, nil } aggregate := make([]map[string]any, 0, len(items)) - statistics := map[string]any{"taskCount": len(items), "sliceCount": 0, "annotationSegmentCount": 0, "headposeFrameCount": 0, "handposeFrameCount": 0, "headposeAvailable": false, "handposeAvailable": false} - seenHeadposeArtifacts := map[string]struct{}{} - seenHandposeArtifacts := map[string]struct{}{} + statistics := map[string]any{ + "taskCount": len(items), "sliceCount": 0, "annotationSegmentCount": 0, + "headposeEnabled": false, "handposeEnabled": false, + "headposeFrameCount": 0, "handposeFrameCount": 0, + "headposeAvailable": false, "handposeAvailable": false, + } for _, humanItem := range items { if strings.TrimSpace(humanItem.LegacyTaskID) == "" { return false, fmt.Errorf("fanyu v5 source item %s has no annotation task", humanItem.ID) @@ -1310,50 +1364,7 @@ func (server *server) ensureFanyuV5PrePackageAcceptance(ctx context.Context, sou } statistics["sliceCount"] = statistics["sliceCount"].(int) + len(latestSlices.Segments) statistics["annotationSegmentCount"] = statistics["annotationSegmentCount"].(int) + len(latestRevision.Segments) - poseSummary := map[string]any{} - if humanItem.Subject.Payload != nil { - if value, ok := humanItem.Subject.Payload["poseSummary"].(map[string]any); ok { - poseSummary = value - } - } - headposeRefs := acceptanceArtifactRefs(poseSummary["headposeArtifacts"]) - invalidHeadpose, _ := poseSummary["headposeInvalid"].(bool) - if available, _ := poseSummary["headposeAvailable"].(bool); !available || invalidHeadpose || acceptanceEvidenceCount(poseSummary["headposeFrameCount"]) <= 0 || len(headposeRefs) == 0 { - return false, fmt.Errorf("fanyu v5 pre-package acceptance task %s is missing required Headpose evidence", task.ID) - } - newHeadposeEvidence := false - for _, raw := range headposeRefs { - artifactID := strings.TrimSpace(fmt.Sprint(raw["artifactId"])) - if artifactID != "" { - if _, seen := seenHeadposeArtifacts[artifactID]; !seen { - seenHeadposeArtifacts[artifactID] = struct{}{} - newHeadposeEvidence = true - } - } - } - if newHeadposeEvidence { - statistics["headposeFrameCount"] = statistics["headposeFrameCount"].(int) + acceptanceEvidenceCount(poseSummary["headposeFrameCount"]) - statistics["headposeAvailable"] = true - } - newHandposeEvidence := false - handposeRefs := acceptanceArtifactRefs(poseSummary["handposeArtifacts"]) - for _, raw := range handposeRefs { - artifactID := strings.TrimSpace(fmt.Sprint(raw["artifactId"])) - if artifactID != "" { - if _, seen := seenHandposeArtifacts[artifactID]; !seen { - seenHandposeArtifacts[artifactID] = struct{}{} - newHandposeEvidence = true - } - } - } - if newHandposeEvidence { - statistics["handposeFrameCount"] = statistics["handposeFrameCount"].(int) + acceptanceEvidenceCount(poseSummary["handposeFrameCount"]) - statistics["handposeAvailable"] = true - } - aggregate = append(aggregate, map[string]any{"taskId": task.ID, "runId": task.RunID, "clipArtifactId": task.ArtifactID, "sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "finalSlices": latestSlices.Segments, "annotation": latestRevision.Segments, "poseSummary": poseSummary}) - } - if headposeAvailable, _ := statistics["headposeAvailable"].(bool); !headposeAvailable { - return false, fmt.Errorf("fanyu v5 pre-package acceptance requires real Headpose evidence for every accepted partition") + aggregate = append(aggregate, map[string]any{"taskId": task.ID, "runId": task.RunID, "clipArtifactId": task.ArtifactID, "sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "finalSlices": latestSlices.Segments, "annotation": latestRevision.Segments}) } now := nowISO() acceptanceItem = &domain.HumanItemRecord{ diff --git a/internal/api/human_loop_test.go b/internal/api/human_loop_test.go index 561705c6..88879151 100644 --- a/internal/api/human_loop_test.go +++ b/internal/api/human_loop_test.go @@ -38,7 +38,7 @@ func TestFanyuV5PrePackageAcceptanceAggregatesMultipleTasksBeforeRelease(t *test if err := dataStore.UpsertAnnotationRevision(ctx, domain.AnnotationRevisionRecord{ID: "annotation-v5-" + string(rune('0'+index)), OrganizationID: domain.DefaultOrganizationID, TaskID: task.ID, Revision: 1, TemplateVersion: 5, Segments: []domain.AnnotationSegment{{ID: "label", LabelID: "01", Label: "鲜食处理", Start: 0, End: 10, Confidence: 1}}, AuthorUserID: "annotator"}); err != nil { t.Fatal(err) } - item := domain.HumanItemRecord{ID: "item-v5-" + string(rune('0'+index)), OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", BatchID: batch.ID, ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, GateNodeID: batch.GateNodeID, LegacyTaskID: task.ID, WorkgroupID: "wg-v5", VendorID: "vendor-v5", Status: domain.HumanItemAccepted, AcceptedAt: nowISO(), Version: 1, Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID, Payload: map[string]any{"poseSummary": map[string]any{"headposeAvailable": true, "headposeFrameCount": 20, "headposeArtifacts": []map[string]any{{"artifactId": "headpose-artifact", "sha256": "abc"}}, "handposeAvailable": false, "handposeArtifacts": []map[string]any{}}}}} + item := domain.HumanItemRecord{ID: "item-v5-" + string(rune('0'+index)), OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", BatchID: batch.ID, ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, GateNodeID: batch.GateNodeID, LegacyTaskID: task.ID, WorkgroupID: "wg-v5", VendorID: "vendor-v5", Status: domain.HumanItemAccepted, AcceptedAt: nowISO(), Version: 1, Subject: domain.HumanSubjectRef{ItemKey: task.ArtifactID}} if err := dataStore.UpsertHumanItem(ctx, item); err != nil { t.Fatal(err) } @@ -65,6 +65,10 @@ func TestFanyuV5PrePackageAcceptanceAggregatesMultipleTasksBeforeRelease(t *test if len(items) != 2 { t.Fatalf("aggregate should contain both clip tasks, got %#v", aggregateItem.Subject.Payload) } + statistics, _ := aggregateItem.Subject.Payload["statistics"].(map[string]any) + if statistics["headposeEnabled"] != false || statistics["handposeEnabled"] != false || acceptanceEvidenceCount(statistics["headposeFrameCount"]) != 0 || acceptanceEvidenceCount(statistics["handposeFrameCount"]) != 0 { + t.Fatalf("pose-free aggregate statistics=%#v", statistics) + } aggregateItem.Status = domain.HumanItemAccepted aggregateItem.AcceptedAt = nowISO() aggregateItem.AcceptedByUserID = "acceptance-reviewer" diff --git a/internal/api/plugins.go b/internal/api/plugins.go index 71350a36..5546e9ef 100644 --- a/internal/api/plugins.go +++ b/internal/api/plugins.go @@ -203,7 +203,7 @@ func (server *server) migrateLegacyAnnotationHumanItems(ctx context.Context, org BatchID: batch.ID, ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, RunID: task.RunID, GateNodeID: task.NodeID, LegacyTaskID: task.ID, Subject: domain.HumanSubjectRef{ArtifactID: task.ArtifactID, ItemKey: task.ID, MediaType: "video/mp4", Payload: map[string]any{ - "sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "templateId": task.TemplateID, + "sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "sourceIsFullVideo": task.SourceIsFullVideo, "sourceDurationSeconds": task.SourceDurationSeconds, "templateId": task.TemplateID, }}, CurrentStageID: stageID, Status: status, AssignedUserID: task.AssignedUserID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, CurrentRevision: task.CurrentRevision, MaxReworkRounds: batch.MaxReworkRounds, diff --git a/internal/api/server.go b/internal/api/server.go index 27ccc905..8ba0c2c1 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1919,7 +1919,7 @@ func (server *server) enrichDashboardHumanWork(ctx context.Context, request *htt } switch item.Status { case domain.HumanItemReady, domain.HumanItemRework: - if item.CurrentStageID == "quality_review" { + if item.CurrentStageID == "quality_review" || item.CurrentStageID == domain.FanyuV5StageCutReview || item.CurrentStageID == domain.FanyuV5StageSemanticReview { summary.Tasks.PendingQuality++ } else { summary.Tasks.PendingAnnotation++ @@ -1927,7 +1927,7 @@ func (server *server) enrichDashboardHumanWork(ctx context.Context, request *htt case domain.HumanItemClaimed, domain.HumanItemInProgress: summary.Tasks.InAnnotation++ case domain.HumanItemSubmitted: - if item.CurrentStageID == "quality_review" { + if item.CurrentStageID == "quality_review" || item.CurrentStageID == domain.FanyuV5StageCutReview || item.CurrentStageID == domain.FanyuV5StageSemanticReview { summary.Tasks.PendingQuality++ } else if item.CurrentStageID == "acceptance" { summary.Tasks.PendingAcceptance++ diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 0034f007..863504c2 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -2215,7 +2215,7 @@ func TestOperatorsListEnsuresRegressionBuiltinCatalog(t *testing.T) { for _, name := range []string{ "真实数据源", "BOS 增量扫描", - "目录扫描", + "目录读取", "模型推理", "X 平台数据集索引", "采集目录校验", diff --git a/internal/domain/annotation_operations.go b/internal/domain/annotation_operations.go index ca2992b0..f3adfd63 100644 --- a/internal/domain/annotation_operations.go +++ b/internal/domain/annotation_operations.go @@ -141,24 +141,26 @@ type SliceRevisionRecord struct { type SemanticAnnotationRevisionRecord = AnnotationRevisionRecord type AnnotationTaskRecord struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - ProjectID string `json:"projectId" bson:"projectId"` - DataTaskID string `json:"dataTaskId,omitempty" bson:"dataTaskId,omitempty"` - WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"` - RunID string `json:"runId,omitempty" bson:"runId,omitempty"` - NodeID string `json:"nodeId,omitempty" bson:"nodeId,omitempty"` - ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` - SourceVideo string `json:"sourceVideo" bson:"sourceVideo"` - ClipStart float64 `json:"clipStart" bson:"clipStart"` - ClipEnd float64 `json:"clipEnd" bson:"clipEnd"` - TemplateID string `json:"templateId" bson:"templateId"` - TemplateVersion int `json:"templateVersion" bson:"templateVersion"` - WorkgroupID string `json:"workgroupId" bson:"workgroupId"` - VendorID string `json:"vendorId" bson:"vendorId"` - AssignedUserID string `json:"assignedUserId,omitempty" bson:"assignedUserId,omitempty"` - Status AnnotationTaskStatus `json:"status" bson:"status"` - CurrentRevision int `json:"currentRevision" bson:"currentRevision"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + ProjectID string `json:"projectId" bson:"projectId"` + DataTaskID string `json:"dataTaskId,omitempty" bson:"dataTaskId,omitempty"` + WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"` + RunID string `json:"runId,omitempty" bson:"runId,omitempty"` + NodeID string `json:"nodeId,omitempty" bson:"nodeId,omitempty"` + ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` + SourceVideo string `json:"sourceVideo" bson:"sourceVideo"` + ClipStart float64 `json:"clipStart" bson:"clipStart"` + ClipEnd float64 `json:"clipEnd" bson:"clipEnd"` + SourceIsFullVideo bool `json:"sourceIsFullVideo,omitempty" bson:"sourceIsFullVideo,omitempty"` + SourceDurationSeconds float64 `json:"sourceDurationSeconds,omitempty" bson:"sourceDurationSeconds,omitempty"` + TemplateID string `json:"templateId" bson:"templateId"` + TemplateVersion int `json:"templateVersion" bson:"templateVersion"` + WorkgroupID string `json:"workgroupId" bson:"workgroupId"` + VendorID string `json:"vendorId" bson:"vendorId"` + AssignedUserID string `json:"assignedUserId,omitempty" bson:"assignedUserId,omitempty"` + Status AnnotationTaskStatus `json:"status" bson:"status"` + CurrentRevision int `json:"currentRevision" bson:"currentRevision"` // PoseSummary is evidence copied from real upstream Headpose/Handpose // manifests for the v5 aggregate acceptance workbench. It contains only // scoped references/counts, never whole untrimmed pose data. diff --git a/internal/domain/human_loop.go b/internal/domain/human_loop.go index 8e522c38..8faf86be 100644 --- a/internal/domain/human_loop.go +++ b/internal/domain/human_loop.go @@ -44,6 +44,17 @@ const ( HumanDecisionRejected HumanDecision = "rejected" ) +// Fanyu v5 keeps the two human editing concerns independently auditable. The +// names are part of the persisted HumanItem contract, so callers must not +// collapse them back into the historical split_annotation/quality_review +// aliases. +const ( + FanyuV5StageCutAdjustment = "cut_adjustment" + FanyuV5StageCutReview = "cut_review" + FanyuV5StageSemanticAnnotation = "semantic_annotation" + FanyuV5StageSemanticReview = "semantic_review" +) + type HumanStageDefinition struct { ID string `json:"id" bson:"id"` Label string `json:"label" bson:"label"` @@ -254,15 +265,33 @@ func DefaultAnnotationHumanStages() []HumanStageDefinition { } } -// DefaultFanyuV5HumanStages is intentionally separate from the historical -// four-stage pipeline. The first task lets one annotator correct final clip -// boundaries and semantic labels in either editing order; quality review is a -// single rework-capable gate. The final acceptance stage is confirm-only and -// releases the downstream package node only after an acceptance reviewer has -// claimed and confirmed the item. +// DefaultFanyuV5HumanStages describes the supplier task-pool loop used after +// automatic rough cutting. A cut reviewer can return only to cut adjustment; +// a semantic reviewer can return only to semantic annotation. The separate +// aggregate pre-package acceptance item remains confirm-only and is created +// only after every item reaches the end of this loop. func DefaultFanyuV5HumanStages() []HumanStageDefinition { return []HumanStageDefinition{ - {ID: "annotation_combined", Label: "切片校正与22类语义标注", Kind: HumanStageAnnotation, RequiredRole: "annotator", EditorSchema: "temporal-semantic-annotation/v2"}, - {ID: "quality_review", Label: "切片与语义统一质检", Kind: HumanStageReview, RequiredRole: "quality_reviewer", RejectTargets: []string{"annotation_combined"}}, + {ID: FanyuV5StageCutAdjustment, Label: "人工剪切调整", Kind: HumanStageAnnotation, RequiredRole: "annotator", EditorSchema: "temporal-segments/v1"}, + {ID: FanyuV5StageCutReview, Label: "剪切审核", Kind: HumanStageReview, RequiredRole: "quality_reviewer", RejectTargets: []string{FanyuV5StageCutAdjustment}}, + {ID: FanyuV5StageSemanticAnnotation, Label: "语义标注", Kind: HumanStageAnnotation, RequiredRole: "annotator", EditorSchema: "temporal-semantic-annotation/v2"}, + {ID: FanyuV5StageSemanticReview, Label: "标注审核", Kind: HumanStageReview, RequiredRole: "quality_reviewer", RejectTargets: []string{FanyuV5StageSemanticAnnotation}}, } } + +// IsFanyuV5FourStageHumanLoop is deliberately strict: historical v5 combined +// batches stay readable through their legacy adapter while newly created v5 +// work uses the independent cut and semantic review loop. +func IsFanyuV5FourStageHumanLoop(stages []HumanStageDefinition) bool { + if len(stages) != 4 { + return false + } + return stages[0].ID == FanyuV5StageCutAdjustment && + stages[0].Kind == HumanStageAnnotation && + stages[1].ID == FanyuV5StageCutReview && + stages[1].Kind == HumanStageReview && + stages[2].ID == FanyuV5StageSemanticAnnotation && + stages[2].Kind == HumanStageAnnotation && + stages[3].ID == FanyuV5StageSemanticReview && + stages[3].Kind == HumanStageReview +} diff --git a/internal/worker/annotation_gate.go b/internal/worker/annotation_gate.go index cf1e5bfe..d79a572a 100644 --- a/internal/worker/annotation_gate.go +++ b/internal/worker/annotation_gate.go @@ -391,9 +391,6 @@ func (runtime *Runtime) createAnnotationGateTasks( clips := annotationClipItems(inputs) prelabels := annotationPrelabels(inputs, *template) poseSummary := annotationPoseSummary(inputs) - if config.RequirePrePackageAcceptance && !annotationPoseSummaryHasRequiredHeadpose(poseSummary) { - return nil, fmt.Errorf("fanyu v5 annotation gate requires a non-empty verified Headpose manifest before human work is created") - } tasks := make([]domain.AnnotationTaskRecord, 0) for _, artifact := range inputs { if artifact.Type != domain.PortTypeVideo { @@ -412,17 +409,25 @@ func (runtime *Runtime) createAnnotationGateTasks( return nil, fmt.Errorf("annotation gate cannot resolve clip duration for %s", artifact.Name) } task := domain.AnnotationTaskRecord{ - ID: domain.NewID("annotation-task"), - OrganizationID: run.OrganizationID, - ProjectID: run.ProjectID, - DataTaskID: run.TaskID, - WorkflowID: run.WorkflowID, - RunID: run.ID, - NodeID: node.ID, - ArtifactID: artifact.ID, - SourceVideo: fmt.Sprintf("/api/runs/%s/artifacts/%s/content", url.PathEscape(run.ID), url.PathEscape(artifact.ID)), - ClipStart: clipStart, - ClipEnd: clipEnd, + ID: domain.NewID("annotation-task"), + OrganizationID: run.OrganizationID, + ProjectID: run.ProjectID, + DataTaskID: run.TaskID, + WorkflowID: run.WorkflowID, + RunID: run.ID, + NodeID: node.ID, + ArtifactID: artifact.ID, + SourceVideo: fmt.Sprintf("/api/runs/%s/artifacts/%s/content", url.PathEscape(run.ID), url.PathEscape(artifact.ID)), + ClipStart: clipStart, + ClipEnd: clipEnd, + SourceIsFullVideo: func() bool { + value, _ := clipItem["sourceIsFullVideo"].(bool) + return value + }(), + SourceDurationSeconds: func() float64 { + value, _ := annotationNumber(clipItem["sourceDurationSeconds"]) + return value + }(), TemplateID: template.ID, TemplateVersion: template.Version, WorkgroupID: group.ID, @@ -568,11 +573,13 @@ func (runtime *Runtime) ensureAnnotationHumanBatch( ItemKey: itemKey, MediaType: "video/mp4", Payload: map[string]any{ - "sourceVideo": task.SourceVideo, - "clipStart": task.ClipStart, - "clipEnd": task.ClipEnd, - "templateId": task.TemplateID, - "poseSummary": task.PoseSummary, + "sourceVideo": task.SourceVideo, + "clipStart": task.ClipStart, + "clipEnd": task.ClipEnd, + "sourceIsFullVideo": task.SourceIsFullVideo, + "sourceDurationSeconds": task.SourceDurationSeconds, + "templateId": task.TemplateID, + "poseSummary": task.PoseSummary, }, }, CurrentStageID: initialStageID, @@ -677,9 +684,16 @@ func annotationArtifactSize(size int64) string { // final MP4 is cut from the already-sliced clip (clip-relative seconds), while // Headpose/Handpose manifests still use original source-video seconds. func fanyuV5FinalSliceCoordinates(task domain.AnnotationTaskRecord, finalSlice domain.SliceSegment) (clipStart, clipEnd, sourceStart, sourceEnd float64, err error) { - if task.ClipEnd <= task.ClipStart || finalSlice.End <= finalSlice.Start || finalSlice.Start < task.ClipStart || finalSlice.End > task.ClipEnd { + allowedStart, allowedEnd := task.ClipStart, task.ClipEnd + if task.SourceIsFullVideo { + allowedStart, allowedEnd = 0, task.SourceDurationSeconds + } + if allowedEnd <= allowedStart || finalSlice.End <= finalSlice.Start || finalSlice.Start < allowedStart || finalSlice.End > allowedEnd { return 0, 0, 0, 0, fmt.Errorf("final slice is outside annotation task clip range") } + if task.SourceIsFullVideo { + return finalSlice.Start, finalSlice.End, finalSlice.Start, finalSlice.End, nil + } return finalSlice.Start - task.ClipStart, finalSlice.End - task.ClipStart, finalSlice.Start, finalSlice.End, nil } @@ -777,6 +791,13 @@ func (runtime *Runtime) releasePartitionArtifacts( if coordinateErr != nil { return nil, fmt.Errorf("invalid final slice for task %s: %w", task.ID, coordinateErr) } + // Qwen-VL semantic rough-cut items are virtual clips backed by the + // complete sanitized source video. Their final cut coordinates are + // therefore already source-relative and must not be shifted back to a + // pre-cut clip timeline before the package operator materializes MP4s. + if sourceIsFullVideo, _ := base["sourceIsFullVideo"].(bool); sourceIsFullVideo { + clipStart, clipEnd = sourceStart, sourceEnd + } finalClipID := clipID if batch.RequirePrePackageAcceptance { finalClipID = fmt.Sprintf("%s_final_%03d", clipID, index+1) diff --git a/internal/worker/human_release_test.go b/internal/worker/human_release_test.go index 93c78925..a1a0d475 100644 --- a/internal/worker/human_release_test.go +++ b/internal/worker/human_release_test.go @@ -139,6 +139,17 @@ func TestFanyuV5FinalSliceCoordinatesSeparateClipAndSourceTimelines(t *testing.T } } +func TestFanyuV5FinalSliceCoordinatesAllowVirtualFullVideoExpansion(t *testing.T) { + task := domain.AnnotationTaskRecord{ClipStart: 10, ClipEnd: 12, SourceIsFullVideo: true, SourceDurationSeconds: 30} + clipStart, clipEnd, sourceStart, sourceEnd, err := fanyuV5FinalSliceCoordinates(task, domain.SliceSegment{ID: "expanded", Start: 8, End: 14}) + if err != nil || clipStart != 8 || clipEnd != 14 || sourceStart != 8 || sourceEnd != 14 { + t.Fatalf("virtual coordinates clip=[%v,%v) source=[%v,%v) err=%v", clipStart, clipEnd, sourceStart, sourceEnd, err) + } + if _, _, _, _, err := fanyuV5FinalSliceCoordinates(task, domain.SliceSegment{ID: "outside", Start: 8, End: 31}); err == nil { + t.Fatal("virtual source slice outside full duration must fail closed") + } +} + func TestCompletedDeliveryPackageCreatesUnassignedAcceptanceHumanItem(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 7bab48c8..1d088382 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -1764,7 +1764,40 @@ func stringList(value any) []string { return result } -func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode) (domain.WorkflowNode, error) { +func inputStorageConnectionID(bindings []domain.NodeInputBinding) (string, error) { + connectionID := "" + consider := func(payload *domain.ArtifactPayload) error { + if payload == nil || payload.Mode != domain.ArtifactTransferModeRef || payload.Ref == nil { + return nil + } + candidate := strings.TrimSpace(payload.Ref.ConnectionID) + if candidate == "" { + return nil + } + if connectionID != "" && connectionID != candidate { + return fmt.Errorf("input_storage_ambiguous: node inputs reference multiple storage connections") + } + connectionID = candidate + return nil + } + for index := range bindings { + if err := consider(bindings[index].Payload); err != nil { + return "", err + } + for artifactIndex := range bindings[index].Artifacts { + if err := consider(bindings[index].Artifacts[artifactIndex].Payload); err != nil { + return "", err + } + } + } + return connectionID, nil +} + +func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *domain.WorkflowRecord, node domain.WorkflowNode, bindingSets ...[]domain.NodeInputBinding) (domain.WorkflowNode, error) { + var inputBindings []domain.NodeInputBinding + if len(bindingSets) > 0 { + inputBindings = bindingSets[0] + } copy, err := runtime.resolveBuiltinNodeRuntime(ctx, node) if err != nil { return node, err @@ -1777,6 +1810,13 @@ func (runtime *Runtime) prepareExecutionNode(ctx context.Context, workflow *doma if err != nil { return node, err } + if strings.TrimSpace(copy.Data.ConnectionID) == "" { + connectionID, inputErr := inputStorageConnectionID(inputBindings) + if inputErr != nil { + return node, inputErr + } + copy.Data.ConnectionID = connectionID + } copy, err = runtime.resolveNodeStorageConnection(ctx, copy) if err != nil { return node, err @@ -2924,7 +2964,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor // Use the node attempt for the Kubernetes Job suffix so a recovered // execution cannot collide with the vanished attempt name. runSnapshot.Attempt = executionAttempt - nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, nodeToPrepare) + nodeSnapshot, err := runtime.prepareExecutionNode(ctx, workflow, nodeToPrepare, inputBindings) if err != nil { results <- nodeExecutionOutcome{ nodeID: node.ID, diff --git a/internal/worker/runtime_test.go b/internal/worker/runtime_test.go index d6413b18..938f73cc 100644 --- a/internal/worker/runtime_test.go +++ b/internal/worker/runtime_test.go @@ -3120,6 +3120,21 @@ func TestRuntimeCommitsIncrementalSourceStateOnlyOnSuccessfulRuns(t *testing.T) }) } +func TestInputStorageConnectionIDUsesDirectoryReferenceAndRejectsAmbiguity(t *testing.T) { + payload := domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeFile, Ref: &domain.ArtifactReference{ConnectionID: "storage-source", Bucket: "bucket", Prefix: "capture", Kind: domain.ArtifactRefKindDirectory}} + connectionID, err := inputStorageConnectionID([]domain.NodeInputBinding{{TargetPortID: "in-1", Payload: &payload}}) + if err != nil || connectionID != "storage-source" { + t.Fatalf("connectionID=%q err=%v", connectionID, err) + } + other := payload + refCopy := *payload.Ref + refCopy.ConnectionID = "storage-other" + other.Ref = &refCopy + if _, err := inputStorageConnectionID([]domain.NodeInputBinding{{Payload: &payload}, {Payload: &other}}); err == nil { + t.Fatal("multiple input storage connections must fail closed") + } +} + func TestPrepareExecutionNodeResolvesStorageConnectionSecrets(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() diff --git a/operator-repos/builtin/directory_scan/README.md b/operator-repos/builtin/directory_scan/README.md index 4c9fd6e4..e934e257 100644 --- a/operator-repos/builtin/directory_scan/README.md +++ b/operator-repos/builtin/directory_scan/README.md @@ -1,4 +1,4 @@ -# 目录扫描 +# 目录读取 独立 builtin 算子目录。该目录是 `operator-builtin-directory-scan` 的生产运行权威来源,包含镜像构建、运行契约、源码和验收输入定义。 diff --git a/operator-repos/builtin/directory_scan/operator.json b/operator-repos/builtin/directory_scan/operator.json index 363fe841..29f9787b 100644 --- a/operator-repos/builtin/directory_scan/operator.json +++ b/operator-repos/builtin/directory_scan/operator.json @@ -9,7 +9,7 @@ "category": "IO", "id": "operator-builtin-directory-scan", "migrationStatus": "migrated", - "name": "目录扫描", + "name": "目录读取", "runtimeHandler": "builtin_directory_scan", "runtimeSource": "operator-repos/builtin/directory_scan/src/operator.py", "type": "builtin" diff --git a/operator-repos/builtin/directory_scan/runtime_contract.json b/operator-repos/builtin/directory_scan/runtime_contract.json index b709fb64..b248c652 100644 --- a/operator-repos/builtin/directory_scan/runtime_contract.json +++ b/operator-repos/builtin/directory_scan/runtime_contract.json @@ -32,7 +32,8 @@ "requiresGpu": false }, "runtimeDependencies": [ - "filesystem" + "filesystem", + "bos-directory-ref" ], "runtimeSource": "src/operator.py", "schemaVersion": "droboticflow.operator.runtime/v1", diff --git a/operator-repos/builtin/directory_scan/src/operator.py b/operator-repos/builtin/directory_scan/src/operator.py index 7173b6d8..04f9b60c 100644 --- a/operator-repos/builtin/directory_scan/src/operator.py +++ b/operator-repos/builtin/directory_scan/src/operator.py @@ -1,11 +1,57 @@ from __future__ import annotations +import os +import pathlib from typing import Any, Callable HelperMap = dict[str, Callable[..., Any]] +def _directory_ref(inputs: list[dict[str, Any]]) -> tuple[str, str] | None: + for artifact in inputs: + payload = artifact.get("payload") if isinstance(artifact.get("payload"), dict) else {} + ref = payload.get("ref") if isinstance(payload.get("ref"), dict) else {} + if str(ref.get("kind", "")).strip().lower() != "directory": + continue + bucket = str(ref.get("bucket", "")).strip() + prefix = str(ref.get("prefix", "")).strip().strip("/") + if bucket: + return bucket, prefix + raw_path = str(artifact.get("path", "")).strip() + if raw_path.startswith("bos://"): + without_scheme = raw_path[len("bos://") :] + parsed_bucket, _, parsed_prefix = without_scheme.partition("/") + if parsed_bucket: + return parsed_bucket, parsed_prefix.strip("/") + return None + + +def _bos_entries( + bucket: str, + prefix: str, + pattern: str, + params: dict[str, str], + helpers: HelperMap, +) -> list[dict[str, Any]]: + endpoint = str(params.get("BOS_ENDPOINT") or os.getenv("BOS_ENDPOINT", "")).strip() + access_key_id = str(params.get("BOS_ACCESS_KEY_ID") or os.getenv("BOS_ACCESS_KEY_ID", "")).strip() + secret_access_key = str(params.get("BOS_SECRET_ACCESS_KEY") or os.getenv("BOS_SECRET_ACCESS_KEY", "")).strip() + if not endpoint or not access_key_id or not secret_access_key: + raise RuntimeError("目录读取需要输入目录对应的 BOS 连接凭证") + entries = helpers["list_bos_entries"]( + bucket, + prefix, + pattern, + endpoint, + access_key_id, + secret_access_key, + ) + if not entries: + raise RuntimeError(f"目录读取未在 bos://{bucket}/{prefix} 找到真实文件") + return entries + + def run( run_id: str, node_id: str, @@ -15,12 +61,56 @@ def run( helpers: HelperMap, ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) + pattern = str(params.get("PATTERN", "")).strip() + directory_ref = _directory_ref(inputs) + if directory_ref is not None: + bucket, prefix = directory_ref + entries = _bos_entries(bucket, prefix, pattern, params, helpers) + source_path = helpers["canonical_bos_path"](bucket, prefix) + payload = { + "schemaVersion": "droboticflow.directory-read/v1", + "datasetType": "directory", + "dataSource": "BOS", + "sourcePath": source_path, + "bucket": bucket, + "prefix": prefix, + "pattern": pattern, + "fileCount": len(entries), + "totalBytes": sum(int(item.get("sizeBytes", 0) or 0) for item in entries), + "entries": entries, + } + return [ + helpers["write_json_artifact"]( + run_id, + node_id, + output_dir, + "directory-manifest.json", + payload, + output=helpers["output_binding"](node_data), + ) + ] + source_path = helpers["resolve_source_path"](params, inputs) if source_path is not None: root = source_path if source_path.is_dir() else source_path.parent + entries = helpers["list_source_entries"](root, pattern) payload = { + "schemaVersion": "droboticflow.directory-read/v1", + "datasetType": "directory", + "dataSource": "FILESYSTEM", "sourcePath": str(source_path), "scanRoot": str(root), + "pattern": pattern, + "fileCount": len(entries), + "totalBytes": sum(int(item.get("sizeBytes", 0) or 0) for item in entries), + "entries": [ + { + **item, + "storagePath": str(root / pathlib.PurePosixPath(str(item.get("path", "")))), + "source": {"path": str(root / pathlib.PurePosixPath(str(item.get("path", ""))))}, + } + for item in entries + ], "summary": helpers["summarize_directory_tree"](root), } elif inputs: @@ -30,13 +120,13 @@ def run( "summary": helpers["summarize_input_artifacts"](inputs), } else: - raise RuntimeError("目录扫描需要 SOURCE_PATH 或上游 artifact 指向真实目录") + raise RuntimeError("目录读取需要必填目录输入或 SOURCE_PATH 指向真实目录") return [ helpers["write_json_artifact"]( run_id, node_id, output_dir, - "directory-scan.json", + "directory-manifest.json", payload, output=helpers["output_binding"](node_data), ) diff --git a/operator-repos/builtin/package/runtime_contract.json b/operator-repos/builtin/package/runtime_contract.json index 9becd340..394659fc 100644 --- a/operator-repos/builtin/package/runtime_contract.json +++ b/operator-repos/builtin/package/runtime_contract.json @@ -36,8 +36,7 @@ "video-files", "accepted-annotation-partitions", "headpose-manifests", - "optional-handpose-manifests", - "qwen-training-provenance" + "optional-handpose-manifests" ], "runtimeSource": "src/operator.py", "schemaVersion": "droboticflow.operator.runtime/v1", diff --git a/operator-repos/builtin/package/src/operator.py b/operator-repos/builtin/package/src/operator.py index a4c3ebe0..442aabd1 100644 --- a/operator-repos/builtin/package/src/operator.py +++ b/operator-repos/builtin/package/src/operator.py @@ -124,7 +124,14 @@ def _materialize_final_mp4(item: dict[str, Any], package_dir: pathlib.Path, help destination = package_dir / name source_path = pathlib.Path(str(item.get("clipPath", "")).strip()) if not source_path.is_file() or source_path.stat().st_size <= 0: - raise RuntimeError(f"final segment source clip is missing or empty: {source_path}") + source = item.get("source") if isinstance(item.get("source"), dict) else {} + source_path = helpers["source_path_from_item"]( + {"source": source or {"path": str(item.get("clipPath", "")).strip()}}, + str(item.get("sourceVideo") or pathlib.PurePosixPath(str(item.get("clipPath", ""))).name or "source.mp4"), + package_dir, + ) + if not source_path.is_file() or source_path.stat().st_size <= 0: + raise RuntimeError(f"final segment source video is missing or empty: {source_path}") start = float(item.get("startSecond", 0.0)) end = float(item.get("endSecond", 0.0)) if not (0 <= start < end): @@ -153,11 +160,8 @@ def _run_fanyu_v5(run_id: str, node_id: str, output_dir: Any, node_data: dict[st payloads = _payload_tree(helpers["payloads_from_inputs"](inputs)) headpose = _first_payload(payloads, "droboticflow.headpose/v1") handpose = _first_payload(payloads, "droboticflow.wilor.hand_pose_result/v1") - training = _first_payload(payloads, "droboticflow.qwen3vl.continuous-training/v2") if require_headpose and (headpose is None or not headpose.get("enabled") or not isinstance(headpose.get("frames"), list)): raise RuntimeError("package requires a real Headpose manifest with frames") - if require_final_acceptance and (training is None or training.get("status") != "succeeded" or not isinstance(training.get("adapter"), dict) or not isinstance(training.get("provenance"), dict)): - raise RuntimeError("v5 package requires successful real Qwen3-VL training evidence before packaging") accepted = next((payload for payload in payloads if isinstance(payload.get("items"), list) and ("annotationCount" in payload or payload.get("schemaVersion") == "droboticflow.annotation.accepted-partition/v1")), None) if accepted is None: raise RuntimeError("package requires an accepted annotation partition") @@ -201,7 +205,7 @@ def _run_fanyu_v5(run_id: str, node_id: str, output_dir: Any, node_data: dict[st package_items.append({"packageName": package_dir.name, "clipId": clip_id, "clipName": video_path.name, "storagePath": str(package_dir), "startSecond": start, "endSecond": end, "duration": end - start, "fileCount": len([path for path in package_dir.iterdir() if path.is_file()]), "quality": quality, "annotation": annotation, "segmentManifest": segment_manifest}) if not package_items: raise RuntimeError("package produced zero final segment packages") - manifest = {"schemaVersion": "droboticflow.delivery.manifest/v2", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "items": package_items, "trainingProvenance": training.get("provenance") if training else None} + manifest = {"schemaVersion": "droboticflow.delivery.manifest/v2", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "items": package_items, "trainingEnabled": False} artifacts = [helpers["write_json_artifact"](run_id, node_id, output, "packages-manifest.json", manifest, output=binding)] artifacts.extend(helpers["collect_output_artifacts"](run_id, node_id, output / "packages", node_data=node_data, output=binding)) return artifacts diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/Dockerfile b/operator-repos/builtin/qwen_behavior_rough_cut/Dockerfile new file mode 100644 index 00000000..a9129c0d --- /dev/null +++ b/operator-repos/builtin/qwen_behavior_rough_cut/Dockerfile @@ -0,0 +1,23 @@ +# Generated by scripts/generate_operator_images.py. Do not edit by hand. +ARG OPERATOR_GPU_BASE_IMAGE=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-gpu-base:builtin +FROM ${OPERATOR_GPU_BASE_IMAGE} + +WORKDIR /app +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app \ + DROBOTICFLOW_OPERATOR_ID=operator-builtin-qwen-behavior-rough-cut \ + DROBOTICFLOW_OPERATOR_SLUG=qwen_behavior_rough_cut \ + DROBOTICFLOW_OPERATOR_SOURCE=/app/operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py \ + DROBOTICFLOW_OPERATOR_REPOSITORY=/app/operator-repos/builtin/qwen_behavior_rough_cut \ + DROBOTICFLOW_OPERATOR_HANDLER=builtin_qwen_behavior_rough_cut \ + USER=droboticflow \ + LOGNAME=droboticflow \ + HOME=/tmp/droboticflow-home + +RUN mkdir -p /tmp/droboticflow-home /workspace /outputs +COPY backend ./backend +RUN python -m pip install --no-cache-dir \ + transformers==4.57.6 accelerate==1.14.0 safetensors==0.8.0 peft==0.19.0 \ + ms-swift==4.5.2 qwen-vl-utils==0.0.14 av==16.0.1 decord==0.6.0 opencv-python-headless==4.11.0.86 +COPY operator-repos/builtin/qwen_behavior_rough_cut ./operator-repos/builtin/qwen_behavior_rough_cut +ENTRYPOINT ["python", "/app/backend/operator-runner/main.py"] diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/README.md b/operator-repos/builtin/qwen_behavior_rough_cut/README.md new file mode 100644 index 00000000..4c92a4a5 --- /dev/null +++ b/operator-repos/builtin/qwen_behavior_rough_cut/README.md @@ -0,0 +1,10 @@ +# Qwen3-VL 语义行为粗切(PTS) + +独立 builtin 算子目录。该目录是 `operator-builtin-qwen-behavior-rough-cut` 的生产运行权威来源,包含镜像构建、运行契约、源码和验收输入定义。 + +- 发布镜像:以 `release.json` 中的 digest-pinned CCR imageId 为准 +- Repository manifest:`repository.json` +- Runtime handler:`builtin_qwen_behavior_rough_cut` +- Source:`src/operator.py` + +生产验收要求真实 fixture、artifact checksum、lineage 证据;不接受 mock、仿真、smoke 或空文件验收。 diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/operator.json b/operator-repos/builtin/qwen_behavior_rough_cut/operator.json new file mode 100644 index 00000000..8bac7efe --- /dev/null +++ b/operator-repos/builtin/qwen_behavior_rough_cut/operator.json @@ -0,0 +1,16 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "id": "operator-builtin-qwen-behavior-rough-cut", + "migrationStatus": "migrated", + "name": "Qwen3-VL 语义行为粗切(PTS)", + "runtimeHandler": "builtin_qwen_behavior_rough_cut", + "runtimeSource": "operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py", + "type": "builtin" +} diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/requirements.txt b/operator-repos/builtin/qwen_behavior_rough_cut/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/runtime_contract.json b/operator-repos/builtin/qwen_behavior_rough_cut/runtime_contract.json new file mode 100644 index 00000000..acfe98b4 --- /dev/null +++ b/operator-repos/builtin/qwen_behavior_rough_cut/runtime_contract.json @@ -0,0 +1,49 @@ +{ + "acceptance": { + "allowsEmptyResultList": true, + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresNonEmptyManifest": true, + "requiresNonEmptyOutput": false, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_qwen_behavior_rough_cut", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "operatorId": "operator-builtin-qwen-behavior-rough-cut", + "operatorSlug": "qwen_behavior_rough_cut", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 1, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": true + }, + "runtimeDependencies": [ + "qwen3-vl-8b-instruct", + "verified-local-model-manifest", + "torch-cuda", + "opencv", + "qwen-vl-utils", + "real-video-files", + "orbbec-pts-csv" + ], + "runtimeSource": "src/operator.py", + "schemaVersion": "droboticflow.operator.runtime/v1", + "sourceRoot": "operator-repos/builtin/qwen_behavior_rough_cut" +} diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/src/__init__.py b/operator-repos/builtin/qwen_behavior_rough_cut/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py b/operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py new file mode 100644 index 00000000..45df192d --- /dev/null +++ b/operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py @@ -0,0 +1,1007 @@ +"""PTS-grounded Qwen3-VL behaviour rough-cut for the Fanyu workflow. + +This is deliberately *not* a duration slicer. It asks a verified local +Qwen3-VL checkpoint to describe hand-action changes (subject-verb-object) and +uses only a PTS value visibly burned into every model input frame as timing +evidence. The bounded technical blocks exist solely to cap a single inference +request; they are never emitted as clip boundaries. + +Candidates are virtual clips: each one retains the original full, face-blurred +video artifact reference. The human cutter therefore works against the full +sanitized source and can correct Qwen's proposed boundaries before any physical +MP4 is materialized by the downstream package stage. +""" +from __future__ import annotations + +import copy +import csv +import hashlib +import json +import math +import pathlib +import re +import uuid +from datetime import datetime, timezone +from typing import Any, Callable + +from backend.operators.builtin.qwen3vl_model_manifest import ( + REVISION as QWEN3VL_8B_INSTRUCT_REVISION, + stage_preverified_model_to_local, + verify as verify_qwen3vl_model_manifest, +) + + +HelperMap = dict[str, Callable[..., Any]] +VIDEO_SUFFIXES = {".mp4", ".mov", ".m4v", ".avi", ".mkv", ".webm"} +PTS_COLUMN = "timestamp_us" +DEFAULT_TECHNICAL_BLOCK_SECONDS = 10.0 +DEFAULT_FRAMES_PER_TECHNICAL_BLOCK = 18 +REFERENCE_GROUP_SCRIPT_SHA256 = "2cf5fdb34fc11911dbcdbd4c51965d9cfe17239f862ebd9cad2e18078f90afc2" +# The operator itself writes exactly ``PTS_US=`` into the top-right +# overlay. Accepting a bare number or another clock format would make it +# impossible to prove that a model response was copied from that overlay. +_PTS_TEXT = re.compile(r"^PTS_US=(\d+)$") + + +def _sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _hyperparams(node_data: dict[str, Any], helpers: HelperMap) -> dict[str, str]: + fn = helpers.get("hyperparams") + if callable(fn): + return {str(key): str(value) for key, value in fn(node_data).items()} + return { + str(item.get("key")): str(item.get("value", "")) + for item in node_data.get("hyperparams", []) + if isinstance(item, dict) and item.get("key") + } + + +def _boolean(params: dict[str, str], key: str, default: bool) -> bool: + raw = str(params.get(key, str(default))).strip().lower() + if raw in {"1", "true", "yes", "on"}: + return True + if raw in {"0", "false", "no", "off"}: + return False + raise RuntimeError(f"{key} must be a boolean") + + +def _positive_float(params: dict[str, str], key: str, default: float) -> float: + try: + value = float(params.get(key, default) or default) + except (TypeError, ValueError) as exc: + raise RuntimeError(f"{key} must be a finite positive number") from exc + if not math.isfinite(value) or value <= 0: + raise RuntimeError(f"{key} must be a finite positive number") + return value + + +def _positive_int(params: dict[str, str], key: str, default: int) -> int: + value = _positive_float(params, key, float(default)) + if not value.is_integer(): + raise RuntimeError(f"{key} must be a positive integer") + return int(value) + + +def _payloads(inputs: list[dict[str, Any]], helpers: HelperMap) -> list[dict[str, Any]]: + fn = helpers.get("payloads_from_inputs") + if callable(fn): + values = fn(inputs) + if isinstance(values, list): + return [value for value in values if isinstance(value, dict)] + values: list[dict[str, Any]] = [] + for item in inputs: + data = item.get("data") if isinstance(item, dict) else None + if isinstance(data, dict): + values.append(data) + return values + + +def _face_blur_manifest(payloads: list[dict[str, Any]]) -> dict[str, Any]: + for payload in payloads: + schema = str(payload.get("schemaVersion", "")).strip() + items = payload.get("items") + if schema.startswith("droboticflow.video_face_blur.") and isinstance(items, list): + return payload + if isinstance(items, list) and any(isinstance(item, dict) and item.get("sanitizedVideoPath") for item in items): + return payload + raise RuntimeError("Qwen 行为粗切需要视频人脸脱敏输出 manifest 与真实脱敏视频 artifact") + + +def _orbbec_manifest(payloads: list[dict[str, Any]]) -> dict[str, Any]: + for payload in payloads: + schema = str(payload.get("schemaVersion", "")).strip() + if schema == "droboticflow.orbbec.normalized/v1" and isinstance(payload.get("entries"), list): + return payload + if payload.get("datasetType") == "orbbec-ego" and isinstance(payload.get("entries"), list): + return payload + raise RuntimeError("Qwen 行为粗切需要 Orbbec 标准化 manifest,以定位每帧 PTS CSV") + + +def _basename(raw: Any) -> str: + return pathlib.PurePosixPath(str(raw or "").replace("\\", "/")).name + + +def _original_video_name(item: dict[str, Any]) -> str: + for key in ("relativePath", "originalFilename", "sourceVideo"): + candidate = _basename(item.get(key)) + if pathlib.PurePosixPath(candidate).suffix.lower() in VIDEO_SUFFIXES: + return candidate + filename = _basename(item.get("filename") or item.get("sanitizedFilename")) + if filename.startswith("blurred_"): + filename = filename[len("blurred_") :] + if pathlib.PurePosixPath(filename).suffix.lower() in VIDEO_SUFFIXES: + return filename + raise RuntimeError("视频人脸脱敏 manifest 缺少原始视频文件名") + + +def _sanitized_video_name(item: dict[str, Any]) -> str: + for key in ("sanitizedFilename", "filename", "sanitizedVideoPath"): + candidate = _basename(item.get(key)) + if pathlib.PurePosixPath(candidate).suffix.lower() in VIDEO_SUFFIXES: + return candidate + raise RuntimeError("视频人脸脱敏 manifest 缺少已脱敏视频文件名") + + +def _artifact_by_names(inputs: list[dict[str, Any]], names: list[str], suffixes: set[str]) -> dict[str, Any] | None: + expected = {name.lower() for name in names if name} + for artifact in inputs: + if not isinstance(artifact, dict): + continue + name = _basename(artifact.get("name") or artifact.get("path")) + if name.lower() in expected and pathlib.PurePosixPath(name).suffix.lower() in suffixes: + return artifact + return None + + +def _entry_by_names(entries: list[dict[str, Any]], names: list[str]) -> dict[str, Any] | None: + expected = {name.lower() for name in names if name} + for entry in entries: + if not isinstance(entry, dict): + continue + name = _basename(entry.get("relativePath") or entry.get("path") or entry.get("name")) + if name.lower() in expected: + return entry + return None + + +def _artifact_source_item(artifact: dict[str, Any], helpers: HelperMap) -> dict[str, Any]: + source_ref = helpers.get("source_ref_from_artifact") + source = source_ref(artifact) if callable(source_ref) else {} + if not isinstance(source, dict): + source = {} + return {"path": str(artifact.get("path", "")).strip(), "source": source} + + +def _materialize_item(item: dict[str, Any], default_name: str, output: pathlib.Path, helpers: HelperMap) -> pathlib.Path: + raw_path = str(item.get("path", "")).strip() + if raw_path: + direct = pathlib.Path(raw_path) + if direct.is_file() and direct.stat().st_size > 0: + return direct + source = item.get("source") if isinstance(item.get("source"), dict) else {} + direct_source = str(source.get("path", "")).strip() + if direct_source: + direct = pathlib.Path(direct_source) + if direct.is_file() and direct.stat().st_size > 0: + return direct + materialize = helpers.get("source_path_from_item") + if not callable(materialize): + raise RuntimeError(f"Qwen 行为粗切无法物化真实输入文件: {default_name}") + path = materialize(item, default_name, output, {}) + if not isinstance(path, pathlib.Path): + path = pathlib.Path(path) + if not path.is_file() or path.stat().st_size <= 0: + raise RuntimeError(f"Qwen 行为粗切物化的输入文件为空: {default_name}") + return path + + +def _pts_candidates( + inputs: list[dict[str, Any]], + normalized_entries: list[dict[str, Any]], + blur_item: dict[str, Any], + original_video_name: str, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + original_pts = pathlib.PurePosixPath(original_video_name).stem + "_pts.csv" + blurred_pts = "blurred_" + original_pts + paired = blur_item.get("pairedFiles") if isinstance(blur_item.get("pairedFiles"), dict) else {} + paired_pts = _basename(paired.get("pts")) + names = [paired_pts, blurred_pts, original_pts] + artifacts: list[dict[str, Any]] = [] + entries: list[dict[str, Any]] = [] + for name in names: + artifact = _artifact_by_names(inputs, [name], {".csv"}) + if artifact is not None and artifact not in artifacts: + artifacts.append(artifact) + entry = _entry_by_names(normalized_entries, [name]) + if entry is not None and entry not in entries: + entries.append(entry) + # A copied PTS artifact is preferred because it travelled with the blur + # output. The standardised source entry is retained as a durable fallback. + return artifacts, entries + + +def _read_pts_csv(path: pathlib.Path) -> list[int]: + try: + handle = path.open("r", encoding="utf-8-sig", newline="") + except OSError as exc: + raise RuntimeError(f"cannot open PTS CSV {path}: {exc}") from exc + with handle: + reader = csv.DictReader(handle) + fields = {str(field).strip(): field for field in (reader.fieldnames or []) if field is not None} + source_field = fields.get(PTS_COLUMN) + if source_field is None: + raise RuntimeError(f"PTS CSV {path.name} must contain exactly-named {PTS_COLUMN} column") + values: list[int] = [] + for line_number, row in enumerate(reader, start=2): + raw = str(row.get(source_field, "")).strip() + if not re.fullmatch(r"\d+", raw): + raise RuntimeError(f"PTS CSV {path.name} line {line_number} has invalid {PTS_COLUMN}") + value = int(raw) + if values and value <= values[-1]: + raise RuntimeError(f"PTS CSV {path.name} {PTS_COLUMN} must be strictly increasing") + values.append(value) + if not values: + raise RuntimeError(f"PTS CSV {path.name} has no timestamp rows") + return values + + +def _decode_frame_count(video: pathlib.Path) -> tuple[int, float, int, int]: + try: + import cv2 + except Exception as exc: # pragma: no cover - image dependency boundary + raise RuntimeError("Qwen 行为粗切需要 OpenCV 解码真实视频") from exc + capture = cv2.VideoCapture(str(video)) + if not capture.isOpened(): + raise RuntimeError(f"cannot decode sanitized video {video}") + try: + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + if not math.isfinite(fps) or fps <= 0 or width <= 0 or height <= 0: + raise RuntimeError("sanitized video has invalid FPS or resolution") + count = 0 + while True: + ok, frame = capture.read() + if not ok or frame is None: + break + count += 1 + if count <= 0: + raise RuntimeError("sanitized video decoded zero frames") + return count, fps, width, height + finally: + capture.release() + + +def _technical_blocks(frame_count: int, fps: float, technical_block_seconds: float, frames_per_block: int) -> list[dict[str, Any]]: + if frame_count <= 0 or fps <= 0: + raise RuntimeError("technical inference planning requires positive frame count and FPS") + frames_per_duration = max(1, int(math.ceil(technical_block_seconds * fps))) + blocks: list[dict[str, Any]] = [] + for offset in range(0, frame_count, frames_per_duration): + end = min(frame_count, offset + frames_per_duration) + available = end - offset + sample_count = min(frames_per_block, available) + if sample_count <= 0: + continue + if sample_count == 1: + sample_indices = [offset] + else: + sample_indices = [offset + int(round((available - 1) * index / (sample_count - 1))) for index in range(sample_count)] + if len(set(sample_indices)) != len(sample_indices): + raise RuntimeError("technical inference frame plan produced duplicate samples") + blocks.append( + { + "technicalOnly": True, + "technicalBlockIndex": len(blocks) + 1, + "startFrameIndex": offset, + "endFrameIndexExclusive": end, + "sampleFrameIndices": sample_indices, + } + ) + if not blocks: + raise RuntimeError("technical inference plan contains no blocks") + return blocks + + +def _overlay_pts(frame: Any, timestamp_us: int) -> Any: + import cv2 + + image = frame.copy() + height, width = image.shape[:2] + text = f"PTS_US={timestamp_us}" + font = cv2.FONT_HERSHEY_SIMPLEX + scale = 0.8 + thickness = 2 + while scale > 0.28: + (text_width, text_height), baseline = cv2.getTextSize(text, font, scale, thickness) + if text_width + 20 <= width: + break + scale -= 0.08 + (text_width, text_height), baseline = cv2.getTextSize(text, font, scale, thickness) + x = max(5, width - text_width - 10) + y = max(text_height + baseline + 8, min(height - 8, text_height + baseline + 8)) + top_left = (max(0, x - 5), max(0, y - text_height - baseline - 5)) + bottom_right = (min(width - 1, x + text_width + 5), min(height - 1, y + baseline + 5)) + cv2.rectangle(image, top_left, bottom_right, (0, 0, 0), -1) + cv2.putText(image, text, (x, y), font, scale, (255, 255, 255), thickness, cv2.LINE_AA) + return image + + +def _write_overlaid_samples( + video: pathlib.Path, + timestamps_us: list[int], + blocks: list[dict[str, Any]], + output: pathlib.Path, +) -> list[dict[str, Any]]: + try: + import cv2 + except Exception as exc: # pragma: no cover - image dependency boundary + raise RuntimeError("Qwen 行为粗切需要 OpenCV 写入带 PTS 叠加的真实帧") from exc + requested: dict[int, tuple[int, int]] = {} + for block_index, block in enumerate(blocks): + for sample_position, frame_index in enumerate(block["sampleFrameIndices"]): + if frame_index in requested: + raise RuntimeError("technical inference blocks must not reuse sampled frames") + requested[frame_index] = (block_index, sample_position) + capture = cv2.VideoCapture(str(video)) + if not capture.isOpened(): + raise RuntimeError(f"cannot decode sanitized video {video}") + sample_root = output / "_qwen_behavior_rough_cut_frames" + sample_root.mkdir(parents=True, exist_ok=True) + materialized: list[list[dict[str, Any] | None]] = [ + [None for _ in block["sampleFrameIndices"]] for block in blocks + ] + try: + index = 0 + while True: + ok, frame = capture.read() + if not ok or frame is None: + break + target = requested.get(index) + if target is not None: + block_index, sample_position = target + timestamp_us = timestamps_us[index] + rendered = _overlay_pts(frame, timestamp_us) + path = sample_root / f"block-{block_index + 1:05d}-frame-{index:08d}.jpg" + if not cv2.imwrite(str(path), rendered) or not path.is_file() or path.stat().st_size <= 0: + raise RuntimeError(f"failed to write PTS-overlaid model frame {index}") + materialized[block_index][sample_position] = { + "path": str(path), + "frameIndex": index, + "timestampUs": timestamp_us, + "overlayText": f"PTS_US={timestamp_us}", + } + index += 1 + finally: + capture.release() + results: list[dict[str, Any]] = [] + for block_index, items in enumerate(materialized): + if any(item is None for item in items): + missing = [blocks[block_index]["sampleFrameIndices"][position] for position, item in enumerate(items) if item is None] + raise RuntimeError(f"unable to decode planned model frame(s): {missing}") + completed = [item for item in items if isinstance(item, dict)] + blocks[block_index]["sampledFrames"] = completed + blocks[block_index]["sampledPTSUs"] = [int(item["timestampUs"]) for item in completed] + results.append(blocks[block_index]) + return results + + +def _load_qwen(model_path: pathlib.Path) -> tuple[Any, Any, Any]: + try: + import torch + from transformers import AutoProcessor, Qwen3VLForConditionalGeneration + except Exception as exc: # pragma: no cover - image dependency boundary + raise RuntimeError("Qwen3-VL runtime requires torch and transformers in the operator image") from exc + if not torch.cuda.is_available(): + raise RuntimeError("Qwen3-VL 行为粗切需要已分配 CUDA GPU;没有 CPU、mock 或 heuristic fallback") + try: + processor = AutoProcessor.from_pretrained(str(model_path), local_files_only=True) + model = Qwen3VLForConditionalGeneration.from_pretrained( + str(model_path), local_files_only=True, torch_dtype="auto", device_map="auto" + ) + model.eval() + except Exception as exc: + raise RuntimeError(f"cannot load verified local Qwen3-VL checkpoint {model_path}: {exc}") from exc + return torch, processor, model + + +def _json_behaviors(text: str) -> dict[str, Any]: + candidate = re.sub(r"```(?:json)?", "", text, flags=re.IGNORECASE).strip() + decoder = json.JSONDecoder() + matches: list[dict[str, Any]] = [] + for start, character in enumerate(candidate): + if character != "{": + continue + try: + payload, _ = decoder.raw_decode(candidate[start:]) + except json.JSONDecodeError: + continue + if isinstance(payload, dict) and isinstance(payload.get("behaviors"), list): + matches.append(payload) + if not matches: + response_sha = hashlib.sha256(text.encode("utf-8")).hexdigest() + raise RuntimeError( + "Qwen3-VL behaviour response must be a JSON object containing behaviors " + f"(sha256={response_sha}, chars={len(text)})" + ) + return matches[-1] + + +def _previous_context(events: list[dict[str, Any]]) -> list[dict[str, str]]: + return [ + { + "behavior": str(event["behavior"]), + "object_change": str(event.get("object_change", "")), + "start_timestamp": str(event["start_timestamp"]), + "end_timestamp": str(event["end_timestamp"]), + } + for event in events + ] + + +def _infer_technical_block( + torch: Any, + processor: Any, + model: Any, + frames: list[dict[str, Any]], + previous_events: list[dict[str, Any]], + max_new_tokens: int, +) -> tuple[dict[str, Any], str]: + prompt = ( + "Analyze the chronological images from a convenience-store egocentric video. " + "Identify every change in hand action as subject-verb-object and describe what happens to the manipulated object. " + "The images are a bounded technical inference request only; they are NOT delivery clips or proposed fixed-duration cuts.\n\n" + "IMPORTANT: every image contains a high-contrast PTS_US timestamp overlay in the top-right corner. " + "That top-right PTS_US overlay is the ONLY source of time. Read it directly. " + "Do NOT derive time from frame ordering, FPS, video duration, request boundaries, or any offset. " + "For every action return exact PTS_US strings visibly printed in the images.\n\n" + "Return JSON only: {\"behaviors\":[{\"behavior\":string,\"object_change\":string," + "\"start_timestamp\":string,\"end_timestamp\":string}]}. " + "Use an empty behaviors array only when no hand-action change is visible. " + "Do not include Markdown, explanation, confidence, invented timestamps, or extra fields." + ) + if previous_events: + prompt += ( + "\n\nPrevious technical-block analysis:\n" + + json.dumps(_previous_context(previous_events), ensure_ascii=False, separators=(",", ":")) + + "\nContinue behavior naming consistently. Timestamps must still come only from the visible PTS_US overlay." + ) + try: + from qwen_vl_utils import process_vision_info + + conversation = [ + { + "role": "user", + "content": [ + *[{"type": "image", "image": str(frame["path"])} for frame in frames], + {"type": "text", "text": prompt}, + ], + } + ] + text = processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True) + image_inputs, video_inputs = process_vision_info(conversation) + model_inputs = processor( + text=[text], images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True + ).to(model.device) + with torch.inference_mode(): + generated = model.generate(**model_inputs, max_new_tokens=max_new_tokens, do_sample=False) + trimmed = [output_ids[len(input_ids) :] for input_ids, output_ids in zip(model_inputs.input_ids, generated)] + response = processor.batch_decode(trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + except Exception as exc: + raise RuntimeError(f"Qwen3-VL behaviour inference failed on real PTS-overlaid frames: {exc}") from exc + return _json_behaviors(response), response + + +def _parse_overlay_timestamp(raw: Any) -> tuple[str, int]: + if not isinstance(raw, str): + raise RuntimeError("Qwen3-VL behavior timestamp must be a string read from the PTS_US overlay") + value = raw.strip() + matched = _PTS_TEXT.fullmatch(value) + if matched is None: + raise RuntimeError("Qwen3-VL behavior timestamp is not an exact PTS_US overlay value") + return value, int(matched.group(1)) + + +def _normalize_behavior( + raw: dict[str, Any], + sampled_timestamps_us: set[int], + timestamp_to_frame: dict[int, int], + pts_origin_us: int, +) -> dict[str, Any]: + required = {"behavior", "object_change", "start_timestamp", "end_timestamp"} + if set(raw) != required: + raise RuntimeError("Qwen3-VL behavior event must contain exactly behavior, object_change, start_timestamp and end_timestamp") + behavior = raw.get("behavior") + object_change = raw.get("object_change") + if not isinstance(behavior, str) or not behavior.strip(): + raise RuntimeError("Qwen3-VL behavior event requires non-empty behavior") + if not isinstance(object_change, str): + raise RuntimeError("Qwen3-VL behavior event object_change must be a string") + start_raw, start_us = _parse_overlay_timestamp(raw.get("start_timestamp")) + end_raw, end_us = _parse_overlay_timestamp(raw.get("end_timestamp")) + if start_us not in sampled_timestamps_us or end_us not in sampled_timestamps_us: + raise RuntimeError("Qwen3-VL behavior timestamps must exactly match PTS_US values in this request's displayed frames") + if start_us >= end_us: + raise RuntimeError("Qwen3-VL behavior event end timestamp must be greater than start timestamp") + if start_us not in timestamp_to_frame or end_us not in timestamp_to_frame: + raise RuntimeError("Qwen3-VL behavior timestamp is not present in the verified PTS CSV") + return { + "behavior": behavior.strip(), + "object_change": object_change.strip(), + "start_timestamp": start_raw, + "end_timestamp": end_raw, + "start_us": start_us, + "end_us": end_us, + "start_frame_index": timestamp_to_frame[start_us], + "end_frame_index": timestamp_to_frame[end_us], + "start_second": round((start_us - pts_origin_us) / 1_000_000.0, 6), + "end_second": round((end_us - pts_origin_us) / 1_000_000.0, 6), + } + + +def _merge_adjacent_behaviors(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Mirror the group reference: adjacent events with exact same behavior merge. + + The original raw events remain in the manifest. We intentionally do not + merge by duration, similarity, time gap, taxonomy, or any heuristic. + """ + merged: list[dict[str, Any]] = [] + for event in events: + if merged and event["behavior"] == merged[-1]["behavior"]: + prior = merged[-1] + prior["end_timestamp"] = event["end_timestamp"] + prior["end_us"] = event["end_us"] + prior["end_frame_index"] = event["end_frame_index"] + prior["end_second"] = event["end_second"] + prior["technical_block_indexes"].extend(event["technical_block_indexes"]) + prior["raw_event_indexes"].extend(event["raw_event_indexes"]) + prior.setdefault("raw_response_sha256s", [prior["raw_response_sha256"]]) + prior["raw_response_sha256s"].extend( + event.get("raw_response_sha256s", [event["raw_response_sha256"]]) + ) + continue + merged.append(copy.deepcopy(event)) + return merged + + +def _artifact_size_bytes(artifact: dict[str, Any], source_path: pathlib.Path) -> int: + raw = artifact.get("sizeBytes") + if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw >= 0: + return int(raw) + payload = artifact.get("payload") if isinstance(artifact.get("payload"), dict) else {} + ref = payload.get("ref") if isinstance(payload.get("ref"), dict) else {} + ref_size = ref.get("sizeBytes") + if isinstance(ref_size, (int, float)) and not isinstance(ref_size, bool) and ref_size >= 0: + return int(ref_size) + return source_path.stat().st_size + + +def _source_ref(artifact: dict[str, Any], helpers: HelperMap) -> dict[str, Any]: + fn = helpers.get("source_ref_from_artifact") + value = fn(artifact) if callable(fn) else {} + if isinstance(value, dict) and value: + return copy.deepcopy(value) + raw_path = str(artifact.get("path", "")).strip() + return {"path": raw_path} if raw_path else {} + + +def _virtual_clip_artifact( + run_id: str, + node_id: str, + binding: dict[str, Any] | None, + source_artifact: dict[str, Any], + source_path: pathlib.Path, + clip: dict[str, Any], + helpers: HelperMap, +) -> dict[str, Any]: + build = helpers.get("build_virtual_artifact") + if not callable(build): + raise RuntimeError("Qwen 行为粗切运行时缺少 virtual artifact builder") + payload = source_artifact.get("payload") if isinstance(source_artifact.get("payload"), dict) else None + payload_ref = payload.get("ref") if isinstance(payload, dict) and isinstance(payload.get("ref"), dict) else None + metadata = source_artifact.get("metadata") if isinstance(source_artifact.get("metadata"), dict) else {} + data: dict[str, Any] = { + "clipId": clip["clipId"], + "clipName": clip["clipName"], + "source": _source_ref(source_artifact, helpers), + "sourceIsFullVideo": True, + "virtualClip": True, + "startSecond": clip["startSecond"], + "endSecond": clip["endSecond"], + "sourceDurationSeconds": clip["sourceDurationSeconds"], + "behavior": clip["behavior"], + "objectChange": clip["objectChange"], + } + if isinstance(payload_ref, dict): + data["ref"] = copy.deepcopy(payload_ref) + result = build( + run_id, + node_id, + str(clip["clipName"]), + str(source_artifact.get("path", "")).strip() or str(source_path), + "video", + _artifact_size_bytes(source_artifact, source_path), + data=data, + metadata=copy.deepcopy(metadata), + output=binding, + ) + if isinstance(payload, dict): + # This is the important virtual-clip invariant: retain the exact + # credential-free ref / payload of the complete face-blurred source, + # rather than uploading or slicing a second media object. + result["payload"] = copy.deepcopy(payload) + return result + + +def _source_contexts( + inputs: list[dict[str, Any]], output: pathlib.Path, helpers: HelperMap +) -> list[dict[str, Any]]: + payloads = _payloads(inputs, helpers) + blur_manifest = _face_blur_manifest(payloads) + normalized = _orbbec_manifest(payloads) + normalized_entries = [entry for entry in normalized.get("entries", []) if isinstance(entry, dict)] + contexts: list[dict[str, Any]] = [] + seen_video_artifacts: set[str] = set() + for blur_item in blur_manifest.get("items", []): + if not isinstance(blur_item, dict): + continue + sanitized_name = _sanitized_video_name(blur_item) + original_name = _original_video_name(blur_item) + video_artifact = _artifact_by_names(inputs, [sanitized_name, _basename(blur_item.get("sanitizedVideoPath"))], VIDEO_SUFFIXES) + if video_artifact is None: + raise RuntimeError(f"Qwen 行为粗切未收到人脸脱敏视频 artifact: {sanitized_name}") + artifact_key = str(video_artifact.get("id") or video_artifact.get("path") or sanitized_name) + if artifact_key in seen_video_artifacts: + continue + seen_video_artifacts.add(artifact_key) + source_path = _materialize_item(_artifact_source_item(video_artifact, helpers), sanitized_name, output, helpers) + pts_artifacts, pts_entries = _pts_candidates(inputs, normalized_entries, blur_item, original_name) + pts_path: pathlib.Path | None = None + pts_origin = "" + for artifact in pts_artifacts: + try: + pts_path = _materialize_item(_artifact_source_item(artifact, helpers), _basename(artifact.get("name")), output, helpers) + except RuntimeError: + continue + pts_origin = "face-blur-output-artifact" + break + if pts_path is None: + for entry in pts_entries: + try: + pts_path = _materialize_item(entry, _basename(entry.get("relativePath") or entry.get("path")), output, helpers) + except RuntimeError: + continue + pts_origin = "orbbec-normalized-manifest" + break + if pts_path is None: + expected = pathlib.PurePosixPath(original_name).stem + "_pts.csv" + raise RuntimeError(f"Qwen 行为粗切无法从 Orbbec manifest 物化对应 PTS CSV: {expected}") + contexts.append( + { + "blurItem": blur_item, + "originalVideoName": original_name, + "sanitizedVideoName": sanitized_name, + "sourceArtifact": video_artifact, + "sourcePath": source_path, + "ptsPath": pts_path, + "ptsOrigin": pts_origin, + } + ) + if not contexts: + raise RuntimeError("视频人脸脱敏 manifest 不包含可供 Qwen 行为粗切的真实视频") + return contexts + + +def run( + run_id: str, + node_id: str, + output_dir: Any, + node_data: dict[str, Any], + inputs: list[dict[str, Any]], + helpers: HelperMap, +) -> list[dict[str, Any]]: + output = pathlib.Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + params = _hyperparams(node_data, helpers) + technical_block_seconds = _positive_float(params, "TECHNICAL_BLOCK_SECONDS", DEFAULT_TECHNICAL_BLOCK_SECONDS) + frames_per_block = _positive_int(params, "FRAMES_PER_TECHNICAL_BLOCK", DEFAULT_FRAMES_PER_TECHNICAL_BLOCK) + if frames_per_block > 96: + raise RuntimeError("FRAMES_PER_TECHNICAL_BLOCK must not exceed 96") + max_new_tokens = _positive_int(params, "MAX_NEW_TOKENS", 2048) + require_non_empty = _boolean(params, "REQUIRE_NON_EMPTY_RESULT", False) + model_path = pathlib.Path(str(params.get("MODEL_PATH", "/workspace/models/Qwen3-VL-8B-Instruct")).strip()) + revision = str(params.get("MODEL_REVISION", QWEN3VL_8B_INSTRUCT_REVISION)).strip() + manifest_raw = str(params.get("MODEL_MANIFEST_PATH", "")).strip() + manifest_path = pathlib.Path(manifest_raw) if manifest_raw else None + trust_preverified = _boolean(params, "TRUST_PREVERIFIED_MODEL", False) + + contexts = _source_contexts(inputs, output, helpers) + model_manifest, verified_model_sha = verify_qwen3vl_model_manifest( + model_path, + revision, + manifest_path, + str(params.get("MODEL_MANIFEST_SHA256", "")).strip(), + trust_preverified, + ) + runtime_model_path, staged_model_bytes = ( + stage_preverified_model_to_local(model_path, model_manifest, verified_model_sha) + if trust_preverified + else (model_path, 0) + ) + torch, processor, model = _load_qwen(runtime_model_path) + + binding_fn = helpers.get("output_binding") + binding = binding_fn(node_data) if callable(binding_fn) else None + raw_behavior_events: list[dict[str, Any]] = [] + merged_behavior_events: list[dict[str, Any]] = [] + technical_blocks: list[dict[str, Any]] = [] + source_provenance: list[dict[str, Any]] = [] + virtual_artifacts: list[dict[str, Any]] = [] + candidate_count = 0 + + for context_index, context in enumerate(contexts, start=1): + source_path = context["sourcePath"] + timestamps_us = _read_pts_csv(context["ptsPath"]) + frame_count, fps, width, height = _decode_frame_count(source_path) + if frame_count != len(timestamps_us): + raise RuntimeError( + "PTS CSV row count must exactly equal decoded sanitized-video frame count " + f"for {context['sanitizedVideoName']}: ptsRows={len(timestamps_us)}, decodedFrames={frame_count}" + ) + timestamp_to_frame = {timestamp: index for index, timestamp in enumerate(timestamps_us)} + decoded_duration_seconds = round(frame_count / fps, 6) + pts_duration_seconds = round((timestamps_us[-1] - timestamps_us[0]) / 1_000_000.0, 6) + if pts_duration_seconds > decoded_duration_seconds + max(1.0 / fps, 0.001): + raise RuntimeError( + "PTS CSV duration is inconsistent with decoded sanitized-video duration " + f"for {context['sanitizedVideoName']}: ptsDuration={pts_duration_seconds}, " + f"decodedDuration={decoded_duration_seconds}" + ) + blocks = _technical_blocks(frame_count, fps, technical_block_seconds, frames_per_block) + blocks = _write_overlaid_samples(source_path, timestamps_us, blocks, output) + previous_events: list[dict[str, Any]] = [] + source_events: list[dict[str, Any]] = [] + for block in blocks: + result, raw_response = _infer_technical_block( + torch, + processor, + model, + block["sampledFrames"], + previous_events, + max_new_tokens, + ) + raw_values = result.get("behaviors") + if not isinstance(raw_values, list): + raise RuntimeError("Qwen3-VL behavior response is missing behaviors array") + accepted: list[dict[str, Any]] = [] + prior_start_us: int | None = None + for model_index, raw in enumerate(raw_values): + if not isinstance(raw, dict): + raise RuntimeError("Qwen3-VL behavior response behaviors must contain objects only") + event = _normalize_behavior( + raw, + set(block["sampledPTSUs"]), + timestamp_to_frame, + timestamps_us[0], + ) + if prior_start_us is not None and event["start_us"] < prior_start_us: + raise RuntimeError("Qwen3-VL behavior events must be ordered by visible PTS_US timestamp") + prior_start_us = event["start_us"] + event.update( + { + "technical_block_indexes": [block["technicalBlockIndex"]], + "raw_event_indexes": [len(raw_behavior_events) + 1], + "model_event_index": model_index, + "source_video_index": context_index, + "raw_response_sha256": hashlib.sha256(raw_response.encode("utf-8")).hexdigest(), + "raw_response_sha256s": [hashlib.sha256(raw_response.encode("utf-8")).hexdigest()], + } + ) + accepted.append(event) + raw_behavior_events.append( + { + "rawEventIndex": len(raw_behavior_events) + 1, + "sourceVideoIndex": context_index, + "technicalBlockIndex": block["technicalBlockIndex"], + "technicalOnly": True, + "modelEventIndex": model_index, + "behavior": event["behavior"], + "objectChange": event["object_change"], + "rawPTS": { + "startTimestamp": event["start_timestamp"], + "endTimestamp": event["end_timestamp"], + "startTimestampUs": event["start_us"], + "endTimestampUs": event["end_us"], + }, + "startSecond": event["start_second"], + "endSecond": event["end_second"], + "rawResponseSHA256": event["raw_response_sha256"], + } + ) + source_events.append(event) + block["previousTechnicalBlockContextCount"] = len(previous_events) + block["rawResponseSHA256"] = hashlib.sha256(raw_response.encode("utf-8")).hexdigest() + block["acceptedBehaviorCount"] = len(accepted) + # Same as the group reference script: only the immediately previous + # technical block is context for the next request. + previous_events = accepted + for merged in _merge_adjacent_behaviors(source_events): + candidate_count += 1 + clip_stem = pathlib.PurePosixPath(context["sanitizedVideoName"]).stem + event_key = f"{context['sanitizedVideoName']}|{merged['start_us']}|{merged['end_us']}|{merged['behavior']}" + suffix = hashlib.sha256(event_key.encode("utf-8")).hexdigest()[:12] + clip_id = f"{clip_stem}_rough_{candidate_count:04d}_{suffix}" + clip_name = clip_id + ".mp4" + source_payload = ( + context["sourceArtifact"].get("payload") + if isinstance(context["sourceArtifact"].get("payload"), dict) + else {} + ) + source_payload_ref = source_payload.get("ref") if isinstance(source_payload.get("ref"), dict) else None + clip = { + "clipId": clip_id, + "clipName": clip_name, + "sourceVideo": context["sanitizedVideoName"], + "sourceIsFullVideo": True, + "virtualClip": True, + "source": _source_ref(context["sourceArtifact"], helpers), + "startSecond": merged["start_second"], + "endSecond": merged["end_second"], + "sourceDurationSeconds": decoded_duration_seconds, + "duration": round(merged["end_second"] - merged["start_second"], 6), + "behavior": merged["behavior"], + "objectChange": merged["object_change"], + "rawPTS": { + "startTimestamp": merged["start_timestamp"], + "endTimestamp": merged["end_timestamp"], + "startTimestampUs": merged["start_us"], + "endTimestampUs": merged["end_us"], + "startFrameIndex": merged["start_frame_index"], + "endFrameIndex": merged["end_frame_index"], + "originTimestampUs": timestamps_us[0], + }, + "provenance": { + "model": "Qwen/Qwen3-VL-8B-Instruct", + "timeSource": "visible upper-right PTS_US overlay only", + "sourceVideoArtifactId": str(context["sourceArtifact"].get("id", "")), + "technicalOnly": True, + "technicalBlockIndexes": merged["technical_block_indexes"], + "rawBehaviorEventIndexes": merged["raw_event_indexes"], + "rawResponseSHA256s": merged["raw_response_sha256s"], + }, + "annotation": { + "schemaVersion": "droboticflow.qwen3vl.behavior-rough-prelabel/v1", + "clipId": clip_id, + "sourceVideo": context["sanitizedVideoName"], + "actions": [ + { + "id": clip_id + "-action-001", + # Qwen's free-form rough-cut behavior is not a + # member of the later 22-class semantic taxonomy. + # Keep this prelabel deliberately unlabeled; the + # semantic annotator makes the taxonomy decision. + "label": "", + "start": merged["start_second"], + "end": merged["end_second"], + "phrases": [ + value + for value in (merged["behavior"], merged["object_change"]) + if value + ], + "source": "qwen3-vl-behavior-rough-cut", + } + ], + "objects": [merged["object_change"]] if merged["object_change"] else [], + }, + } + if isinstance(source_payload_ref, dict): + # The manifest mirrors the same durable reference as the + # matching virtual artifact; no candidate has its own MP4. + clip["ref"] = copy.deepcopy(source_payload_ref) + for raw_event_index in merged["raw_event_indexes"]: + raw_behavior_events[raw_event_index - 1]["mergedCandidateClipId"] = clip_id + merged_behavior_events.append(clip) + virtual_artifacts.append( + _virtual_clip_artifact( + run_id, + node_id, + binding if isinstance(binding, dict) else None, + context["sourceArtifact"], + source_path, + clip, + helpers, + ) + ) + source_provenance.append( + { + "sourceVideoIndex": context_index, + "originalVideoName": context["originalVideoName"], + "sanitizedVideoName": context["sanitizedVideoName"], + "sanitizedVideoSHA256": _sha256(source_path), + "sourceVideoArtifactId": str(context["sourceArtifact"].get("id", "")), + "sourceVideoArtifactPath": str(context["sourceArtifact"].get("path", "")), + "ptsCSVName": context["ptsPath"].name, + "ptsSHA256": _sha256(context["ptsPath"]), + "ptsSource": context["ptsOrigin"], + "ptsRows": len(timestamps_us), + "decodedFrameCount": frame_count, + "ptsOriginTimestampUs": timestamps_us[0], + "decodedDurationSeconds": decoded_duration_seconds, + "ptsDurationSeconds": pts_duration_seconds, + "fps": fps, + "width": width, + "height": height, + } + ) + technical_blocks.extend( + [ + { + "sourceVideoIndex": context_index, + "sourceVideo": context["sanitizedVideoName"], + "technicalOnly": True, + "technicalBlockIndex": block["technicalBlockIndex"], + "startFrameIndex": block["startFrameIndex"], + "endFrameIndexExclusive": block["endFrameIndexExclusive"], + "sampleFrameIndices": block["sampleFrameIndices"], + "sampledPTSUs": block["sampledPTSUs"], + "previousTechnicalBlockContextCount": block["previousTechnicalBlockContextCount"], + "acceptedBehaviorCount": block["acceptedBehaviorCount"], + "rawResponseSHA256": block["rawResponseSHA256"], + } + for block in blocks + ] + ) + if require_non_empty and not merged_behavior_events: + raise RuntimeError("Qwen3-VL returned no accepted behavior rough-cut candidates while REQUIRE_NON_EMPTY_RESULT=true") + + manifest = { + "schemaVersion": "droboticflow.qwen3vl.behavior-rough-cut/v1", + "datasetType": "qwen3vl-behavior-rough-cut", + "status": "succeeded", + "clipCount": len(merged_behavior_events), + "itemCount": len(merged_behavior_events), + "sourceIsFullVideo": True, + "physicalMP4Slicing": False, + "items": merged_behavior_events, + "rawBehaviorEvents": raw_behavior_events, + "technicalBlocks": technical_blocks, + "parameters": { + "technicalBlockSeconds": technical_block_seconds, + "framesPerTechnicalBlock": frames_per_block, + "technicalOnly": True, + "technicalBlocksAreClipBoundaries": False, + "modelTimeSource": "visible upper-right PTS_US overlay only", + "requireNonEmptyResult": require_non_empty, + }, + "sourceVideos": source_provenance, + "provenance": { + "inferenceId": f"{run_id}:{node_id}:{uuid.uuid4().hex}", + "semanticReference": { + "source": "Feishu 云平台-泛宇交流群 video_segmentation.py", + "sha256": REFERENCE_GROUP_SCRIPT_SHA256, + }, + "modelId": model_manifest["modelId"], + "modelRevision": model_manifest["revision"], + "modelSha256": verified_model_sha, + "modelManifestPath": str(manifest_path or model_path / "MODEL_MANIFEST.json"), + "modelRuntimePath": str(runtime_model_path), + "modelStagedBytes": staged_model_bytes, + "device": torch.cuda.get_device_name(0), + "executedAt": datetime.now(timezone.utc).isoformat(), + }, + } + write = helpers.get("write_json_artifact") + if not callable(write): + path = output / "clips-manifest.json" + path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + return [{"name": path.name, "path": str(path), "type": "json", "data": manifest}, *virtual_artifacts] + return [ + write(run_id, node_id, output, "clips-manifest.json", manifest, output=binding if isinstance(binding, dict) else None), + *virtual_artifacts, + ] diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/tests/acceptance/fixture_contract.json b/operator-repos/builtin/qwen_behavior_rough_cut/tests/acceptance/fixture_contract.json new file mode 100644 index 00000000..f97b8e3c --- /dev/null +++ b/operator-repos/builtin/qwen_behavior_rough_cut/tests/acceptance/fixture_contract.json @@ -0,0 +1,21 @@ +{ + "fixtures": [], + "minimumEvidence": [ + "input checksum", + "artifact checksum", + "lineage", + "schema validation" + ], + "operatorId": "operator-builtin-qwen-behavior-rough-cut", + "operatorSlug": "qwen_behavior_rough_cut", + "rules": { + "forbidsMock": true, + "forbidsSimulationOnly": true, + "forbidsSmokeOnly": true, + "forbidsSynthetic": true, + "requiresLicense": true, + "requiresPublicSourceUrl": true, + "requiresSha256": true + }, + "schemaVersion": "droboticflow.operator.fixture/v1" +} diff --git a/scripts/generate_operator_images.py b/scripts/generate_operator_images.py index f1970dd4..a00f58da 100644 --- a/scripts/generate_operator_images.py +++ b/scripts/generate_operator_images.py @@ -38,11 +38,13 @@ GPU_SLUGS = { "yolo_video_target_clip", # Fanyu v5 always executes Qwen3-VL on an allocated CUDA GPU. These # operators intentionally have no CPU, heuristic, or mock fallback. + "qwen_behavior_rough_cut", "qwen_temporal_localization", "wilor_valid_event_detection", "qwen_label_crop_classification", } QWEN3VL_SLUGS = { + "qwen_behavior_rough_cut", "qwen_temporal_localization", "wilor_valid_event_detection", "qwen_label_crop_classification", @@ -51,6 +53,7 @@ FANYU_V5_ALLOW_EMPTY_SEMANTIC_RESULT_SLUGS = { # A real inference can legitimately find no action/event, and an accepted # annotation can be an explicit negative sample. Their manifests and # provenance must still be non-empty and cryptographically verifiable. + "qwen_behavior_rough_cut", "qwen_temporal_localization", "wilor_valid_event_detection", "ms_swift_dataset_export", diff --git a/scripts/generate_operator_repository_manifests.py b/scripts/generate_operator_repository_manifests.py index 18568ce3..3ca084db 100644 --- a/scripts/generate_operator_repository_manifests.py +++ b/scripts/generate_operator_repository_manifests.py @@ -126,6 +126,7 @@ RUNTIME_DEPENDENCIES: dict[str, list[str]] = { "xplatform_dataset_index": ["xplatform-dataset-files"], "xplatform_dataset_validate": ["xplatform-dataset-index"], "qwen_temporal_localization": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files"], + "qwen_behavior_rough_cut": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "torch-cuda", "opencv", "qwen-vl-utils", "real-video-files", "orbbec-pts-csv"], "qwen_label_crop_classification": ["qwen3-vl-8b-instruct", "verified-local-model-manifest", "ms-swift", "peft", "torch-cuda", "accepted-annotation-dataset"], "yolo_detection_validate": ["yolo-label-files", "image-files", "pyyaml"], "yolo_embodied_detection_train": ["real-yolo-training-command", "yolo-or-coco-dataset", "gpu-a800"], diff --git a/src/App.tsx b/src/App.tsx index 1533c961..0c5a8f1c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -119,7 +119,7 @@ function RoleRouteGuard({ children }: { children: React.ReactNode }) { } if ( effectiveRole === "annotator" && - !(path.includes("/plugins/annotation-qa/workflows/") && path.endsWith("/annotate")) && + !(path.includes("/plugins/annotation-qa/workflows/") && (path.endsWith("/annotate") || path.endsWith("/clip"))) && !itemPath ) { return ; diff --git a/src/components/AnnotationTimelineEditor.tsx b/src/components/AnnotationTimelineEditor.tsx index 9f1bfc5d..d8bfee43 100644 --- a/src/components/AnnotationTimelineEditor.tsx +++ b/src/components/AnnotationTimelineEditor.tsx @@ -39,6 +39,8 @@ type AnnotationTimelineEditorProps = { videoSource: string; clipStart: number; clipEnd: number; + sourceIsFullVideo?: boolean; + initialTime?: number; segments: AnnotationSegment[]; prelabels?: AnnotationSegment[]; labels?: SemanticLabelDefinition[]; @@ -53,6 +55,8 @@ export default function AnnotationTimelineEditor({ videoSource, clipStart, clipEnd, + sourceIsFullVideo = false, + initialTime, segments, prelabels = [], labels = [], @@ -64,7 +68,7 @@ export default function AnnotationTimelineEditor({ const timelineRef = useRef(null); const dragRef = useRef(null); const [selectedSegmentId, setSelectedSegmentId] = useState(""); - const [currentTime, setCurrentTime] = useState(clipStart); + const [currentTime, setCurrentTime] = useState(initialTime ?? clipStart); const [isPlaying, setIsPlaying] = useState(false); const [zoom, setZoom] = useState(1); const [showHotkeysHelp, setShowHotkeysHelp] = useState(false); @@ -75,6 +79,10 @@ export default function AnnotationTimelineEditor({ const isRerun = Boolean(videoSource && (videoSource.toLowerCase().endsWith(".rrd") || videoSource.includes(".rrd"))); const playableSource = isRerun ? videoSource : buildBrowserPlayableVideoURL(videoSource); + useEffect(() => { + setCurrentTime(Math.min(clipEnd, Math.max(clipStart, initialTime ?? clipStart))); + }, [clipStart, clipEnd, initialTime, videoSource]); + // Build taxonomy labels merged with convenience store 22 defaults const effectiveLabels = useMemo(() => { if (labels && labels.length > 0) return labels; @@ -153,7 +161,7 @@ export default function AnnotationTimelineEditor({ const next = clipStart + ((event.clientX - rect.left) / Math.max(rect.width, 1)) * duration; const clamped = Math.max(clipStart, Math.min(clipEnd, next)); setCurrentTime(clamped); - if (videoRef.current) videoRef.current.currentTime = Math.max(0, clamped - clipStart); + if (videoRef.current) videoRef.current.currentTime = Math.max(0, sourceIsFullVideo ? clamped : clamped - clipStart); }; const togglePlay = useCallback(() => { @@ -294,7 +302,7 @@ export default function AnnotationTimelineEditor({ const step = event.shiftKey ? snapSeconds * 10 : snapSeconds; const next = Math.max(clipStart, currentTime - step); setCurrentTime(next); - if (videoRef.current) videoRef.current.currentTime = Math.max(0, next - clipStart); + if (videoRef.current) videoRef.current.currentTime = Math.max(0, sourceIsFullVideo ? next : next - clipStart); return; } if (key === "arrowright") { @@ -302,7 +310,7 @@ export default function AnnotationTimelineEditor({ const step = event.shiftKey ? snapSeconds * 10 : snapSeconds; const next = Math.min(clipEnd, currentTime + step); setCurrentTime(next); - if (videoRef.current) videoRef.current.currentTime = Math.max(0, next - clipStart); + if (videoRef.current) videoRef.current.currentTime = Math.max(0, sourceIsFullVideo ? next : next - clipStart); return; } @@ -330,6 +338,7 @@ export default function AnnotationTimelineEditor({ currentTime, clipStart, clipEnd, + sourceIsFullVideo, snapSeconds, togglePlay, onChange, @@ -363,9 +372,21 @@ export default function AnnotationTimelineEditor({ const mediaDuration = Number.isFinite(event.currentTarget.duration) ? event.currentTarget.duration : duration; - setCurrentTime(clipStart + Math.min(mediaDuration, duration, event.currentTarget.currentTime)); + if (sourceIsFullVideo) { + const target = Math.min(Math.max(initialTime ?? clipStart, clipStart, 0), clipEnd, mediaDuration); + event.currentTarget.currentTime = target; + setCurrentTime(target); + } else { + setCurrentTime(clipStart + Math.min(mediaDuration, duration, event.currentTarget.currentTime)); + } + }} + onTimeUpdate={(event) => { + const sourceTime = sourceIsFullVideo ? event.currentTarget.currentTime : clipStart + event.currentTarget.currentTime; + if (sourceIsFullVideo && sourceTime >= clipEnd) { + event.currentTarget.pause(); + } + setCurrentTime(Math.min(clipEnd, sourceTime)); }} - onTimeUpdate={(event) => setCurrentTime(Math.min(clipEnd, clipStart + event.currentTarget.currentTime))} /> ) : (
diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index b6071594..8605b4ac 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -93,14 +93,14 @@ const navItems: readonly NavigationItem[] = [ icon: Scissors, label: "人工剪切", path: "/plugins/annotation-qa/workflows/all/clip", - roles: ["admin"], + roles: ["admin", "annotator"], pluginId: "annotation-qa", }, { icon: ShieldCheck, label: "剪切质检", path: "/plugins/annotation-qa/workflows/all/clip-quality", - roles: ["admin"], + roles: ["admin", "quality_reviewer"], pluginId: "annotation-qa", }, { diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 4f4b3ff6..c846051d 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -107,6 +107,13 @@ const inputClass = "w-full rounded-lg border border-[#2a2b2f] bg-[#0a0a0a] px-3 py-2 text-sm text-white outline-none focus:border-blue-500"; function taskVisibleInMode(task: AnnotationTaskRecord, mode: WorkspaceMode) { + if (task.humanStageId) { + if (mode === "clip") return task.humanStageId === "split_annotation" || task.humanStageId === "cut_adjustment"; + if (mode === "clip-quality") + return task.humanStageId === "split_quality_review" || task.humanStageId === "cut_review"; + if (mode === "annotate") return task.humanStageId === "semantic_annotation"; + if (mode === "quality") return task.humanStageId === "quality_review" || task.humanStageId === "semantic_review"; + } if (mode === "clip") { return ( (task.status === "pending_annotation" && task.currentRevision === 0) || @@ -365,6 +372,10 @@ export default function AnnotationOperations() { null, [detail?.task.templateId, templates], ); + const editableTimelineStart = detail?.task.sourceIsFullVideo ? 0 : detail?.task.clipStart || 0; + const editableTimelineEnd = detail?.task.sourceIsFullVideo + ? detail?.task.sourceDurationSeconds || detail?.task.clipEnd || 0 + : detail?.task.clipEnd || 0; const workloadTotals = useMemo(() => summarizeAnnotationWorkload(workload.items), [workload.items]); const annotationStage = detail?.humanItem?.currentStageId || @@ -384,6 +395,8 @@ export default function AnnotationOperations() { [tasks, workspaceMode], ); const selectedHumanItem = detail?.humanItem; + // Old combined v5 batches remain readable while all newly created v5 work + // uses the four independent stage IDs above. const isV5Combined = Boolean( detail && detail.task.templateVersion >= 5 && selectedHumanItem?.currentStageId === "annotation_combined", ); @@ -422,8 +435,8 @@ export default function AnnotationOperations() { source: "manual", }, ], - detail.task.clipStart, - detail.task.clipEnd, + editableTimelineStart, + editableTimelineEnd, ); setSegments(next); }; @@ -590,11 +603,15 @@ export default function AnnotationOperations() { comment: reviewComment, rejectTargets: decision === "rejected" - ? stage === "split_quality" - ? ["split_annotation"] - : stage === "quality" + ? selectedHumanItem?.currentStageId === "cut_review" + ? ["cut_adjustment"] + : selectedHumanItem?.currentStageId === "semantic_review" ? ["semantic_annotation"] - : ["quality_review"] + : stage === "split_quality" + ? ["split_annotation"] + : stage === "quality" + ? ["semantic_annotation"] + : ["quality_review"] : undefined, reasonCodes: decision === "rejected" ? ["manual_review_reject"] : undefined, videoQuality: qualityDraft.videoQuality, @@ -1453,13 +1470,15 @@ export default function AnnotationOperations() {
- {annotationStage === "split_annotation" - ? "人工剪切" - : annotationStage === "split_quality_review" - ? "剪切质检" + {annotationStage === "split_annotation" || annotationStage === "cut_adjustment" + ? "人工剪切调整" + : annotationStage === "split_quality_review" || annotationStage === "cut_review" + ? "剪切审核" : annotationStage === "semantic_annotation" - ? "人工标注" - : "标注质检"}{" "} + ? "语义标注" + : annotationStage === "semantic_review" + ? "标注审核" + : "标注质检"}{" "} · Revision {detail.task.currentRevision}
{selectedHumanItem && @@ -1501,6 +1520,11 @@ export default function AnnotationOperations() { src={buildBrowserPlayableVideoURL(detail.task.sourceVideo)} controls preload="metadata" + onLoadedMetadata={(event) => { + if (detail.task.sourceIsFullVideo) { + event.currentTarget.currentTime = Math.max(0, detail.task.clipStart); + } + }} /> )}
@@ -1599,8 +1623,10 @@ export default function AnnotationOperations() { ) : ( ; createdAt: string; updatedAt: string; diff --git a/tests/backend/test_delivery_pose_contracts.py b/tests/backend/test_delivery_pose_contracts.py index 6e713183..6cc9c3a1 100644 --- a/tests/backend/test_delivery_pose_contracts.py +++ b/tests/backend/test_delivery_pose_contracts.py @@ -163,15 +163,9 @@ def test_v5_pose_free_package_writes_only_video_semantic_and_manifests(tmp_path) "annotation": {"actions": [{"labelId": "06", "label": "热食加工与出餐", "start": 0, "end": 1}]}, }], } - training = { - "schemaVersion": "droboticflow.qwen3vl.continuous-training/v2", - "status": "succeeded", - "adapter": {}, - "provenance": {}, - } package_helpers = helpers( params={"REQUIRE_HEADPOSE": "false", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}, - payloads=[accepted, training], + payloads=[accepted], ) def write_json(run_id, node_id, output_dir, name, data, output=None): @@ -196,3 +190,61 @@ def test_v5_pose_free_package_writes_only_video_semantic_and_manifests(tmp_path) segment = json.loads((package_dir / "segment.json").read_text(encoding="utf-8")) assert segment["semantic"]["file"] == "semantic.json" assert "headpose" not in segment and "handpose" not in segment + package_manifest = json.loads((tmp_path / "output" / "packages-manifest.json").read_text(encoding="utf-8")) + assert package_manifest["trainingEnabled"] is False + assert "trainingProvenance" not in package_manifest + + +def test_v5_virtual_rough_cut_materializes_full_source_then_cuts_final_range(tmp_path): + full_source = tmp_path / "full-sanitized.mp4" + full_source.write_bytes(b"real-full-sanitized-video") + canonical = '{"confirmed":true}' + accepted = { + "schemaVersion": "droboticflow.annotation.accepted-partition/v1", + "annotationCount": 1, + "items": [{ + "clipId": "rough-final-001", + "clipName": "rough-final-001.mp4", + "clipPath": "bos://testxyz/sanitized/full.mp4", + "source": {"path": "bos://testxyz/sanitized/full.mp4"}, + "sourceIsFullVideo": True, + "startSecond": 8.5, + "endSecond": 11.25, + "sourceStartSecond": 8.5, + "sourceEndSecond": 11.25, + "finalAcceptanceConfirmed": True, + "finalAcceptance": { + "canonical": canonical, + "sha256": hashlib.sha256(canonical.encode("utf-8")).hexdigest(), + "confirmedByUserId": "acceptance-user", + }, + "annotation": {"actions": []}, + }], + } + observed = {} + package_helpers = helpers( + params={"REQUIRE_HEADPOSE": "false", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}, + payloads=[accepted], + ) + + def write_json(run_id, node_id, output_dir, name, data, output=None): + path = pathlib.Path(output_dir) / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + return {"name": name, "data": data, "path": str(path)} + + def cut(source, destination, start, end): + observed.update(source=pathlib.Path(source), start=start, end=end) + pathlib.Path(destination).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + return {"duration": end - start} + + package_helpers.update({ + "write_json_artifact": write_json, + "source_path_from_item": lambda item, name, output, archives=None: full_source, + "write_video_segment_with_ffmpeg": cut, + "write_video_segment": cut, + "inspect_video_file": lambda path: {"duration": 2.75}, + }) + package_operator.run("run", "package", tmp_path / "virtual-output", {}, [], package_helpers) + assert observed == {"source": full_source, "start": 8.5, "end": 11.25} diff --git a/tests/backend/test_qwen_behavior_rough_cut.py b/tests/backend/test_qwen_behavior_rough_cut.py new file mode 100644 index 00000000..48ad678b --- /dev/null +++ b/tests/backend/test_qwen_behavior_rough_cut.py @@ -0,0 +1,242 @@ +"""Unit contracts for the PTS-grounded Qwen behaviour rough-cut builtin. + +These tests deliberately exercise only parsing, matching and virtual-artifact +invariants. They do not stand in for the real CUDA Qwen3-VL acceptance run. +""" +from __future__ import annotations + +import json +import pathlib +import tempfile +import unittest + +from backend.operators.builtin.qwen_behavior_rough_cut import operator + + +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +class TestQwenBehaviorRoughCut(unittest.TestCase): + def test_reference_script_sha_and_no_legacy_temporal_parameters(self) -> None: + source = (ROOT / "backend/operators/builtin/qwen_behavior_rough_cut/operator.py").read_text(encoding="utf-8") + self.assertIn("2cf5fdb34fc11911dbcdbd4c51965d9cfe17239f862ebd9cad2e18078f90afc2", source) + self.assertNotIn('"WINDOW_SECONDS"', source) + self.assertNotIn('"STRIDE_SECONDS"', source) + self.assertNotIn('"OVERLAP_SECONDS"', source) + self.assertIn("technicalOnly", source) + self.assertIn("sourceIsFullVideo", source) + + def test_pts_csv_requires_strictly_increasing_timestamp_us(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + valid = root / "capture_camera_left_part0001_pts.csv" + valid.write_text("timestamp_us\n1000000\n1040000\n1080000\n", encoding="utf-8") + self.assertEqual(operator._read_pts_csv(valid), [1000000, 1040000, 1080000]) + + duplicate = root / "duplicate_pts.csv" + duplicate.write_text("timestamp_us\n1000000\n1000000\n", encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, "strictly increasing"): + operator._read_pts_csv(duplicate) + + def test_model_times_must_be_visible_pts_and_map_from_pts_origin(self) -> None: + mapped = operator._normalize_behavior( + { + "behavior": "员工拿起商品", + "object_change": "商品从货架移到手中", + "start_timestamp": "PTS_US=1040000", + "end_timestamp": "PTS_US=1120000", + }, + {1040000, 1120000}, + {1000000: 0, 1040000: 1, 1080000: 2, 1120000: 3}, + 1000000, + ) + self.assertEqual((mapped["start_second"], mapped["end_second"]), (0.04, 0.12)) + self.assertEqual((mapped["start_frame_index"], mapped["end_frame_index"]), (1, 3)) + with self.assertRaisesRegex(RuntimeError, "displayed frames"): + operator._normalize_behavior( + { + "behavior": "员工拿起商品", + "object_change": "", + "start_timestamp": "PTS_US=1000000", + "end_timestamp": "PTS_US=1120000", + }, + {1040000, 1120000}, + {1000000: 0, 1040000: 1, 1080000: 2, 1120000: 3}, + 1000000, + ) + with self.assertRaisesRegex(RuntimeError, "exact PTS_US overlay"): + operator._parse_overlay_timestamp("1120000") + + def test_adjacent_same_behavior_merges_without_duration_heuristic(self) -> None: + events = [ + { + "behavior": "拿起商品", + "object_change": "商品离开货架", + "start_timestamp": "PTS_US=1000000", + "end_timestamp": "PTS_US=1100000", + "start_us": 1000000, + "end_us": 1100000, + "start_frame_index": 0, + "end_frame_index": 1, + "start_second": 0.0, + "end_second": 0.1, + "technical_block_indexes": [1], + "raw_event_indexes": [1], + "raw_response_sha256": "a", + }, + { + "behavior": "拿起商品", + "object_change": "商品在手中", + "start_timestamp": "PTS_US=1200000", + "end_timestamp": "PTS_US=1300000", + "start_us": 1200000, + "end_us": 1300000, + "start_frame_index": 2, + "end_frame_index": 3, + "start_second": 0.2, + "end_second": 0.3, + "technical_block_indexes": [2], + "raw_event_indexes": [2], + "raw_response_sha256": "b", + }, + { + "behavior": "放下商品", + "object_change": "商品回到台面", + "start_timestamp": "PTS_US=1400000", + "end_timestamp": "PTS_US=1500000", + "start_us": 1400000, + "end_us": 1500000, + "start_frame_index": 4, + "end_frame_index": 5, + "start_second": 0.4, + "end_second": 0.5, + "technical_block_indexes": [2], + "raw_event_indexes": [3], + "raw_response_sha256": "b", + }, + ] + merged = operator._merge_adjacent_behaviors(events) + self.assertEqual(len(merged), 2) + self.assertEqual(merged[0]["object_change"], "商品离开货架") + self.assertEqual(merged[0]["end_us"], 1300000) + self.assertEqual(merged[0]["technical_block_indexes"], [1, 2]) + self.assertEqual(merged[0]["raw_event_indexes"], [1, 2]) + + def test_technical_blocks_are_explicitly_not_clip_boundaries(self) -> None: + blocks = operator._technical_blocks(251, 25.0, 10.0, 18) + self.assertEqual(len(blocks), 2) + self.assertTrue(all(block["technicalOnly"] for block in blocks)) + self.assertTrue(all("startSecond" not in block and "endSecond" not in block for block in blocks)) + self.assertEqual(len(blocks[0]["sampleFrameIndices"]), 18) + + def test_face_blur_video_pairs_pts_from_orbbec_metadata(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + video = root / "blurred_capture_camera_left_part0001.mp4" + video.write_bytes(b"real-sanitized-video-bytes") + pts = root / "capture_camera_left_part0001_pts.csv" + pts.write_text("timestamp_us\n1\n2\n", encoding="utf-8") + inputs = [ + { + "id": "blurred-video-artifact", + "name": video.name, + "path": str(video), + }, + { + "name": "blurred-manifest.json", + "data": { + "schemaVersion": "droboticflow.video_face_blur.manifest/v1", + "items": [ + { + "filename": video.name, + "sanitizedFilename": video.name, + "relativePath": "capture_camera_left_part0001.mp4", + "pairedFiles": {"pts": "blurred_capture_camera_left_part0001_pts.csv"}, + } + ], + }, + }, + { + "name": "orbbec-normalized-manifest.json", + "data": { + "schemaVersion": "droboticflow.orbbec.normalized/v1", + "datasetType": "orbbec-ego", + "entries": [{"relativePath": pts.name, "path": str(pts), "source": {"path": str(pts)}}], + }, + }, + ] + helpers = { + "payloads_from_inputs": lambda values: [item["data"] for item in values if isinstance(item.get("data"), dict)], + "source_ref_from_artifact": lambda artifact: {"path": artifact["path"]}, + "source_path_from_item": lambda item, _name, _output, _archive: pathlib.Path(item.get("path") or item["source"]["path"]), + } + contexts = operator._source_contexts(inputs, root, helpers) + self.assertEqual(len(contexts), 1) + self.assertEqual(contexts[0]["sourcePath"], video) + self.assertEqual(contexts[0]["ptsPath"], pts) + self.assertEqual(contexts[0]["ptsOrigin"], "orbbec-normalized-manifest") + + def test_virtual_clip_keeps_full_source_path_and_ref(self) -> None: + with tempfile.TemporaryDirectory() as directory: + source = pathlib.Path(directory) / "blurred_full.mp4" + source.write_bytes(b"sanitized-video") + source_artifact = { + "id": "face-blur-video", + "name": source.name, + "path": "bos://accepted-bucket/outputs/blurred_full.mp4", + "sizeBytes": source.stat().st_size, + "metadata": {"sha256": "a" * 64, "mimeType": "video/mp4"}, + "payload": { + "mode": "ref", + "type": "video", + "ref": {"provider": "BOS", "connectionId": "conn-1", "bucket": "accepted-bucket", "key": "outputs/blurred_full.mp4", "kind": "object"}, + }, + } + captured: dict[str, object] = {} + + def build(*args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + return {"name": args[2], "path": args[3], "data": kwargs["data"]} + + clip = { + "clipId": "rough-1", + "clipName": "rough-1.mp4", + "startSecond": 1.0, + "endSecond": 2.0, + "sourceDurationSeconds": 25.0, + "behavior": "拿起商品", + "objectChange": "商品离开货架", + } + artifact = operator._virtual_clip_artifact( + "run-1", + "rough", + {"id": "out-1"}, + source_artifact, + source, + clip, + {"build_virtual_artifact": build, "source_ref_from_artifact": lambda value: {"path": value["path"]}}, + ) + self.assertEqual(artifact["path"], source_artifact["path"]) + self.assertTrue(artifact["data"]["sourceIsFullVideo"]) + self.assertEqual(artifact["data"]["sourceDurationSeconds"], 25.0) + self.assertEqual(artifact["data"]["ref"], source_artifact["payload"]["ref"]) + self.assertEqual(artifact["payload"], source_artifact["payload"]) + + def test_catalog_has_only_technical_block_parameters_and_expected_ports(self) -> None: + catalog = json.loads((ROOT / "internal/api/builtin_operators_catalog.json").read_text(encoding="utf-8")) + item = next(value for value in catalog if value.get("id") == "operator-builtin-qwen-behavior-rough-cut") + params = {str(value["key"]) for value in item["hyperparams"]} + self.assertEqual(item["publicationStatus"], "draft") + self.assertEqual([value["id"] for value in item["inputs"]], ["video", "metadata"]) + self.assertEqual(item["outputs"][0]["id"], "out-1") + self.assertTrue({"TECHNICAL_BLOCK_SECONDS", "FRAMES_PER_TECHNICAL_BLOCK"}.issubset(params)) + self.assertFalse({"WINDOW_SECONDS", "STRIDE_SECONDS", "OVERLAP_SECONDS", "SEGMENT_DURATION_SECONDS"} & params) + + def test_runner_registers_dedicated_handler(self) -> None: + runner = (ROOT / "backend/operator-runner/main.py").read_text(encoding="utf-8") + self.assertIn('"qwen_behavior_rough_cut": operator_handler(\'backend.operators.builtin.qwen_behavior_rough_cut.operator\')', runner) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/frontend/annotation-operations.test.mjs b/tests/frontend/annotation-operations.test.mjs index 2c8d121c..d2242870 100644 --- a/tests/frontend/annotation-operations.test.mjs +++ b/tests/frontend/annotation-operations.test.mjs @@ -64,4 +64,6 @@ test("annotation operations uses the draggable video timeline and workflow deep assert.match(timeline, /resize-end/); assert.match(timeline, /拖动片段/); assert.match(timeline, /buildBrowserPlayableVideoURL/); + assert.match(timeline, /sourceIsFullVideo/); + assert.match(page, /sourceIsFullVideo=\{detail\.task\.sourceIsFullVideo\}/); }); diff --git a/tests/frontend/fanyu-v5-four-stage.test.mjs b/tests/frontend/fanyu-v5-four-stage.test.mjs new file mode 100644 index 00000000..7cd1425c --- /dev/null +++ b/tests/frontend/fanyu-v5-four-stage.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +const annotationURL = new URL("../../src/pages/AnnotationOperations.tsx", import.meta.url); +const sidebarURL = new URL("../../src/components/Sidebar.tsx", import.meta.url); +const appURL = new URL("../../src/App.tsx", import.meta.url); + +test("v5 queues each supplier task in exactly its configured human stage", async () => { + const source = await readFile(annotationURL, "utf8"); + assert.match(source, /task\.humanStageId === "cut_adjustment"/); + assert.match(source, /task\.humanStageId === "cut_review"/); + assert.match(source, /task\.humanStageId === "semantic_annotation"/); + assert.match(source, /task\.humanStageId === "semantic_review"/); + assert.match(source, /selectedHumanItem\?\.currentStageId === "cut_review"[\s\S]*\["cut_adjustment"\]/); + assert.match(source, /selectedHumanItem\?\.currentStageId === "semantic_review"[\s\S]*\["semantic_annotation"\]/); +}); + +test("annotator and reviewer each have the required cut workbench route", async () => { + const sidebar = await readFile(sidebarURL, "utf8"); + const app = await readFile(appURL, "utf8"); + assert.match(sidebar, /label: "人工剪切",[\s\S]*roles: \["admin", "annotator"\]/); + assert.match(sidebar, /label: "剪切质检",[\s\S]*roles: \["admin", "quality_reviewer"\]/); + assert.match(app, /path\.endsWith\("\/annotate"\) \|\| path\.endsWith\("\/clip"\)/); +});