diff --git a/Dockerfile.headpose-service b/Dockerfile.headpose-service index c3ab558b..fbcb13d5 100644 --- a/Dockerfile.headpose-service +++ b/Dockerfile.headpose-service @@ -13,6 +13,8 @@ ENV HEADPOSE_MODEL_PATH=/opt/models/6DRepNet_300W_LP_AFLW2000.pth \ PYTHONUNBUFFERED=1 USER root +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \ + && rm -rf /var/lib/apt/lists/* COPY services/headpose/requirements.txt /app/services/headpose/requirements.txt RUN if [ "$HEADPOSE_REUSE_BASE_RUNTIME" = "true" ]; then \ HEADPOSE_MODEL_PATH="$HEADPOSE_MODEL_PATH" HEADPOSE_MODEL_SHA256="$HEADPOSE_MODEL_SHA256" python -c 'import hashlib, os, pathlib; path=pathlib.Path(os.environ["HEADPOSE_MODEL_PATH"]); assert path.is_file(), f"base runtime model missing: {path}"; actual=hashlib.sha256(path.read_bytes()).hexdigest(); expected=os.environ["HEADPOSE_MODEL_SHA256"].removeprefix("sha256:"); assert actual == expected, f"base runtime weight digest mismatch: {actual}"; import cv2, torch, sixdrepnet'; \ diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py index afe9c6ff..113af001 100644 --- a/backend/operator-runner/test_main.py +++ b/backend/operator-runner/test_main.py @@ -359,6 +359,7 @@ class RuntimeFileContractTests(unittest.TestCase): self.assertEqual(MODULE.select_handler("DexYCB 数据集校验"), "dexycb_dataset_validate") self.assertEqual(MODULE.select_handler("HO3D/H2O3D 数据集校验"), "ho3d_dataset_validate") self.assertEqual(MODULE.select_handler("WiLoR 3D手部姿态估计"), "wilor_hand_pose") + self.assertEqual(MODULE.select_handler("WiLoR 手部预标(标手辅助)"), "wilor_hand_pose") self.assertEqual(MODULE.select_handler("Hand Removal 视频去手"), "hand_removal") self.assertEqual(MODULE.select_handler("手-物体姿态标注校验"), "hand_object_pose_validate") self.assertEqual(MODULE.select_handler("PartNet-Mobility 资产校验"), "partnet_mobility_asset_validate") diff --git a/backend/operators/builtin/head_pose/operator.py b/backend/operators/builtin/head_pose/operator.py index 9505365e..74768dfe 100644 --- a/backend/operators/builtin/head_pose/operator.py +++ b/backend/operators/builtin/head_pose/operator.py @@ -249,6 +249,10 @@ def run( "frames": normalized_frames, "summary": _summary(normalized_frames), "annotatedMedia": annotated_media.name, + "rendering": response.get("rendering") or { + "kind": "headpose_axis_overlay", + "browserPlayable": annotated_media.suffix.lower() in {".mp4", ".jpg", ".jpeg", ".png"}, + }, "runtimeEvidence": runtime_evidence, } binding = helpers["output_binding"](node_data) diff --git a/backend/operators/builtin/package/operator.py b/backend/operators/builtin/package/operator.py index 11a2a23b..89569663 100644 --- a/backend/operators/builtin/package/operator.py +++ b/backend/operators/builtin/package/operator.py @@ -423,13 +423,14 @@ def _run_fanyu_v5(run_id: str, node_id: str, output_dir: Any, node_data: dict[st if handpose_segment is not None: helpers["write_json_artifact"](run_id, node_id, package_dir, "handpose.json", handpose_segment, output=binding) segment_manifest["handpose"] = {"file": "handpose.json", "frameCount": handpose_segment["frameCount"]} + segment_manifest["handPrelabel"] = {"file": "handpose.json", "frameCount": handpose_segment["frameCount"], "reviewRequired": True} helpers["write_json_artifact"](run_id, node_id, package_dir, "segment.json", segment_manifest, output=binding) source_video = _source_video_name(item) quality = {"clip_id": clip_id, "source_video": source_video, **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {})} package_items.append({"packageName": package_dir.name, "clipId": clip_id, "clipName": video_path.name, "rightClipName": right_video_manifest["name"] if right_video_manifest else "", "sourceVideo": source_video, "sourceRightVideo": right_video_manifest["sourceVideo"] if right_video_manifest else "", "stereo": right_video_manifest is not None, "storagePath": str(package_dir), "startSecond": start, "endSecond": end, "clipDuration": end - start, "duration": end - start, "fileCount": len([path for path in package_dir.iterdir() if path.is_file()]), "quality": quality, "annotation": annotation, "segmentManifest": segment_manifest}) if not package_items: raise RuntimeError("package produced zero final segment packages") - manifest = {"schemaVersion": "droboticflow.delivery.manifest/v3", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "stereoDelivery": all(item["stereo"] for item in package_items), "headposeIncluded": headpose is not None, "handposeIncluded": handpose is not None and handpose.get("enabled", True), "items": package_items, "trainingEnabled": False} + manifest = {"schemaVersion": "droboticflow.delivery.manifest/v3", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "stereoDelivery": all(item["stereo"] for item in package_items), "headposeIncluded": headpose is not None, "handposeIncluded": handpose is not None and handpose.get("enabled", True), "handPrelabelIncluded": handpose is not None and handpose.get("enabled", True), "items": package_items, "trainingEnabled": False} artifacts = [helpers["write_json_artifact"](run_id, node_id, output, "packages-manifest.json", manifest, output=binding)] artifacts.extend(_collect_unique_package_artifacts(run_id, node_id, output / "packages", node_data, binding, helpers)) return artifacts @@ -594,7 +595,7 @@ def _run_multiport( manifest_path = output / "packages-manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else {} params = helpers["hyperparams"](node_data) - summary = {"packageCount": int(manifest.get("packageCount", 0) or 0), "trainingEnabled": False, "poseFree": _bool(params, "POSE_FREE", False), "stereoDelivery": bool(manifest.get("stereoDelivery")), "headposeIncluded": bool(manifest.get("headposeIncluded")), "handposeIncluded": bool(manifest.get("handposeIncluded")), "finalAcceptanceRequired": _bool(params, "REQUIRE_FINAL_ACCEPTANCE", False)} + summary = {"packageCount": int(manifest.get("packageCount", 0) or 0), "trainingEnabled": False, "poseFree": _bool(params, "POSE_FREE", False), "stereoDelivery": bool(manifest.get("stereoDelivery")), "headposeIncluded": bool(manifest.get("headposeIncluded")), "handposeIncluded": bool(manifest.get("handposeIncluded")), "handPrelabelIncluded": bool(manifest.get("handPrelabelIncluded", manifest.get("handposeIncluded"))), "finalAcceptanceRequired": _bool(params, "REQUIRE_FINAL_ACCEPTANCE", False)} result.append(helpers["write_json_artifact"](run_id, node_id, output, "delivery-summary.json", summary, output=_output(node_data, helpers, "deliverySummary"))) return result diff --git a/backend/operators/builtin/wilor_hand_pose/operator.py b/backend/operators/builtin/wilor_hand_pose/operator.py index 08940184..5e6eda38 100644 --- a/backend/operators/builtin/wilor_hand_pose/operator.py +++ b/backend/operators/builtin/wilor_hand_pose/operator.py @@ -655,6 +655,9 @@ def run( { "schemaVersion": "droboticflow.wilor.hand_pose_result/v1", "operator": "wilor_hand_pose", + "presentation": "hand_annotation_prelabel", + "taskType": "hand_annotation", + "requiresHumanReview": True, "enabled": False, "status": "skipped_by_delivery_options", "items": [], @@ -736,6 +739,9 @@ def run( manifest = { "schemaVersion": "droboticflow.wilor.hand_pose_result/v1", "operator": "wilor_hand_pose", + "presentation": "hand_annotation_prelabel", + "taskType": "hand_annotation", + "requiresHumanReview": True, "enabled": True, "status": result.get("status"), "mediaKind": media_kind, diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index f1f5a8b9..d11e6713 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -3615,7 +3615,7 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "调用 digest 固定的 6DRepNet 服务,对图片或视频输出逐帧头部 yaw/pitch/roll、置信度和运行证据。", + "description": "调用 digest 固定的 6DRepNet 服务,输出逐帧头部 yaw/pitch/roll、置信度及 H.264 姿态轴叠加视频,作为人工质检的 HeadPose 预标证据。", "hyperparams": [ { "description": "6DRepNet 服务地址", @@ -3696,10 +3696,10 @@ "type": "json" } ], - "name": "6DRepNet Headpose", + "name": "6DRepNet HeadPose 预标(带渲染)", "outputs": [ { - "description": "droboticflow.headpose/v1 manifest", + "description": "droboticflow.headpose/v1 预标 manifest 与姿态轴叠加媒体", "id": "out-1", "name": "headpose", "transmission": "bus", @@ -3752,7 +3752,7 @@ ], "runtimeHandler": "builtin_head_pose", "runtimeSource": "operator-repos/builtin/head_pose/src/operator.py", - "skillDefinition": "真实调用 6DRepNet 服务,输出逐帧 yaw/pitch/roll、置信度、标注媒体引用和服务镜像证据。", + "skillDefinition": "真实调用 6DRepNet 服务,输出逐帧 yaw/pitch/roll、置信度、H.264 姿态轴叠加媒体和服务镜像证据,并将可视化结果交给人工质检。", "type": "builtin" }, { @@ -7679,7 +7679,7 @@ "value": "false" }, { - "description": "是否强制要求 WiLoR HandPose", + "description": "是否强制要求已经进入人工质检的 WiLoR 手部预标(兼容键名)", "key": "REQUIRE_HANDPOSE", "type": "boolean", "value": "false" @@ -7691,7 +7691,7 @@ "value": "false" }, { - "description": "是否启用 pose-free 契约;启用后拒绝 HeadPose 和 HandPose 输入", + "description": "是否启用 pose-free 契约;启用后拒绝 HeadPose 和手部预标输入", "key": "POSE_FREE", "type": "boolean", "value": "false" @@ -7747,7 +7747,7 @@ }, { "container": "single", - "description": "6DRepNet HeadPose 逐帧结果", + "description": "已经进入人工质检的 6DRepNet HeadPose 预标结果", "id": "headpose", "name": "headpose", "payloadMode": "ref", @@ -7757,7 +7757,7 @@ }, { "container": "single", - "description": "WiLoR HandPose 逐帧结果", + "description": "已经进入人工质检的 WiLoR 手部预标(标手辅助,兼容端口名)", "id": "handpose", "name": "handpose", "payloadMode": "ref", @@ -7830,7 +7830,7 @@ }, { "container": "collection", - "description": "按最终片段裁切的 HandPose JSON 集合", + "description": "按最终片段裁切的手部预标 JSON 集合(兼容端口名)", "id": "handposeFiles", "name": "handposeFiles", "payloadMode": "ref", @@ -12800,7 +12800,7 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "调用 WiLoR GPU 推理服务,对图片或视频执行 3D 手部姿态估计,并输出标注图、标注视频、关键点 JSON 和运行证据。", + "description": "调用 WiLoR GPU 推理服务生成手部位置与姿态预标、叠加视频和关键点证据,作为标手人工质检的辅助输入,不可绕过人工审核直接视为最终标注。", "hyperparams": [ { "description": "WiLoR 推理服务地址", @@ -12863,7 +12863,7 @@ "value": "119" }, { - "description": "分块覆盖完整视频并拒绝任何截断分块,确保后段候选区间也有 HandPose 数据", + "description": "分块覆盖完整视频并拒绝任何截断分块,确保后段候选区间也有手部预标数据", "key": "REQUIRE_FULL_VIDEO_COVERAGE", "type": "boolean", "value": "true" @@ -12922,10 +12922,10 @@ "type": "json" } ], - "name": "WiLoR 3D手部姿态估计", + "name": "WiLoR 手部预标(标手辅助)", "outputs": [ { - "description": "WiLoR 结果 manifest、标注媒体和关键点 JSON", + "description": "供标手质检使用的 WiLoR 预标 manifest、叠加媒体和关键点 JSON", "id": "out-1", "name": "wilorResult", "transmission": "bus", @@ -12980,7 +12980,7 @@ }, "runtimeHandler": "builtin_wilor_hand_pose", "runtimeSource": "operator-repos/builtin/wilor_hand_pose/src/operator.py", - "skillDefinition": "WiLoR 3D 手部姿态估计算子;读取图片或视频 URL/上游媒体 artifact,调用已部署的 WiLoR GPU 服务,输出标注图或标注视频、关键点 JSON、mesh OBJ 和服务端 CUDA/A800 运行证据。", + "skillDefinition": "WiLoR 手部预标(标手辅助)算子;读取图片或视频引用,调用已部署的 WiLoR GPU 服务生成手部位置、姿态、关键点和叠加媒体,结果必须进入人工标手质检后才能交付。", "type": "builtin" }, { diff --git a/internal/api/data_platform.go b/internal/api/data_platform.go index 883df4de..a02423e0 100644 --- a/internal/api/data_platform.go +++ b/internal/api/data_platform.go @@ -1308,11 +1308,12 @@ func wilorPackageItem(run domain.RunRecord, manifest domain.ExecutionArtifact, p } } annotation := map[string]any{ - "type": "wilor_hand_pose", - "mediaKind": mediaKind, - "handCount": intValue(result["handCount"]), - "frameCount": intValue(result["frameCount"]), - "status": stringValue(payload["status"]), + "type": "hand_annotation_prelabel", + "mediaKind": mediaKind, + "handCount": intValue(result["handCount"]), + "frameCount": intValue(result["frameCount"]), + "status": stringValue(payload["status"]), + "requiresHumanReview": true, } if task := mapValue(result["wilorTask"]); task != nil { annotation["task"] = task @@ -1330,7 +1331,7 @@ func wilorPackageItem(run domain.RunRecord, manifest domain.ExecutionArtifact, p "sourceVideo": sourceName, "clipDuration": 0, "fileCount": len(rawFiles), - "pipelineTags": []any{"wilor", "hand_pose", mediaKind}, + "pipelineTags": []any{"wilor", "hand_prelabel", mediaKind}, "storagePath": storagePath, "quality": quality, "annotation": annotation, diff --git a/internal/api/fanyu_template.go b/internal/api/fanyu_template.go index 3c53bae2..948e1fcd 100644 --- a/internal/api/fanyu_template.go +++ b/internal/api/fanyu_template.go @@ -19,6 +19,7 @@ const fanyuDeliveryTemplateV5ID = "template-fanyu-left-camera-delivery-v5" const fanyuDeliveryTemplateV54ID = "template-fanyu-left-camera-delivery-v5-4" const fanyuDeliveryTemplateV55ID = "template-fanyu-left-camera-delivery-v5-5" const fanyuDeliveryTemplateV56ID = "template-fanyu-stereo-pose-delivery-v5-6" +const fanyuDeliveryTemplateV57ID = "template-fanyu-stereo-prelabel-delivery-v5-7" func fanyuLegacyPorts(operatorID string, inputs []domain.PortDefinition, outputs []domain.PortDefinition) ([]domain.PortDefinition, []domain.PortDefinition) { legacy := func(id, name string, typ domain.PortType, required bool) domain.PortDefinition { @@ -176,6 +177,9 @@ func (server *server) ensureFanyuDeliveryTemplate(ctx context.Context, organizat if err := server.ensureFanyuDeliveryTemplateV56(ctx, organizationID); err != nil { return err } + if err := server.ensureFanyuDeliveryTemplateV57(ctx, organizationID); err != nil { + return err + } return server.ensureFanyuConvenienceStoreAnnotationTemplates(ctx, organizationID) } @@ -730,6 +734,107 @@ func (server *server) ensureFanyuDeliveryTemplateV56(ctx context.Context, organi return server.store.UpsertWorkflowRelease(ctx, release) } +// ensureFanyuDeliveryTemplateV57 corrects the business meaning of the pose +// branches: WiLoR is hand-annotation prelabel evidence (标手辅助), while +// 6DRepNet is a rendered HeadPose prelabel. Both are reviewed in the human gate +// before their per-segment data is included in the delivery package. +func (server *server) ensureFanyuDeliveryTemplateV57(ctx context.Context, organizationID string) error { + organizationID = normalizedOrganizationID(organizationID) + base, err := server.store.GetTemplate(ctx, fanyuDeliveryTemplateV56ID+"-"+organizationID) + if err != nil || base == nil { + return err + } + templateID := fanyuDeliveryTemplateV57ID + "-" + organizationID + releaseID := "workflow-release-fanyu-v5-7-" + organizationID + nodes := append([]domain.WorkflowNode(nil), base.Nodes...) + for index := range nodes { + node := &nodes[index] + switch node.ID { + case "fanyu-v56-headpose": + node.Data.Label = "6DRepNet HeadPose 预标(带渲染)" + node.Data.Description = "生成逐帧 HeadPose 角度和浏览器可播放的姿态轴叠加视频,作为人工质检证据。" + if operator, operatorErr := server.store.GetOperator(ctx, "operator-builtin-head-pose"); operatorErr != nil { + return operatorErr + } else if operator != nil { + node.Data.OperatorImageID = operator.ImageID + node.Data.Image = operator.Image + node.Data.SkillDefinition = operator.SkillDefinition + node.Data.PublicationStatus = operator.PublicationStatus + } + case "fanyu-v56-handpose": + node.Data.Label = "WiLoR 手部预标(标手辅助)" + node.Data.Description = "使用 WiLoR 生成手部位置与姿态预标和叠加视频,结果必须进入人工质检,不能直接作为最终标手结果。" + if operator, operatorErr := server.store.GetOperator(ctx, "operator-builtin-wilor-hand-pose"); operatorErr != nil { + return operatorErr + } else if operator != nil { + node.Data.OperatorImageID = operator.ImageID + node.Data.Image = operator.Image + node.Data.SkillDefinition = operator.SkillDefinition + node.Data.PublicationStatus = operator.PublicationStatus + } + case "fanyu-v56-human": + node.Data.Label = "人工调切→语义/标手质检→聚合确认" + node.Data.Description = "人工完成调切、剪切审核和语义标注,并对 HeadPose 与手部预标渲染结果进行质检后聚合确认。" + case "fanyu-v56-package": + node.Data.Label = "双目同步裁切 + 已质检预标交付" + } + } + edges := append([]domain.WorkflowEdge(nil), base.Edges...) + edges = append(edges, + fanyuEdge("fanyu-v57-e22", "fanyu-v56-headpose", "out-1", "fanyu-v56-human", "headpose"), + fanyuEdge("fanyu-v57-e23", "fanyu-v56-handpose", "out-1", "fanyu-v56-human", "handpose"), + ) + now := nowISO() + createdAt := now + if existing, existingErr := server.store.GetTemplate(ctx, templateID); existingErr != nil { + return existingErr + } else if existing != nil && strings.TrimSpace(existing.CreatedAt) != "" { + createdAt = existing.CreatedAt + } + template := domain.TemplateRecord{ + ID: templateID, OrganizationID: organizationID, Name: "01-双目裁切与手/头预标质检交付-v5.7", + Description: "左右相机去畸变和脱敏后同步裁切;6DRepNet HeadPose 与 WiLoR 手部预标(标手辅助)生成可视化证据并进入人工质检,通过后再按最终边界裁切预标数据进入交付包。", + Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindStandard, ExecutionScope: domain.WorkflowExecutionScopeProduction, + DefinitionState: domain.WorkflowDefinitionStateDraft, DefinitionVersion: 7, DefinitionFamilyID: fanyuDeliveryTemplateV5ID + "-" + organizationID, SupersedesTemplateID: fanyuDeliveryTemplateV56ID + "-" + organizationID, + MaxConcurrentRuns: 1, WorkflowInputs: append([]domain.WorkflowInputDefinition(nil), base.WorkflowInputs...), Nodes: nodes, Edges: edges, CreatedAt: createdAt, UpdatedAt: now, + } + if issues := validateWorkflowTopology(nodes, edges); len(issues) > 0 { + return fmt.Errorf("invalid fanyu v5.7 template: %s", strings.TrimSpace(issues[0].Message)) + } + if issues := validateWorkflowInputContract(nodes, edges, template.WorkflowInputs, true); len(issues) > 0 { + return fmt.Errorf("invalid fanyu v5.7 input contract: %s", strings.TrimSpace(issues[0].Message)) + } + if err := server.store.UpsertTemplate(ctx, template); err != nil { + return err + } + if existingRelease, releaseErr := server.store.GetWorkflowRelease(ctx, releaseID); releaseErr != nil || existingRelease != nil { + return releaseErr + } + if len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { + return nil + } + for _, node := range nodes { + if node.ID != "fanyu-v56-human" { + continue + } + for _, parameter := range node.Data.Hyperparams { + if parameter.Key == "ANNOTATION_WORKGROUP_ID" && strings.TrimSpace(fmt.Sprint(parameter.Value)) == "" { + return nil + } + } + } + release := domain.WorkflowReleaseRecord{ + ID: releaseID, OrganizationID: organizationID, TemplateID: template.ID, Version: 7, Name: template.Name, Description: template.Description, + Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, + WorkflowInputs: template.WorkflowInputs, Nodes: nodes, Edges: edges, + Capabilities: []string{"directory_input", "multiport_artifact_contract", "stereo_opencv_undistort", "stereo_yunet_face_anonymization", "qwen3vl_behavior_rough_cut", "human_cut_adjustment", "human_cut_review", "human_semantic_annotation", "headpose_prelabel_render", "hand_annotation_prelabel", "human_prelabel_review", "human_semantic_review", "human_acceptance", "stereo_synchronized_crop", "stereo_reviewed_prelabel_package"}, + RequiredPluginIDs: []string{pluginregistry.HumanLoopCoreID, pluginregistry.AnnotationQAID}, ContentDigest: workflowReleaseDigest(template), + ReleaseNotes: "v5.7 reclassifies WiLoR as hand-annotation prelabel evidence, adds browser-rendered HeadPose evidence, and routes both prelabels through human quality review before packaging", + PublishedByUserID: "system", PublishedAt: now, CreatedAt: now, + } + return server.store.UpsertWorkflowRelease(ctx, release) +} + func (server *server) ensureFanyuDeliveryTemplateV3(ctx context.Context, organizationID string) error { templateID := fanyuDeliveryTemplateV3ID + "-" + normalizedOrganizationID(organizationID) releaseID := "workflow-release-fanyu-v3-" + normalizedOrganizationID(organizationID) diff --git a/internal/api/fanyu_template_test.go b/internal/api/fanyu_template_test.go index 01656867..d21ee6d5 100644 --- a/internal/api/fanyu_template_test.go +++ b/internal/api/fanyu_template_test.go @@ -485,4 +485,28 @@ func TestEnsureFanyuDeliveryTemplateV56BuildsStereoCropAndPoseTopology(t *testin t.Fatalf("v5.6 missing edge %s", name) } } + v57, v57Err := dataStore.GetTemplate(ctx, fanyuDeliveryTemplateV57ID+"-org-v56") + if v57Err != nil || v57 == nil { + t.Fatalf("v5.7 template=%#v err=%v", v57, v57Err) + } + if v57.DefinitionVersion != 7 || v57.SupersedesTemplateID != fanyuDeliveryTemplateV56ID+"-org-v56" || len(v57.Nodes) != 10 || len(v57.Edges) != 23 { + t.Fatalf("unexpected v5.7 prelabel topology: %#v", v57) + } + v57HandPrelabel := fanyuTestNode(v57.Nodes, "fanyu-v56-handpose") + v57Headpose := fanyuTestNode(v57.Nodes, "fanyu-v56-headpose") + if v57HandPrelabel == nil || !strings.Contains(v57HandPrelabel.Data.Label, "手部预标") || v57Headpose == nil || !strings.Contains(v57Headpose.Data.Label, "带渲染") { + t.Fatalf("v5.7 must present hand/head outputs as reviewed prelabels: hand=%#v head=%#v", v57HandPrelabel, v57Headpose) + } + reviewEdges := map[string]bool{"head": false, "hand": false} + for _, edge := range v57.Edges { + if edge.Source == "fanyu-v56-headpose" && edge.Target == "fanyu-v56-human" && edge.TargetHandle == "headpose" { + reviewEdges["head"] = true + } + if edge.Source == "fanyu-v56-handpose" && edge.Target == "fanyu-v56-human" && edge.TargetHandle == "handpose" { + reviewEdges["hand"] = true + } + } + if !reviewEdges["head"] || !reviewEdges["hand"] { + t.Fatalf("v5.7 prelabels must enter human review: %#v", reviewEdges) + } } diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index 2ce236f3..bd195fda 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -1464,8 +1464,8 @@ func acceptanceArtifactRefs(value any) []map[string]any { // ensureFanyuV5PrePackageAcceptance creates exactly one vendor-scoped, // partition-level acceptance item after every source item has passed the cut -// and semantic-review loop. Pose stages are intentionally out of scope for -// this workflow; aggregate acceptance covers only final cuts and annotations. +// and semantic-review loop. Optional hand/head prelabels and their rendered +// evidence are carried into this confirmation step for human quality review. func (server *server) ensureFanyuV5PrePackageAcceptance(ctx context.Context, source *domain.HumanBatchRecord) (bool, error) { if source == nil || !source.RequirePrePackageAcceptance { return true, nil @@ -1512,6 +1512,31 @@ func (server *server) ensureFanyuV5PrePackageAcceptance(ctx context.Context, sou "headposeEnabled": false, "handposeEnabled": false, "headposeFrameCount": 0, "handposeFrameCount": 0, "headposeAvailable": false, "handposeAvailable": false, + "handPrelabelEnabled": false, "handPrelabelAvailable": false, "handPrelabelFrameCount": 0, + "headposeArtifacts": []map[string]any{}, "handPrelabelArtifacts": []map[string]any{}, + "headposeRenderArtifacts": []map[string]any{}, "handPrelabelRenderArtifacts": []map[string]any{}, + } + seenPoseArtifacts := map[string]map[string]struct{}{ + "headposeArtifacts": {}, + "handPrelabelArtifacts": {}, + "headposeRenderArtifacts": {}, + "handPrelabelRenderArtifacts": {}, + } + mergePoseRefs := func(poseSummary map[string]any, sourceKey string, targetKey string, countKey string) { + for _, ref := range acceptanceArtifactRefs(poseSummary[sourceKey]) { + artifactID := strings.TrimSpace(fmt.Sprint(ref["artifactId"])) + if artifactID == "" { + continue + } + if _, exists := seenPoseArtifacts[targetKey][artifactID]; exists { + continue + } + seenPoseArtifacts[targetKey][artifactID] = struct{}{} + statistics[targetKey] = append(statistics[targetKey].([]map[string]any), ref) + if countKey != "" { + statistics[countKey] = acceptanceEvidenceCount(statistics[countKey]) + acceptanceEvidenceCount(ref["frameCount"]) + } + } } for _, humanItem := range items { if strings.TrimSpace(humanItem.LegacyTaskID) == "" { @@ -1539,7 +1564,24 @@ func (server *server) ensureFanyuV5PrePackageAcceptance(ctx context.Context, sou } statistics["sliceCount"] = statistics["sliceCount"].(int) + len(latestSlices.Segments) statistics["annotationSegmentCount"] = statistics["annotationSegmentCount"].(int) + len(latestRevision.Segments) - aggregate = append(aggregate, map[string]any{"taskId": task.ID, "runId": task.RunID, "clipArtifactId": task.ArtifactID, "sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "finalSlices": latestSlices.Segments, "annotation": latestRevision.Segments}) + poseSummary := task.PoseSummary + for _, key := range []string{"headposeEnabled", "headposeAvailable", "handposeEnabled", "handposeAvailable", "handPrelabelEnabled", "handPrelabelAvailable"} { + if enabled, _ := poseSummary[key].(bool); enabled { + statistics[key] = true + } + } + mergePoseRefs(poseSummary, "headposeArtifacts", "headposeArtifacts", "headposeFrameCount") + handPrelabelKey := "handPrelabelArtifacts" + if len(acceptanceArtifactRefs(poseSummary[handPrelabelKey])) == 0 { + handPrelabelKey = "handposeArtifacts" + } + mergePoseRefs(poseSummary, handPrelabelKey, "handPrelabelArtifacts", "handPrelabelFrameCount") + mergePoseRefs(poseSummary, "headposeRenderArtifacts", "headposeRenderArtifacts", "") + mergePoseRefs(poseSummary, "handPrelabelRenderArtifacts", "handPrelabelRenderArtifacts", "") + statistics["handposeEnabled"] = statistics["handPrelabelEnabled"] + statistics["handposeAvailable"] = statistics["handPrelabelAvailable"] + statistics["handposeFrameCount"] = statistics["handPrelabelFrameCount"] + aggregate = append(aggregate, map[string]any{"taskId": task.ID, "runId": task.RunID, "clipArtifactId": task.ArtifactID, "sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "finalSlices": latestSlices.Segments, "annotation": latestRevision.Segments, "poseSummary": poseSummary}) } now := nowISO() acceptanceItem = &domain.HumanItemRecord{ diff --git a/internal/api/human_loop_test.go b/internal/api/human_loop_test.go index efd7b53e..9a4e06c8 100644 --- a/internal/api/human_loop_test.go +++ b/internal/api/human_loop_test.go @@ -45,8 +45,17 @@ func TestFanyuV5PrePackageAcceptanceAggregatesMultipleTasksBeforeRelease(t *test if err := dataStore.UpsertHumanBatch(ctx, batch); err != nil { t.Fatal(err) } + poseSummary := map[string]any{ + "headposeEnabled": true, "headposeAvailable": true, "headposeFrameCount": 2, + "headposeArtifacts": []map[string]any{{"artifactId": "headpose-manifest", "frameCount": 2}}, + "headposeRenderArtifacts": []map[string]any{{"artifactId": "headpose-render", "name": "headpose-annotated.mp4"}}, + "handposeEnabled": true, "handposeAvailable": true, "handposeFrameCount": 3, + "handPrelabelEnabled": true, "handPrelabelAvailable": true, "handPrelabelFrameCount": 3, + "handPrelabelArtifacts": []map[string]any{{"artifactId": "hand-prelabel-manifest", "frameCount": 3}}, + "handPrelabelRenderArtifacts": []map[string]any{{"artifactId": "hand-prelabel-render", "name": "wilor-annotated-video.mp4"}}, + } for index := 1; index <= 2; index++ { - task := domain.AnnotationTaskRecord{ID: "task-v5-" + string(rune('0'+index)), OrganizationID: domain.DefaultOrganizationID, ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, NodeID: batch.GateNodeID, ArtifactID: "clip-artifact-" + string(rune('0'+index)), SourceVideo: "clip.mp4", ClipStart: 0, ClipEnd: 10, TemplateID: "tpl-v5", TemplateVersion: 5, WorkgroupID: "wg-v5", VendorID: "vendor-v5", Status: domain.AnnotationTaskStatusAccepted, CurrentRevision: 1} + task := domain.AnnotationTaskRecord{ID: "task-v5-" + string(rune('0'+index)), OrganizationID: domain.DefaultOrganizationID, ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, NodeID: batch.GateNodeID, ArtifactID: "clip-artifact-" + string(rune('0'+index)), SourceVideo: "clip.mp4", ClipStart: 0, ClipEnd: 10, TemplateID: "tpl-v5", TemplateVersion: 5, WorkgroupID: "wg-v5", VendorID: "vendor-v5", Status: domain.AnnotationTaskStatusAccepted, CurrentRevision: 1, PoseSummary: poseSummary} if err := dataStore.UpsertAnnotationTask(ctx, task); err != nil { t.Fatal(err) } @@ -84,8 +93,16 @@ func TestFanyuV5PrePackageAcceptanceAggregatesMultipleTasksBeforeRelease(t *test t.Fatalf("aggregate should contain both clip tasks, got %#v", aggregateItem.Subject.Payload) } statistics, _ := aggregateItem.Subject.Payload["statistics"].(map[string]any) - if statistics["headposeEnabled"] != false || statistics["handposeEnabled"] != false || acceptanceEvidenceCount(statistics["headposeFrameCount"]) != 0 || acceptanceEvidenceCount(statistics["handposeFrameCount"]) != 0 { - t.Fatalf("pose-free aggregate statistics=%#v", statistics) + if statistics["headposeEnabled"] != true || statistics["handPrelabelEnabled"] != true || acceptanceEvidenceCount(statistics["headposeFrameCount"]) != 2 || acceptanceEvidenceCount(statistics["handPrelabelFrameCount"]) != 3 { + t.Fatalf("prelabel aggregate statistics=%#v", statistics) + } + if len(acceptanceArtifactRefs(statistics["headposeRenderArtifacts"])) != 1 || len(acceptanceArtifactRefs(statistics["handPrelabelRenderArtifacts"])) != 1 { + t.Fatalf("rendered prelabel evidence must be de-duplicated: %#v", statistics) + } + firstEvidence, _ := items[0].(map[string]any) + firstPose, _ := firstEvidence["poseSummary"].(map[string]any) + if firstEvidence == nil || firstPose == nil || len(acceptanceArtifactRefs(firstPose["headposeArtifacts"])) != 1 || len(acceptanceArtifactRefs(firstPose["headposeRenderArtifacts"])) != 1 || len(acceptanceArtifactRefs(firstPose["handPrelabelArtifacts"])) != 1 || len(acceptanceArtifactRefs(firstPose["handPrelabelRenderArtifacts"])) != 1 { + t.Fatalf("per-task pose evidence must be preserved: %#v", items) } aggregateItem.Status = domain.HumanItemAccepted aggregateItem.AcceptedAt = nowISO() diff --git a/internal/worker/annotation_gate.go b/internal/worker/annotation_gate.go index 1fa7196a..c1bb4ad2 100644 --- a/internal/worker/annotation_gate.go +++ b/internal/worker/annotation_gate.go @@ -257,19 +257,46 @@ func annotationPrelabels(inputs []domain.ExecutionArtifact, template domain.Anno return result } -// annotationPoseSummary keeps only scoped evidence references/counts in the -// human task. The full manifests remain artifacts and are later cropped by the -// package operator; acceptance never receives an unbounded pose stream. +// annotationPoseSummary keeps scoped prelabel evidence references/counts in the +// human task. The full manifests remain artifacts and are later cropped by the +// package operator; reviewers receive browser-renderable evidence rather than +// an unbounded pose stream. func annotationPoseSummary(inputs []domain.ExecutionArtifact) map[string]any { summary := map[string]any{ - "headposeAvailable": false, "handposeAvailable": false, - "headposeFrameCount": 0, "handposeFrameCount": 0, + "headposeEnabled": false, "headposeAvailable": false, "headposeFrameCount": 0, + "handposeEnabled": false, "handposeAvailable": false, "handposeFrameCount": 0, + "handPrelabelEnabled": false, "handPrelabelAvailable": false, "handPrelabelFrameCount": 0, "headposeInvalid": false, "headposeArtifacts": []map[string]any{}, "handposeArtifacts": []map[string]any{}, + "handPrelabelArtifacts": []map[string]any{}, + "headposeRenderArtifacts": []map[string]any{}, "handPrelabelRenderArtifacts": []map[string]any{}, } seenHeadposeArtifacts := map[string]struct{}{} seenHandposeArtifacts := map[string]struct{}{} for _, artifact := range inputs { + identity := strings.ToLower(strings.TrimSpace(artifact.NodeID + " " + artifact.Name + " " + artifact.Path)) + mimeType := "" + if artifact.Metadata != nil { + mimeType = strings.ToLower(strings.TrimSpace(artifact.Metadata.MimeType)) + } + isVideo := artifact.Type == domain.PortTypeVideo || strings.HasPrefix(mimeType, "video/") || strings.HasSuffix(identity, ".mp4") + if isVideo && (strings.Contains(identity, "headpose") || strings.Contains(identity, "head-pose")) { + ref := map[string]any{"artifactId": artifact.ID, "name": artifact.Name, "path": artifact.Path, "kind": "rendered_video"} + if artifact.Metadata != nil && strings.TrimSpace(artifact.Metadata.SHA256) != "" { + ref["sha256"] = artifact.Metadata.SHA256 + } + summary["headposeEnabled"] = true + summary["headposeRenderArtifacts"] = append(summary["headposeRenderArtifacts"].([]map[string]any), ref) + } + if isVideo && (strings.Contains(identity, "wilor") || strings.Contains(identity, "handpose") || strings.Contains(identity, "hand-pose")) { + ref := map[string]any{"artifactId": artifact.ID, "name": artifact.Name, "path": artifact.Path, "kind": "rendered_video"} + if artifact.Metadata != nil && strings.TrimSpace(artifact.Metadata.SHA256) != "" { + ref["sha256"] = artifact.Metadata.SHA256 + } + summary["handposeEnabled"] = true + summary["handPrelabelEnabled"] = true + summary["handPrelabelRenderArtifacts"] = append(summary["handPrelabelRenderArtifacts"].([]map[string]any), ref) + } payload := annotationArtifactPayload(artifact) if payload == nil { continue @@ -281,6 +308,7 @@ func annotationPoseSummary(inputs []domain.ExecutionArtifact) map[string]any { } switch schema { case "droboticflow.headpose/v1": + summary["headposeEnabled"] = true if enabled, _ := payload["enabled"].(bool); !enabled { continue } @@ -299,6 +327,8 @@ func annotationPoseSummary(inputs []domain.ExecutionArtifact) map[string]any { summary["headposeFrameCount"] = summary["headposeFrameCount"].(int) + len(frames) summary["headposeArtifacts"] = append(summary["headposeArtifacts"].([]map[string]any), ref) case "droboticflow.wilor.hand_pose_result/v1": + summary["handposeEnabled"] = true + summary["handPrelabelEnabled"] = true if enabled, exists := payload["enabled"].(bool); exists && !enabled { continue } @@ -309,11 +339,16 @@ func annotationPoseSummary(inputs []domain.ExecutionArtifact) map[string]any { summary["handposeAvailable"] = true result, _ := payload["result"].(map[string]any) if count, ok := annotationNumber(result["frameCount"]); ok { - summary["handposeFrameCount"] = summary["handposeFrameCount"].(int) + int(count) + frameCount := int(count) + ref["frameCount"] = frameCount + summary["handposeFrameCount"] = summary["handposeFrameCount"].(int) + frameCount } summary["handposeArtifacts"] = append(summary["handposeArtifacts"].([]map[string]any), ref) } } + summary["handPrelabelAvailable"] = summary["handposeAvailable"] + summary["handPrelabelFrameCount"] = summary["handposeFrameCount"] + summary["handPrelabelArtifacts"] = summary["handposeArtifacts"] return summary } diff --git a/internal/worker/human_release_test.go b/internal/worker/human_release_test.go index ba2b789a..1ae1f0eb 100644 --- a/internal/worker/human_release_test.go +++ b/internal/worker/human_release_test.go @@ -131,13 +131,24 @@ func TestAnnotationPoseSummaryRequiresNonEmptyVerifiedHeadposeFrames(t *testing. if annotationPoseSummaryHasRequiredHeadpose(empty) { t.Fatalf("empty Headpose evidence must fail closed: %#v", empty) } - valid := annotationPoseSummary([]domain.ExecutionArtifact{{ - ID: "headpose-valid", - Data: map[string]any{"schemaVersion": "droboticflow.headpose/v1", "enabled": true, "frameCount": 2, "frames": []any{map[string]any{"timestamp": 0.0}, map[string]any{"timestamp": 1.0}}}, - }}) + valid := annotationPoseSummary([]domain.ExecutionArtifact{ + { + ID: "headpose-valid", NodeID: "headpose", Name: "headpose-manifest.json", + Data: map[string]any{"schemaVersion": "droboticflow.headpose/v1", "enabled": true, "frameCount": 2, "frames": []any{map[string]any{"timestamp": 0.0}, map[string]any{"timestamp": 1.0}}}, + }, + {ID: "headpose-render", NodeID: "headpose", Name: "headpose-annotated.mp4", Type: domain.PortTypeVideo}, + { + ID: "hand-prelabel", NodeID: "handpose", Name: "wilor-result.json", + Data: map[string]any{"schemaVersion": "droboticflow.wilor.hand_pose_result/v1", "enabled": true, "result": map[string]any{"frameCount": 3}}, + }, + {ID: "hand-prelabel-render", NodeID: "handpose", Name: "wilor-annotated-video.mp4", Type: domain.PortTypeVideo}, + }) if !annotationPoseSummaryHasRequiredHeadpose(valid) || valid["headposeFrameCount"] != 2 { t.Fatalf("non-empty verified Headpose evidence must be retained: %#v", valid) } + if valid["handPrelabelAvailable"] != true || valid["handPrelabelFrameCount"] != 3 || len(valid["headposeRenderArtifacts"].([]map[string]any)) != 1 || len(valid["handPrelabelRenderArtifacts"].([]map[string]any)) != 1 { + t.Fatalf("rendered hand/head prelabel evidence must be retained: %#v", valid) + } } func TestFanyuV5FinalSliceCoordinatesSeparateClipAndSourceTimelines(t *testing.T) { diff --git a/operator-repos/builtin/head_pose/README.md b/operator-repos/builtin/head_pose/README.md index bea078ce..1ccb2744 100644 --- a/operator-repos/builtin/head_pose/README.md +++ b/operator-repos/builtin/head_pose/README.md @@ -1,4 +1,4 @@ -# 6DRepNet Headpose +# 6DRepNet HeadPose 预标(带渲染) 独立 builtin 算子目录。该目录是 `operator-builtin-head-pose` 的生产运行权威来源,包含镜像构建、运行契约、源码和验收输入定义。 diff --git a/operator-repos/builtin/head_pose/src/operator.py b/operator-repos/builtin/head_pose/src/operator.py index 9505365e..74768dfe 100644 --- a/operator-repos/builtin/head_pose/src/operator.py +++ b/operator-repos/builtin/head_pose/src/operator.py @@ -249,6 +249,10 @@ def run( "frames": normalized_frames, "summary": _summary(normalized_frames), "annotatedMedia": annotated_media.name, + "rendering": response.get("rendering") or { + "kind": "headpose_axis_overlay", + "browserPlayable": annotated_media.suffix.lower() in {".mp4", ".jpg", ".jpeg", ".png"}, + }, "runtimeEvidence": runtime_evidence, } binding = helpers["output_binding"](node_data) diff --git a/operator-repos/builtin/package/src/operator.py b/operator-repos/builtin/package/src/operator.py index 11a2a23b..89569663 100644 --- a/operator-repos/builtin/package/src/operator.py +++ b/operator-repos/builtin/package/src/operator.py @@ -423,13 +423,14 @@ def _run_fanyu_v5(run_id: str, node_id: str, output_dir: Any, node_data: dict[st if handpose_segment is not None: helpers["write_json_artifact"](run_id, node_id, package_dir, "handpose.json", handpose_segment, output=binding) segment_manifest["handpose"] = {"file": "handpose.json", "frameCount": handpose_segment["frameCount"]} + segment_manifest["handPrelabel"] = {"file": "handpose.json", "frameCount": handpose_segment["frameCount"], "reviewRequired": True} helpers["write_json_artifact"](run_id, node_id, package_dir, "segment.json", segment_manifest, output=binding) source_video = _source_video_name(item) quality = {"clip_id": clip_id, "source_video": source_video, **(item.get("quality", {}) if isinstance(item.get("quality"), dict) else {})} package_items.append({"packageName": package_dir.name, "clipId": clip_id, "clipName": video_path.name, "rightClipName": right_video_manifest["name"] if right_video_manifest else "", "sourceVideo": source_video, "sourceRightVideo": right_video_manifest["sourceVideo"] if right_video_manifest else "", "stereo": right_video_manifest is not None, "storagePath": str(package_dir), "startSecond": start, "endSecond": end, "clipDuration": end - start, "duration": end - start, "fileCount": len([path for path in package_dir.iterdir() if path.is_file()]), "quality": quality, "annotation": annotation, "segmentManifest": segment_manifest}) if not package_items: raise RuntimeError("package produced zero final segment packages") - manifest = {"schemaVersion": "droboticflow.delivery.manifest/v3", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "stereoDelivery": all(item["stereo"] for item in package_items), "headposeIncluded": headpose is not None, "handposeIncluded": handpose is not None and handpose.get("enabled", True), "items": package_items, "trainingEnabled": False} + manifest = {"schemaVersion": "droboticflow.delivery.manifest/v3", "datasetType": "fanyu-v5-final-segments", "packageCount": len(package_items), "stereoDelivery": all(item["stereo"] for item in package_items), "headposeIncluded": headpose is not None, "handposeIncluded": handpose is not None and handpose.get("enabled", True), "handPrelabelIncluded": handpose is not None and handpose.get("enabled", True), "items": package_items, "trainingEnabled": False} artifacts = [helpers["write_json_artifact"](run_id, node_id, output, "packages-manifest.json", manifest, output=binding)] artifacts.extend(_collect_unique_package_artifacts(run_id, node_id, output / "packages", node_data, binding, helpers)) return artifacts @@ -594,7 +595,7 @@ def _run_multiport( manifest_path = output / "packages-manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.exists() else {} params = helpers["hyperparams"](node_data) - summary = {"packageCount": int(manifest.get("packageCount", 0) or 0), "trainingEnabled": False, "poseFree": _bool(params, "POSE_FREE", False), "stereoDelivery": bool(manifest.get("stereoDelivery")), "headposeIncluded": bool(manifest.get("headposeIncluded")), "handposeIncluded": bool(manifest.get("handposeIncluded")), "finalAcceptanceRequired": _bool(params, "REQUIRE_FINAL_ACCEPTANCE", False)} + summary = {"packageCount": int(manifest.get("packageCount", 0) or 0), "trainingEnabled": False, "poseFree": _bool(params, "POSE_FREE", False), "stereoDelivery": bool(manifest.get("stereoDelivery")), "headposeIncluded": bool(manifest.get("headposeIncluded")), "handposeIncluded": bool(manifest.get("handposeIncluded")), "handPrelabelIncluded": bool(manifest.get("handPrelabelIncluded", manifest.get("handposeIncluded"))), "finalAcceptanceRequired": _bool(params, "REQUIRE_FINAL_ACCEPTANCE", False)} result.append(helpers["write_json_artifact"](run_id, node_id, output, "delivery-summary.json", summary, output=_output(node_data, helpers, "deliverySummary"))) return result diff --git a/operator-repos/builtin/wilor_hand_pose/README.md b/operator-repos/builtin/wilor_hand_pose/README.md index 0994eb83..0d5927d8 100644 --- a/operator-repos/builtin/wilor_hand_pose/README.md +++ b/operator-repos/builtin/wilor_hand_pose/README.md @@ -1,4 +1,4 @@ -# WiLoR 3D手部姿态估计 +# WiLoR 手部预标(标手辅助) 独立 builtin 算子目录。该目录是 `operator-builtin-wilor-hand-pose` 的生产运行权威来源,包含镜像构建、运行契约、源码和验收输入定义。 diff --git a/operator-repos/builtin/wilor_hand_pose/src/operator.py b/operator-repos/builtin/wilor_hand_pose/src/operator.py index 08940184..5e6eda38 100644 --- a/operator-repos/builtin/wilor_hand_pose/src/operator.py +++ b/operator-repos/builtin/wilor_hand_pose/src/operator.py @@ -655,6 +655,9 @@ def run( { "schemaVersion": "droboticflow.wilor.hand_pose_result/v1", "operator": "wilor_hand_pose", + "presentation": "hand_annotation_prelabel", + "taskType": "hand_annotation", + "requiresHumanReview": True, "enabled": False, "status": "skipped_by_delivery_options", "items": [], @@ -736,6 +739,9 @@ def run( manifest = { "schemaVersion": "droboticflow.wilor.hand_pose_result/v1", "operator": "wilor_hand_pose", + "presentation": "hand_annotation_prelabel", + "taskType": "hand_annotation", + "requiresHumanReview": True, "enabled": True, "status": result.get("status"), "mediaKind": media_kind, diff --git a/services/headpose/app.py b/services/headpose/app.py index facc3301..880d0276 100644 --- a/services/headpose/app.py +++ b/services/headpose/app.py @@ -6,6 +6,7 @@ import math import os import pathlib import shutil +import subprocess import tempfile import threading import time @@ -128,6 +129,34 @@ def predict_frame(frame: np.ndarray) -> tuple[float, float, float, float, tuple[ return float(pitch[0]), float(yaw[0]), float(roll[0]), confidence, box +def encode_browser_video(source: pathlib.Path, destination: pathlib.Path) -> None: + command = [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(source), + "-an", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(destination), + ] + completed = subprocess.run(command, capture_output=True, text=True, check=False) + if completed.returncode != 0 or not destination.is_file() or destination.stat().st_size <= 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(f"failed to encode browser-playable HeadPose render: {detail[:800]}") + + def process_video(source: pathlib.Path, destination: pathlib.Path, stride: int, max_frames: int) -> list[dict[str, Any]]: capture = cv2.VideoCapture(str(source)) if not capture.isOpened(): @@ -138,7 +167,8 @@ def process_video(source: pathlib.Path, destination: pathlib.Path, stride: int, if fps <= 0 or width <= 0 or height <= 0: capture.release() raise RuntimeError("input video metadata is invalid") - writer = cv2.VideoWriter(str(destination), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) + working_video = destination.with_name(destination.stem + "-working.mp4") + writer = cv2.VideoWriter(str(working_video), cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) if not writer.isOpened(): capture.release() raise RuntimeError("annotated video writer could not be opened") @@ -174,7 +204,12 @@ def process_video(source: pathlib.Path, destination: pathlib.Path, stride: int, capture.release() writer.release() if not results: + working_video.unlink(missing_ok=True) raise RuntimeError("no sampled video frame produced a headpose result") + try: + encode_browser_video(working_video, destination) + finally: + working_video.unlink(missing_ok=True) return results @@ -225,6 +260,12 @@ def infer(payload: InferenceRequest, request: Request) -> dict[str, Any]: return { "frames": frames, "annotatedMediaUrl": artifact_url, + "rendering": { + "kind": "headpose_axis_overlay", + "videoCodec": "h264", + "pixelFormat": "yuv420p", + "browserPlayable": True, + }, "runtimeEvidence": { "service": "droboticflow-headpose", "serviceVersion": "1.0.0", @@ -237,6 +278,8 @@ def infer(payload: InferenceRequest, request: Request) -> dict[str, Any]: "sampledFrameCount": len(frames), "latencyMs": elapsed_ms, "serviceUptimeSeconds": int(time.time() - STARTED_AT), + "renderCodec": "h264" if not is_image else "jpeg", + "browserPlayable": True, }, } diff --git a/src/lib/artifact-groups.js b/src/lib/artifact-groups.js index fcd37643..6ce17a6a 100644 --- a/src/lib/artifact-groups.js +++ b/src/lib/artifact-groups.js @@ -1,4 +1,5 @@ import { artifactExtension } from "./artifact-download.js"; +import { isPoseRenderArtifact } from "./pose-artifacts.js"; import { summarizeQualityReportArtifact } from "./report-summary.js"; export const ARTIFACT_GROUP_ORDER = ["final", "intermediate", "input"]; @@ -10,6 +11,7 @@ export const ARTIFACT_GROUP_LABELS = { }; const artifactPriority = (item) => { + if (isPoseRenderArtifact(item)) return -1; if (summarizeQualityReportArtifact(item)) return 0; const extension = artifactExtension(item); if (extension === "html" || extension === "htm" || extension === "pdf") return 1; diff --git a/src/lib/pose-artifacts.js b/src/lib/pose-artifacts.js new file mode 100644 index 00000000..25693f7c --- /dev/null +++ b/src/lib/pose-artifacts.js @@ -0,0 +1,19 @@ +const identity = (item) => `${item?.nodeId || ""} ${item?.name || ""} ${item?.path || ""}`.toLowerCase(); + +export function isPoseRenderArtifact(item) { + if (!item || typeof item !== "object") return false; + const mimeType = String(item?.metadata?.mimeType || "").toLowerCase(); + const value = identity(item); + const isVideo = item.type === "video" || mimeType.startsWith("video/") || value.includes(".mp4"); + if (!isVideo) return false; + return ( + value.includes("headpose-annotated") || + value.includes("head-pose-annotated") || + value.includes("wilor-annotated") || + value.includes("hand-prelabel") + ); +} + +export function preferredPoseRenderArtifact(items = []) { + return (Array.isArray(items) ? items : []).find(isPoseRenderArtifact) || null; +} diff --git a/src/lib/video-preview.js b/src/lib/video-preview.js index 2de43b8f..63d1ef3b 100644 --- a/src/lib/video-preview.js +++ b/src/lib/video-preview.js @@ -2,6 +2,7 @@ const SOURCE_DATA_CONTENT_PATH_PREFIX = "/api/source-data/collections/"; const PACKAGE_CONTENT_PATH_PREFIX = "/api/packages/"; const OBJECT_STORAGE_CONTENT_PATH = "/api/object-storage/content"; const RUN_ARTIFACT_CONTENT_PATH_PREFIX = "/api/runs/"; +const HUMAN_EVIDENCE_CONTENT_PATH_PREFIX = "/api/plugins/human-loop/items/"; function normalizeURL(input) { if (!input || typeof input !== "string") { @@ -19,10 +20,7 @@ export function isSourceDataContentURL(input) { if (!normalized) { return false; } - return ( - normalized.pathname.startsWith(SOURCE_DATA_CONTENT_PATH_PREFIX) && - normalized.pathname.endsWith("/content") - ); + return normalized.pathname.startsWith(SOURCE_DATA_CONTENT_PATH_PREFIX) && normalized.pathname.endsWith("/content"); } export function isPackageContentURL(input) { @@ -30,10 +28,7 @@ export function isPackageContentURL(input) { if (!normalized) { return false; } - return ( - normalized.pathname.startsWith(PACKAGE_CONTENT_PATH_PREFIX) && - normalized.pathname.endsWith("/content") - ); + return normalized.pathname.startsWith(PACKAGE_CONTENT_PATH_PREFIX) && normalized.pathname.endsWith("/content"); } export function isObjectStorageContentURL(input) { @@ -56,6 +51,16 @@ export function isRunArtifactContentURL(input) { ); } +export function isHumanEvidenceContentURL(input) { + const normalized = normalizeURL(input); + if (!normalized) return false; + return ( + normalized.pathname.startsWith(HUMAN_EVIDENCE_CONTENT_PATH_PREFIX) && + normalized.pathname.includes("/evidence/") && + normalized.pathname.endsWith("/content") + ); +} + /** * Detect a `bos://bucket/key` URL and return true. */ @@ -119,7 +124,8 @@ export function buildBrowserPlayableVideoURL(input) { (!isSourceDataContentURL(input) && !isPackageContentURL(input) && !isObjectStorageContentURL(input) && - !isRunArtifactContentURL(input)) + !isRunArtifactContentURL(input) && + !isHumanEvidenceContentURL(input)) ) { return typeof input === "string" ? input : ""; } diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 79e997db..52370bff 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -51,6 +51,7 @@ import { getActionErrorMessage } from "../lib/action-errors.js"; import { artifactContentPath } from "../lib/artifact-download.js"; import AnnotationTimelineEditor from "../components/AnnotationTimelineEditor"; import RerunArtifactPreview from "../components/RerunArtifactPreview"; +import { LazyVideoPreview } from "../components/LazyVideoPreview"; import { PageSubnav } from "../components/PageSubnav"; import { usePlugins } from "../context/PluginContext"; import { buildBrowserPlayableVideoURL } from "../lib/video-preview.js"; @@ -127,6 +128,54 @@ function deliveryActions(item: DeliveryPackageManifestItem) { return Array.isArray(item.annotation?.actions) ? item.annotation.actions : []; } +function poseEvidenceRefs(summary: Record, key: string, fallbackKey?: string) { + const direct = Array.isArray(summary[key]) ? summary[key] : []; + if (direct.length > 0 || !fallbackKey) return direct as Array>; + return (Array.isArray(summary[fallbackKey]) ? summary[fallbackKey] : []) as Array>; +} + +function PoseEvidencePreview({ + itemId, + artifact, + title, + frameCount, +}: { + itemId: string; + artifact?: Record; + title: string; + frameCount: number; +}) { + const artifactId = String(artifact?.artifactId || "").trim(); + if (!artifactId) { + return ( +
+ {title}:没有可视化预标证据 +
+ ); + } + const evidenceURL = humanLoopApi.evidenceContentURL(itemId, artifactId); + return ( +
+
+
+
{title}
+
+ {frameCount} 帧 · {String(artifact?.name || artifactId)} +
+
+ 人工质检证据 +
+ +
+ ); +} + const panelClass = "rounded-2xl border border-[#2a2b2f] bg-[#151619]"; const inputClass = "w-full rounded-lg border border-[#2a2b2f] bg-[#0a0a0a] px-3 py-2 text-sm text-white outline-none focus:border-blue-500"; @@ -1265,6 +1314,13 @@ export default function AnnotationOperations() { const evidenceItems = Array.isArray(payload.items) ? payload.items : []; const selectedIndex = selectedAcceptanceEvidence.itemId === item.id ? selectedAcceptanceEvidence.index : 0; const selectedEvidence = evidenceItems[selectedIndex] as Record | undefined; + const selectedPoseSummary = (selectedEvidence?.poseSummary || {}) as Record; + const headposeRender = poseEvidenceRefs(selectedPoseSummary, "headposeRenderArtifacts")[0]; + const handPrelabelRender = poseEvidenceRefs( + selectedPoseSummary, + "handPrelabelRenderArtifacts", + "handposeRenderArtifacts", + )[0]; const selectedClipArtifactId = String(selectedEvidence?.clipArtifactId || ""); const claimedByMe = item.assignedUserId === currentUser?.id; const acceptanceStatistics = prepackageAcceptanceDrafts[item.id] || emptyPrepackageAcceptanceStatistics; @@ -1323,7 +1379,7 @@ export default function AnnotationOperations() {
- Headpose: + HeadPose 预标: {statistics.headposeEnabled === false ? "本次未启用" : statistics.headposeAvailable @@ -1331,11 +1387,11 @@ export default function AnnotationOperations() { : "缺失"}
- Handpose: - {statistics.handposeEnabled === false + 手部预标(标手): + {statistics.handPrelabelEnabled === false && statistics.handposeEnabled === false ? "本次未启用" - : statistics.handposeAvailable - ? `${statistics.handposeFrameCount || 0} 帧` + : statistics.handPrelabelAvailable || statistics.handposeAvailable + ? `${statistics.handPrelabelFrameCount || statistics.handposeFrameCount || 0} 帧` : "缺失"}
@@ -1348,9 +1404,25 @@ export default function AnnotationOperations() { controls preload="metadata" className="max-h-72 w-full rounded-lg bg-black object-contain" - src={humanLoopApi.evidenceContentURL(item.id, selectedClipArtifactId)} + src={buildBrowserPlayableVideoURL(humanLoopApi.evidenceContentURL(item.id, selectedClipArtifactId))} /> ) : null} +
+ + +
{evidenceItems.map((evidence, index) => { const entry = evidence as Record; @@ -1363,9 +1435,9 @@ export default function AnnotationOperations() { onClick={() => setSelectedAcceptanceEvidence({ itemId: item.id, index })} className={`block w-full rounded border p-2 text-left ${selectedIndex === index ? "border-blue-500 bg-blue-500/10" : "border-[#2a2b2f]"}`} > - 切片任务 {index + 1} · {slices.length} 段 · {annotations.length} 条标签 · Headpose{" "} - {entry.poseSummary?.headposeFrameCount || 0} 帧 · Handpose{" "} - {entry.poseSummary?.handposeFrameCount || 0} 帧 + 切片任务 {index + 1} · {slices.length} 段 · {annotations.length} 条标签 · HeadPose 预标{" "} + {entry.poseSummary?.headposeFrameCount || 0} 帧 · 标手预标{" "} + {entry.poseSummary?.handPrelabelFrameCount || entry.poseSummary?.handposeFrameCount || 0} 帧 ); })} @@ -1566,8 +1638,17 @@ export default function AnnotationOperations() { ["标注准确性", acceptanceStatistics?.annotationAccuracy ?? "-"], ["标注完整性", acceptanceStatistics?.annotationCompleteness ?? "-"], ["总体等级", acceptanceStatistics?.overallGrade ?? overallGrade], - ["HeadPose", acceptanceStatistics?.headposeEnabled === false ? "未启用" : "-"], - ["HandPose", acceptanceStatistics?.handposeEnabled === false ? "未启用" : "-"], + [ + "HeadPose 预标", + acceptanceStatistics?.headposeEnabled === false ? "未启用" : "已进入人工质检", + ], + [ + "手部预标(标手)", + acceptanceStatistics?.handPrelabelEnabled === false && + acceptanceStatistics?.handposeEnabled === false + ? "未启用" + : "已进入人工质检", + ], ].map(([label, value]) => (
{label}
diff --git a/src/pages/ProjectDetail.tsx b/src/pages/ProjectDetail.tsx index 892f74d3..4d7e2569 100644 --- a/src/pages/ProjectDetail.tsx +++ b/src/pages/ProjectDetail.tsx @@ -1776,7 +1776,9 @@ export default function ProjectDetail() { {input.id === "deliveryOptions" ? (
@@ -562,7 +574,8 @@ const OperatorNode = memo(({ data, selected }: NodeProps) => {
{port.name} - {portDisplayLabel(port)} · {effectivePortContract(port).payloadMode}{port.role ? ` · ${port.role}` : ""} + {portDisplayLabel(port)} · {effectivePortContract(port).payloadMode} + {port.role ? ` · ${port.role}` : ""}
selectedEdgeArtifacts.find((artifact) => artifactSelectionKey(artifact) === selectedEdgeArtifactId) || + preferredPoseRenderArtifact(selectedEdgeArtifacts) || selectedEdgeArtifacts[0] || null, [selectedEdgeArtifactId, selectedEdgeArtifacts], @@ -1709,10 +1723,7 @@ export default function WorkflowEditor() { const selectedEdgeArtifactIsManifest = isCollectionArtifact(selectedEdgeArtifact); const selectedEdgeArtifactTransferMode = artifactTransferMode(selectedEdgeArtifact); const selectedEdgeManifestArtifactCount = useMemo( - () => - selectedEdgeArtifacts.filter( - (artifact) => isCollectionArtifact(artifact), - ).length, + () => selectedEdgeArtifacts.filter((artifact) => isCollectionArtifact(artifact)).length, [selectedEdgeArtifacts], ); useEffect(() => { @@ -1726,7 +1737,9 @@ export default function WorkflowEditor() { ) { return; } - setSelectedEdgeArtifactId(artifactSelectionKey(selectedEdgeArtifacts[0])); + setSelectedEdgeArtifactId( + artifactSelectionKey(preferredPoseRenderArtifact(selectedEdgeArtifacts) || selectedEdgeArtifacts[0]), + ); }, [selectedEdge, selectedEdgeArtifactId, selectedEdgeArtifacts]); useEffect(() => { if (!selectedEdgeArtifact) { @@ -7837,7 +7850,8 @@ export default function WorkflowEditor() { newInputs[idx] = { ...port, type: e.target.value, - payloadMode: e.target.value === "json" ? effectivePortContract(port).payloadMode : "ref", + payloadMode: + e.target.value === "json" ? effectivePortContract(port).payloadMode : "ref", }; updateNodeData({ inputs: newInputs }); }} @@ -7947,7 +7961,8 @@ export default function WorkflowEditor() { newOutputs[idx] = { ...port, type: e.target.value, - payloadMode: e.target.value === "json" ? effectivePortContract(port).payloadMode : "ref", + payloadMode: + e.target.value === "json" ? effectivePortContract(port).payloadMode : "ref", }; updateNodeData({ outputs: newOutputs, @@ -8092,8 +8107,12 @@ export default function WorkflowEditor() {
{activeRun?.id || "未加载"}
-
{selectedEdgeArtifactIsManifest ? "实际内容" : "顶层产物"}
-
{selectedEdgeArtifactIsManifest ? "引用集合" : selectedEdgeArtifacts.length}
+
+ {selectedEdgeArtifactIsManifest ? "实际内容" : "顶层产物"} +
+
+ {selectedEdgeArtifactIsManifest ? "引用集合" : selectedEdgeArtifacts.length} +
传输方式
@@ -8114,15 +8133,21 @@ export default function WorkflowEditor() { onClick={() => setSelectedEdgeArtifactId(artifactSelectionKey(artifact))} className={cn( "w-full rounded-lg border px-3 py-2 text-left transition-colors", - artifactSelectionKey(selectedEdgeArtifact) === artifactSelectionKey(artifact) + artifactSelectionKey(selectedEdgeArtifact) === + artifactSelectionKey(artifact) ? "border-blue-500/50 bg-blue-500/10" : "border-[#2a2b2f] bg-[#0a0a0a] hover:border-blue-500/30", )} >
-
{artifact.name}
-
{artifact.portName || selectedEdgePortLabel} · {artifact.type.toUpperCase()}
+
+ {artifact.name} +
+
+ {artifact.portName || selectedEdgePortLabel} ·{" "} + {artifact.type.toUpperCase()} +
{artifact.size}
@@ -8133,38 +8158,42 @@ export default function WorkflowEditor() { )}
- {selectedEdgeArtifactIsManifest && selectedEdgeArtifact && selectedEdgeArtifactRunID ? ( -
- - - -
清单成员已作为引用内容展示,技术元数据默认收起。
-
- ) : selectedEdgeArtifact && selectedEdgeArtifactTransferMode === "inline" ? ( - - ) : ( -
-
-
引用内容
- + {selectedEdgeArtifactIsManifest && + selectedEdgeArtifact && + selectedEdgeArtifactRunID ? ( +
+ + + +
清单成员已作为引用内容展示,技术元数据默认收起。
- -
暂不支持预览的引用可通过受保护下载操作获取。
-
- )} + ) : selectedEdgeArtifact && selectedEdgeArtifactTransferMode === "inline" ? ( + + ) : ( +
+
+
引用内容
+ +
+ +
+ 暂不支持预览的引用可通过受保护下载操作获取。 +
+
+ )}
) : ( @@ -8416,7 +8445,7 @@ export default function WorkflowEditor() {