diff --git a/backend/operators/builtin/annotation_gate/operator.py b/backend/operators/builtin/annotation_gate/operator.py index 438bff28..d57bc213 100644 --- a/backend/operators/builtin/annotation_gate/operator.py +++ b/backend/operators/builtin/annotation_gate/operator.py @@ -16,9 +16,9 @@ def _run_legacy( ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) template_id = params.get("ANNOTATION_TEMPLATE_ID", "").strip() - workgroup_id = params.get("ANNOTATION_WORKGROUP_ID", "").strip() - if not template_id or not workgroup_id: - raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID 和 ANNOTATION_WORKGROUP_ID") + supplier_id = params.get("ANNOTATION_SUPPLIER_ID", "").strip() + if not template_id: + raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID") payloads = helpers["payloads_from_inputs"](inputs) items = helpers["collect_items"](payloads) if not items: @@ -34,7 +34,7 @@ def _run_legacy( "runId": run_id, "nodeId": node_id, "templateId": template_id, - "workgroupId": workgroup_id, + "supplierId": supplier_id, "itemCount": len(items), "items": items, "interceptedByWorker": False, @@ -62,9 +62,9 @@ def _run_multiport( ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) template_id = params.get("ANNOTATION_TEMPLATE_ID", "").strip() - workgroup_id = params.get("ANNOTATION_WORKGROUP_ID", "").strip() - if not template_id or not workgroup_id: - raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID 和 ANNOTATION_WORKGROUP_ID") + supplier_id = params.get("ANNOTATION_SUPPLIER_ID", "").strip() + if not template_id: + raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID") # Only the Qwen ``clips`` port creates human work. HeadPose rendering is # also a video input in v5.7, but it is review evidence, not another clip # to cut, annotate, accept, and package. @@ -86,12 +86,12 @@ def _run_multiport( annotations = { "datasetType": "annotation-gate-input", "templateId": template_id, - "workgroupId": workgroup_id, + "supplierId": supplier_id, "itemCount": len(clips), "items": prelabels, } rejected = {"datasetType": "annotation-gate-rejected", "itemCount": 0, "items": []} - summary = {"clipCount": len(clips), "prelabelCount": len(prelabels), "boundaryCount": len(boundaries), "templateId": template_id, "workgroupId": workgroup_id} + summary = {"clipCount": len(clips), "prelabelCount": len(prelabels), "boundaryCount": len(boundaries), "templateId": template_id, "supplierId": supplier_id} return [ *clips, write(run_id, node_id, output_dir, "annotation-gate-annotations-dataset.json", annotations, output=_output(node_data, helpers, "annotations")), diff --git a/deploy/fanyu-v5-s100/values.env.example b/deploy/fanyu-v5-s100/values.env.example index d3f6ac5b..a18ea499 100644 --- a/deploy/fanyu-v5-s100/values.env.example +++ b/deploy/fanyu-v5-s100/values.env.example @@ -1,5 +1,5 @@ # No secrets belong in this file. Inject credentials via the platform secret store. -FANYU_V5_ANNOTATION_WORKGROUP_ID= +FANYU_V5_SUPPLIER_ID= FANYU_HEADPOSE_API_BASE_URL= FANYU_HEADPOSE_SERVICE_IMAGE_ID= FANYU_HEADPOSE_MODEL_SHA256= diff --git a/docs/annotation-operations.md b/docs/annotation-operations.md index dd206660..ae30961c 100644 --- a/docs/annotation-operations.md +++ b/docs/annotation-operations.md @@ -10,7 +10,7 @@ - `cutter`:只保留历史 v5.7 任务兼容,新建 v5.8 项目不再分配该角色。 - `vendor_manager`:只聚合查看所管供应商/工作组的工作量和质量,不能读取任务媒体或执行审核。 -每个成员关系同时绑定 `projectId`、`workgroupId` 和 `vendorId`。除管理员、流程工程师和兼容管理员外,后端只返回当前用户所在分组的任务、模板、分组和工时事件;前端隐藏不是安全边界。 +每个成员关系绑定 `projectId`、可选 `supplierId` 和三类项目角色。除组织管理员外,后端只返回当前用户所属项目和供应商范围内的任务、模板和工时事件;前端隐藏不是安全边界。 每个工作台均有“可领取”“我的任务”池;管理员额外拥有“全部任务”,并可通过带乐观锁版本的 `PATCH /api/human-items/{id}/assignment` 将任务改派给符合当前阶段角色和工作组成员关系的用户,或退回共享池。所有领取、释放、改派、提交、审核和打回都会追加 HumanEvent/StageAttempt 审计记录。 @@ -38,7 +38,7 @@ 人工审核节点配置 `REVIEW_MODE=annotation` 后成为原生标注门,另外需要: - `ANNOTATION_TEMPLATE_ID`:当前项目中已发布的模板。 -- `ANNOTATION_WORKGROUP_ID`:目标供应商分组。 +- `ANNOTATION_SUPPLIER_ID`:可选供应商 ID;留空时按项目成员关系确定。 - 新任务不接受 `ANNOTATION_ASSIGNED_USER_ID` 预分配;成员从 HumanItem 待领取池主动领取。 Worker 到达该节点后不会启动 Kubernetes Job,也不会创建通用 AuditTask。它会从上游真实视频与 Qwen 边界创建 AnnotationTask,并写入 `projectId`、`dataTaskId`、`workflowId`、`runId`、`nodeId`、`artifactId`。Run 保持 `waiting_for_audit`,直到该节点下所有任务依次完成:标注员真实切分与语义标注 → 审核员统一审核 → 最终验收员整包验收。验收通过后系统自动释放 Package child Run。 diff --git a/docs/deployment.md b/docs/deployment.md index 9b1a989d..c0123b69 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -77,7 +77,7 @@ export HEADPOSE_SERVICE_IMAGE_ID=ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow/d export HEADPOSE_MODEL_SHA256=sha256:9bdcd0e072df0d21e92288b40ea1d1b3cf7173c44728ab9c45c4d673f361e018 # Required to publish the Fanyu v5 left-camera workflow release. Obtain this # from the verified POC annotation workgroup; do not invent an ID. -export FANYU_V5_ANNOTATION_WORKGROUP_ID= +export FANYU_V5_SUPPLIER_ID= export OPERATOR_GPU_NODE_SELECTOR=accelerator=nvidia export OPERATOR_GPU_TOLERATIONS_JSON='[{"key":"nvidia.com/gpu","operator":"Exists","effect":"NoSchedule"}]' export OPERATOR_GPU_RUNTIME_CLASS=nvidia diff --git a/docs/fanyu-v5-workflow.md b/docs/fanyu-v5-workflow.md index f7b36603..eb673bb6 100644 --- a/docs/fanyu-v5-workflow.md +++ b/docs/fanyu-v5-workflow.md @@ -49,5 +49,5 @@ python3 scripts/verify_qwen3vl_model_manifest.py --model-root /workspace/models/ 1. 独立构建并发布 `directory-scan`、`orbbec-data-normalize`、`orbbec-left-undistort`、`video-face-blur`、`qwen-behavior-rough-cut` 与 `package` 镜像,写入 digest-pinned release manifest;YuNet 资产必须随 `video-face-blur` 镜像离线携带并通过 SHA/size/license 门禁。 2. 预置并校验 Qwen3-VL 本地快照;当前无模型或未校验 manifest 即为预期 fail-closed 状态。 3. 为 Qwen3-VL 语义粗切腾出 1 张 GPU;目录读取、人工阶段与打包不申请 GPU。 -4. 提供 `FANYU_V5_ANNOTATION_WORKGROUP_ID`;本版本不读取 `FANYU_HEADPOSE_*`。 +4. 可选提供 `FANYU_V5_SUPPLIER_ID`;留空时按项目成员关系确定供应商。本版本不读取 `FANYU_HEADPOSE_*`。 5. 仅在 `dev/poc` 验收环境中部署 `dev`,S100 使用同一镜像 digest 和以上 PVC/模型配置,不把草稿算子直接标为已发布。 diff --git a/docs/operators/原生视频标注-Gate.md b/docs/operators/原生视频标注-Gate.md index e3b55c05..7428d166 100644 --- a/docs/operators/原生视频标注-Gate.md +++ b/docs/operators/原生视频标注-Gate.md @@ -65,7 +65,7 @@ flowchart LR ## 关键超参数 - `ANNOTATION_TEMPLATE_ID`:类型 `string`,默认值 空字符串或由工作流显式填写。当前项目中已发布的语义模板 ID 通常建议在工作流中显式配置。 -- `ANNOTATION_WORKGROUP_ID`:类型 `string`,默认值 空字符串或由工作流显式填写。目标供应商工作组 ID 通常建议在工作流中显式配置。 +- `ANNOTATION_SUPPLIER_ID`:类型 `string`,可选。留空时由项目成员关系确定供应商。 - `REQUIRE_PREPACKAGE_ACCEPTANCE`:类型 `boolean`,默认值 `false`。全部切片统一质检通过后创建一个聚合的包前验收任务;验收仅允许确认 可直接使用默认值,也可在节点级覆盖。 ## 执行前准备 @@ -93,7 +93,7 @@ flowchart LR "category": "HUMAN", "hyperparams": { "ANNOTATION_TEMPLATE_ID": "<按节点填写>", - "ANNOTATION_WORKGROUP_ID": "<按节点填写>", + "ANNOTATION_SUPPLIER_ID": "<可选供应商 ID>", "REQUIRE_PREPACKAGE_ACCEPTANCE": "false" } } diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index 5721c583..f1155c2c 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -30,7 +30,7 @@ func annotationManager(principal *authenticatedPrincipal) bool { if principal == nil { return false } - return principal.User.Role == domain.UserRoleAdmin + return isOrganizationAdmin(principal) } func annotationPrincipalUserID(principal *authenticatedPrincipal) string { @@ -71,15 +71,43 @@ func (server *server) annotationMembershipsForUser(ctx context.Context, organiza filtered = append(filtered, item) } } + projectMembers, err := server.store.ListProjectMembers(ctx, organizationID, projectID) + if err != nil { + return nil, err + } + for _, member := range projectMembers { + if member.UserID != userID { + continue + } + for _, role := range member.Roles { + filtered = append(filtered, domain.ProjectAnnotationMembershipRecord{ + ID: "project-member:" + member.ID + ":" + string(role), + OrganizationID: member.OrganizationID, + ProjectID: member.ProjectID, + UserID: member.UserID, + VendorID: member.SupplierID, + AssignmentRole: role, + CreatedAt: member.CreatedAt, + UpdatedAt: member.UpdatedAt, + }) + } + } return filtered, nil } func membershipAllowsTask(memberships []domain.ProjectAnnotationMembershipRecord, task domain.AnnotationTaskRecord) bool { for _, membership := range memberships { - if membership.ProjectID != task.ProjectID || membership.WorkgroupID != task.WorkgroupID { + if membership.ProjectID != task.ProjectID { continue } - if membership.VendorID != "" && task.VendorID != "" && membership.VendorID != task.VendorID { + taskSupplierID := task.SupplierID + if taskSupplierID == "" { + taskSupplierID = task.VendorID + } + if membership.VendorID != "" && taskSupplierID != "" && membership.VendorID != taskSupplierID { + continue + } + if membership.WorkgroupID != "" && task.WorkgroupID != "" && membership.WorkgroupID != task.WorkgroupID { continue } return true @@ -764,7 +792,7 @@ func (server *server) handleAnnotationTaskCreate(writer http.ResponseWriter, req taskMode, isTemp := server.resolveTaskRunMode(request.Context(), item.RunID) _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ ID: domain.NewID("annotation-worklog"), OrganizationID: item.OrganizationID, ProjectID: item.ProjectID, - TaskID: item.ID, UserID: annotationPrincipalUserID(principal), WorkgroupID: item.WorkgroupID, VendorID: item.VendorID, + TaskID: item.ID, UserID: annotationPrincipalUserID(principal), SupplierID: item.SupplierID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID, EventType: "created", Mode: taskMode, Temporary: isTemp, TemplateVersion: item.TemplateVersion, OutputDelta: 0, CreatedAt: nowISO(), }) renderJSON(writer, http.StatusCreated, item) @@ -1049,7 +1077,7 @@ func (server *server) handleCombinedAnnotationSubmission(writer http.ResponseWri if annotationRevision.CountsAsOutput { outputDelta = 1 } - _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "combined_submitted", Mode: taskMode, Temporary: isTemp, Revision: annotationRevision.Revision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: outputDelta, CreatedAt: now}) + _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "combined_submitted", Mode: taskMode, Temporary: isTemp, Revision: annotationRevision.Revision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: outputDelta, CreatedAt: now}) status := http.StatusCreated if resumed { status = http.StatusOK @@ -1295,7 +1323,7 @@ func (server *server) handleSliceRevisionCreate(writer http.ResponseWriter, requ _ = server.appendHumanEvent(request.Context(), *item, "submitted", principal.User.ID, map[string]any{"submissionRef": revision.ID, "sliceRevision": revision.Revision}) } taskMode, isTemp := server.resolveTaskRunMode(request.Context(), task.RunID) - _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "slice_submitted", Mode: taskMode, Temporary: isTemp, Revision: revision.Revision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO()}) + _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "slice_submitted", Mode: taskMode, Temporary: isTemp, Revision: revision.Revision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO()}) renderJSON(writer, http.StatusCreated, map[string]any{"task": task, "sliceRevision": revision}) } @@ -1385,7 +1413,7 @@ func (server *server) handleAnnotationRevisionCreate(writer http.ResponseWriter, taskMode, isTemp := server.resolveTaskRunMode(request.Context(), task.RunID) _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, - TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, + TaskID: task.ID, UserID: principal.User.ID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: eventType, Mode: taskMode, Temporary: isTemp, Revision: revisionNumber, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: outputDelta, CreatedAt: nowISO(), }) @@ -1670,7 +1698,7 @@ func (server *server) handleFanyuV5FourStageReview(writer http.ResponseWriter, r taskMode, isTemp := server.resolveTaskRunMode(request.Context(), task.RunID) _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, - UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, + UserID: principal.User.ID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: eventType, Mode: taskMode, Temporary: isTemp, Revision: review.Revision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: now, }) @@ -1777,7 +1805,7 @@ func (server *server) handleFanyuV5CombinedQualityReview(writer http.ResponseWri _ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: "quality_review", Round: item.ReviewRound, Status: item.Status, AssigneeUserID: principal.User.ID, ReviewedRevision: review.Revision, Decision: domain.HumanDecision(payload.Decision), RejectTargets: review.RejectTargets, ReasonCodes: review.ReasonCodes, Comment: review.Comment}) _ = server.appendHumanEvent(request.Context(), *item, "combined_quality_"+string(payload.Decision), principal.User.ID, map[string]any{"reviewedRevision": review.Revision, "reasonCodes": review.ReasonCodes, "comment": review.Comment}) taskMode, isTemp := server.resolveTaskRunMode(request.Context(), task.RunID) - _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "quality_" + string(payload.Decision), Mode: taskMode, Temporary: isTemp, Revision: review.Revision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: now}) + _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "quality_" + string(payload.Decision), Mode: taskMode, Temporary: isTemp, Revision: review.Revision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: now}) if payload.Decision == domain.AnnotationReviewApproved { if err := server.flushHumanReleasePartitions(request.Context(), batch, false); err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -1966,7 +1994,7 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r taskMode, isTemp := server.resolveTaskRunMode(request.Context(), task.RunID) _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, - TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, + TaskID: task.ID, UserID: principal.User.ID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: eventType, Mode: taskMode, Temporary: isTemp, Revision: task.CurrentRevision, TemplateVersion: task.TemplateVersion, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO(), }) diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index 6809773f..ddd64b39 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -520,8 +520,8 @@ "value": "" }, { - "description": "目标供应商工作组 ID", - "key": "ANNOTATION_WORKGROUP_ID", + "description": "可选的供应商 ID;留空时按项目成员关系自动确定", + "key": "ANNOTATION_SUPPLIER_ID", "type": "string", "value": "" }, diff --git a/internal/api/builtin_operators_test.go b/internal/api/builtin_operators_test.go index 447054a6..a5af814f 100644 --- a/internal/api/builtin_operators_test.go +++ b/internal/api/builtin_operators_test.go @@ -128,7 +128,7 @@ func TestBuiltinCatalogIncludesNativeAnnotationGateContract(t *testing.T) { for _, hyperparam := range item.Hyperparams { keys[hyperparam.Key] = true } - for _, key := range []string{"ANNOTATION_TEMPLATE_ID", "ANNOTATION_WORKGROUP_ID", "REQUIRE_PREPACKAGE_ACCEPTANCE"} { + for _, key := range []string{"ANNOTATION_TEMPLATE_ID", "ANNOTATION_SUPPLIER_ID", "REQUIRE_PREPACKAGE_ACCEPTANCE"} { if !keys[key] { t.Fatalf("annotation gate is missing hyperparameter %s", key) } diff --git a/internal/api/capture_automation.go b/internal/api/capture_automation.go index d5cb8480..797a30c9 100644 --- a/internal/api/capture_automation.go +++ b/internal/api/capture_automation.go @@ -42,6 +42,9 @@ func normalizeCaptureAutomationConfig(config domain.CaptureAutomationConfig) dom if config.PollIntervalSeconds <= 0 { config.PollIntervalSeconds = 30 } + if config.StabilityWindowSeconds <= 0 { + config.StabilityWindowSeconds = 30 + } return config } @@ -51,6 +54,8 @@ func captureAutomationCounts(records []domain.CaptureRecord) domain.CaptureRecor switch record.Status { case domain.CaptureRecordIncomplete: counts.Incomplete++ + case domain.CaptureRecordStabilizing: + counts.Ready++ case domain.CaptureRecordReady, domain.CaptureRecordLaunching: counts.Ready++ case domain.CaptureRecordRequeueing, domain.CaptureRecordQueued: @@ -136,6 +141,9 @@ func validateCaptureAutomationConfig(workflow *domain.WorkflowRecord, config dom if config.PollIntervalSeconds < 5 || config.PollIntervalSeconds > 3600 { issues = append(issues, validationIssue{Field: "captureAutomation.pollIntervalSeconds", Message: "检查间隔必须在5到3600秒之间"}) } + if config.StabilityWindowSeconds < 5 || config.StabilityWindowSeconds > 3600 { + issues = append(issues, validationIssue{Field: "captureAutomation.stabilityWindowSeconds", Message: "稳定等待时间必须在5到3600秒之间"}) + } for index, pattern := range config.RequiredPatterns { if _, err := regexp.Compile(pattern); err != nil { issues = append(issues, validationIssue{Field: "captureAutomation.requiredPatterns", Message: "第" + strconv.Itoa(index+1) + "个必需文件规则不是有效正则表达式"}) diff --git a/internal/api/dashboard_workforce.go b/internal/api/dashboard_workforce.go index 14cb7299..6f245795 100644 --- a/internal/api/dashboard_workforce.go +++ b/internal/api/dashboard_workforce.go @@ -70,26 +70,20 @@ func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrg return nil } scope := "self" - allowedVendors := map[string]struct{}{} - if principal == nil || principal.User.Role == domain.UserRoleAdmin { + allowedSuppliers := map[string]struct{}{} + if principal == nil || isOrganizationAdmin(principal) { scope = "platform" } else if principal.User.Role == domain.UserRoleSupplierManager { scope = "vendor" - memberships, err := server.store.ListProjectAnnotationMemberships(ctx, requestOrganizationID, "") - if err != nil { - return err - } - for _, membership := range memberships { - if membership.UserID == principal.User.ID && membership.AssignmentRole == domain.AnnotationRoleVendorManager { - allowedVendors[membership.VendorID] = struct{}{} - } + if principal.User.SupplierID != "" { + allowedSuppliers[principal.User.SupplierID] = struct{}{} } } events, err := server.store.ListAnnotationWorklogEvents(ctx, requestOrganizationID, "") if err != nil { return err } - type aggregateKey struct{ userID, vendorID, workgroupID string } + type aggregateKey struct{ userID, supplierID string } aggregates := map[aggregateKey]*domain.DashboardWorkforceMetric{} totals := domain.DashboardWorkforceMetric{} for _, event := range events { @@ -104,14 +98,14 @@ func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrg continue } if scope == "vendor" { - if _, ok := allowedVendors[event.VendorID]; !ok { + if _, ok := allowedSuppliers[event.SupplierID]; !ok { continue } } - key := aggregateKey{event.UserID, event.VendorID, event.WorkgroupID} + key := aggregateKey{event.UserID, event.SupplierID} item := aggregates[key] if item == nil { - item = &domain.DashboardWorkforceMetric{UserID: event.UserID, VendorID: event.VendorID, WorkgroupID: event.WorkgroupID} + item = &domain.DashboardWorkforceMetric{UserID: event.UserID, SupplierID: event.SupplierID} aggregates[key] = item } addDashboardWorkforceEvent(item, event) @@ -128,8 +122,8 @@ func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrg items = append(items, *item) } sort.Slice(items, func(left int, right int) bool { - leftKey := items[left].VendorID + "\x00" + items[left].WorkgroupID + "\x00" + items[left].UserName + "\x00" + items[left].UserID - rightKey := items[right].VendorID + "\x00" + items[right].WorkgroupID + "\x00" + items[right].UserName + "\x00" + items[right].UserID + leftKey := items[left].SupplierID + "\x00" + items[left].UserName + "\x00" + items[left].UserID + rightKey := items[right].SupplierID + "\x00" + items[right].UserName + "\x00" + items[right].UserID return leftKey < rightKey }) finalizeDashboardWorkforceMetric(&totals) diff --git a/internal/api/delivery_packages.go b/internal/api/delivery_packages.go index 3b491cb9..2363c9aa 100644 --- a/internal/api/delivery_packages.go +++ b/internal/api/delivery_packages.go @@ -175,7 +175,7 @@ func (server *server) deliveryPackageReviewer(request *http.Request, item domain if err != nil { return false } - legacy := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID} + legacy := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, SupplierID: item.SupplierID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID} return membershipHasRole(memberships, legacy, domain.AnnotationRoleQualityReviewer) } @@ -386,7 +386,7 @@ func (server *server) handleDeliveryPackageDecision(writer http.ResponseWriter, } _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ ID: domain.NewID("annotation-worklog"), OrganizationID: item.OrganizationID, ProjectID: item.ProjectID, - TaskID: item.ID, UserID: principal.User.ID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID, + TaskID: item.ID, UserID: principal.User.ID, SupplierID: item.SupplierID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID, EventType: "acceptance_" + string(payload.Decision), Mode: item.Mode, Temporary: item.Temporary || item.Mode == domain.RunModeDebug, OutputDelta: 0, CreatedAt: nowISO(), }) if payload.Decision == domain.HumanDecisionRejected { diff --git a/internal/api/fanyu_comment_followup_test.go b/internal/api/fanyu_comment_followup_test.go index 68b98fb9..1f5c7273 100644 --- a/internal/api/fanyu_comment_followup_test.go +++ b/internal/api/fanyu_comment_followup_test.go @@ -148,6 +148,14 @@ func TestSupplierManagerDashboardIsVendorScopedAndMonitoringIsForbidden(t *testi rootCookie := bootstrapSuperAdminSession(t, handler) organization := adminCreateOrganization(t, handler, rootCookie, "Vendor Dashboard", "vendor-dashboard") manager := adminCreateUser(t, handler, rootCookie, organization.ID, "Supplier Manager", "supplier-manager@example.com", domain.UserRoleSupplierManager, "correct-horse-battery-staple-manager") + storedManager, err := dataStore.GetUser(ctx, manager.ID) + if err != nil || storedManager == nil { + t.Fatalf("load supplier manager: %v", err) + } + storedManager.SupplierID = "vendor-a" + if err := dataStore.UpsertUser(ctx, *storedManager); err != nil { + t.Fatalf("set supplier manager scope: %v", err) + } managerCookie := loginSession(t, handler, manager.Email, "correct-horse-battery-staple-manager") project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{OrganizationID: organization.ID, Name: "Vendor Project"}) if err != nil { @@ -175,8 +183,8 @@ func TestSupplierManagerDashboardIsVendorScopedAndMonitoringIsForbidden(t *testi } now := time.Now().UTC().Format(time.RFC3339Nano) for _, event := range []domain.AnnotationWorklogEventRecord{ - {ID: "event-a", OrganizationID: organization.ID, ProjectID: project.ID, UserID: "worker-a", VendorID: "vendor-a", WorkgroupID: "group-a", EventType: "annotation_submitted", OutputDelta: 1, CreatedAt: now}, - {ID: "event-b", OrganizationID: organization.ID, ProjectID: project.ID, UserID: "worker-b", VendorID: "vendor-b", WorkgroupID: "group-b", EventType: "annotation_submitted", OutputDelta: 99, CreatedAt: now}, + {ID: "event-a", OrganizationID: organization.ID, ProjectID: project.ID, UserID: "worker-a", SupplierID: "vendor-a", EventType: "annotation_submitted", OutputDelta: 1, CreatedAt: now}, + {ID: "event-b", OrganizationID: organization.ID, ProjectID: project.ID, UserID: "worker-b", SupplierID: "vendor-b", EventType: "annotation_submitted", OutputDelta: 99, CreatedAt: now}, } { if err := dataStore.UpsertAnnotationWorklogEvent(ctx, event); err != nil { t.Fatalf("create worklog event: %v", err) @@ -193,7 +201,7 @@ func TestSupplierManagerDashboardIsVendorScopedAndMonitoringIsForbidden(t *testi if err := json.Unmarshal(dashboard.Body.Bytes(), &summary); err != nil { t.Fatalf("decode dashboard: %v", err) } - if summary.AudienceScope != "vendor" || len(summary.Workforce.Items) != 1 || summary.Workforce.Items[0].VendorID != "vendor-a" || summary.Workforce.Totals.OutputCount != 1 { + if summary.AudienceScope != "vendor" || len(summary.Workforce.Items) != 1 || summary.Workforce.Items[0].SupplierID != "vendor-a" || summary.Workforce.Totals.OutputCount != 1 { t.Fatalf("vendor scope leaked cross-vendor metrics: %#v", summary.Workforce) } monitoring := serveJSONWithCookie(handler, http.MethodGet, "/api/runs", nil, managerCookie) diff --git a/internal/api/fanyu_template.go b/internal/api/fanyu_template.go index 7154ad83..a5bf3d39 100644 --- a/internal/api/fanyu_template.go +++ b/internal/api/fanyu_template.go @@ -396,7 +396,7 @@ func (server *server) ensureFanyuDeliveryTemplateV58(ctx context.Context, organi node.Position.Y, map[string]string{ "ANNOTATION_TEMPLATE_ID": annotationTemplateID, - "ANNOTATION_WORKGROUP_ID": strings.TrimSpace(os.Getenv("FANYU_V5_ANNOTATION_WORKGROUP_ID")), + "ANNOTATION_SUPPLIER_ID": strings.TrimSpace(os.Getenv("FANYU_V5_SUPPLIER_ID")), "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", @@ -530,16 +530,6 @@ func (server *server) ensureFanyuDeliveryTemplateV58(ctx context.Context, organi if len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { return nil } - for _, node := range nodes { - if node.ID != "fanyu-v56-human" { - continue - } - for _, parameter := range node.Data.Hyperparams { - if parameter.Key == "ANNOTATION_WORKGROUP_ID" && strings.TrimSpace(fmt.Sprint(parameter.Value)) == "" { - return nil - } - } - } release := domain.WorkflowReleaseRecord{ ID: releaseID, OrganizationID: organizationID, TemplateID: template.ID, Version: 8, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, @@ -580,13 +570,13 @@ func (server *server) ensureFanyuDeliveryTemplateV5(ctx context.Context, organiz operators[operatorID] = *operator } annotationTemplateID := fmt.Sprintf("tpl-fanyu-convenience-store-v5-%s", normalizedOrganizationID(organizationID)) - annotationWorkgroupID := strings.TrimSpace(os.Getenv("FANYU_V5_ANNOTATION_WORKGROUP_ID")) + annotationSupplierID := strings.TrimSpace(os.Getenv("FANYU_V5_SUPPLIER_ID")) nodes := []domain.WorkflowNode{ fanyuNode("fanyu-v5-directory", "目录读取(必填)", operators[operatorIDs[0]], 0, 260), fanyuNodeWithHyperparams("fanyu-v5-normalize", "Orbbec Ego 左相机标准化", operators[operatorIDs[1]], 240, 260, map[string]string{"CAMERA_SIDE": "left", "REQUIRE_STEREO": "true"}), fanyuNodeWithHyperparams("fanyu-v5-face-blur", "人脸脱敏(真实视频)", operators[operatorIDs[2]], 500, 260, map[string]string{"PRESERVE_AUDIO": "true", "PRESERVE_PTS": "true", "REQUIRE_DETECTIONS": "true"}), fanyuNodeWithHyperparams("fanyu-v5-rough-cut", "Qwen3-VL 手部行为语义粗切", operators[operatorIDs[3]], 760, 260, map[string]string{"MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "TECHNICAL_BLOCK_SECONDS": "10", "FRAMES_PER_TECHNICAL_BLOCK": "18", "PTS_OCR_TOLERANCE_FRAMES": "3", "REQUIRE_NON_EMPTY_RESULT": "true"}), - fanyuNodeWithHyperparams("fanyu-v5-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认", operators[operatorIDs[4]], 1040, 260, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_WORKGROUP_ID": annotationWorkgroupID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), + fanyuNodeWithHyperparams("fanyu-v5-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认", operators[operatorIDs[4]], 1040, 260, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_SUPPLIER_ID": annotationSupplierID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), fanyuNodeWithHyperparams("fanyu-v5-package", "人工聚合确认后打包最终 MP4、语义 JSON 与 manifest", operators[operatorIDs[5]], 1320, 260, map[string]string{"REQUIRE_HEADPOSE": "false", "POSE_FREE": "true", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}), } for nodeIndex := range nodes { @@ -653,9 +643,6 @@ func (server *server) ensureFanyuDeliveryTemplateV5(ctx context.Context, organiz return nil } } - if strings.TrimSpace(annotationWorkgroupID) == "" { - return nil - } if issues := server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction); len(issues) > 0 { return nil } @@ -708,13 +695,13 @@ func (server *server) ensureFanyuDeliveryTemplateV54(ctx context.Context, organi } } annotationTemplateID := fmt.Sprintf("tpl-fanyu-convenience-store-v5-%s", organizationID) - annotationWorkgroupID := strings.TrimSpace(os.Getenv("FANYU_V5_ANNOTATION_WORKGROUP_ID")) + annotationSupplierID := strings.TrimSpace(os.Getenv("FANYU_V5_SUPPLIER_ID")) nodes := []domain.WorkflowNode{ fanyuMultiportNode("fanyu-v54-directory", "目录读取(多端口)", operators[operatorIDs[0]], 0, 260), fanyuMultiportNodeWithHyperparams("fanyu-v54-normalize", "Orbbec Ego 左相机标准化(多端口)", operators[operatorIDs[1]], 260, 260, map[string]string{"CAMERA_SIDE": "left", "REQUIRE_STEREO": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v54-face-blur", "人脸脱敏(多端口)", operators[operatorIDs[2]], 550, 260, map[string]string{"PRESERVE_AUDIO": "true", "PRESERVE_PTS": "true", "REQUIRE_DETECTIONS": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v54-rough-cut", "Qwen3-VL 手部行为语义粗切(多端口)", operators[operatorIDs[3]], 820, 260, map[string]string{"MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "TECHNICAL_BLOCK_SECONDS": "10", "FRAMES_PER_TECHNICAL_BLOCK": "18", "PTS_OCR_TOLERANCE_FRAMES": "3", "REQUIRE_NON_EMPTY_RESULT": "true"}), - fanyuMultiportNodeWithHyperparams("fanyu-v54-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认(多端口)", operators[operatorIDs[4]], 1110, 260, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_WORKGROUP_ID": annotationWorkgroupID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), + fanyuMultiportNodeWithHyperparams("fanyu-v54-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认(多端口)", operators[operatorIDs[4]], 1110, 260, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_SUPPLIER_ID": annotationSupplierID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v54-package", "人工聚合确认后打包(多端口)", operators[operatorIDs[5]], 1400, 260, map[string]string{"REQUIRE_HEADPOSE": "false", "POSE_FREE": "true", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}), } for index := range nodes { @@ -771,7 +758,7 @@ func (server *server) ensureFanyuDeliveryTemplateV54(ctx context.Context, organi return nil } } - if annotationWorkgroupID == "" || len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { + if len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { return nil } release := domain.WorkflowReleaseRecord{ID: releaseID, OrganizationID: organizationID, TemplateID: template.ID, Version: 4, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, WorkflowInputs: workflowInputs, Nodes: nodes, Edges: edges, Capabilities: []string{"directory_input", "multiport_artifact_contract", "qwen3vl_behavior_rough_cut", "human_cut_adjustment", "human_cut_review", "human_semantic_annotation", "human_semantic_review", "human_acceptance", "final_segment_package"}, RequiredPluginIDs: []string{pluginregistry.HumanLoopCoreID, pluginregistry.AnnotationQAID}, ContentDigest: workflowReleaseDigest(template), ReleaseNotes: "v5.4 explicit multiport data contract; PTS-grounded Qwen and four-stage human loop; pose-free direct package", PublishedByUserID: "system", PublishedAt: now, CreatedAt: now} @@ -822,14 +809,14 @@ func (server *server) ensureFanyuDeliveryTemplateV55(ctx context.Context, organi } } annotationTemplateID := fmt.Sprintf("tpl-fanyu-convenience-store-v5-%s", organizationID) - annotationWorkgroupID := strings.TrimSpace(os.Getenv("FANYU_V5_ANNOTATION_WORKGROUP_ID")) + annotationSupplierID := strings.TrimSpace(os.Getenv("FANYU_V5_SUPPLIER_ID")) nodes := []domain.WorkflowNode{ fanyuMultiportNode("fanyu-v55-directory", "目录读取(多端口)", operators[operatorIDs[0]], 0, 260), fanyuMultiportNodeWithHyperparams("fanyu-v55-normalize", "Orbbec Ego 左相机标准化(多端口)", operators[operatorIDs[1]], 240, 260, map[string]string{"CAMERA_SIDE": "left", "REQUIRE_STEREO": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v55-undistort", "左相机 OpenCV 真实去畸变(多端口)", operators[operatorIDs[2]], 480, 260, map[string]string{"UNDISTORT_ALPHA": "0", "REQUIRE_CALIBRATION": "true", "PRESERVE_AUDIO": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v55-face-blur", "YuNet 人脸真实脱敏(多端口)", operators[operatorIDs[3]], 720, 260, map[string]string{"DETECTION_METHOD": "yunet", "MODEL_PATH": "/app/operator-repos/builtin/video_face_blur/assets/face_detection_yunet_2023mar.onnx", "MODEL_SHA256": "8f2383e4dd3cfbb4553ea8718107fc0423210dc964f9f4280604804ed2552fa4", "CONFIDENCE_THRESHOLD": "0.5", "NMS_THRESHOLD": "0.3", "TOP_K": "5000", "BLUR_TYPE": "mosaic", "BLUR_EXPANSION_RATIO": "1.35", "TRACK_HOLD_FRAMES": "4", "TRACK_IOU_THRESHOLD": "0.2", "PRESERVE_AUDIO": "true", "PRESERVE_PTS": "true", "REQUIRE_PTS": "true", "REQUIRE_DETECTIONS": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v55-rough-cut", "Qwen3-VL 手部行为语义粗切(多端口)", operators[operatorIDs[4]], 960, 260, map[string]string{"MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "TECHNICAL_BLOCK_SECONDS": "10", "FRAMES_PER_TECHNICAL_BLOCK": "6", "MAX_NEW_TOKENS": "128", "PTS_OCR_TOLERANCE_FRAMES": "3", "REQUIRE_NON_EMPTY_RESULT": "true"}), - fanyuMultiportNodeWithHyperparams("fanyu-v55-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认(多端口)", operators[operatorIDs[5]], 1200, 260, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_WORKGROUP_ID": annotationWorkgroupID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), + fanyuMultiportNodeWithHyperparams("fanyu-v55-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认(多端口)", operators[operatorIDs[5]], 1200, 260, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_SUPPLIER_ID": annotationSupplierID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v55-package", "人工聚合确认后打包(多端口)", operators[operatorIDs[6]], 1440, 260, map[string]string{"REQUIRE_HEADPOSE": "false", "POSE_FREE": "true", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}), } for index := range nodes { @@ -889,7 +876,7 @@ func (server *server) ensureFanyuDeliveryTemplateV55(ctx context.Context, organi return nil } } - if annotationWorkgroupID == "" || len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { + if len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { return nil } release := domain.WorkflowReleaseRecord{ID: releaseID, OrganizationID: organizationID, TemplateID: template.ID, Version: 5, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, WorkflowInputs: workflowInputs, Nodes: nodes, Edges: edges, Capabilities: []string{"directory_input", "multiport_artifact_contract", "left_camera_opencv_undistort", "yunet_face_anonymization", "qwen3vl_behavior_rough_cut", "human_cut_adjustment", "human_cut_review", "human_semantic_annotation", "human_semantic_review", "human_acceptance", "final_segment_package"}, RequiredPluginIDs: []string{pluginregistry.HumanLoopCoreID, pluginregistry.AnnotationQAID}, ContentDigest: workflowReleaseDigest(template), ReleaseNotes: "v5.5 materializes OpenCV rectification and fixed-SHA MIT YuNet face anonymization before PTS rough cutting; virtual cutting and four-stage human loop unchanged", PublishedByUserID: "system", PublishedAt: now, CreatedAt: now} @@ -951,7 +938,7 @@ func (server *server) ensureFanyuDeliveryTemplateV56(ctx context.Context, organi } annotationTemplateID := fmt.Sprintf("tpl-fanyu-convenience-store-v5-%s", organizationID) - annotationWorkgroupID := strings.TrimSpace(os.Getenv("FANYU_V5_ANNOTATION_WORKGROUP_ID")) + annotationSupplierID := strings.TrimSpace(os.Getenv("FANYU_V5_SUPPLIER_ID")) headposeOverrides := map[string]string{ "HEADPOSE_API_BASE_URL": strings.TrimSpace(os.Getenv("FANYU_HEADPOSE_API_BASE_URL")), "HEADPOSE_SERVICE_IMAGE_ID": strings.TrimSpace(os.Getenv("FANYU_HEADPOSE_SERVICE_IMAGE_ID")), @@ -985,7 +972,7 @@ func (server *server) ensureFanyuDeliveryTemplateV56(ctx context.Context, organi fanyuMultiportNodeWithHyperparams("fanyu-v56-rough-cut", "Qwen3-VL 左相机语义粗切", operators[operatorIDs[4]], 1060, 80, map[string]string{"MODEL_PATH": "/workspace/models/Qwen3-VL-8B-Instruct", "MODEL_REVISION": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", "MODEL_MANIFEST_PATH": "/workspace/models/Qwen3-VL-8B-Instruct/MODEL_MANIFEST.json", "MODEL_MANIFEST_SHA256": "3c00bb1b1ada0542de91785001395fc6d4252e3d8a0844ec5fdfe05b306a481a", "TRUST_PREVERIFIED_MODEL": "true", "TECHNICAL_BLOCK_SECONDS": "10", "FRAMES_PER_TECHNICAL_BLOCK": "6", "MAX_NEW_TOKENS": "128", "PTS_OCR_TOLERANCE_FRAMES": "3", "REQUIRE_NON_EMPTY_RESULT": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v56-headpose", "6DRepNet HeadPose(开源)", operators[operatorIDs[6]], 1060, 320, headposeOverrides), fanyuMultiportNodeWithHyperparams("fanyu-v56-handpose", "WiLoR HandPose(开源)", operators[operatorIDs[7]], 1060, 560, map[string]string{"INPUT_URL": "", "INPUT_PATH": "", "MEDIA_KIND": "video", "FRAME_STRIDE": "3", "MAX_FRAMES": "120", "VIDEO_CHUNK_FRAMES": "119", "REQUIRE_FULL_VIDEO_COVERAGE": "true", "DOWNLOAD_OUTPUTS": "true", "REQUEST_TIMEOUT_SECONDS": "1800"}), - fanyuMultiportNodeWithHyperparams("fanyu-v56-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认", operators[operatorIDs[5]], 1340, 80, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_WORKGROUP_ID": annotationWorkgroupID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), + fanyuMultiportNodeWithHyperparams("fanyu-v56-human", "人工调切→剪切审核→语义标注→标注审核→聚合确认", operators[operatorIDs[5]], 1340, 80, map[string]string{"ANNOTATION_TEMPLATE_ID": annotationTemplateID, "ANNOTATION_SUPPLIER_ID": annotationSupplierID, "RELEASE_POLICY": "all_accepted", "PARTITION_MAX_ITEMS": "32", "MAX_REWORK_ROUNDS": "5", "REQUIRE_PREPACKAGE_ACCEPTANCE": "true"}), fanyuMultiportNodeWithHyperparams("fanyu-v56-package", "双目同步裁切 + 姿态结果打包交付", operators[operatorIDs[8]], 1640, 320, map[string]string{"REQUIRE_HEADPOSE": "true", "REQUIRE_HANDPOSE": "true", "REQUIRE_STEREO_DELIVERY": "true", "POSE_FREE": "false", "REQUIRE_FINAL_ACCEPTANCE": "true", "GENERATE_SEGMENT_MP4": "true"}), } edges := []domain.WorkflowEdge{ @@ -1053,7 +1040,7 @@ func (server *server) ensureFanyuDeliveryTemplateV56(ctx context.Context, organi return nil } } - if annotationWorkgroupID == "" || len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { + if len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { return nil } release := domain.WorkflowReleaseRecord{ @@ -1144,16 +1131,6 @@ func (server *server) ensureFanyuDeliveryTemplateV57(ctx context.Context, organi if len(server.validateWorkflowOperatorPublication(ctx, nodes, domain.WorkflowExecutionScopeProduction)) > 0 { return nil } - for _, node := range nodes { - if node.ID != "fanyu-v56-human" { - continue - } - for _, parameter := range node.Data.Hyperparams { - if parameter.Key == "ANNOTATION_WORKGROUP_ID" && strings.TrimSpace(fmt.Sprint(parameter.Value)) == "" { - return nil - } - } - } release := domain.WorkflowReleaseRecord{ ID: releaseID, OrganizationID: organizationID, TemplateID: template.ID, Version: 7, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index 80c32440..14c712b6 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -408,7 +408,7 @@ func (server *server) canAccessHumanItem(ctx context.Context, request *http.Requ if err != nil { return false } - legacy := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID} + legacy := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, SupplierID: item.SupplierID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID} if requiredRole != "" { return membershipHasRole(memberships, legacy, requiredRole) } @@ -799,7 +799,7 @@ func (server *server) handleHumanItemRelease(writer http.ResponseWriter, request } func userRoleSupportsAnnotationAssignment(role domain.UserRole, assignmentRole domain.AnnotationAssignmentRole) bool { - if role == domain.UserRoleAdmin { + if role == domain.UserRoleAdmin || role == domain.UserRoleOrganizationAdmin || role == domain.UserRoleMember { return true } switch assignmentRole { @@ -879,7 +879,7 @@ func (server *server) handleHumanItemAssignment(writer http.ResponseWriter, requ renderError(writer, http.StatusInternalServerError, membershipErr) return } - legacy := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID} + legacy := domain.AnnotationTaskRecord{ProjectID: item.ProjectID, SupplierID: item.SupplierID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID} if !membershipHasRole(memberships, legacy, requiredRole) { renderValidationError(writer, validationIssue{Field: "assignedUserId", Message: "assigned user lacks the required project workgroup role"}) return @@ -1494,6 +1494,98 @@ func (server *server) handleHumanRejectReopen(writer http.ResponseWriter, reques renderJSON(writer, http.StatusOK, map[string]any{"item": item, "reject": reject}) } +type humanEscalationDecisionRequest struct { + Version int `json:"version"` + Action string `json:"action"` + TargetStageID string `json:"targetStageId,omitempty"` + Comment string `json:"comment"` +} + +func (server *server) handleHumanEscalationDecision(writer http.ResponseWriter, request *http.Request) { + principal, ok := currentPrincipal(request) + if !ok || !isOrganizationAdmin(principal) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "organization admin is required"}) + return + } + item, ok := server.loadHumanItemForRequest(writer, request) + if !ok { + return + } + var payload humanEscalationDecisionRequest + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Version != item.Version { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "human item version conflict", "code": "version_conflict"}) + return + } + if item.Status != domain.HumanItemEscalated { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "human item is not escalated"}) + return + } + payload.Action = strings.ToLower(strings.TrimSpace(payload.Action)) + payload.Comment = strings.TrimSpace(payload.Comment) + if payload.Comment == "" || (payload.Action != "reopen" && payload.Action != "close") { + renderValidationError(writer, validationIssue{Field: "escalation", Message: "action (reopen or close) and comment are required"}) + return + } + previousVersion := item.Version + if payload.Action == "reopen" { + batch, err := server.store.GetHumanBatch(request.Context(), item.BatchID) + if err != nil || batch == nil { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "human batch not found"}) + } + return + } + target := strings.TrimSpace(payload.TargetStageID) + if target == "" { + target = batch.Stages[0].ID + } + if _, _, exists := humanStage(batch, target); !exists { + renderValidationError(writer, validationIssue{Field: "targetStageId", Message: "target stage is not configured for this batch"}) + return + } + item.CurrentStageID = target + item.Status = domain.HumanItemRework + item.AssignedUserID = "" + item.ClaimedAt = "" + if item.MaxReworkRounds <= item.ReviewRound { + item.MaxReworkRounds = item.ReviewRound + 1 + } + batch.Status = domain.HumanBatchActive + _ = server.store.UpsertHumanBatch(request.Context(), *batch) + } else { + item.Status = domain.HumanItemRejected + item.RejectedAt = nowISO() + } + item.Version++ + updated, err := server.store.UpdateHumanItemVersioned(request.Context(), *item, previousVersion) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !updated { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "human item version conflict", "code": "version_conflict"}) + return + } + _ = server.appendHumanEvent(request.Context(), *item, "escalation_"+payload.Action, principal.User.ID, map[string]any{"comment": payload.Comment, "targetStageId": item.CurrentStageID}) + rejects, _ := server.store.ListHumanRejects(request.Context(), item.OrganizationID, domain.HumanRejectQuery{ItemID: item.ID}) + for _, reject := range rejects { + if reject.Status != domain.HumanRejectEscalated { + continue + } + reject.Status = domain.HumanRejectResolved + reject.Version++ + reject.ReopenComment = payload.Comment + reject.UpdatedAt = nowISO() + _ = server.store.UpsertHumanReject(request.Context(), reject) + } + renderJSON(writer, http.StatusOK, item) +} + func (server *server) reopenRejectedDeliveryPackage(writer http.ResponseWriter, request *http.Request, reject *domain.HumanRejectRecord, payload humanReopenRequest) { item, err := server.store.GetDeliveryPackage(request.Context(), reject.ItemID) if err != nil { diff --git a/internal/api/object_storage_upload_sessions.go b/internal/api/object_storage_upload_sessions.go new file mode 100644 index 00000000..993901d1 --- /dev/null +++ b/internal/api/object_storage_upload_sessions.go @@ -0,0 +1,496 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + "time" + + "droboticflow/internal/domain" + + "github.com/go-chi/chi/v5" +) + +const ( + uploadSessionDirectThresholdBytes int64 = 100 * 1024 * 1024 + uploadSessionActiveRetention = 7 * 24 * time.Hour + uploadSessionCompletedRetention = 30 * 24 * time.Hour +) + +type objectStorageUploadSessionCreateInput struct { + Bucket string `json:"bucket"` + Prefix string `json:"prefix,omitempty"` + ConflictPolicy string `json:"conflictPolicy,omitempty"` + Items []objectStorageUploadSessionCreateItemInput `json:"items"` +} + +type objectStorageUploadSessionCreateItemInput struct { + RelativePath string `json:"relativePath"` + Kind string `json:"kind"` + SizeBytes int64 `json:"sizeBytes,omitempty"` + ContentType string `json:"contentType,omitempty"` + LastModified int64 `json:"lastModified,omitempty"` +} + +type objectStorageUploadSessionPartInput struct { + PartNumber int `json:"partNumber"` + ETag string `json:"eTag"` + UploadedBytes int64 `json:"uploadedBytes"` +} + +type objectStorageUploadSessionPartURLInput struct { + PartNumber int `json:"partNumber"` +} + +func uploadSessionExpiry(status domain.ObjectStorageUploadSessionStatus, now time.Time) string { + if status == domain.ObjectStorageUploadSessionCompleted || status == domain.ObjectStorageUploadSessionCancelled { + return now.Add(uploadSessionCompletedRetention).UTC().Format(time.RFC3339Nano) + } + return now.Add(uploadSessionActiveRetention).UTC().Format(time.RFC3339Nano) +} + +func (server *server) ensureUploadSessionAccess(writer http.ResponseWriter, request *http.Request, session *domain.ObjectStorageUploadSessionRecord) (*authenticatedPrincipal, bool) { + principal, ok := currentPrincipal(request) + if !ok || principal == nil { + renderJSON(writer, http.StatusUnauthorized, map[string]string{"error": "login is required"}) + return nil, false + } + if session == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "upload session not found"}) + return nil, false + } + if !server.ensureTenantAccess(writer, request, session.OrganizationID, "Upload session") { + return nil, false + } + if !isOrganizationAdmin(principal) && session.UserID != principal.User.ID { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "upload session access is restricted to its creator"}) + return nil, false + } + return principal, true +} + +func uploadSessionResponse(session domain.ObjectStorageUploadSessionRecord) domain.ObjectStorageUploadSessionSummary { + return domain.SummarizeObjectStorageUploadSession(session) +} + +func uploadSessionRefreshStatus(session *domain.ObjectStorageUploadSessionRecord) { + if session == nil || session.Status == domain.ObjectStorageUploadSessionPaused || session.Status == domain.ObjectStorageUploadSessionCancelled { + return + } + allResolved := true + hasFailure := false + for _, item := range session.Items { + if item.Status == domain.ObjectStorageUploadItemFailed { + hasFailure = true + } + if item.Status != domain.ObjectStorageUploadItemSuccess && item.Status != domain.ObjectStorageUploadItemSkipped && item.Status != domain.ObjectStorageUploadItemCancelled { + allResolved = false + } + } + if allResolved && !hasFailure { + session.Status = domain.ObjectStorageUploadSessionCompleted + session.CompletedAt = nowISO() + } else if hasFailure { + session.Status = domain.ObjectStorageUploadSessionFailed + } else { + session.Status = domain.ObjectStorageUploadSessionRunning + } +} + +func uploadSessionItem(session *domain.ObjectStorageUploadSessionRecord, itemID string) (*domain.ObjectStorageUploadItemRecord, int) { + if session == nil { + return nil, -1 + } + for index := range session.Items { + if session.Items[index].ID == itemID { + return &session.Items[index], index + } + } + return nil, -1 +} + +func (server *server) saveUploadSession(request *http.Request, session *domain.ObjectStorageUploadSessionRecord) error { + if session == nil { + return fmt.Errorf("upload session is required") + } + session.Version++ + session.UpdatedAt = nowISO() + session.ExpiresAt = uploadSessionExpiry(session.Status, time.Now()) + return server.store.UpsertObjectStorageUploadSession(request.Context(), *session) +} + +func (server *server) handleObjectStorageUploadSessionsList(writer http.ResponseWriter, request *http.Request) { + principal, ok := currentPrincipal(request) + if !ok || principal == nil { + renderJSON(writer, http.StatusUnauthorized, map[string]string{"error": "login is required"}) + return + } + userID := principal.User.ID + if isOrganizationAdmin(principal) && strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("all")), "true") { + userID = "" + } + items, err := server.store.ListObjectStorageUploadSessions(request.Context(), server.requestOrganizationID(request), userID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + summaries := make([]domain.ObjectStorageUploadSessionSummary, 0, len(items)) + for _, item := range items { + summaries = append(summaries, uploadSessionResponse(item)) + } + renderJSON(writer, http.StatusOK, map[string]any{"items": summaries}) +} + +func (server *server) handleObjectStorageUploadSessionGet(writer http.ResponseWriter, request *http.Request) { + session, err := server.store.GetObjectStorageUploadSession(request.Context(), chi.URLParam(request, "sessionId")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if _, ok := server.ensureUploadSessionAccess(writer, request, session); !ok { + return + } + renderJSON(writer, http.StatusOK, uploadSessionResponse(*session)) +} + +func (server *server) handleObjectStorageUploadSessionCreate(writer http.ResponseWriter, request *http.Request) { + principal, ok := currentPrincipal(request) + if !ok || principal == nil { + renderJSON(writer, http.StatusUnauthorized, map[string]string{"error": "login is required"}) + return + } + var payload objectStorageUploadSessionCreateInput + if !decodeJSON(writer, request, &payload) { + return + } + batchInput := domain.ObjectStorageBatchUploadPrepareInput{Bucket: payload.Bucket, Prefix: payload.Prefix, ConflictPolicy: payload.ConflictPolicy, Items: make([]domain.ObjectStorageBatchUploadItemInput, 0, len(payload.Items))} + for _, item := range payload.Items { + batchInput.Items = append(batchInput.Items, domain.ObjectStorageBatchUploadItemInput{RelativePath: item.RelativePath, Kind: item.Kind, SizeBytes: item.SizeBytes, ContentType: item.ContentType}) + } + prepared, prefix, policy, issue := prepareObjectStorageBatchItems(batchInput) + if issue != nil { + renderValidationError(writer, *issue) + return + } + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if cleanup != nil { + defer cleanup() + } + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if browser == nil || connection == nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + return + } + bucket, err := resolveObjectStorageBucket(connection, payload.Bucket) + if err != nil || bucket == "" { + renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}) + return + } + if err := inspectObjectStorageBatchConflicts(request.Context(), browser, bucket, prepared); err != nil { + renderObjectStorageWriteError(writer, err) + return + } + plan := finalizeObjectStorageBatchPlan(prepared, policy) + now := time.Now() + session := domain.ObjectStorageUploadSessionRecord{ID: domain.NewID("upload-session"), OrganizationID: server.requestOrganizationID(request), UserID: principal.User.ID, ConnectionID: connectionID, Bucket: bucket, Prefix: prefix, ConflictPolicy: policy, Status: domain.ObjectStorageUploadSessionPrepared, Version: 1, CreatedAt: now.UTC().Format(time.RFC3339Nano), ExpiresAt: uploadSessionExpiry(domain.ObjectStorageUploadSessionPrepared, now)} + for index, item := range plan.Items { + input := payload.Items[index] + status := domain.ObjectStorageUploadItemPending + if item.Action == "skip" { + status = domain.ObjectStorageUploadItemSkipped + } + if item.Action == "conflict" { + status = domain.ObjectStorageUploadItemFailed + } + session.Items = append(session.Items, domain.ObjectStorageUploadItemRecord{ID: domain.NewID("upload-item"), RelativePath: item.RelativePath, Key: item.Key, Kind: item.Kind, ContentType: item.ContentType, SizeBytes: item.SizeBytes, LastModified: input.LastModified, Action: item.Action, Status: status, Error: item.Message, UpdatedAt: session.CreatedAt}) + } + uploadSessionRefreshStatus(&session) + if len(session.Items) > 0 && session.Status == domain.ObjectStorageUploadSessionCompleted { + session.CompletedAt = session.CreatedAt + } + if err := server.store.UpsertObjectStorageUploadSession(request.Context(), session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, uploadSessionResponse(session)) +} + +func (server *server) mutateObjectStorageUploadSession(writer http.ResponseWriter, request *http.Request, target domain.ObjectStorageUploadSessionStatus) { + session, err := server.store.GetObjectStorageUploadSession(request.Context(), chi.URLParam(request, "sessionId")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if _, ok := server.ensureUploadSessionAccess(writer, request, session); !ok { + return + } + if target == domain.ObjectStorageUploadSessionCancelled { + browser, _, cleanup, resolveErr := server.resolveObjectStorageBrowser(request.Context(), session.ConnectionID) + if cleanup != nil { + defer cleanup() + } + if resolveErr != nil { + renderError(writer, http.StatusInternalServerError, resolveErr) + return + } + for index := range session.Items { + item := &session.Items[index] + if item.Status == domain.ObjectStorageUploadItemSuccess || item.Status == domain.ObjectStorageUploadItemSkipped { + continue + } + if browser != nil && item.UploadID != "" { + _ = browser.AbortMultipartUpload(request.Context(), session.Bucket, item.Key, item.UploadID) + } + item.Status = domain.ObjectStorageUploadItemCancelled + item.UpdatedAt = nowISO() + } + session.CancelledAt = nowISO() + } + if target == domain.ObjectStorageUploadSessionRunning && session.Status != domain.ObjectStorageUploadSessionPaused && session.Status != domain.ObjectStorageUploadSessionFailed && session.Status != domain.ObjectStorageUploadSessionPrepared { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "upload session cannot be resumed from its current state"}) + return + } + session.Status = target + if err := server.saveUploadSession(request, session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, uploadSessionResponse(*session)) +} + +func (server *server) handleObjectStorageUploadSessionPause(writer http.ResponseWriter, request *http.Request) { + server.mutateObjectStorageUploadSession(writer, request, domain.ObjectStorageUploadSessionPaused) +} +func (server *server) handleObjectStorageUploadSessionResume(writer http.ResponseWriter, request *http.Request) { + server.mutateObjectStorageUploadSession(writer, request, domain.ObjectStorageUploadSessionRunning) +} +func (server *server) handleObjectStorageUploadSessionCancel(writer http.ResponseWriter, request *http.Request) { + server.mutateObjectStorageUploadSession(writer, request, domain.ObjectStorageUploadSessionCancelled) +} + +func (server *server) loadUploadSessionItem(writer http.ResponseWriter, request *http.Request) (*domain.ObjectStorageUploadSessionRecord, *domain.ObjectStorageUploadItemRecord, bool) { + session, err := server.store.GetObjectStorageUploadSession(request.Context(), chi.URLParam(request, "sessionId")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return nil, nil, false + } + if _, ok := server.ensureUploadSessionAccess(writer, request, session); !ok { + return nil, nil, false + } + item, _ := uploadSessionItem(session, chi.URLParam(request, "itemId")) + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "upload item not found"}) + return nil, nil, false + } + return session, item, true +} + +func (server *server) handleObjectStorageUploadSessionItemStart(writer http.ResponseWriter, request *http.Request) { + session, item, ok := server.loadUploadSessionItem(writer, request) + if !ok { + return + } + if session.Status == domain.ObjectStorageUploadSessionPaused || session.Status == domain.ObjectStorageUploadSessionCancelled || session.Status == domain.ObjectStorageUploadSessionCompleted { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "upload session is not runnable"}) + return + } + if item.Status == domain.ObjectStorageUploadItemSkipped || item.Status == domain.ObjectStorageUploadItemSuccess { + renderJSON(writer, http.StatusOK, uploadSessionResponse(*session)) + return + } + browser, _, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), session.ConnectionID) + if cleanup != nil { + defer cleanup() + } + if err != nil || browser == nil { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "storage connection unavailable"}) + } + return + } + if item.Kind == "directory" { + result, putErr := browser.PutObject(request.Context(), session.Bucket, item.Key, []byte{}, "application/x-directory") + if putErr != nil { + renderObjectStorageWriteError(writer, putErr) + return + } + item.Status, item.UploadedBytes, item.ETag, item.UpdatedAt = domain.ObjectStorageUploadItemSuccess, 0, result.ETag, nowISO() + uploadSessionRefreshStatus(session) + if err := server.saveUploadSession(request, session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, uploadSessionResponse(*session)) + return + } + session.Status = domain.ObjectStorageUploadSessionRunning + item.Status, item.Error, item.UpdatedAt = domain.ObjectStorageUploadItemUploading, "", nowISO() + if item.SizeBytes >= uploadSessionDirectThresholdBytes { + if item.UploadID == "" { + initiated, initiateErr := browser.InitiateMultipartUpload(request.Context(), session.Bucket, item.Key, item.ContentType, objectStoragePresignDefaultExpires) + if initiateErr != nil { + renderObjectStorageWriteError(writer, initiateErr) + return + } + item.UploadID, item.Mode = initiated.UploadID, "multipart" + } + if err := server.saveUploadSession(request, session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"session": uploadSessionResponse(*session), "mode": "multipart", "uploadId": item.UploadID}) + return + } + presigned, presignErr := browser.PresignPutObject(request.Context(), session.Bucket, item.Key, item.ContentType, objectStoragePresignDefaultExpires) + if presignErr != nil { + renderObjectStorageWriteError(writer, presignErr) + return + } + item.Mode = "direct" + if err := server.saveUploadSession(request, session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"session": uploadSessionResponse(*session), "mode": "direct", "upload": presigned}) +} + +func (server *server) handleObjectStorageUploadSessionPartURL(writer http.ResponseWriter, request *http.Request) { + session, item, ok := server.loadUploadSessionItem(writer, request) + if !ok { + return + } + if session.Status != domain.ObjectStorageUploadSessionRunning || item.UploadID == "" { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "multipart upload is not running"}) + return + } + var payload objectStorageUploadSessionPartURLInput + if !decodeJSON(writer, request, &payload) { + return + } + if payload.PartNumber < 1 || payload.PartNumber > 10000 { + renderValidationError(writer, validationIssue{Field: "partNumber", Message: "partNumber must be between 1 and 10000"}) + return + } + browser, _, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), session.ConnectionID) + if cleanup != nil { + defer cleanup() + } + if err != nil || browser == nil { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "storage connection unavailable"}) + } + return + } + result, err := browser.PresignMultipartUploadPart(request.Context(), session.Bucket, item.Key, item.UploadID, payload.PartNumber, objectStoragePresignDefaultExpires) + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, result) +} + +func (server *server) handleObjectStorageUploadSessionPartComplete(writer http.ResponseWriter, request *http.Request) { + session, item, ok := server.loadUploadSessionItem(writer, request) + if !ok { + return + } + var payload objectStorageUploadSessionPartInput + if !decodeJSON(writer, request, &payload) { + return + } + parts := append([]domain.ObjectStorageMultipartUploadPart(nil), item.UploadedParts...) + parts = append(parts, domain.ObjectStorageMultipartUploadPart{PartNumber: payload.PartNumber, ETag: strings.Trim(payload.ETag, `"`)}) + normalized, err := normalizeObjectStorageMultipartParts(parts) + if err != nil { + renderValidationError(writer, validationIssue{Field: "part", Message: err.Error()}) + return + } + item.UploadedParts = normalized + if payload.UploadedBytes > item.UploadedBytes { + item.UploadedBytes = payload.UploadedBytes + } + item.UpdatedAt = nowISO() + if err := server.saveUploadSession(request, session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, uploadSessionResponse(*session)) +} + +func (server *server) handleObjectStorageUploadSessionItemComplete(writer http.ResponseWriter, request *http.Request) { + session, item, ok := server.loadUploadSessionItem(writer, request) + if !ok { + return + } + browser, _, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), session.ConnectionID) + if cleanup != nil { + defer cleanup() + } + if err != nil || browser == nil { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "storage connection unavailable"}) + } + return + } + if item.Mode == "multipart" { + parts, partsErr := normalizeObjectStorageMultipartParts(item.UploadedParts) + if partsErr != nil { + renderValidationError(writer, validationIssue{Field: "parts", Message: partsErr.Error()}) + return + } + result, completeErr := browser.CompleteMultipartUpload(request.Context(), session.Bucket, item.Key, item.UploadID, parts) + if completeErr != nil { + renderObjectStorageWriteError(writer, completeErr) + return + } + item.ETag = result.ETag + } else { + result, summaryErr := browser.GetObjectSummary(request.Context(), session.Bucket, item.Key) + if summaryErr != nil { + renderObjectStorageWriteError(writer, summaryErr) + return + } + if result.SizeBytes != item.SizeBytes { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "uploaded object size does not match the selected file"}) + return + } + item.ETag = result.ETag + } + item.Status, item.UploadedBytes, item.Error, item.UpdatedAt = domain.ObjectStorageUploadItemSuccess, item.SizeBytes, "", nowISO() + uploadSessionRefreshStatus(session) + if err := server.saveUploadSession(request, session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, uploadSessionResponse(*session)) +} + +func (server *server) handleObjectStorageUploadSessionItemFailure(writer http.ResponseWriter, request *http.Request) { + session, item, ok := server.loadUploadSessionItem(writer, request) + if !ok { + return + } + var payload struct { + Error string `json:"error"` + } + if !decodeJSON(writer, request, &payload) { + return + } + item.Status, item.Error, item.UpdatedAt = domain.ObjectStorageUploadItemFailed, strings.TrimSpace(payload.Error), nowISO() + uploadSessionRefreshStatus(session) + if err := server.saveUploadSession(request, session); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, uploadSessionResponse(*session)) +} diff --git a/internal/api/object_storage_upload_sessions_test.go b/internal/api/object_storage_upload_sessions_test.go new file mode 100644 index 00000000..979a7d85 --- /dev/null +++ b/internal/api/object_storage_upload_sessions_test.go @@ -0,0 +1,67 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func TestObjectStorageUploadSessionPersistsSkipAndMultipartResumeState(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + organization, err := dataStore.EnsureDefaultOrganization(ctx) + if err != nil { + t.Fatal(err) + } + passwordHash, err := hashPassword("upload-admin-password") + if err != nil { + t.Fatal(err) + } + admin, err := dataStore.CreateUser(ctx, domain.CreateUserInput{OrganizationID: organization.ID, Name: "Upload Admin", Email: "upload-admin@example.com", Role: domain.UserRoleAdmin, PasswordHash: passwordHash}) + if err != nil { + t.Fatal(err) + } + connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{Name: "BOS", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test", Bucket: "datastorage", AccessKeyID: "ak", SecretAccessKey: "sk"}) + if err != nil { + t.Fatal(err) + } + fakeBrowser := newFakeObjectStorageBrowser() + fakeBrowser.objects["datastorage|dataset/already.bin"] = domain.ObjectStorageObjectSummary{Bucket: "datastorage", Key: "dataset/already.bin", SizeBytes: 10} + original := storageConnectionBrowserResolver + storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + return fakeBrowser, func() {}, nil + } + t.Cleanup(func() { storageConnectionBrowserResolver = original }) + handler := NewHandler(dataStore, nil) + cookie := loginSession(t, handler, admin.Email, "upload-admin-password") + body := `{"bucket":"datastorage","prefix":"dataset","conflictPolicy":"skip","items":[{"relativePath":"already.bin","kind":"file","sizeBytes":10},{"relativePath":"large.bin","kind":"file","sizeBytes":104857600,"contentType":"application/octet-stream"}]}` + created := serveJSONWithCookie(handler, http.MethodPost, "/api/object-storage/upload-sessions?connectionId="+connection.ID, []byte(body), cookie) + if created.Code != http.StatusCreated { + t.Fatalf("create=%d body=%s", created.Code, created.Body.String()) + } + var summary domain.ObjectStorageUploadSessionSummary + if err := json.Unmarshal(created.Body.Bytes(), &summary); err != nil { + t.Fatal(err) + } + if summary.SkippedCount != 1 || summary.ProcessedBytes != 10 || len(summary.Session.Items) != 2 { + t.Fatalf("summary=%#v", summary) + } + itemID := summary.Session.Items[1].ID + started := serveJSONWithCookie(handler, http.MethodPost, "/api/object-storage/upload-sessions/"+summary.Session.ID+"/items/"+itemID+"/start", nil, cookie) + if started.Code != http.StatusOK || !strings.Contains(started.Body.String(), `"mode":"multipart"`) { + t.Fatalf("start=%d body=%s", started.Code, started.Body.String()) + } + paused := serveJSONWithCookie(handler, http.MethodPost, "/api/object-storage/upload-sessions/"+summary.Session.ID+"/pause", nil, cookie) + if paused.Code != http.StatusOK || !strings.Contains(paused.Body.String(), `"status":"paused"`) { + t.Fatalf("pause=%d body=%s", paused.Code, paused.Body.String()) + } + cancelled := serveJSONWithCookie(handler, http.MethodPost, "/api/object-storage/upload-sessions/"+summary.Session.ID+"/cancel", nil, cookie) + if cancelled.Code != http.StatusOK || !strings.Contains(cancelled.Body.String(), `"status":"cancelled"`) { + t.Fatalf("cancel=%d body=%s", cancelled.Code, cancelled.Body.String()) + } +} diff --git a/internal/api/plugins.go b/internal/api/plugins.go index 14c30a61..5021858e 100644 --- a/internal/api/plugins.go +++ b/internal/api/plugins.go @@ -171,7 +171,7 @@ func (server *server) migrateLegacyAnnotationHumanItems(ctx context.Context, org } batch = domain.HumanBatchRecord{ ID: legacyHumanBatchID(task), OrganizationID: task.OrganizationID, PluginID: pluginregistry.AnnotationQAID, - ProjectID: task.ProjectID, WorkflowID: task.WorkflowID, RunID: task.RunID, GateNodeID: task.NodeID, + ProjectID: task.ProjectID, SupplierID: task.SupplierID, WorkflowID: task.WorkflowID, RunID: task.RunID, GateNodeID: task.NodeID, Status: domain.HumanBatchActive, Stages: stages, ReleasePolicy: pluginReleasePolicy(config), PartitionMaxItems: pluginConfigInt(config, "partitionMaxItems", 32), @@ -205,7 +205,7 @@ func (server *server) migrateLegacyAnnotationHumanItems(ctx context.Context, org Subject: domain.HumanSubjectRef{ArtifactID: task.ArtifactID, ItemKey: task.ID, MediaType: "video/mp4", Payload: map[string]any{ "sourceVideo": task.SourceVideo, "clipStart": task.ClipStart, "clipEnd": task.ClipEnd, "sourceIsFullVideo": task.SourceIsFullVideo, "sourceDurationSeconds": task.SourceDurationSeconds, "templateId": task.TemplateID, }}, - CurrentStageID: stageID, Status: status, AssignedUserID: task.AssignedUserID, WorkgroupID: task.WorkgroupID, + CurrentStageID: stageID, Status: status, AssignedUserID: task.AssignedUserID, SupplierID: task.SupplierID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, CurrentRevision: task.CurrentRevision, MaxReworkRounds: batch.MaxReworkRounds, Version: 1, LatestAuthorUserID: authorID, CreatedAt: task.CreatedAt, } @@ -432,6 +432,23 @@ func (server *server) handleOrganizationPluginUpdate(writer http.ResponseWriter, return } config = normalized + if !payload.Enabled { + items, err := server.store.ListHumanItems(request.Context(), organizationID, domain.HumanItemQuery{PluginID: pluginregistry.AnnotationQAID}) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + active := 0 + for _, item := range items { + if humanItemPending(item.Status) || item.Status == domain.HumanItemEscalated { + active++ + } + } + if active > 0 { + renderJSON(writer, http.StatusConflict, map[string]any{"error": "插件仍有未完成的人工任务,不能关闭", "code": "plugin_has_active_human_work", "activeItemCount": active}) + return + } + } } principal, _ := currentPrincipal(request) updatedBy := "" diff --git a/internal/api/project_members.go b/internal/api/project_members.go new file mode 100644 index 00000000..8fac086c --- /dev/null +++ b/internal/api/project_members.go @@ -0,0 +1,333 @@ +package api + +import ( + "context" + "net/http" + "sort" + "strings" + + "droboticflow/internal/domain" + + "github.com/go-chi/chi/v5" +) + +type supplierInput struct { + Name string `json:"name"` +} + +type projectMemberInput struct { + UserID string `json:"userId"` + SupplierID string `json:"supplierId,omitempty"` + Roles []domain.AnnotationAssignmentRole `json:"roles"` +} + +type projectMemberBulkInput struct { + Action string `json:"action"` + MemberIDs []string `json:"memberIds"` + Roles []domain.AnnotationAssignmentRole `json:"roles,omitempty"` +} + +func isOrganizationAdmin(principal *authenticatedPrincipal) bool { + return principal != nil && (principal.User.IsSuperAdmin || principal.User.Role == domain.UserRoleAdmin || principal.User.Role == domain.UserRoleOrganizationAdmin) +} + +func normalizedMemberRoles(values []domain.AnnotationAssignmentRole) ([]domain.AnnotationAssignmentRole, bool) { + roles := domain.NormalizeProjectMemberRoles(values) + return roles, len(roles) > 0 +} + +func (server *server) projectMemberVisibleTo(principal *authenticatedPrincipal, member domain.ProjectMemberRecord) bool { + if isOrganizationAdmin(principal) { + return true + } + if principal == nil { + return false + } + if principal.User.Role == domain.UserRoleSupplierManager { + return strings.TrimSpace(principal.User.SupplierID) != "" && principal.User.SupplierID == member.SupplierID + } + return principal.User.ID == member.UserID +} + +func (server *server) handleSuppliersList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListSuppliers(request.Context(), server.requestOrganizationID(request)) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + principal, _ := currentPrincipal(request) + if principal != nil && principal.User.Role == domain.UserRoleSupplierManager { + filtered := items[:0] + for _, item := range items { + if item.ID == principal.User.SupplierID { + filtered = append(filtered, item) + } + } + items = filtered + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleSupplierCreate(writer http.ResponseWriter, request *http.Request) { + principal, _ := currentPrincipal(request) + if !isOrganizationAdmin(principal) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "organization admin is required"}) + return + } + var payload supplierInput + if !decodeJSON(writer, request, &payload) { + return + } + payload.Name = strings.TrimSpace(payload.Name) + if payload.Name == "" { + renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"}) + return + } + item := domain.SupplierRecord{ID: domain.NewID("supplier"), OrganizationID: server.requestOrganizationID(request), Name: payload.Name, Status: "active", CreatedAt: nowISO()} + if err := server.store.UpsertSupplier(request.Context(), item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, item) +} + +func (server *server) handleProjectMembersList(writer http.ResponseWriter, request *http.Request) { + projectID := strings.TrimSpace(chi.URLParam(request, "id")) + project, err := server.store.GetProject(request.Context(), projectID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if project == nil || !server.filterByTenant(request, project.OrganizationID) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "project not found"}) + return + } + items, err := server.store.ListProjectMembers(request.Context(), project.OrganizationID, project.ID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + principal, _ := currentPrincipal(request) + filtered := items[:0] + for _, item := range items { + if server.projectMemberVisibleTo(principal, item) { + filtered = append(filtered, item) + } + } + sort.Slice(filtered, func(left, right int) bool { return filtered[left].CreatedAt < filtered[right].CreatedAt }) + renderJSON(writer, http.StatusOK, map[string]any{"items": filtered}) +} + +func (server *server) saveProjectMember(writer http.ResponseWriter, request *http.Request, project *domain.ProjectRecord, member *domain.ProjectMemberRecord, payload projectMemberInput, created bool) { + payload.UserID = strings.TrimSpace(payload.UserID) + payload.SupplierID = strings.TrimSpace(payload.SupplierID) + roles, valid := normalizedMemberRoles(payload.Roles) + if payload.UserID == "" || !valid { + renderValidationError(writer, validationIssue{Field: "member", Message: "userId and at least one project role are required"}) + return + } + user, err := server.store.GetUser(request.Context(), payload.UserID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if user == nil || !annotationOrganizationEqual(user.OrganizationID, project.OrganizationID) { + renderValidationError(writer, validationIssue{Field: "userId", Message: "user must exist in the current organization"}) + return + } + if payload.SupplierID == "" { + payload.SupplierID = strings.TrimSpace(user.SupplierID) + } + if payload.SupplierID != "" { + supplier, supplierErr := server.store.GetSupplier(request.Context(), payload.SupplierID) + if supplierErr != nil { + renderError(writer, http.StatusInternalServerError, supplierErr) + return + } + if supplier == nil || !annotationOrganizationEqual(supplier.OrganizationID, project.OrganizationID) { + renderValidationError(writer, validationIssue{Field: "supplierId", Message: "supplier must exist in the current organization"}) + return + } + } + if member == nil { + member = &domain.ProjectMemberRecord{ID: domain.NewID("project-member"), OrganizationID: project.OrganizationID, ProjectID: project.ID, UserID: user.ID, CreatedAt: nowISO()} + } + member.SupplierID = payload.SupplierID + member.Roles = roles + if err := server.store.UpsertProjectMember(request.Context(), *member); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if created { + renderJSON(writer, http.StatusCreated, member) + return + } + renderJSON(writer, http.StatusOK, member) +} + +func (server *server) handleProjectMemberCreate(writer http.ResponseWriter, request *http.Request) { + principal, _ := currentPrincipal(request) + if !isOrganizationAdmin(principal) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "organization admin is required"}) + return + } + projectID := strings.TrimSpace(chi.URLParam(request, "id")) + project, err := server.store.GetProject(request.Context(), projectID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if project == nil || !server.filterByTenant(request, project.OrganizationID) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "project not found"}) + return + } + var payload projectMemberInput + if !decodeJSON(writer, request, &payload) { + return + } + existing, err := server.store.GetProjectMemberByProjectUser(request.Context(), project.OrganizationID, project.ID, strings.TrimSpace(payload.UserID)) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if existing != nil { + renderJSON(writer, http.StatusConflict, map[string]any{"error": "project member already exists", "member": existing}) + return + } + server.saveProjectMember(writer, request, project, nil, payload, true) +} + +func (server *server) handleProjectMemberUpdate(writer http.ResponseWriter, request *http.Request) { + principal, _ := currentPrincipal(request) + if !isOrganizationAdmin(principal) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "organization admin is required"}) + return + } + member, err := server.store.GetProjectMember(request.Context(), strings.TrimSpace(chi.URLParam(request, "memberId"))) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if member == nil || !server.filterByTenant(request, member.OrganizationID) || member.ProjectID != strings.TrimSpace(chi.URLParam(request, "id")) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "project member not found"}) + return + } + project, err := server.store.GetProject(request.Context(), member.ProjectID) + if err != nil || project == nil { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "project not found"}) + } + return + } + var payload projectMemberInput + if !decodeJSON(writer, request, &payload) { + return + } + payload.UserID = member.UserID + server.saveProjectMember(writer, request, project, member, payload, false) +} + +func (server *server) releaseProjectMemberTasks(ctx context.Context, organizationID string, member domain.ProjectMemberRecord, actorID string) (int, error) { + items, err := server.store.ListHumanItems(ctx, organizationID, domain.HumanItemQuery{ProjectID: member.ProjectID}) + if err != nil { + return 0, err + } + released := 0 + for _, item := range items { + if item.AssignedUserID != member.UserID || (member.SupplierID != "" && item.SupplierID != "" && item.SupplierID != member.SupplierID) || !humanItemPending(item.Status) { + continue + } + expectedVersion := item.Version + item.AssignedUserID = "" + item.ClaimedAt = "" + item.Version++ + if item.Status == domain.HumanItemClaimed || item.Status == domain.HumanItemInProgress { + item.Status = domain.HumanItemReady + } + updated, updateErr := server.store.UpdateHumanItemVersioned(ctx, item, expectedVersion) + if updateErr != nil { + return released, updateErr + } + if !updated { + return released, nil + } + _ = server.store.AppendHumanEvent(ctx, domain.HumanEventRecord{OrganizationID: item.OrganizationID, BatchID: item.BatchID, ItemID: item.ID, StageID: item.CurrentStageID, Round: item.ReviewRound, EventType: "project_member_removed_release", ActorUserID: actorID, ItemVersion: item.Version, Payload: map[string]any{"projectMemberId": member.ID}, CreatedAt: nowISO()}) + released++ + } + return released, nil +} + +func (server *server) handleProjectMemberDelete(writer http.ResponseWriter, request *http.Request) { + principal, _ := currentPrincipal(request) + if !isOrganizationAdmin(principal) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "organization admin is required"}) + return + } + member, err := server.store.GetProjectMember(request.Context(), strings.TrimSpace(chi.URLParam(request, "memberId"))) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if member == nil || member.ProjectID != strings.TrimSpace(chi.URLParam(request, "id")) || !server.filterByTenant(request, member.OrganizationID) { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "project member not found"}) + return + } + releaseTasks := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("releaseTasks")), "true") + if releaseTasks { + if _, err := server.releaseProjectMemberTasks(request.Context(), member.OrganizationID, *member, principal.User.ID); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } + deleted, err := server.store.DeleteProjectMember(request.Context(), member.ID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if !deleted { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "project member not found"}) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (server *server) handleProjectMembersBulk(writer http.ResponseWriter, request *http.Request) { + principal, _ := currentPrincipal(request) + if !isOrganizationAdmin(principal) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "organization admin is required"}) + return + } + projectID := strings.TrimSpace(chi.URLParam(request, "id")) + var payload projectMemberBulkInput + if !decodeJSON(writer, request, &payload) { + return + } + roles, valid := normalizedMemberRoles(payload.Roles) + if len(payload.MemberIDs) == 0 || (payload.Action == "set_roles" && !valid) || (payload.Action != "set_roles" && payload.Action != "remove") { + renderValidationError(writer, validationIssue{Field: "bulk", Message: "memberIds and a valid bulk action are required"}) + return + } + updated := 0 + for _, id := range payload.MemberIDs { + member, err := server.store.GetProjectMember(request.Context(), strings.TrimSpace(id)) + if err != nil || member == nil || member.ProjectID != projectID || !server.filterByTenant(request, member.OrganizationID) { + continue + } + if payload.Action == "remove" { + if _, err = server.releaseProjectMemberTasks(request.Context(), member.OrganizationID, *member, principal.User.ID); err == nil { + _, err = server.store.DeleteProjectMember(request.Context(), member.ID) + } + } else { + member.Roles = roles + err = server.store.UpsertProjectMember(request.Context(), *member) + } + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + updated++ + } + renderJSON(writer, http.StatusOK, map[string]int{"updated": updated}) +} diff --git a/internal/api/project_members_test.go b/internal/api/project_members_test.go new file mode 100644 index 00000000..6c28083b --- /dev/null +++ b/internal/api/project_members_test.go @@ -0,0 +1,62 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func TestProjectMemberEndpointsUseProjectRolesWithoutWorkgroups(t *testing.T) { + t.Setenv("DROBOTICFLOW_REQUIRE_SESSION_AUTH", "true") + ctx := context.Background() + dataStore := store.NewMemoryStore() + handler := NewHandler(dataStore, nil) + rootCookie := bootstrapSuperAdminSession(t, handler) + organization := adminCreateOrganization(t, handler, rootCookie, "Project members", "project-members") + admin := adminCreateUser(t, handler, rootCookie, organization.ID, "Org Admin", "org-admin@example.com", domain.UserRoleAdmin, "org-admin-password") + member := adminCreateUser(t, handler, rootCookie, organization.ID, "Worker", "worker@example.com", domain.UserRoleMember, "worker-password") + adminCookie := loginSession(t, handler, admin.Email, "org-admin-password") + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{OrganizationID: organization.ID, Name: "测试项目"}) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(map[string]any{"userId": member.ID, "roles": []string{"annotator", "quality_reviewer"}}) + created := serveJSONWithCookie(handler, http.MethodPost, "/api/projects/"+project.ID+"/members", body, adminCookie) + if created.Code != http.StatusCreated { + t.Fatalf("create member=%d body=%s", created.Code, created.Body.String()) + } + var stored domain.ProjectMemberRecord + if err := json.Unmarshal(created.Body.Bytes(), &stored); err != nil { + t.Fatal(err) + } + if len(stored.Roles) != 2 || stored.SupplierID != "" { + t.Fatalf("member=%#v", stored) + } + memberCookie := loginSession(t, handler, member.Email, "worker-password") + listed := serveJSONWithCookie(handler, http.MethodGet, "/api/projects/"+project.ID+"/members", nil, memberCookie) + if listed.Code != http.StatusOK || !jsonContains(listed.Body.String(), member.ID) { + t.Fatalf("member list=%d body=%s", listed.Code, listed.Body.String()) + } + updatedBody, _ := json.Marshal(map[string]any{"roles": []string{"acceptance_reviewer"}}) + updated := serveJSONWithCookie(handler, http.MethodPatch, "/api/projects/"+project.ID+"/members/"+stored.ID, updatedBody, adminCookie) + if updated.Code != http.StatusOK || !jsonContains(updated.Body.String(), "acceptance_reviewer") { + t.Fatalf("update=%d body=%s", updated.Code, updated.Body.String()) + } +} + +func jsonContains(value string, expected string) bool { + return len(expected) == 0 || (len(value) >= len(expected) && containsText(value, expected)) +} + +func containsText(value string, expected string) bool { + for index := 0; index+len(expected) <= len(value); index++ { + if value[index:index+len(expected)] == expected { + return true + } + } + return false +} diff --git a/internal/api/server.go b/internal/api/server.go index 2040ed78..72a8a8b6 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1348,6 +1348,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Post("/api/human-items/{id}/release", instance.handleHumanItemRelease) router.Post("/api/human-items/{id}/submissions", instance.handleHumanItemSubmission) router.Post("/api/human-items/{id}/decisions", instance.handleHumanItemDecision) + router.Post("/api/human-items/{id}/escalation", instance.handleHumanEscalationDecision) }) router.Group(func(router chi.Router) { router.Use(instance.requireBusinessRole(domain.UserRoleAdmin)) @@ -1366,8 +1367,6 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/api/annotation-tasks", deprecatedAnnotationEndpoint(instance.handleAnnotationTasksList)) router.Get("/api/annotation-tasks/{id}", deprecatedAnnotationEndpoint(instance.handleAnnotationTaskGet)) router.Get("/api/annotation-workload-summary", deprecatedAnnotationEndpoint(instance.handleAnnotationWorkloadSummary)) - router.Get("/api/plugins/annotation-qa/workgroups", instance.handleAnnotationWorkgroupsList) - router.Get("/api/plugins/annotation-qa/memberships", instance.handleAnnotationMembershipsList) router.Get("/api/plugins/annotation-qa/templates", instance.handleAnnotationTemplatesList) router.Get("/api/plugins/annotation-qa/tasks", instance.handleAnnotationTasksList) router.Get("/api/plugins/annotation-qa/tasks/{id}", instance.handleAnnotationTaskGet) @@ -1394,9 +1393,6 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Post("/api/annotation-memberships", deprecatedAnnotationEndpoint(instance.handleAnnotationMembershipCreate)) router.Post("/api/annotation-templates", deprecatedAnnotationEndpoint(instance.handleAnnotationTemplateCreate)) router.Post("/api/annotation-templates/{id}/publish", deprecatedAnnotationEndpoint(instance.handleAnnotationTemplatePublish)) - router.Post("/api/plugins/annotation-qa/workgroups", instance.handleAnnotationWorkgroupCreate) - router.Post("/api/plugins/annotation-qa/memberships", instance.handleAnnotationMembershipCreate) - router.Delete("/api/plugins/annotation-qa/memberships/{id}", instance.handleAnnotationMembershipDelete) router.Post("/api/plugins/annotation-qa/templates", instance.handleAnnotationTemplateCreate) router.Post("/api/plugins/annotation-qa/templates/{id}/publish", instance.handleAnnotationTemplatePublish) }) @@ -1412,6 +1408,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/api/projects", instance.handleProjectsList) router.Get("/api/projects/{id}", instance.handleProjectsGet) + router.Get("/api/projects/{id}/members", instance.handleProjectMembersList) router.Get("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesGet) router.Get("/api/projects/{id}/workflows", instance.handleProjectWorkflowsList) router.Get("/api/projects/{id}/workflow-instances", instance.handleProjectWorkflowInstancesList) @@ -1453,6 +1450,15 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Delete("/api/scan-configs/{id}", instance.handleScanConfigsDelete) router.Post("/api/scan-configs/{id}/run", instance.handleScanConfigsRun) }) + router.Group(func(router chi.Router) { + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleOrganizationAdmin)) + router.Post("/api/suppliers", instance.handleSupplierCreate) + router.Post("/api/projects/{id}/members", instance.handleProjectMemberCreate) + router.Patch("/api/projects/{id}/members/{memberId}", instance.handleProjectMemberUpdate) + router.Delete("/api/projects/{id}/members/{memberId}", instance.handleProjectMemberDelete) + router.Post("/api/projects/{id}/members/bulk", instance.handleProjectMembersBulk) + }) + router.Get("/api/suppliers", instance.handleSuppliersList) router.Get("/api/source-data/collections", instance.handleSourceCollectionsList) router.Get("/api/source-data/collections/{runId}/{collectionId}", instance.handleSourceCollectionDetail) @@ -1618,6 +1624,17 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Post("/api/object-storage/objects", instance.handleObjectStorageObjectCreate) router.Post("/api/object-storage/objects/upload", instance.handleObjectStorageObjectUpload) router.Post("/api/object-storage/uploads/presign-put", instance.handleObjectStoragePresignPut) + router.Get("/api/object-storage/upload-sessions", instance.handleObjectStorageUploadSessionsList) + router.Post("/api/object-storage/upload-sessions", instance.handleObjectStorageUploadSessionCreate) + router.Get("/api/object-storage/upload-sessions/{sessionId}", instance.handleObjectStorageUploadSessionGet) + router.Post("/api/object-storage/upload-sessions/{sessionId}/pause", instance.handleObjectStorageUploadSessionPause) + router.Post("/api/object-storage/upload-sessions/{sessionId}/resume", instance.handleObjectStorageUploadSessionResume) + router.Post("/api/object-storage/upload-sessions/{sessionId}/cancel", instance.handleObjectStorageUploadSessionCancel) + router.Post("/api/object-storage/upload-sessions/{sessionId}/items/{itemId}/start", instance.handleObjectStorageUploadSessionItemStart) + router.Post("/api/object-storage/upload-sessions/{sessionId}/items/{itemId}/part-url", instance.handleObjectStorageUploadSessionPartURL) + router.Post("/api/object-storage/upload-sessions/{sessionId}/items/{itemId}/parts", instance.handleObjectStorageUploadSessionPartComplete) + router.Post("/api/object-storage/upload-sessions/{sessionId}/items/{itemId}/complete", instance.handleObjectStorageUploadSessionItemComplete) + router.Post("/api/object-storage/upload-sessions/{sessionId}/items/{itemId}/failed", instance.handleObjectStorageUploadSessionItemFailure) router.Post("/api/object-storage/uploads/batch/prepare", instance.handleObjectStorageBatchUploadPrepare) router.Post("/api/object-storage/uploads/multipart/initiate", instance.handleObjectStorageMultipartInitiate) router.Post("/api/object-storage/uploads/multipart/part-url", instance.handleObjectStorageMultipartPartURL) @@ -8071,6 +8088,8 @@ type userPatch struct { func validUserRole(value domain.UserRole) bool { switch value { case domain.UserRoleAdmin, + domain.UserRoleOrganizationAdmin, + domain.UserRoleMember, domain.UserRoleCutter, domain.UserRoleAnnotator, domain.UserRoleQualityReviewer, diff --git a/internal/api/tenant.go b/internal/api/tenant.go index eaa2381b..d12af389 100644 --- a/internal/api/tenant.go +++ b/internal/api/tenant.go @@ -379,7 +379,19 @@ func (server *server) requireBusinessRole(roles ...domain.UserRole) func(http.Ha renderJSON(writer, http.StatusUnauthorized, map[string]string{"error": "login is required"}) return } - if _, ok := allowed[principal.User.Role]; !ok { + _, permitted := allowed[principal.User.Role] + if !permitted && principal.User.Role == domain.UserRoleOrganizationAdmin { + _, permitted = allowed[domain.UserRoleAdmin] + } + if !permitted && principal.User.Role == domain.UserRoleMember { + for _, humanRole := range []domain.UserRole{domain.UserRoleCutter, domain.UserRoleAnnotator, domain.UserRoleQualityReviewer, domain.UserRoleAcceptanceReviewer} { + if _, enabled := allowed[humanRole]; enabled { + permitted = true + break + } + } + } + if !permitted { renderJSON(writer, http.StatusForbidden, map[string]string{"error": "insufficient role permissions"}) return } @@ -399,7 +411,7 @@ func (server *server) requireAdminMonitoringAccess(next http.Handler) http.Handl renderJSON(writer, http.StatusUnauthorized, map[string]string{"error": "login is required"}) return } - if principal.User.Role != domain.UserRoleAdmin { + if principal.User.Role != domain.UserRoleAdmin && principal.User.Role != domain.UserRoleOrganizationAdmin { renderJSON(writer, http.StatusForbidden, map[string]string{"error": "run monitoring is restricted to administrators"}) return } diff --git a/internal/domain/annotation_operations.go b/internal/domain/annotation_operations.go index 86a65312..8d1c5d3c 100644 --- a/internal/domain/annotation_operations.go +++ b/internal/domain/annotation_operations.go @@ -142,29 +142,33 @@ type SliceRevisionRecord struct { type SemanticAnnotationRevisionRecord = AnnotationRevisionRecord type AnnotationTaskRecord struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - ProjectID string `json:"projectId" bson:"projectId"` - DataTaskID string `json:"dataTaskId,omitempty" bson:"dataTaskId,omitempty"` - WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"` - RunID string `json:"runId,omitempty" bson:"runId,omitempty"` - NodeID string `json:"nodeId,omitempty" bson:"nodeId,omitempty"` - ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` - SourceVideo string `json:"sourceVideo" bson:"sourceVideo"` - ClipStart float64 `json:"clipStart" bson:"clipStart"` - ClipEnd float64 `json:"clipEnd" bson:"clipEnd"` - SourceIsFullVideo bool `json:"sourceIsFullVideo,omitempty" bson:"sourceIsFullVideo,omitempty"` - SourceDurationSeconds float64 `json:"sourceDurationSeconds,omitempty" bson:"sourceDurationSeconds,omitempty"` - CoverageStartSecond float64 `json:"coverageStartSecond,omitempty" bson:"coverageStartSecond,omitempty"` - CoverageEndSecond float64 `json:"coverageEndSecond,omitempty" bson:"coverageEndSecond,omitempty"` - CoverageMode string `json:"coverageMode,omitempty" bson:"coverageMode,omitempty"` - TemplateID string `json:"templateId" bson:"templateId"` - TemplateVersion int `json:"templateVersion" bson:"templateVersion"` - WorkgroupID string `json:"workgroupId" bson:"workgroupId"` - VendorID string `json:"vendorId" bson:"vendorId"` - AssignedUserID string `json:"assignedUserId,omitempty" bson:"assignedUserId,omitempty"` - Status AnnotationTaskStatus `json:"status" bson:"status"` - CurrentRevision int `json:"currentRevision" bson:"currentRevision"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + ProjectID string `json:"projectId" bson:"projectId"` + DataTaskID string `json:"dataTaskId,omitempty" bson:"dataTaskId,omitempty"` + WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"` + RunID string `json:"runId,omitempty" bson:"runId,omitempty"` + NodeID string `json:"nodeId,omitempty" bson:"nodeId,omitempty"` + ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` + SourceVideo string `json:"sourceVideo" bson:"sourceVideo"` + ClipStart float64 `json:"clipStart" bson:"clipStart"` + ClipEnd float64 `json:"clipEnd" bson:"clipEnd"` + SourceIsFullVideo bool `json:"sourceIsFullVideo,omitempty" bson:"sourceIsFullVideo,omitempty"` + SourceDurationSeconds float64 `json:"sourceDurationSeconds,omitempty" bson:"sourceDurationSeconds,omitempty"` + CoverageStartSecond float64 `json:"coverageStartSecond,omitempty" bson:"coverageStartSecond,omitempty"` + CoverageEndSecond float64 `json:"coverageEndSecond,omitempty" bson:"coverageEndSecond,omitempty"` + CoverageMode string `json:"coverageMode,omitempty" bson:"coverageMode,omitempty"` + TemplateID string `json:"templateId" bson:"templateId"` + TemplateVersion int `json:"templateVersion" bson:"templateVersion"` + SupplierID string `json:"supplierId,omitempty" bson:"supplierId,omitempty"` + LegacyAssignment map[string]string `json:"legacyAssignment,omitempty" bson:"legacyAssignment,omitempty"` + // Deprecated compatibility fields. Mongo migration copies their historical + // value into LegacyAssignment and unsets them for production records. + WorkgroupID string `json:"workgroupId,omitempty" bson:"workgroupId,omitempty"` + VendorID string `json:"vendorId,omitempty" bson:"vendorId,omitempty"` + AssignedUserID string `json:"assignedUserId,omitempty" bson:"assignedUserId,omitempty"` + Status AnnotationTaskStatus `json:"status" bson:"status"` + CurrentRevision int `json:"currentRevision" bson:"currentRevision"` // PoseSummary is evidence copied from real upstream Headpose/Handpose // manifests for the v5 aggregate acceptance workbench. It contains only // scoped references/counts, never whole untrimmed pose data. @@ -209,21 +213,23 @@ type AnnotationReviewRecord struct { } type AnnotationWorklogEventRecord struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - ProjectID string `json:"projectId" bson:"projectId"` - TaskID string `json:"taskId" bson:"taskId"` - UserID string `json:"userId" bson:"userId"` - WorkgroupID string `json:"workgroupId" bson:"workgroupId"` - VendorID string `json:"vendorId" bson:"vendorId"` - EventType string `json:"eventType" bson:"eventType"` - Mode RunMode `json:"mode,omitempty" bson:"mode,omitempty"` - Temporary bool `json:"temporary,omitempty" bson:"temporary,omitempty"` - Revision int `json:"revision,omitempty" bson:"revision,omitempty"` - TemplateVersion int `json:"templateVersion,omitempty" bson:"templateVersion,omitempty"` - DurationSeconds int64 `json:"durationSeconds,omitempty" bson:"durationSeconds,omitempty"` - OutputDelta int64 `json:"outputDelta" bson:"outputDelta"` - CreatedAt string `json:"createdAt" bson:"createdAt"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + ProjectID string `json:"projectId" bson:"projectId"` + TaskID string `json:"taskId" bson:"taskId"` + UserID string `json:"userId" bson:"userId"` + SupplierID string `json:"supplierId,omitempty" bson:"supplierId,omitempty"` + LegacyAssignment map[string]string `json:"legacyAssignment,omitempty" bson:"legacyAssignment,omitempty"` + WorkgroupID string `json:"workgroupId,omitempty" bson:"workgroupId,omitempty"` + VendorID string `json:"vendorId,omitempty" bson:"vendorId,omitempty"` + EventType string `json:"eventType" bson:"eventType"` + Mode RunMode `json:"mode,omitempty" bson:"mode,omitempty"` + Temporary bool `json:"temporary,omitempty" bson:"temporary,omitempty"` + Revision int `json:"revision,omitempty" bson:"revision,omitempty"` + TemplateVersion int `json:"templateVersion,omitempty" bson:"templateVersion,omitempty"` + DurationSeconds int64 `json:"durationSeconds,omitempty" bson:"durationSeconds,omitempty"` + OutputDelta int64 `json:"outputDelta" bson:"outputDelta"` + CreatedAt string `json:"createdAt" bson:"createdAt"` } type AnnotationWorkloadAggregate struct { diff --git a/internal/domain/capture_automation.go b/internal/domain/capture_automation.go index f666cd13..480a5e77 100644 --- a/internal/domain/capture_automation.go +++ b/internal/domain/capture_automation.go @@ -1,34 +1,36 @@ package domain type CaptureAutomationConfig struct { - Enabled bool `json:"enabled" bson:"enabled"` - ConnectionID string `json:"connectionId" bson:"connectionId"` - Bucket string `json:"bucket" bson:"bucket"` - Prefix string `json:"prefix" bson:"prefix"` - Schema string `json:"schema,omitempty" bson:"schema,omitempty"` - CompletionMode string `json:"completionMode,omitempty" bson:"completionMode,omitempty"` - ManifestName string `json:"manifestName,omitempty" bson:"manifestName,omitempty"` - WorkflowInputID string `json:"workflowInputId,omitempty" bson:"workflowInputId,omitempty"` - PollIntervalSeconds int `json:"pollIntervalSeconds,omitempty" bson:"pollIntervalSeconds,omitempty"` - RequiredPatterns []string `json:"requiredPatterns,omitempty" bson:"requiredPatterns,omitempty"` - MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` - RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` + Enabled bool `json:"enabled" bson:"enabled"` + ConnectionID string `json:"connectionId" bson:"connectionId"` + Bucket string `json:"bucket" bson:"bucket"` + Prefix string `json:"prefix" bson:"prefix"` + Schema string `json:"schema,omitempty" bson:"schema,omitempty"` + CompletionMode string `json:"completionMode,omitempty" bson:"completionMode,omitempty"` + ManifestName string `json:"manifestName,omitempty" bson:"manifestName,omitempty"` + WorkflowInputID string `json:"workflowInputId,omitempty" bson:"workflowInputId,omitempty"` + PollIntervalSeconds int `json:"pollIntervalSeconds,omitempty" bson:"pollIntervalSeconds,omitempty"` + StabilityWindowSeconds int `json:"stabilityWindowSeconds,omitempty" bson:"stabilityWindowSeconds,omitempty"` + RequiredPatterns []string `json:"requiredPatterns,omitempty" bson:"requiredPatterns,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` } type CaptureRecordStatus string const ( - CaptureRecordIncomplete CaptureRecordStatus = "incomplete" - CaptureRecordReady CaptureRecordStatus = "ready" - CaptureRecordLaunching CaptureRecordStatus = "launching" - CaptureRecordRequeueing CaptureRecordStatus = "requeueing" - CaptureRecordQueued CaptureRecordStatus = "queued" - CaptureRecordRunning CaptureRecordStatus = "running" - CaptureRecordWaiting CaptureRecordStatus = "waiting_for_human" - CaptureRecordCompleted CaptureRecordStatus = "completed" - CaptureRecordFailed CaptureRecordStatus = "failed" - CaptureRecordCancelled CaptureRecordStatus = "cancelled" - CaptureRecordDiscarded CaptureRecordStatus = "discarded" + CaptureRecordIncomplete CaptureRecordStatus = "incomplete" + CaptureRecordStabilizing CaptureRecordStatus = "stabilizing" + CaptureRecordReady CaptureRecordStatus = "ready" + CaptureRecordLaunching CaptureRecordStatus = "launching" + CaptureRecordRequeueing CaptureRecordStatus = "requeueing" + CaptureRecordQueued CaptureRecordStatus = "queued" + CaptureRecordRunning CaptureRecordStatus = "running" + CaptureRecordWaiting CaptureRecordStatus = "waiting_for_human" + CaptureRecordCompleted CaptureRecordStatus = "completed" + CaptureRecordFailed CaptureRecordStatus = "failed" + CaptureRecordCancelled CaptureRecordStatus = "cancelled" + CaptureRecordDiscarded CaptureRecordStatus = "discarded" ) type CaptureActionType string @@ -65,6 +67,8 @@ type CaptureRecord struct { Fingerprint string `json:"fingerprint,omitempty" bson:"fingerprint,omitempty"` Status CaptureRecordStatus `json:"status" bson:"status"` MissingTypes []string `json:"missingTypes,omitempty" bson:"missingTypes,omitempty"` + StabilityChecks int `json:"stabilityChecks,omitempty" bson:"stabilityChecks,omitempty"` + StabilityObservedAt string `json:"stabilityObservedAt,omitempty" bson:"stabilityObservedAt,omitempty"` FileCount int `json:"fileCount" bson:"fileCount"` TotalBytes int64 `json:"totalBytes" bson:"totalBytes"` RunID string `json:"runId,omitempty" bson:"runId,omitempty"` diff --git a/internal/domain/delivery_package.go b/internal/domain/delivery_package.go index 4a96f36d..93064cb0 100644 --- a/internal/domain/delivery_package.go +++ b/internal/domain/delivery_package.go @@ -28,6 +28,8 @@ type DeliveryPackageRecord struct { OrganizationID string `json:"organizationId" bson:"organizationId"` ProjectID string `json:"projectId" bson:"projectId"` WorkflowID string `json:"workflowId" bson:"workflowId"` + SupplierID string `json:"supplierId,omitempty" bson:"supplierId,omitempty"` + LegacyAssignment map[string]string `json:"legacyAssignment,omitempty" bson:"legacyAssignment,omitempty"` WorkgroupID string `json:"workgroupId,omitempty" bson:"workgroupId,omitempty"` VendorID string `json:"vendorId,omitempty" bson:"vendorId,omitempty"` ParentRunID string `json:"parentRunId" bson:"parentRunId"` diff --git a/internal/domain/human_loop.go b/internal/domain/human_loop.go index 3dfa466a..9b5812cb 100644 --- a/internal/domain/human_loop.go +++ b/internal/domain/human_loop.go @@ -109,6 +109,7 @@ type HumanBatchRecord struct { OrganizationID string `json:"organizationId" bson:"organizationId"` PluginID string `json:"pluginId" bson:"pluginId"` ProjectID string `json:"projectId" bson:"projectId"` + SupplierID string `json:"supplierId,omitempty" bson:"supplierId,omitempty"` WorkflowID string `json:"workflowId" bson:"workflowId"` RunID string `json:"runId" bson:"runId"` GateNodeID string `json:"gateNodeId" bson:"gateNodeId"` @@ -134,6 +135,7 @@ type HumanItemRecord struct { PluginID string `json:"pluginId" bson:"pluginId"` BatchID string `json:"batchId" bson:"batchId"` ProjectID string `json:"projectId" bson:"projectId"` + SupplierID string `json:"supplierId,omitempty" bson:"supplierId,omitempty"` WorkflowID string `json:"workflowId" bson:"workflowId"` RunID string `json:"runId" bson:"runId"` GateNodeID string `json:"gateNodeId" bson:"gateNodeId"` @@ -143,6 +145,7 @@ type HumanItemRecord struct { CurrentStageID string `json:"currentStageId" bson:"currentStageId"` Status HumanItemStatus `json:"status" bson:"status"` AssignedUserID string `json:"assignedUserId,omitempty" bson:"assignedUserId,omitempty"` + LegacyAssignment map[string]string `json:"legacyAssignment,omitempty" bson:"legacyAssignment,omitempty"` WorkgroupID string `json:"workgroupId,omitempty" bson:"workgroupId,omitempty"` VendorID string `json:"vendorId,omitempty" bson:"vendorId,omitempty"` CurrentRevision int `json:"currentRevision" bson:"currentRevision"` diff --git a/internal/domain/project_members.go b/internal/domain/project_members.go new file mode 100644 index 00000000..ba69c2c0 --- /dev/null +++ b/internal/domain/project_members.go @@ -0,0 +1,77 @@ +package domain + +import "strings" + +// AccountRole scopes organization-wide administration. Human production duties +// are deliberately assigned on ProjectMember instead of being global user roles. +type AccountRole string + +const ( + AccountRoleOrganizationAdmin AccountRole = "organization_admin" + AccountRoleSupplierManager AccountRole = "supplier_manager" + AccountRoleMember AccountRole = "member" +) + +type SupplierRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + Name string `json:"name" bson:"name"` + Status string `json:"status" bson:"status"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type ProjectMemberRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + ProjectID string `json:"projectId" bson:"projectId"` + UserID string `json:"userId" bson:"userId"` + SupplierID string `json:"supplierId,omitempty" bson:"supplierId,omitempty"` + Roles []AnnotationAssignmentRole `json:"roles" bson:"roles"` + LegacySnapshot map[string]string `json:"legacySnapshot,omitempty" bson:"legacySnapshot,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +func NormalizeProjectMemberRoles(values []AnnotationAssignmentRole) []AnnotationAssignmentRole { + seen := map[AnnotationAssignmentRole]struct{}{} + roles := make([]AnnotationAssignmentRole, 0, len(values)) + for _, role := range values { + switch role { + case AnnotationRoleCutter: + role = AnnotationRoleAnnotator + case AnnotationRoleAnnotator, AnnotationRoleQualityReviewer, AnnotationRoleAcceptanceReviewer: + default: + continue + } + if _, exists := seen[role]; exists { + continue + } + seen[role] = struct{}{} + roles = append(roles, role) + } + return roles +} + +func (member ProjectMemberRecord) HasRole(role AnnotationAssignmentRole) bool { + for _, assigned := range member.Roles { + if assigned == role { + return true + } + } + return false +} + +func AccountRoleForUser(user UserRecord) AccountRole { + if user.IsSuperAdmin || user.Role == UserRoleAdmin || user.Role == UserRoleOrganizationAdmin { + return AccountRoleOrganizationAdmin + } + if user.Role == UserRoleSupplierManager { + return AccountRoleSupplierManager + } + return AccountRoleMember +} + +func NormalizedSupplierID(value string) string { + return strings.TrimSpace(value) +} diff --git a/internal/domain/project_members_test.go b/internal/domain/project_members_test.go new file mode 100644 index 00000000..8c2369ca --- /dev/null +++ b/internal/domain/project_members_test.go @@ -0,0 +1,21 @@ +package domain + +import "testing" + +func TestNormalizeProjectMemberRolesMapsLegacyCutterAndDeduplicates(t *testing.T) { + roles := NormalizeProjectMemberRoles([]AnnotationAssignmentRole{AnnotationRoleCutter, AnnotationRoleAnnotator, AnnotationRoleQualityReviewer, AnnotationRoleVendorManager}) + if len(roles) != 2 || roles[0] != AnnotationRoleAnnotator || roles[1] != AnnotationRoleQualityReviewer { + t.Fatalf("roles = %#v", roles) + } +} + +func TestUploadSessionSummaryCountsSkippedAsProcessedWithoutAddingNetworkBytes(t *testing.T) { + summary := SummarizeObjectStorageUploadSession(ObjectStorageUploadSessionRecord{Items: []ObjectStorageUploadItemRecord{ + {SizeBytes: 100, Status: ObjectStorageUploadItemSkipped}, + {SizeBytes: 200, UploadedBytes: 200, Status: ObjectStorageUploadItemSuccess}, + {SizeBytes: 50, Status: ObjectStorageUploadItemPending}, + }}) + if summary.TotalBytes != 350 || summary.ProcessedBytes != 300 || summary.UploadedBytes != 200 || summary.SkippedCount != 1 || summary.SuccessCount != 1 || summary.PendingCount != 1 { + t.Fatalf("summary = %#v", summary) + } +} diff --git a/internal/domain/types.go b/internal/domain/types.go index 3501b00d..836a2f66 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -23,6 +23,8 @@ const ( UserRoleQualityReviewer UserRole = "quality_reviewer" UserRoleAcceptanceReviewer UserRole = "acceptance_reviewer" UserRoleSupplierManager UserRole = "supplier_manager" + UserRoleOrganizationAdmin UserRole = "organization_admin" + UserRoleMember UserRole = "member" ) type UserStatus string @@ -961,6 +963,7 @@ type UserRecord struct { Name string `json:"name" bson:"name"` Email string `json:"email" bson:"email"` Role UserRole `json:"role" bson:"role"` + SupplierID string `json:"supplierId,omitempty" bson:"supplierId,omitempty"` IsSuperAdmin bool `json:"isSuperAdmin,omitempty" bson:"isSuperAdmin,omitempty"` PasswordHash string `json:"passwordHash,omitempty" bson:"passwordHash,omitempty"` Status UserStatus `json:"status,omitempty" bson:"status,omitempty"` @@ -1166,8 +1169,7 @@ type DashboardSummary struct { type DashboardWorkforceMetric struct { UserID string `json:"userId,omitempty"` UserName string `json:"userName,omitempty"` - VendorID string `json:"vendorId,omitempty"` - WorkgroupID string `json:"workgroupId,omitempty"` + SupplierID string `json:"supplierId,omitempty"` OutputCount int64 `json:"outputCount"` SubmittedCount int64 `json:"submittedCount"` ReworkCount int64 `json:"reworkCount"` @@ -1386,6 +1388,7 @@ type CreateUserInput struct { Name string `json:"name"` Email string `json:"email"` Role UserRole `json:"role"` + SupplierID string `json:"supplierId,omitempty"` Password string `json:"password,omitempty"` PasswordHash string `json:"passwordHash,omitempty"` IsSuperAdmin bool `json:"isSuperAdmin,omitempty"` diff --git a/internal/domain/upload_sessions.go b/internal/domain/upload_sessions.go new file mode 100644 index 00000000..1f85fc85 --- /dev/null +++ b/internal/domain/upload_sessions.go @@ -0,0 +1,92 @@ +package domain + +type ObjectStorageUploadSessionStatus string + +const ( + ObjectStorageUploadSessionPrepared ObjectStorageUploadSessionStatus = "prepared" + ObjectStorageUploadSessionRunning ObjectStorageUploadSessionStatus = "running" + ObjectStorageUploadSessionPaused ObjectStorageUploadSessionStatus = "paused" + ObjectStorageUploadSessionCompleted ObjectStorageUploadSessionStatus = "completed" + ObjectStorageUploadSessionFailed ObjectStorageUploadSessionStatus = "failed" + ObjectStorageUploadSessionCancelled ObjectStorageUploadSessionStatus = "cancelled" +) + +type ObjectStorageUploadItemStatus string + +const ( + ObjectStorageUploadItemPending ObjectStorageUploadItemStatus = "pending" + ObjectStorageUploadItemUploading ObjectStorageUploadItemStatus = "uploading" + ObjectStorageUploadItemSuccess ObjectStorageUploadItemStatus = "success" + ObjectStorageUploadItemSkipped ObjectStorageUploadItemStatus = "skipped" + ObjectStorageUploadItemFailed ObjectStorageUploadItemStatus = "failed" + ObjectStorageUploadItemCancelled ObjectStorageUploadItemStatus = "cancelled" +) + +type ObjectStorageUploadItemRecord struct { + ID string `json:"id" bson:"id"` + RelativePath string `json:"relativePath" bson:"relativePath"` + Key string `json:"key" bson:"key"` + Kind string `json:"kind" bson:"kind"` + ContentType string `json:"contentType,omitempty" bson:"contentType,omitempty"` + SizeBytes int64 `json:"sizeBytes" bson:"sizeBytes"` + LastModified int64 `json:"lastModified,omitempty" bson:"lastModified,omitempty"` + Action string `json:"action" bson:"action"` + Status ObjectStorageUploadItemStatus `json:"status" bson:"status"` + Mode string `json:"mode,omitempty" bson:"mode,omitempty"` + UploadID string `json:"uploadId,omitempty" bson:"uploadId,omitempty"` + UploadedParts []ObjectStorageMultipartUploadPart `json:"uploadedParts,omitempty" bson:"uploadedParts,omitempty"` + UploadedBytes int64 `json:"uploadedBytes,omitempty" bson:"uploadedBytes,omitempty"` + ETag string `json:"eTag,omitempty" bson:"eTag,omitempty"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type ObjectStorageUploadSessionRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + UserID string `json:"userId" bson:"userId"` + ConnectionID string `json:"connectionId" bson:"connectionId"` + Bucket string `json:"bucket" bson:"bucket"` + Prefix string `json:"prefix,omitempty" bson:"prefix,omitempty"` + ConflictPolicy string `json:"conflictPolicy" bson:"conflictPolicy"` + Status ObjectStorageUploadSessionStatus `json:"status" bson:"status"` + Items []ObjectStorageUploadItemRecord `json:"items" bson:"items"` + Version int `json:"version" bson:"version"` + ExpiresAt string `json:"expiresAt" bson:"expiresAt"` + CompletedAt string `json:"completedAt,omitempty" bson:"completedAt,omitempty"` + CancelledAt string `json:"cancelledAt,omitempty" bson:"cancelledAt,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type ObjectStorageUploadSessionSummary struct { + Session ObjectStorageUploadSessionRecord `json:"session"` + TotalBytes int64 `json:"totalBytes"` + ProcessedBytes int64 `json:"processedBytes"` + UploadedBytes int64 `json:"uploadedBytes"` + SuccessCount int `json:"successCount"` + SkippedCount int `json:"skippedCount"` + FailedCount int `json:"failedCount"` + PendingCount int `json:"pendingCount"` +} + +func SummarizeObjectStorageUploadSession(session ObjectStorageUploadSessionRecord) ObjectStorageUploadSessionSummary { + summary := ObjectStorageUploadSessionSummary{Session: session} + for _, item := range session.Items { + summary.TotalBytes += item.SizeBytes + summary.UploadedBytes += item.UploadedBytes + switch item.Status { + case ObjectStorageUploadItemSuccess: + summary.SuccessCount++ + summary.ProcessedBytes += item.SizeBytes + case ObjectStorageUploadItemSkipped: + summary.SkippedCount++ + summary.ProcessedBytes += item.SizeBytes + case ObjectStorageUploadItemFailed: + summary.FailedCount++ + default: + summary.PendingCount++ + } + } + return summary +} diff --git a/internal/plugins/registry.go b/internal/plugins/registry.go index 671486cb..4a922a92 100644 --- a/internal/plugins/registry.go +++ b/internal/plugins/registry.go @@ -22,17 +22,17 @@ var builtin = map[string]domain.PluginManifest{ AnnotationQAID: { ID: AnnotationQAID, Name: "Annotation QA", Version: "1.0.0", RequiredCoreVersion: ">=0.1.0", Routes: []domain.PluginRouteContribution{ - {ID: "clip", Path: "/plugins/annotation-qa/workflows/:workflowId/clip", Label: "人工调切", RequiredRoles: []string{"admin", "cutter"}}, - {ID: "clip-quality", Path: "/plugins/annotation-qa/workflows/:workflowId/clip-quality", Label: "剪切审核", RequiredRoles: []string{"admin", "annotator"}}, - {ID: "annotate", Path: "/plugins/annotation-qa/workflows/:workflowId/annotate", Label: "真实切分与语义标注", RequiredRoles: []string{"admin", "annotator"}}, - {ID: "quality", Path: "/plugins/annotation-qa/workflows/:workflowId/quality", Label: "切分与标注审核", RequiredRoles: []string{"admin", "quality_reviewer"}}, - {ID: "acceptance", Path: "/plugins/annotation-qa/workflows/:workflowId/acceptance", Label: "最终验收", RequiredRoles: []string{"admin", "acceptance_reviewer"}}, + {ID: "clip", Path: "/plugins/annotation-qa/workflows/:workflowId/clip", Label: "人工调切", RequiredRoles: []string{"admin", "organization_admin", "member", "cutter"}}, + {ID: "clip-quality", Path: "/plugins/annotation-qa/workflows/:workflowId/clip-quality", Label: "剪切审核", RequiredRoles: []string{"admin", "organization_admin", "member", "annotator"}}, + {ID: "annotate", Path: "/plugins/annotation-qa/workflows/:workflowId/annotate", Label: "真实切分与语义标注", RequiredRoles: []string{"admin", "organization_admin", "member", "annotator"}}, + {ID: "quality", Path: "/plugins/annotation-qa/workflows/:workflowId/quality", Label: "切分与标注审核", RequiredRoles: []string{"admin", "organization_admin", "member", "quality_reviewer"}}, + {ID: "acceptance", Path: "/plugins/annotation-qa/workflows/:workflowId/acceptance", Label: "最终验收", RequiredRoles: []string{"admin", "organization_admin", "member", "acceptance_reviewer"}}, {ID: "item", Path: "/plugins/annotation-qa/items/:itemId", Label: "标注任务详情"}, }, Navigation: []domain.PluginNavigationContribution{ - {ID: "annotate", Label: "切分与语义标注", Path: "/plugins/annotation-qa/workflows/all/annotate", Icon: "tags", Order: 10, GroupID: "annotation-center", GroupLabel: "标注中心", GroupIcon: "tags", RequiredRoles: []string{"admin", "annotator"}}, - {ID: "quality", Label: "切分与标注审核", Path: "/plugins/annotation-qa/workflows/all/quality", Icon: "clipboard-check", Order: 20, GroupID: "annotation-center", GroupLabel: "标注中心", GroupIcon: "tags", RequiredRoles: []string{"admin", "quality_reviewer"}}, - {ID: "acceptance", Label: "最终验收", Path: "/plugins/annotation-qa/workflows/all/acceptance", Icon: "shield-check", Order: 50, GroupID: "annotation-center", GroupLabel: "标注中心", GroupIcon: "tags", RequiredRoles: []string{"admin", "acceptance_reviewer"}}, + {ID: "annotate", Label: "切分与语义标注", Path: "/plugins/annotation-qa/workflows/all/annotate", Icon: "tags", Order: 10, GroupID: "annotation-center", GroupLabel: "标注中心", GroupIcon: "tags", RequiredRoles: []string{"admin", "organization_admin", "member", "annotator"}}, + {ID: "quality", Label: "切分与标注审核", Path: "/plugins/annotation-qa/workflows/all/quality", Icon: "clipboard-check", Order: 20, GroupID: "annotation-center", GroupLabel: "标注中心", GroupIcon: "tags", RequiredRoles: []string{"admin", "organization_admin", "member", "quality_reviewer"}}, + {ID: "acceptance", Label: "最终验收", Path: "/plugins/annotation-qa/workflows/all/acceptance", Icon: "shield-check", Order: 50, GroupID: "annotation-center", GroupLabel: "标注中心", GroupIcon: "tags", RequiredRoles: []string{"admin", "organization_admin", "member", "acceptance_reviewer"}}, }, WorkflowCapabilities: []string{"human_cut_adjustment", "human_cut_review", "human_annotation", "human_quality_review", "human_acceptance"}, OperatorIDs: []string{ diff --git a/internal/plugins/registry_test.go b/internal/plugins/registry_test.go index 8ab72cfd..a16d0618 100644 --- a/internal/plugins/registry_test.go +++ b/internal/plugins/registry_test.go @@ -6,6 +6,15 @@ import ( "droboticflow/internal/domain" ) +func stringSliceContains(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} + func TestAnnotationCapabilitiesAreDerivedFromGateNode(t *testing.T) { workflow := domain.WorkflowRecord{ID: "workflow-1", Nodes: []domain.WorkflowNode{{ ID: "gate", @@ -62,8 +71,8 @@ func TestAnnotationManifestContributesWorkbenchesAndOperators(t *testing.T) { for _, item := range manifest.Navigation { rolesByID[item.ID] = item.RequiredRoles } - if len(rolesByID["acceptance"]) != 2 || rolesByID["acceptance"][1] != "acceptance_reviewer" { - t.Fatalf("final acceptance must belong only to the independent acceptance reviewer: %#v", rolesByID["acceptance"]) + if !stringSliceContains(rolesByID["acceptance"], "acceptance_reviewer") || !stringSliceContains(rolesByID["acceptance"], "member") { + t.Fatalf("final acceptance must be exposed to project-scoped members and independently enforced by membership role: %#v", rolesByID["acceptance"]) } wanted := map[string]bool{ "operator-builtin-annotation-gate": true, diff --git a/internal/store/memory.go b/internal/store/memory.go index de33e4b9..0ee9ccca 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -31,6 +31,8 @@ type MemoryStore struct { samplingTasks map[string]domain.SamplingTaskRecord annotationWorkgroups map[string]domain.AnnotationWorkgroupRecord annotationMemberships map[string]domain.ProjectAnnotationMembershipRecord + suppliers map[string]domain.SupplierRecord + projectMembers map[string]domain.ProjectMemberRecord annotationTemplates map[string]domain.AnnotationTemplateRecord annotationTasks map[string]domain.AnnotationTaskRecord annotationRevisions map[string]domain.AnnotationRevisionRecord @@ -48,6 +50,7 @@ type MemoryStore struct { users map[string]domain.UserRecord sessions map[string]domain.SessionRecord storageConnections map[string]domain.StorageConnectionRecord + uploadSessions map[string]domain.ObjectStorageUploadSessionRecord mongoConnections map[string]domain.MongoConnectionRecord objectSyncSpaces map[string]domain.ObjectSyncSpaceRecord objectSyncDevices map[string]domain.ObjectSyncDeviceRecord @@ -91,6 +94,8 @@ func NewMemoryStore() *MemoryStore { samplingTasks: map[string]domain.SamplingTaskRecord{}, annotationWorkgroups: map[string]domain.AnnotationWorkgroupRecord{}, annotationMemberships: map[string]domain.ProjectAnnotationMembershipRecord{}, + suppliers: map[string]domain.SupplierRecord{}, + projectMembers: map[string]domain.ProjectMemberRecord{}, annotationTemplates: map[string]domain.AnnotationTemplateRecord{}, annotationTasks: map[string]domain.AnnotationTaskRecord{}, annotationRevisions: map[string]domain.AnnotationRevisionRecord{}, @@ -108,6 +113,7 @@ func NewMemoryStore() *MemoryStore { users: map[string]domain.UserRecord{}, sessions: map[string]domain.SessionRecord{}, storageConnections: map[string]domain.StorageConnectionRecord{}, + uploadSessions: map[string]domain.ObjectStorageUploadSessionRecord{}, mongoConnections: map[string]domain.MongoConnectionRecord{}, objectSyncSpaces: map[string]domain.ObjectSyncSpaceRecord{}, objectSyncDevices: map[string]domain.ObjectSyncDeviceRecord{}, @@ -389,6 +395,16 @@ func (store *MemoryStore) DeleteOrganizationCascade(_ context.Context, id string delete(store.annotationMemberships, key) } } + for key, item := range store.suppliers { + if organizationMatches(item.OrganizationID, organizationID) { + delete(store.suppliers, key) + } + } + for key, item := range store.projectMembers { + if organizationMatches(item.OrganizationID, organizationID) { + delete(store.projectMembers, key) + } + } for key, item := range store.annotationTemplates { if organizationMatches(item.OrganizationID, organizationID) { delete(store.annotationTemplates, key) @@ -449,6 +465,11 @@ func (store *MemoryStore) DeleteOrganizationCascade(_ context.Context, id string delete(store.storageConnections, key) } } + for key, item := range store.uploadSessions { + if organizationMatches(item.OrganizationID, organizationID) { + delete(store.uploadSessions, key) + } + } for key, item := range store.mongoConnections { if organizationMatches(item.OrganizationID, organizationID) { delete(store.mongoConnections, key) @@ -1924,6 +1945,7 @@ func (store *MemoryStore) CreateUser(_ context.Context, input domain.CreateUserI Name: input.Name, Email: strings.TrimSpace(strings.ToLower(input.Email)), Role: input.Role, + SupplierID: strings.TrimSpace(input.SupplierID), IsSuperAdmin: input.IsSuperAdmin, PasswordHash: input.PasswordHash, Status: status, diff --git a/internal/store/mongo.go b/internal/store/mongo.go index ccf48aa4..434d1aeb 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -38,6 +38,8 @@ type MongoStore struct { samplingTasks *mongo.Collection annotationWorkgroups *mongo.Collection annotationMemberships *mongo.Collection + suppliers *mongo.Collection + projectMembers *mongo.Collection annotationTemplates *mongo.Collection annotationTasks *mongo.Collection annotationRevisions *mongo.Collection @@ -55,6 +57,7 @@ type MongoStore struct { users *mongo.Collection sessions *mongo.Collection storageConnections *mongo.Collection + uploadSessions *mongo.Collection mongoConnections *mongo.Collection objectSyncSpaces *mongo.Collection objectSyncDevices *mongo.Collection @@ -97,6 +100,8 @@ func NewMongoStore(db *mongo.Database) *MongoStore { samplingTasks: db.Collection("samplingTasks"), annotationWorkgroups: db.Collection("annotationWorkgroups"), annotationMemberships: db.Collection("annotationMemberships"), + suppliers: db.Collection("suppliers"), + projectMembers: db.Collection("projectMembers"), annotationTemplates: db.Collection("annotationTemplates"), annotationTasks: db.Collection("annotationTasks"), annotationRevisions: db.Collection("annotationRevisions"), @@ -114,6 +119,7 @@ func NewMongoStore(db *mongo.Database) *MongoStore { users: db.Collection("users"), sessions: db.Collection("sessions"), storageConnections: db.Collection("storageConnections"), + uploadSessions: db.Collection("objectStorageUploadSessions"), mongoConnections: db.Collection("mongoConnections"), objectSyncSpaces: db.Collection("objectSyncSpaces"), objectSyncDevices: db.Collection("objectSyncDevices"), @@ -274,6 +280,8 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin store.samplingTasks, store.annotationWorkgroups, store.annotationMemberships, + store.suppliers, + store.projectMembers, store.annotationTemplates, store.annotationTasks, store.annotationRevisions, @@ -289,6 +297,7 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin store.releasePartitions, store.deliveryPackages, store.storageConnections, + store.uploadSessions, store.mongoConnections, store.objectSyncSpaces, store.objectSyncDevices, @@ -398,6 +407,9 @@ func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, er if _, err := store.samplingTasks.DeleteMany(ctx, bson.M{"projectId": id}); err != nil { return false, err } + if _, err := store.projectMembers.DeleteMany(ctx, bson.M{"projectId": id}); err != nil { + return false, err + } if len(workflowIDs) > 0 { if _, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": bson.M{"$in": workflowIDs}}); err != nil { return false, err @@ -1863,6 +1875,7 @@ func (store *MongoStore) CreateUser(ctx context.Context, input domain.CreateUser Name: input.Name, Email: strings.TrimSpace(strings.ToLower(input.Email)), Role: input.Role, + SupplierID: strings.TrimSpace(input.SupplierID), IsSuperAdmin: input.IsSuperAdmin, PasswordHash: input.PasswordHash, Status: input.Status, diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 7796ec5a..bfbf83a9 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -146,14 +146,14 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { index("status_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}), }, - store.annotationWorkgroups: { + store.suppliers: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), - index("organizationId_vendorId_name", bson.D{{Key: "organizationId", Value: 1}, {Key: "vendorId", Value: 1}, {Key: "name", Value: 1}}), + index("organizationId_name", bson.D{{Key: "organizationId", Value: 1}, {Key: "name", Value: 1}}), }, - store.annotationMemberships: { + store.projectMembers: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), - uniqueIndex("organizationId_projectId_userId_role_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "userId", Value: 1}, {Key: "assignmentRole", Value: 1}}), - index("organizationId_projectId_workgroupId", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "workgroupId", Value: 1}}), + uniqueIndex("organizationId_projectId_userId_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "userId", Value: 1}}), + index("organizationId_projectId_supplierId", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "supplierId", Value: 1}}), }, store.annotationTemplates: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), @@ -161,7 +161,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { }, store.annotationTasks: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), - index("organizationId_projectId_workgroupId_status", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "workgroupId", Value: 1}, {Key: "status", Value: 1}}), + index("organizationId_projectId_supplierId_status", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "supplierId", Value: 1}, {Key: "status", Value: 1}}), index("runId_artifactId", bson.D{{Key: "runId", Value: 1}, {Key: "artifactId", Value: 1}}), }, store.annotationRevisions: { @@ -179,7 +179,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { store.annotationWorklogEvents: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organizationId_projectId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "createdAt", Value: -1}}), - index("organizationId_vendorId_workgroupId_userId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "vendorId", Value: 1}, {Key: "workgroupId", Value: 1}, {Key: "userId", Value: 1}, {Key: "createdAt", Value: -1}}), + index("organizationId_supplierId_userId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "supplierId", Value: 1}, {Key: "userId", Value: 1}, {Key: "createdAt", Value: -1}}), }, store.organizationPlugins: { uniqueIndex("organizationId_pluginId_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "pluginId", Value: 1}}), @@ -194,7 +194,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), uniqueIndex("organizationId_batchId_subjectItemKey_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "batchId", Value: 1}, {Key: "subject.itemKey", Value: 1}}), index("organizationId_workflowId_stage_status", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "status", Value: 1}}), - index("organizationId_vendorId_workgroupId_stage_status_assignee", bson.D{{Key: "organizationId", Value: 1}, {Key: "vendorId", Value: 1}, {Key: "workgroupId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "status", Value: 1}, {Key: "assignedUserId", Value: 1}}), + index("organizationId_supplierId_stage_status_assignee", bson.D{{Key: "organizationId", Value: 1}, {Key: "supplierId", Value: 1}, {Key: "currentStageId", Value: 1}, {Key: "status", Value: 1}, {Key: "assignedUserId", Value: 1}}), }, store.humanStageAttempts: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), @@ -218,7 +218,7 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), uniqueIndex("organizationId_releasePartitionId_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "releasePartitionId", Value: 1}}), index("organizationId_projectId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), - index("organizationId_vendorId_workgroupId_status_assignee", bson.D{{Key: "organizationId", Value: 1}, {Key: "vendorId", Value: 1}, {Key: "workgroupId", Value: 1}, {Key: "status", Value: 1}, {Key: "assignedUserId", Value: 1}}), + index("organizationId_supplierId_status_assignee", bson.D{{Key: "organizationId", Value: 1}, {Key: "supplierId", Value: 1}, {Key: "status", Value: 1}, {Key: "assignedUserId", Value: 1}}), index("organizationId_mode_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "mode", Value: 1}, {Key: "createdAt", Value: -1}}), index("organizationId_projectId_mode_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "mode", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), index("organizationId_parentRunId", bson.D{{Key: "organizationId", Value: 1}, {Key: "parentRunId", Value: 1}}), @@ -240,6 +240,11 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { index("organizationId_name", bson.D{{Key: "organizationId", Value: 1}, {Key: "name", Value: 1}}), index("name", bson.D{{Key: "name", Value: 1}}), }, + store.uploadSessions: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + index("organizationId_userId_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "userId", Value: 1}, {Key: "updatedAt", Value: -1}}), + index("expiresAt", bson.D{{Key: "expiresAt", Value: 1}}), + }, store.mongoConnections: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organizationId_name", bson.D{{Key: "organizationId", Value: 1}, {Key: "name", Value: 1}}), diff --git a/internal/store/mongo_migrations.go b/internal/store/mongo_migrations.go index 2c73f937..10c09d2f 100644 --- a/internal/store/mongo_migrations.go +++ b/internal/store/mongo_migrations.go @@ -2,6 +2,8 @@ package store import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "sort" @@ -35,6 +37,12 @@ func (store *MongoStore) RunMigrations(ctx context.Context) error { if err := store.backfillRunCaptureDirectories(ctx); err != nil { return err } + if err := store.migrateLegacyWorkgroupsToProjectMembers(ctx); err != nil { + return err + } + if _, err := store.DeleteExpiredObjectStorageUploadSessions(ctx, nowISO()); err != nil { + return err + } if err := store.deduplicateOperatorRecordsByID(ctx); err != nil { return err } @@ -44,6 +52,311 @@ func (store *MongoStore) RunMigrations(ctx context.Context) error { return nil } +func legacySupplierID(group domain.AnnotationWorkgroupRecord) string { + if value := strings.TrimSpace(group.VendorID); value != "" { + return value + } + return "supplier-legacy-" + strings.TrimSpace(group.ID) +} + +func legacyProjectMemberID(organizationID string, projectID string, userID string) string { + sum := sha256.Sum256([]byte(normalizedOrganizationID(organizationID) + "\x00" + strings.TrimSpace(projectID) + "\x00" + strings.TrimSpace(userID))) + return "project-member-migrated-" + hex.EncodeToString(sum[:12]) +} + +// migrateLegacyWorkgroupsToProjectMembers is deliberately additive. The +// legacy records remain readable until the rollout has proved every human item +// and audit event is scoped by supplierId; API responses already omit the old +// fields for new records. +func (store *MongoStore) migrateLegacyWorkgroupsToProjectMembers(ctx context.Context) error { + groupCursor, err := store.annotationWorkgroups.Find(ctx, bson.M{}) + if err != nil { + return err + } + groups, err := decodeMany[domain.AnnotationWorkgroupRecord](ctx, groupCursor) + if err != nil { + return err + } + groupSupplier := map[string]string{} + for _, group := range groups { + supplierID := legacySupplierID(group) + groupSupplier[group.ID] = supplierID + if supplier, getErr := store.GetSupplier(ctx, supplierID); getErr != nil { + return getErr + } else if supplier == nil { + if err := store.UpsertSupplier(ctx, domain.SupplierRecord{ + ID: supplierID, OrganizationID: normalizedOrganizationID(group.OrganizationID), + Name: strings.TrimSpace(group.Name), Status: "active", CreatedAt: group.CreatedAt, + }); err != nil { + return err + } + } + } + membershipCursor, err := store.annotationMemberships.Find(ctx, bson.M{}) + if err != nil { + return err + } + legacyMemberships, err := decodeMany[domain.ProjectAnnotationMembershipRecord](ctx, membershipCursor) + if err != nil { + return err + } + byProjectUser := map[string]domain.ProjectMemberRecord{} + for _, legacy := range legacyMemberships { + if legacy.AssignmentRole == domain.AnnotationRoleVendorManager { + if user, userErr := store.GetUser(ctx, legacy.UserID); userErr != nil { + return userErr + } else if user != nil && user.SupplierID == "" { + user.SupplierID = strings.TrimSpace(legacy.VendorID) + if user.SupplierID == "" { + user.SupplierID = groupSupplier[legacy.WorkgroupID] + } + if err := store.UpsertUser(ctx, *user); err != nil { + return err + } + } + continue + } + key := normalizedOrganizationID(legacy.OrganizationID) + "\x00" + legacy.ProjectID + "\x00" + legacy.UserID + member, exists := byProjectUser[key] + if !exists { + member = domain.ProjectMemberRecord{ + ID: legacyProjectMemberID(legacy.OrganizationID, legacy.ProjectID, legacy.UserID), OrganizationID: normalizedOrganizationID(legacy.OrganizationID), + ProjectID: legacy.ProjectID, UserID: legacy.UserID, SupplierID: strings.TrimSpace(legacy.VendorID), + LegacySnapshot: map[string]string{"workgroupId": legacy.WorkgroupID, "vendorId": legacy.VendorID}, CreatedAt: legacy.CreatedAt, + } + if member.SupplierID == "" { + member.SupplierID = groupSupplier[legacy.WorkgroupID] + } + } + member.Roles = append(member.Roles, legacy.AssignmentRole) + byProjectUser[key] = member + } + for _, member := range byProjectUser { + member.Roles = domain.NormalizeProjectMemberRoles(member.Roles) + if len(member.Roles) == 0 { + continue + } + existing, err := store.GetProjectMemberByProjectUser(ctx, member.OrganizationID, member.ProjectID, member.UserID) + if err != nil { + return err + } + if existing != nil { + member.ID = existing.ID + member.CreatedAt = existing.CreatedAt + if member.SupplierID == "" { + member.SupplierID = existing.SupplierID + } + } + if err := store.UpsertProjectMember(ctx, member); err != nil { + return err + } + if user, userErr := store.GetUser(ctx, member.UserID); userErr != nil { + return userErr + } else if user != nil && user.SupplierID == "" && member.SupplierID != "" { + user.SupplierID = member.SupplierID + if user.Role == domain.UserRoleCutter || user.Role == domain.UserRoleAnnotator || user.Role == domain.UserRoleQualityReviewer || user.Role == domain.UserRoleAcceptanceReviewer { + user.Role = domain.UserRoleMember + } + if err := store.UpsertUser(ctx, *user); err != nil { + return err + } + } + } + if err := store.migrateAnnotationSupplierHyperparams(ctx, groupSupplier); err != nil { + return err + } + collections := []*mongo.Collection{store.annotationTasks, store.annotationWorklogEvents, store.humanItems, store.deliveryPackages} + for _, collection := range collections { + pipeline := mongo.Pipeline{bson.D{{Key: "$set", Value: bson.D{ + {Key: "supplierId", Value: bson.D{{Key: "$ifNull", Value: bson.A{"$vendorId", ""}}}}, + {Key: "legacyAssignment", Value: bson.D{{Key: "workgroupId", Value: "$workgroupId"}, {Key: "vendorId", Value: "$vendorId"}}}, + }}}} + if _, err := collection.UpdateMany(ctx, bson.M{"$or": bson.A{ + bson.M{"supplierId": bson.M{"$exists": false}}, + bson.M{"supplierId": ""}, + }}, pipeline); err != nil { + return err + } + } + // A few older records only carried a workgroup id. Resolve them through the + // archived group mapping before the legacy columns are removed. + for workgroupID, supplierID := range groupSupplier { + if strings.TrimSpace(workgroupID) == "" || strings.TrimSpace(supplierID) == "" { + continue + } + for _, collection := range collections { + if _, err := collection.UpdateMany(ctx, bson.M{ + "workgroupId": workgroupID, + "$or": bson.A{bson.M{"supplierId": ""}, bson.M{"supplierId": bson.M{"$exists": false}}}, + }, bson.M{"$set": bson.M{"supplierId": supplierID}}); err != nil { + return err + } + } + } + for _, collection := range collections { + legacyAssignmentPipeline := mongo.Pipeline{ + bson.D{{Key: "$set", Value: bson.D{ + {Key: "legacyAssignment", Value: bson.D{{Key: "workgroupId", Value: "$workgroupId"}, {Key: "vendorId", Value: "$vendorId"}}}, + }}}, + } + if _, err := collection.UpdateMany(ctx, bson.M{ + "workgroupId": bson.M{"$exists": true}, + "legacyAssignment": bson.M{"$exists": false}, + }, legacyAssignmentPipeline); err != nil { + return err + } + } + itemCursor, err := store.humanItems.Find(ctx, bson.M{"supplierId": bson.M{"$ne": ""}}, options.Find().SetProjection(bson.M{"batchId": 1, "supplierId": 1})) + if err != nil { + return err + } + items, err := decodeMany[domain.HumanItemRecord](ctx, itemCursor) + if err != nil { + return err + } + for _, item := range items { + if item.BatchID == "" || item.SupplierID == "" { + continue + } + _, err := store.humanBatches.UpdateOne(ctx, bson.M{"id": item.BatchID, "supplierId": bson.M{"$in": bson.A{"", nil}}}, bson.M{"$set": bson.M{"supplierId": item.SupplierID}}) + if err != nil { + return err + } + } + if _, err := store.users.UpdateMany(ctx, bson.M{"role": bson.M{"$in": bson.A{domain.UserRoleCutter, domain.UserRoleAnnotator, domain.UserRoleQualityReviewer, domain.UserRoleAcceptanceReviewer}}}, bson.M{"$set": bson.M{"role": domain.UserRoleMember}}); err != nil { + return err + } + if len(groups) > 0 || len(legacyMemberships) > 0 { + if err := store.archiveAndDropLegacyWorkgroupCollections(ctx, groups, legacyMemberships); err != nil { + return err + } + if err := store.dropLegacyWorkgroupIndexes(ctx); err != nil { + return err + } + for _, collection := range collections { + if _, err := collection.UpdateMany(ctx, bson.M{}, bson.M{"$unset": bson.M{"workgroupId": "", "vendorId": ""}}); err != nil { + return err + } + } + } + return nil +} + +func (store *MongoStore) dropLegacyWorkgroupIndexes(ctx context.Context) error { + legacyIndexes := map[*mongo.Collection][]string{ + store.annotationTasks: {"organizationId_projectId_workgroupId_status"}, + store.annotationWorklogEvents: {"organizationId_vendorId_workgroupId_userId_createdAt"}, + store.humanItems: {"organizationId_vendorId_workgroupId_stage_status_assignee"}, + store.deliveryPackages: {"organizationId_vendorId_workgroupId_status_assignee"}, + } + for collection, names := range legacyIndexes { + for _, name := range names { + if _, err := collection.Indexes().DropOne(ctx, name); err != nil && !strings.Contains(strings.ToLower(err.Error()), "index not found") { + return err + } + } + } + return nil +} + +func migrateAnnotationSupplierNodes(nodes []domain.WorkflowNode, groupSupplier map[string]string) (bool, []domain.WorkflowNode) { + changed := false + copy := append([]domain.WorkflowNode(nil), nodes...) + for nodeIndex := range copy { + params := copy[nodeIndex].Data.Hyperparams + for paramIndex := range params { + if params[paramIndex].Key != "ANNOTATION_WORKGROUP_ID" { + continue + } + legacyID := strings.TrimSpace(fmt.Sprint(params[paramIndex].Value)) + params[paramIndex].Key = "ANNOTATION_SUPPLIER_ID" + if supplierID := groupSupplier[legacyID]; supplierID != "" { + params[paramIndex].Value = supplierID + } + changed = true + } + copy[nodeIndex].Data.Hyperparams = params + } + return changed, copy +} + +func (store *MongoStore) migrateAnnotationSupplierHyperparams(ctx context.Context, groupSupplier map[string]string) error { + workflowCursor, err := store.workflows.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"_id": 1, "nodes": 1})) + if err != nil { + return err + } + for workflowCursor.Next(ctx) { + var item struct { + DocumentID any `bson:"_id"` + Nodes []domain.WorkflowNode `bson:"nodes"` + } + if err := workflowCursor.Decode(&item); err != nil { + workflowCursor.Close(ctx) + return err + } + if changed, nodes := migrateAnnotationSupplierNodes(item.Nodes, groupSupplier); changed { + if _, err := store.workflows.UpdateOne(ctx, bson.M{"_id": item.DocumentID}, bson.M{"$set": bson.M{"nodes": nodes, "updatedAt": nowISO()}}); err != nil { + workflowCursor.Close(ctx) + return err + } + } + } + if err := workflowCursor.Err(); err != nil { + workflowCursor.Close(ctx) + return err + } + workflowCursor.Close(ctx) + templateCursor, err := store.templates.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"_id": 1, "nodes": 1})) + if err != nil { + return err + } + defer templateCursor.Close(ctx) + for templateCursor.Next(ctx) { + var item struct { + DocumentID any `bson:"_id"` + Nodes []domain.WorkflowNode `bson:"nodes"` + } + if err := templateCursor.Decode(&item); err != nil { + return err + } + if changed, nodes := migrateAnnotationSupplierNodes(item.Nodes, groupSupplier); changed { + if _, err := store.templates.UpdateOne(ctx, bson.M{"_id": item.DocumentID}, bson.M{"$set": bson.M{"nodes": nodes, "updatedAt": nowISO()}}); err != nil { + return err + } + } + } + return templateCursor.Err() +} + +func (store *MongoStore) archiveAndDropLegacyWorkgroupCollections(ctx context.Context, groups []domain.AnnotationWorkgroupRecord, memberships []domain.ProjectAnnotationMembershipRecord) error { + audit := store.db.Collection("legacyWorkgroupAudit") + archive := func(source string, id string, payload any) error { + _, err := audit.UpdateOne(ctx, bson.M{"source": source, "sourceId": id}, bson.M{"$set": bson.M{"source": source, "sourceId": id, "payload": payload, "archivedAt": nowISO()}}, options.Update().SetUpsert(true)) + return err + } + for _, group := range groups { + if err := archive("annotationWorkgroups", group.ID, group); err != nil { + return err + } + } + for _, membership := range memberships { + if err := archive("annotationMemberships", membership.ID, membership); err != nil { + return err + } + } + if len(groups) > 0 { + if err := store.annotationWorkgroups.Drop(ctx); err != nil && !strings.Contains(strings.ToLower(err.Error()), "namespace") { + return err + } + } + if len(memberships) > 0 { + if err := store.annotationMemberships.Drop(ctx); err != nil && !strings.Contains(strings.ToLower(err.Error()), "namespace") { + return err + } + } + return nil +} + func (store *MongoStore) backfillRunCaptureDirectories(ctx context.Context) error { cursor, err := store.runs.Find( ctx, diff --git a/internal/store/mongo_migrations_test.go b/internal/store/mongo_migrations_test.go new file mode 100644 index 00000000..a68b97e7 --- /dev/null +++ b/internal/store/mongo_migrations_test.go @@ -0,0 +1,15 @@ +package store + +import ( + "testing" + + "droboticflow/internal/domain" +) + +func TestMigrateAnnotationSupplierNodesReplacesLegacyWorkgroupHyperparameter(t *testing.T) { + nodes := []domain.WorkflowNode{{ID: "human", Data: domain.WorkflowNodeData{Hyperparams: []domain.HyperParam{{Key: "ANNOTATION_WORKGROUP_ID", Value: "legacy-group"}}}}} + changed, migrated := migrateAnnotationSupplierNodes(nodes, map[string]string{"legacy-group": "supplier-a"}) + if !changed || migrated[0].Data.Hyperparams[0].Key != "ANNOTATION_SUPPLIER_ID" || migrated[0].Data.Hyperparams[0].Value != "supplier-a" { + t.Fatalf("migrated nodes = %#v", migrated) + } +} diff --git a/internal/store/project_members.go b/internal/store/project_members.go new file mode 100644 index 00000000..c2e6e7a3 --- /dev/null +++ b/internal/store/project_members.go @@ -0,0 +1,217 @@ +package store + +import ( + "context" + "sort" + "strings" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func (store *MemoryStore) ListSuppliers(_ context.Context, organizationID string) ([]domain.SupplierRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.SupplierRecord, 0) + for _, item := range store.suppliers { + if annotationOrganizationMatches(item.OrganizationID, organizationID) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(left, right int) bool { return items[left].Name < items[right].Name }) + return items, nil +} + +func (store *MemoryStore) GetSupplier(_ context.Context, id string) (*domain.SupplierRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.suppliers[strings.TrimSpace(id)] + if !ok { + return nil, nil + } + copy := clone(item) + return ©, nil +} + +func (store *MemoryStore) UpsertSupplier(_ context.Context, item domain.SupplierRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if item.ID == "" { + item.ID = domain.NewID("supplier") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.Status == "" { + item.Status = "active" + } + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + item.UpdatedAt = nowISO() + store.suppliers[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) ListProjectMembers(_ context.Context, organizationID string, projectID string) ([]domain.ProjectMemberRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.ProjectMemberRecord, 0) + for _, item := range store.projectMembers { + if annotationOrganizationMatches(item.OrganizationID, organizationID) && (projectID == "" || item.ProjectID == projectID) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(left, right int) bool { + if items[left].ProjectID != items[right].ProjectID { + return items[left].ProjectID < items[right].ProjectID + } + return items[left].CreatedAt < items[right].CreatedAt + }) + return items, nil +} + +func (store *MemoryStore) GetProjectMember(_ context.Context, id string) (*domain.ProjectMemberRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.projectMembers[strings.TrimSpace(id)] + if !ok { + return nil, nil + } + copy := clone(item) + return ©, nil +} + +func (store *MemoryStore) GetProjectMemberByProjectUser(_ context.Context, organizationID string, projectID string, userID string) (*domain.ProjectMemberRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + for _, item := range store.projectMembers { + if annotationOrganizationMatches(item.OrganizationID, organizationID) && item.ProjectID == projectID && item.UserID == userID { + copy := clone(item) + return ©, nil + } + } + return nil, nil +} + +func (store *MemoryStore) UpsertProjectMember(_ context.Context, item domain.ProjectMemberRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if item.ID == "" { + item.ID = domain.NewID("project-member") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + item.Roles = domain.NormalizeProjectMemberRoles(item.Roles) + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + item.UpdatedAt = nowISO() + store.projectMembers[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) DeleteProjectMember(_ context.Context, id string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + id = strings.TrimSpace(id) + if _, exists := store.projectMembers[id]; !exists { + return false, nil + } + delete(store.projectMembers, id) + return true, nil +} + +func (store *MongoStore) ListSuppliers(ctx context.Context, organizationID string) ([]domain.SupplierRecord, error) { + cursor, err := store.suppliers.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "name", Value: 1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.SupplierRecord](ctx, cursor) +} + +func (store *MongoStore) GetSupplier(ctx context.Context, id string) (*domain.SupplierRecord, error) { + var item domain.SupplierRecord + err := store.suppliers.FindOne(ctx, bson.M{"id": strings.TrimSpace(id)}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) UpsertSupplier(ctx context.Context, item domain.SupplierRecord) error { + if item.ID == "" { + item.ID = domain.NewID("supplier") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.Status == "" { + item.Status = "active" + } + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + item.UpdatedAt = nowISO() + _, err := store.suppliers.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListProjectMembers(ctx context.Context, organizationID string, projectID string) ([]domain.ProjectMemberRecord, error) { + filter := orgCompatibilityFilter(organizationID) + if strings.TrimSpace(projectID) != "" { + filter["projectId"] = strings.TrimSpace(projectID) + } + cursor, err := store.projectMembers.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.ProjectMemberRecord](ctx, cursor) +} + +func (store *MongoStore) GetProjectMember(ctx context.Context, id string) (*domain.ProjectMemberRecord, error) { + var item domain.ProjectMemberRecord + err := store.projectMembers.FindOne(ctx, bson.M{"id": strings.TrimSpace(id)}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) GetProjectMemberByProjectUser(ctx context.Context, organizationID string, projectID string, userID string) (*domain.ProjectMemberRecord, error) { + filter := orgCompatibilityFilter(organizationID) + filter["projectId"] = strings.TrimSpace(projectID) + filter["userId"] = strings.TrimSpace(userID) + var item domain.ProjectMemberRecord + err := store.projectMembers.FindOne(ctx, filter).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) UpsertProjectMember(ctx context.Context, item domain.ProjectMemberRecord) error { + if item.ID == "" { + item.ID = domain.NewID("project-member") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + item.Roles = domain.NormalizeProjectMemberRoles(item.Roles) + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + item.UpdatedAt = nowISO() + _, err := store.projectMembers.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) DeleteProjectMember(ctx context.Context, id string) (bool, error) { + result, err := store.projectMembers.DeleteOne(ctx, bson.M{"id": strings.TrimSpace(id)}) + return result != nil && result.DeletedCount == 1, err +} diff --git a/internal/store/project_members_test.go b/internal/store/project_members_test.go new file mode 100644 index 00000000..cc165142 --- /dev/null +++ b/internal/store/project_members_test.go @@ -0,0 +1,21 @@ +package store + +import ( + "context" + "testing" + + "droboticflow/internal/domain" +) + +func TestMemoryStoreProjectMemberUpsertNormalizesRolesAndFindsByProjectUser(t *testing.T) { + ctx := context.Background() + dataStore := NewMemoryStore() + member := domain.ProjectMemberRecord{ID: "member-a", OrganizationID: "org-a", ProjectID: "project-a", UserID: "user-a", Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleCutter, domain.AnnotationRoleQualityReviewer}} + if err := dataStore.UpsertProjectMember(ctx, member); err != nil { + t.Fatal(err) + } + stored, err := dataStore.GetProjectMemberByProjectUser(ctx, "org-a", "project-a", "user-a") + if err != nil || stored == nil || len(stored.Roles) != 2 || stored.Roles[0] != domain.AnnotationRoleAnnotator { + t.Fatalf("member = %#v err=%v", stored, err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index 287693d1..4dd1f365 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -146,6 +146,14 @@ type Store interface { ListProjectAnnotationMemberships(context.Context, string, string) ([]domain.ProjectAnnotationMembershipRecord, error) UpsertProjectAnnotationMembership(context.Context, domain.ProjectAnnotationMembershipRecord) error DeleteProjectAnnotationMembership(context.Context, string) (bool, error) + ListSuppliers(context.Context, string) ([]domain.SupplierRecord, error) + GetSupplier(context.Context, string) (*domain.SupplierRecord, error) + UpsertSupplier(context.Context, domain.SupplierRecord) error + ListProjectMembers(context.Context, string, string) ([]domain.ProjectMemberRecord, error) + GetProjectMember(context.Context, string) (*domain.ProjectMemberRecord, error) + GetProjectMemberByProjectUser(context.Context, string, string, string) (*domain.ProjectMemberRecord, error) + UpsertProjectMember(context.Context, domain.ProjectMemberRecord) error + DeleteProjectMember(context.Context, string) (bool, error) ListAnnotationTemplates(context.Context, string, string) ([]domain.AnnotationTemplateRecord, error) GetAnnotationTemplate(context.Context, string) (*domain.AnnotationTemplateRecord, error) UpsertAnnotationTemplate(context.Context, domain.AnnotationTemplateRecord) error @@ -207,6 +215,10 @@ type Store interface { CreateStorageConnection(context.Context, domain.CreateStorageConnectionInput) (*domain.StorageConnectionRecord, error) DeleteStorageConnection(context.Context, string) (bool, error) UpsertStorageConnection(context.Context, domain.StorageConnectionRecord) error + ListObjectStorageUploadSessions(context.Context, string, string) ([]domain.ObjectStorageUploadSessionRecord, error) + GetObjectStorageUploadSession(context.Context, string) (*domain.ObjectStorageUploadSessionRecord, error) + UpsertObjectStorageUploadSession(context.Context, domain.ObjectStorageUploadSessionRecord) error + DeleteExpiredObjectStorageUploadSessions(context.Context, string) (int64, error) ListMongoConnections(context.Context) ([]domain.MongoConnectionRecord, error) ListMongoConnectionsByOrganization(context.Context, string) ([]domain.MongoConnectionRecord, error) diff --git a/internal/store/upload_sessions.go b/internal/store/upload_sessions.go new file mode 100644 index 00000000..c988707d --- /dev/null +++ b/internal/store/upload_sessions.go @@ -0,0 +1,110 @@ +package store + +import ( + "context" + "sort" + "strings" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func (store *MemoryStore) ListObjectStorageUploadSessions(_ context.Context, organizationID string, userID string) ([]domain.ObjectStorageUploadSessionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.ObjectStorageUploadSessionRecord, 0) + for _, item := range store.uploadSessions { + if annotationOrganizationMatches(item.OrganizationID, organizationID) && (strings.TrimSpace(userID) == "" || item.UserID == userID) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(left, right int) bool { return items[left].UpdatedAt > items[right].UpdatedAt }) + return items, nil +} + +func (store *MemoryStore) GetObjectStorageUploadSession(_ context.Context, id string) (*domain.ObjectStorageUploadSessionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.uploadSessions[strings.TrimSpace(id)] + if !ok { + return nil, nil + } + copy := clone(item) + return ©, nil +} + +func (store *MemoryStore) UpsertObjectStorageUploadSession(_ context.Context, item domain.ObjectStorageUploadSessionRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if item.ID == "" { + item.ID = domain.NewID("upload-session") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + item.UpdatedAt = nowISO() + store.uploadSessions[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) DeleteExpiredObjectStorageUploadSessions(_ context.Context, before string) (int64, error) { + store.mu.Lock() + defer store.mu.Unlock() + deleted := int64(0) + for id, item := range store.uploadSessions { + if item.ExpiresAt != "" && item.ExpiresAt < before { + delete(store.uploadSessions, id) + deleted++ + } + } + return deleted, nil +} + +func (store *MongoStore) ListObjectStorageUploadSessions(ctx context.Context, organizationID string, userID string) ([]domain.ObjectStorageUploadSessionRecord, error) { + filter := orgCompatibilityFilter(organizationID) + if strings.TrimSpace(userID) != "" { + filter["userId"] = strings.TrimSpace(userID) + } + cursor, err := store.uploadSessions.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.ObjectStorageUploadSessionRecord](ctx, cursor) +} + +func (store *MongoStore) GetObjectStorageUploadSession(ctx context.Context, id string) (*domain.ObjectStorageUploadSessionRecord, error) { + var item domain.ObjectStorageUploadSessionRecord + err := store.uploadSessions.FindOne(ctx, bson.M{"id": strings.TrimSpace(id)}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) UpsertObjectStorageUploadSession(ctx context.Context, item domain.ObjectStorageUploadSessionRecord) error { + if item.ID == "" { + item.ID = domain.NewID("upload-session") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + item.UpdatedAt = nowISO() + _, err := store.uploadSessions.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) DeleteExpiredObjectStorageUploadSessions(ctx context.Context, before string) (int64, error) { + result, err := store.uploadSessions.DeleteMany(ctx, bson.M{"expiresAt": bson.M{"$lt": before}}) + if err != nil { + return 0, err + } + return result.DeletedCount, nil +} diff --git a/internal/worker/annotation_gate.go b/internal/worker/annotation_gate.go index cf752925..593584ca 100644 --- a/internal/worker/annotation_gate.go +++ b/internal/worker/annotation_gate.go @@ -18,7 +18,10 @@ import ( ) type annotationGateConfig struct { - TemplateID string + TemplateID string + SupplierID string + // WorkgroupID is retained only to read pre-v6 workflow definitions during + // migration. New templates must use ANNOTATION_SUPPLIER_ID or project members. WorkgroupID string ReleasePolicy domain.HumanReleasePolicy PartitionMaxItems int @@ -78,6 +81,7 @@ func annotationGateConfigFromNode(node domain.WorkflowNode) annotationGateConfig } return annotationGateConfig{ TemplateID: strings.TrimSpace(params["ANNOTATION_TEMPLATE_ID"]), + SupplierID: strings.TrimSpace(params["ANNOTATION_SUPPLIER_ID"]), WorkgroupID: strings.TrimSpace(params["ANNOTATION_WORKGROUP_ID"]), ReleasePolicy: releasePolicy, PartitionMaxItems: positiveAnnotationGateInt(params["PARTITION_MAX_ITEMS"], 32), @@ -87,6 +91,42 @@ func annotationGateConfigFromNode(node domain.WorkflowNode) annotationGateConfig } } +func (runtime *Runtime) annotationGateSupplierID(ctx context.Context, run *domain.RunRecord, config *annotationGateConfig) (string, error) { + if config == nil || run == nil { + return "", nil + } + if config.SupplierID != "" { + return config.SupplierID, nil + } + if config.WorkgroupID != "" { + group, err := runtime.store.GetAnnotationWorkgroup(ctx, config.WorkgroupID) + if err != nil { + return "", err + } + if group != nil { + config.SupplierID = strings.TrimSpace(group.VendorID) + return config.SupplierID, nil + } + } + members, err := runtime.store.ListProjectMembers(ctx, run.OrganizationID, run.ProjectID) + if err != nil { + return "", err + } + suppliers := map[string]struct{}{} + for _, member := range members { + if strings.TrimSpace(member.SupplierID) != "" { + suppliers[member.SupplierID] = struct{}{} + } + } + if len(suppliers) == 1 { + for supplierID := range suppliers { + config.SupplierID = supplierID + return supplierID, nil + } + } + return "", nil +} + func annotationNumber(value any) (float64, bool) { switch typed := value.(type) { case float64: @@ -585,20 +625,19 @@ func (runtime *Runtime) createAnnotationGateTasks( } else if pluginConfig != nil && pluginConfig.Enabled { config = applyAnnotationPluginConfig(config, pluginConfig.Config) } - if config.TemplateID == "" || config.WorkgroupID == "" { - return nil, fmt.Errorf("annotation gate requires ANNOTATION_TEMPLATE_ID and ANNOTATION_WORKGROUP_ID") + if config.TemplateID == "" { + return nil, fmt.Errorf("annotation gate requires ANNOTATION_TEMPLATE_ID") } template, err := runtime.store.GetAnnotationTemplate(ctx, config.TemplateID) if err != nil || template == nil { return nil, fmt.Errorf("load annotation template %s: %w", config.TemplateID, err) } config.HumanStages = template.HumanStages - group, err := runtime.store.GetAnnotationWorkgroup(ctx, config.WorkgroupID) - if err != nil || group == nil { - return nil, fmt.Errorf("load annotation workgroup %s: %w", config.WorkgroupID, err) + if (template.ProjectID != "" && template.ProjectID != run.ProjectID) || template.Status != domain.AnnotationTemplatePublished || template.OrganizationID != run.OrganizationID { + return nil, fmt.Errorf("annotation gate template does not belong to the run project and organization") } - if (template.ProjectID != "" && template.ProjectID != run.ProjectID) || template.Status != domain.AnnotationTemplatePublished || template.OrganizationID != run.OrganizationID || group.OrganizationID != run.OrganizationID { - return nil, fmt.Errorf("annotation gate template/workgroup does not belong to the run project and organization") + if _, err := runtime.annotationGateSupplierID(ctx, run, &config); err != nil { + return nil, fmt.Errorf("resolve annotation supplier: %w", err) } existing, err := annotationGateTasksForRun(ctx, runtime, run, node) if err != nil { @@ -632,7 +671,7 @@ func (runtime *Runtime) createAnnotationGateTasks( if duration <= 0 { return nil, fmt.Errorf("human gate cannot resolve full source duration for %s", artifact.Name) } - task := domain.AnnotationTaskRecord{ID: domain.NewID("annotation-task"), OrganizationID: run.OrganizationID, ProjectID: run.ProjectID, DataTaskID: run.TaskID, WorkflowID: run.WorkflowID, RunID: run.ID, NodeID: node.ID, ArtifactID: artifact.ID, SourceVideo: fmt.Sprintf("/api/runs/%s/artifacts/%s/content", url.PathEscape(run.ID), url.PathEscape(artifact.ID)), ClipStart: 0, ClipEnd: duration, SourceIsFullVideo: true, SourceDurationSeconds: duration, CoverageStartSecond: 0, CoverageEndSecond: duration, CoverageMode: "full_source_pending_common_range", TemplateID: template.ID, TemplateVersion: template.Version, WorkgroupID: group.ID, VendorID: group.VendorID, Status: domain.AnnotationTaskStatusPendingAnnotation, PoseSummary: poseSummary, CreatedAt: nowISO(), UpdatedAt: nowISO()} + task := domain.AnnotationTaskRecord{ID: domain.NewID("annotation-task"), OrganizationID: run.OrganizationID, ProjectID: run.ProjectID, DataTaskID: run.TaskID, WorkflowID: run.WorkflowID, RunID: run.ID, NodeID: node.ID, ArtifactID: artifact.ID, SourceVideo: fmt.Sprintf("/api/runs/%s/artifacts/%s/content", url.PathEscape(run.ID), url.PathEscape(artifact.ID)), ClipStart: 0, ClipEnd: duration, SourceIsFullVideo: true, SourceDurationSeconds: duration, CoverageStartSecond: 0, CoverageEndSecond: duration, CoverageMode: "full_source_pending_common_range", TemplateID: template.ID, TemplateVersion: template.Version, SupplierID: config.SupplierID, Status: domain.AnnotationTaskStatusPendingAnnotation, PoseSummary: poseSummary, CreatedAt: nowISO(), UpdatedAt: nowISO()} if err := runtime.store.UpsertAnnotationTask(ctx, task); err != nil { return nil, err } @@ -713,8 +752,7 @@ func (runtime *Runtime) createAnnotationGateTasks( }(), TemplateID: template.ID, TemplateVersion: template.Version, - WorkgroupID: group.ID, - VendorID: group.VendorID, + SupplierID: config.SupplierID, AssignedUserID: "", Status: domain.AnnotationTaskStatusPendingAnnotation, CurrentRevision: 0, @@ -740,7 +778,7 @@ func (runtime *Runtime) createAnnotationGateTasks( } _ = runtime.store.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ ID: domain.NewID("annotation-worklog"), OrganizationID: run.OrganizationID, ProjectID: run.ProjectID, - TaskID: task.ID, UserID: "system", WorkgroupID: group.ID, VendorID: group.VendorID, + TaskID: task.ID, UserID: "system", SupplierID: config.SupplierID, EventType: "created", Mode: eventMode, Temporary: eventMode == domain.RunModeDebug, TemplateVersion: template.Version, OutputDelta: 0, CreatedAt: nowISO(), }) tasks = append(tasks, task) @@ -789,6 +827,7 @@ func (runtime *Runtime) ensureAnnotationHumanBatch( OrganizationID: run.OrganizationID, PluginID: "annotation-qa", ProjectID: run.ProjectID, + SupplierID: config.SupplierID, WorkflowID: run.WorkflowID, RunID: run.ID, GateNodeID: node.ID, @@ -868,8 +907,7 @@ func (runtime *Runtime) ensureAnnotationHumanBatch( CurrentStageID: initialStageID, Status: domain.HumanItemReady, AssignedUserID: "", - WorkgroupID: task.WorkgroupID, - VendorID: task.VendorID, + SupplierID: task.SupplierID, MaxReworkRounds: batch.MaxReworkRounds, Version: 1, } diff --git a/internal/worker/capture_automation.go b/internal/worker/capture_automation.go index bbd7cc69..55362b01 100644 --- a/internal/worker/capture_automation.go +++ b/internal/worker/capture_automation.go @@ -84,6 +84,9 @@ func normalizedCaptureAutomation(config *domain.CaptureAutomationConfig) domain. if next.PollIntervalSeconds < 5 { next.PollIntervalSeconds = 5 } + if next.StabilityWindowSeconds <= 0 { + next.StabilityWindowSeconds = 30 + } return next } @@ -538,14 +541,29 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d newCount++ } if len(missing) == 0 { - record.Status = domain.CaptureRecordReady - if existing != nil && existing.ReadyAt != "" { - record.ReadyAt = existing.ReadyAt + if existing == nil || existing.Fingerprint != group.Fingerprint { + record.Status = domain.CaptureRecordStabilizing + record.StabilityChecks = 1 + record.StabilityObservedAt = nowText } else { - record.ReadyAt = nowText + record.StabilityChecks = existing.StabilityChecks + 1 + record.StabilityObservedAt = existing.StabilityObservedAt + if record.StabilityObservedAt == "" { + record.StabilityObservedAt = nowText + } + stableSince, stableSinceOK := parseRFC3339Nano(record.StabilityObservedAt) + stableEnough := stableSinceOK && now.Sub(stableSince) >= time.Duration(config.StabilityWindowSeconds)*time.Second + if record.StabilityChecks >= 2 && stableEnough { + record.Status = domain.CaptureRecordReady + record.ReadyAt = nowText + } else { + record.Status = domain.CaptureRecordStabilizing + } } } else { record.Status = domain.CaptureRecordIncomplete + record.StabilityChecks = 0 + record.StabilityObservedAt = "" } if err := runtime.store.UpsertCaptureRecord(ctx, record); err != nil { return err diff --git a/internal/worker/capture_automation_test.go b/internal/worker/capture_automation_test.go index eeb54ef7..be06819b 100644 --- a/internal/worker/capture_automation_test.go +++ b/internal/worker/capture_automation_test.go @@ -205,3 +205,10 @@ func TestLaunchCaptureRunBindsOneCompleteDirectory(t *testing.T) { t.Fatalf("unexpected directory binding: %#v", input) } } + +func TestNormalizedCaptureAutomationDefaultsToThirtySecondStabilityWindow(t *testing.T) { + config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Enabled: true, PollIntervalSeconds: 10}) + if config.StabilityWindowSeconds != 30 { + t.Fatalf("stability window = %d", config.StabilityWindowSeconds) + } +} diff --git a/internal/worker/human_release.go b/internal/worker/human_release.go index 7ccc7401..b0af4bd8 100644 --- a/internal/worker/human_release.go +++ b/internal/worker/human_release.go @@ -221,15 +221,15 @@ func (runtime *Runtime) syncDeliveryPackage(ctx context.Context, batch domain.Hu if len(packages) > 0 { item = packages[0] } else { - workgroupID, vendorID := "", "" + workgroupID, vendorID, supplierID := "", "", "" if len(partition.ItemIDs) > 0 { if humanItem, itemErr := runtime.store.GetHumanItem(ctx, partition.ItemIDs[0]); itemErr == nil && humanItem != nil { - workgroupID, vendorID = humanItem.WorkgroupID, humanItem.VendorID + workgroupID, vendorID, supplierID = humanItem.WorkgroupID, humanItem.VendorID, humanItem.SupplierID } } item = domain.DeliveryPackageRecord{ ID: "delivery-package-" + partition.ID, OrganizationID: batch.OrganizationID, ProjectID: batch.ProjectID, - WorkflowID: batch.WorkflowID, WorkgroupID: workgroupID, VendorID: vendorID, ParentRunID: batch.RunID, ChildRunID: partition.ChildRunID, + WorkflowID: batch.WorkflowID, SupplierID: supplierID, WorkgroupID: workgroupID, VendorID: vendorID, ParentRunID: batch.RunID, ChildRunID: partition.ChildRunID, ReleasePartitionID: partition.ID, ManifestDigest: partition.ManifestDigest, ItemIDs: partition.ItemIDs, Status: domain.DeliveryPackageBuilding, Mode: mode, Temporary: mode == domain.RunModeDebug, Version: 1, CreatedAt: nowISO(), UpdatedAt: nowISO(), } @@ -386,7 +386,7 @@ func (runtime *Runtime) ensureDeliveryPackageAcceptanceItem(ctx context.Context, ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.ChildRunID, GateNodeID: "delivery-package-acceptance", Subject: domain.HumanSubjectRef{ArtifactID: artifactID, ItemKey: item.ID, MediaType: "application/json", Payload: map[string]any{ "deliveryPackageId": item.ID, "artifactIds": item.ArtifactIDs, "manifestDigest": item.ManifestDigest, - }}, CurrentStageID: "acceptance", Status: status, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID, + }}, CurrentStageID: "acceptance", Status: status, SupplierID: item.SupplierID, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID, Version: 1, MaxReworkRounds: sourceBatch.MaxReworkRounds, CreatedAt: now, UpdatedAt: now, } if err := runtime.store.UpsertHumanItem(ctx, *humanItem); err != nil { diff --git a/k8s/droboticflow.template.yaml b/k8s/droboticflow.template.yaml index 19db3a3c..466d2658 100644 --- a/k8s/droboticflow.template.yaml +++ b/k8s/droboticflow.template.yaml @@ -175,7 +175,7 @@ data: FANYU_HEADPOSE_LICENSE_ID: "MIT" # Empty keeps the v5 template draft/fail-closed. Set the real published # annotation workgroup ID in the POC deployment environment before release. - FANYU_V5_ANNOTATION_WORKGROUP_ID: "__FANYU_V5_ANNOTATION_WORKGROUP_ID__" + FANYU_V5_SUPPLIER_ID: "__FANYU_V5_SUPPLIER_ID__" --- apiVersion: v1 kind: Service @@ -371,11 +371,11 @@ spec: configMapKeyRef: name: droboticflow-config key: FANYU_HEADPOSE_LICENSE_ID - - name: FANYU_V5_ANNOTATION_WORKGROUP_ID + - name: FANYU_V5_SUPPLIER_ID valueFrom: configMapKeyRef: name: droboticflow-config - key: FANYU_V5_ANNOTATION_WORKGROUP_ID + key: FANYU_V5_SUPPLIER_ID - name: AI_API_KEY valueFrom: secretKeyRef: diff --git a/operator-repos/builtin/annotation_gate/src/operator.py b/operator-repos/builtin/annotation_gate/src/operator.py index 438bff28..d57bc213 100644 --- a/operator-repos/builtin/annotation_gate/src/operator.py +++ b/operator-repos/builtin/annotation_gate/src/operator.py @@ -16,9 +16,9 @@ def _run_legacy( ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) template_id = params.get("ANNOTATION_TEMPLATE_ID", "").strip() - workgroup_id = params.get("ANNOTATION_WORKGROUP_ID", "").strip() - if not template_id or not workgroup_id: - raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID 和 ANNOTATION_WORKGROUP_ID") + supplier_id = params.get("ANNOTATION_SUPPLIER_ID", "").strip() + if not template_id: + raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID") payloads = helpers["payloads_from_inputs"](inputs) items = helpers["collect_items"](payloads) if not items: @@ -34,7 +34,7 @@ def _run_legacy( "runId": run_id, "nodeId": node_id, "templateId": template_id, - "workgroupId": workgroup_id, + "supplierId": supplier_id, "itemCount": len(items), "items": items, "interceptedByWorker": False, @@ -62,9 +62,9 @@ def _run_multiport( ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) template_id = params.get("ANNOTATION_TEMPLATE_ID", "").strip() - workgroup_id = params.get("ANNOTATION_WORKGROUP_ID", "").strip() - if not template_id or not workgroup_id: - raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID 和 ANNOTATION_WORKGROUP_ID") + supplier_id = params.get("ANNOTATION_SUPPLIER_ID", "").strip() + if not template_id: + raise RuntimeError("原生视频标注 Gate 需要 ANNOTATION_TEMPLATE_ID") # Only the Qwen ``clips`` port creates human work. HeadPose rendering is # also a video input in v5.7, but it is review evidence, not another clip # to cut, annotate, accept, and package. @@ -86,12 +86,12 @@ def _run_multiport( annotations = { "datasetType": "annotation-gate-input", "templateId": template_id, - "workgroupId": workgroup_id, + "supplierId": supplier_id, "itemCount": len(clips), "items": prelabels, } rejected = {"datasetType": "annotation-gate-rejected", "itemCount": 0, "items": []} - summary = {"clipCount": len(clips), "prelabelCount": len(prelabels), "boundaryCount": len(boundaries), "templateId": template_id, "workgroupId": workgroup_id} + summary = {"clipCount": len(clips), "prelabelCount": len(prelabels), "boundaryCount": len(boundaries), "templateId": template_id, "supplierId": supplier_id} return [ *clips, write(run_id, node_id, output_dir, "annotation-gate-annotations-dataset.json", annotations, output=_output(node_data, helpers, "annotations")), diff --git a/scripts/build_fanyu_v5_s100_bundle.py b/scripts/build_fanyu_v5_s100_bundle.py index 3635e9e9..9c444281 100644 --- a/scripts/build_fanyu_v5_s100_bundle.py +++ b/scripts/build_fanyu_v5_s100_bundle.py @@ -45,7 +45,6 @@ REQUIRED_VALUES = { "FANYU_S100_MODE", "FANYU_REMOTE_API_BASE_URL", "FANYU_REMOTE_BOS_ENDPOINT", - "FANYU_V5_ANNOTATION_WORKGROUP_ID", "FANYU_HEADPOSE_API_BASE_URL", "FANYU_HEADPOSE_SERVICE_IMAGE_ID", "FANYU_HEADPOSE_MODEL_SHA256", diff --git a/scripts/deploy-envs/poc.env b/scripts/deploy-envs/poc.env index 68cd3205..d2e959ae 100644 --- a/scripts/deploy-envs/poc.env +++ b/scripts/deploy-envs/poc.env @@ -57,4 +57,4 @@ export AI_MODEL="${AI_MODEL:-}" export HEADPOSE_MODEL_SHA256="${HEADPOSE_MODEL_SHA256:-sha256:9bdcd0e072df0d21e92288b40ea1d1b3cf7173c44728ab9c45c4d673f361e018}" # Required only to publish/run the Fanyu v5 release. Keep blank until the # live POC annotation workgroup has been created and verified. -export FANYU_V5_ANNOTATION_WORKGROUP_ID="${FANYU_V5_ANNOTATION_WORKGROUP_ID:-}" +export FANYU_V5_SUPPLIER_ID="${FANYU_V5_SUPPLIER_ID:-}" diff --git a/scripts/deploy-k8s.sh b/scripts/deploy-k8s.sh index cdf2233b..51ed15d6 100755 --- a/scripts/deploy-k8s.sh +++ b/scripts/deploy-k8s.sh @@ -56,7 +56,7 @@ fi HEADPOSE_SERVICE_IMAGE="${HEADPOSE_SERVICE_IMAGE:-droboticflow-headpose-service:latest}" HEADPOSE_SERVICE_IMAGE_ID="${HEADPOSE_SERVICE_IMAGE_ID:-}" HEADPOSE_MODEL_SHA256="${HEADPOSE_MODEL_SHA256:-sha256:9bdcd0e072df0d21e92288b40ea1d1b3cf7173c44728ab9c45c4d673f361e018}" -FANYU_V5_ANNOTATION_WORKGROUP_ID="${FANYU_V5_ANNOTATION_WORKGROUP_ID:-}" +FANYU_V5_SUPPLIER_ID="${FANYU_V5_SUPPLIER_ID:-}" DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}" DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}" OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}" @@ -226,7 +226,7 @@ env \ HEADPOSE_SERVICE_IMAGE="$HEADPOSE_SERVICE_IMAGE" \ HEADPOSE_SERVICE_IMAGE_ID="$HEADPOSE_SERVICE_IMAGE_ID" \ HEADPOSE_MODEL_SHA256="$HEADPOSE_MODEL_SHA256" \ - FANYU_V5_ANNOTATION_WORKGROUP_ID="$FANYU_V5_ANNOTATION_WORKGROUP_ID" \ + FANYU_V5_SUPPLIER_ID="$FANYU_V5_SUPPLIER_ID" \ OPERATOR_RUNNER_BUILD_COMMIT="$OPERATOR_RUNNER_BUILD_COMMIT" \ OPERATOR_RUNNER_BUILD_TIME="$OPERATOR_RUNNER_BUILD_TIME" \ DROBOTICFLOW_BUILD_COMMIT="$DROBOTICFLOW_BUILD_COMMIT" \ diff --git a/scripts/lib/deploy-validation.sh b/scripts/lib/deploy-validation.sh index 224e10d5..22d41d73 100644 --- a/scripts/lib/deploy-validation.sh +++ b/scripts/lib/deploy-validation.sh @@ -153,7 +153,7 @@ validate_manifest_startup_configuration() { validate_http_service_url WORKER_API_BASE_URL "$WORKER_API_BASE_URL" validate_ai_runtime_configuration validate_optional_http_service_url BOS_ENDPOINT "$BOS_ENDPOINT" - validate_optional_token_value FANYU_V5_ANNOTATION_WORKGROUP_ID "${FANYU_V5_ANNOTATION_WORKGROUP_ID:-}" + validate_optional_token_value FANYU_V5_SUPPLIER_ID "${FANYU_V5_SUPPLIER_ID:-}" validate_integer_min MONGO_CONNECT_RETRIES "$MONGO_CONNECT_RETRIES" 1 validate_integer_min MONGO_CONNECT_RETRY_DELAY_MS "$MONGO_CONNECT_RETRY_DELAY_MS" 1 validate_integer_min WORKER_POLL_INTERVAL_MS "$WORKER_POLL_INTERVAL_MS" 1 diff --git a/scripts/render-manifests.sh b/scripts/render-manifests.sh index 12d8146b..23d430f8 100755 --- a/scripts/render-manifests.sh +++ b/scripts/render-manifests.sh @@ -70,7 +70,7 @@ DROBOTICFLOW_TRUSTED_PROXY_CIDRS="${DROBOTICFLOW_TRUSTED_PROXY_CIDRS:-}" HEADPOSE_SERVICE_IMAGE="${HEADPOSE_SERVICE_IMAGE:-droboticflow-headpose-service:latest}" HEADPOSE_SERVICE_IMAGE_ID="${HEADPOSE_SERVICE_IMAGE_ID:-}" HEADPOSE_MODEL_SHA256="${HEADPOSE_MODEL_SHA256:-sha256:9bdcd0e072df0d21e92288b40ea1d1b3cf7173c44728ab9c45c4d673f361e018}" -FANYU_V5_ANNOTATION_WORKGROUP_ID="${FANYU_V5_ANNOTATION_WORKGROUP_ID:-}" +FANYU_V5_SUPPLIER_ID="${FANYU_V5_SUPPLIER_ID:-}" # shellcheck source=scripts/lib/image-config.sh source "$ROOT_DIR/scripts/lib/image-config.sh" REGISTRY_HOST="${REGISTRY_HOST:-}" @@ -178,7 +178,7 @@ export \ HEADPOSE_SERVICE_IMAGE \ HEADPOSE_SERVICE_IMAGE_ID \ HEADPOSE_MODEL_SHA256 \ - FANYU_V5_ANNOTATION_WORKGROUP_ID + FANYU_V5_SUPPLIER_ID python3 - <<'PY' import os @@ -242,7 +242,7 @@ replacements = { "__HEADPOSE_SERVICE_IMAGE__": os.environ["HEADPOSE_SERVICE_IMAGE"], "__HEADPOSE_SERVICE_IMAGE_ID__": os.environ["HEADPOSE_SERVICE_IMAGE_ID"], "__HEADPOSE_MODEL_SHA256__": os.environ["HEADPOSE_MODEL_SHA256"], - "__FANYU_V5_ANNOTATION_WORKGROUP_ID__": os.environ["FANYU_V5_ANNOTATION_WORKGROUP_ID"], + "__FANYU_V5_SUPPLIER_ID__": os.environ["FANYU_V5_SUPPLIER_ID"], } rendered = template diff --git a/src/components/PluginRouteGuard.tsx b/src/components/PluginRouteGuard.tsx index 0fa3b4db..36bb0b53 100644 --- a/src/components/PluginRouteGuard.tsx +++ b/src/components/PluginRouteGuard.tsx @@ -3,6 +3,7 @@ import { Blocks, Loader2 } from "lucide-react"; import { Link } from "react-router-dom"; import { usePlugins } from "../context/PluginContext"; import { useUser } from "../context/UserContext"; +import { isOrganizationAdminRole } from "../lib/user-roles"; export function PluginRouteGuard({ pluginId, children }: { pluginId: string; children: React.ReactNode }) { const { loading, isEnabled } = usePlugins(); @@ -23,7 +24,7 @@ export function PluginRouteGuard({ pluginId, children }: { pluginId: string; chi

当前组织未安装 {pluginId},相关工作台、路由和算子均不可用。

- {role === "admin" ? ( + {isOrganizationAdminRole(role) ? ( 前往系统配置安装 diff --git a/src/components/PluginSettingsPanel.tsx b/src/components/PluginSettingsPanel.tsx index 335c33fe..43a54286 100644 --- a/src/components/PluginSettingsPanel.tsx +++ b/src/components/PluginSettingsPanel.tsx @@ -1,25 +1,42 @@ import { useEffect, useState } from "react"; -import { Blocks, Loader2, Save } from "lucide-react"; +import { AlertTriangle, Blocks, CheckCircle2, Loader2, Save } from "lucide-react"; import { useUser } from "../context/UserContext"; import { pluginsApi } from "../services/api/plugins"; import type { PluginStatus } from "../types"; import { getActionErrorMessage } from "../lib/action-errors.js"; import { usePlugins } from "../context/PluginContext"; +import { humanLoopApi } from "../services/api/human-loop"; +import type { HumanItem } from "../types"; const inputClass = "rounded-md border border-[#2a2b2f] bg-[#0a0a0a] px-3 py-2 text-sm text-white"; export default function PluginSettingsPanel() { - const { organization } = useUser(); + const { organization, role } = useUser(); const { statuses, loading, error: loadError, refresh: refreshPlugins } = usePlugins(); const [status, setStatus] = useState(null); const [saving, setSaving] = useState(false); const [error, setError] = useState(""); + const [escalated, setEscalated] = useState([]); + const [escalationComment, setEscalationComment] = useState(""); useEffect(() => { const annotationStatus = statuses.find((item) => item.manifest.id === "annotation-qa"); if (annotationStatus) setStatus(annotationStatus); }, [statuses]); + const refreshEscalations = async () => { + if (role !== "admin" && role !== "organization_admin") return; + try { + setEscalated(await humanLoopApi.listItems({ pluginId: "annotation-qa", status: "escalated", pool: "all" })); + } catch (cause) { + setError(getActionErrorMessage(cause, "加载升级任务失败")); + } + }; + + useEffect(() => { + void refreshEscalations(); + }, [role]); + if (!organization || !status) { if (error || loadError) return
{error || loadError}
; return loading ? ( @@ -111,6 +128,104 @@ export default function PluginSettingsPanel() { ))} +
+
+
通过条目分区释放
+

达到条数或等待时间后,已通过条目会先进入下游,未完成任务不会阻塞它们。

+
+
+
全批通过后释放
+

所有任务完成并通过后一次释放;有未解决任务时整批保持等待。

+
+
+
最大返工轮次
+

超过限制后任务进入升级区并阻塞批次,必须由组织管理员强制重开或关闭。

+
+
+ {role === "admin" || role === "organization_admin" ? ( +
+
+
+ + 升级任务({escalated.length}) +
+ +
+