diff --git a/backend/operator-runner/main.py b/backend/operator-runner/main.py index 553c338e..6b3f3da9 100644 --- a/backend/operator-runner/main.py +++ b/backend/operator-runner/main.py @@ -71,7 +71,7 @@ FALSE_ENV_VALUES = {"0", "false", "no", "off"} MAX_SAFE_INTEGER = 9_007_199_254_740_991 EXACT_POSITIVE_INT_RE = re.compile(r"^[1-9][0-9]*$") INTEGERISH_RE = re.compile(r"^[+-]?[0-9]+$") -SUPPORTED_COMPARISON_OPERATORS = {"contains", "eq", "exists", "gt", "gte", "in", "lt", "lte", "missing", "ne"} +SUPPORTED_COMPARISON_OPERATORS = {"contains", "eq", "exists", "gt", "gte", "in", "lt", "lte", "missing", "ne", "regex"} VALID_SOURCE_DATA_SOURCES = {"INLINE", "LOCAL", "BOS", "UPSTREAM"} RESERVED_MONGO_COLLECTIONS = { "projects", @@ -3201,6 +3201,11 @@ def compare_values(actual: Any, operator: str, expected: Any) -> bool: if isinstance(expected, list): return actual in expected return str(actual) in str(expected) + if operator == "regex": + try: + return re.search(str(expected), str(actual or "")) is not None + except re.error as exc: + raise RuntimeError(f"invalid regular expression: {exc}") from exc if operator in {"gt", "gte", "lt", "lte"}: try: actual_number = float(actual) @@ -3219,6 +3224,41 @@ def compare_values(actual: Any, operator: str, expected: Any) -> bool: return actual == expected +def parse_filter_dsl(value: str) -> list[tuple[str, str, Any]]: + expression = str(value or "").strip() + if not expression: + return [] + clauses: list[tuple[str, str, Any]] = [] + operator_map = {"==": "eq", "!=": "ne", ">": "gt", ">=": "gte", "<": "lt", "<=": "lte", "~": "regex"} + pattern = re.compile(r"^\s*([\w.\-\u4e00-\u9fff]+)\s*(<=|>=|==|!=|<|>|~)\s*(.+?)\s*$") + for index, raw_clause in enumerate(expression.split("&")): + match = pattern.match(raw_clause) + if match is None: + raise RuntimeError(f"DSL clause {index + 1} is invalid: {raw_clause.strip()}") + field_path, operator_token, raw_expected = match.groups() + expected_text = raw_expected.strip() + if expected_text.endswith("s") and re.fullmatch(r"-?\d+(?:\.\d+)?s", expected_text): + expected_text = expected_text[:-1] + try: + expected = json.loads(expected_text) + except Exception: + expected = expected_text.strip("\"'") + clauses.append((field_path, operator_map[operator_token], expected)) + return clauses + + +def filter_rows(rows: list[dict[str, Any]], params: dict[str, str]) -> tuple[list[dict[str, Any]], dict[str, Any]]: + dsl = params.get("DSL", "").strip() + if dsl: + clauses = parse_filter_dsl(dsl) + filtered = [row for row in rows if all(compare_values(get_field_value(row, field), operator, expected) for field, operator, expected in clauses)] + return filtered, {"dsl": dsl, "clauses": [{"field": field, "operator": operator, "expected": expected} for field, operator, expected in clauses]} + field_path = params.get("FIELD_PATH", "").strip() + operator = normalize_comparison_operator(params.get("OPERATOR", "eq")) + expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", "")) + return [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)], {"fieldPath": field_path, "operator": operator, "expected": expected} + + def flatten_context(value: Any, prefix: str = "") -> dict[str, str]: context: dict[str, str] = {} if isinstance(value, dict): @@ -4020,15 +4060,10 @@ def builtin_json_filter( inputs: list[dict[str, Any]], ) -> list[dict[str, Any]]: params = hyperparams(node_data) - field_path = params.get("FIELD_PATH", "").strip() - operator = normalize_comparison_operator(params.get("OPERATOR", "eq")) - expected = parse_json_param(params, "EXPECTED_JSON", params.get("EXPECTED", "")) rows = collect_items(payloads_from_inputs(inputs)) - filtered = [row for row in rows if compare_values(get_field_value(row, field_path), operator, expected)] + filtered, filter_metadata = filter_rows(rows, params) payload = { - "fieldPath": field_path, - "operator": operator, - "expected": expected, + **filter_metadata, "items": filtered, "count": len(filtered), } @@ -5052,6 +5087,7 @@ OPERATOR_HELPERS = { "collect_output_artifacts": collect_output_artifacts, "common_input_root": common_input_root, "compare_values": compare_values, + "filter_rows": filter_rows, "date_in_range": date_in_range, "documents_from_inputs": documents_from_inputs, "execute_command": execute_command, diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py index 7efdb6b4..be37d168 100644 --- a/backend/operator-runner/test_main.py +++ b/backend/operator-runner/test_main.py @@ -1593,6 +1593,27 @@ class OperatorHyperparamParsingTests(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "OPERATOR must be one of"): MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [{"data": {"items": [{"status": "ready"}]}}]) + def test_json_filter_supports_safe_dsl_and_regex(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + node_data = { + "label": "JSON 过滤", + "hyperparams": [{"key": "DSL", "value": "quality.occlusion<0.1 & action~搬.*箱"}], + } + inputs = {"data": {"items": [ + {"id": "keep", "quality": {"occlusion": 0.05}, "action": "搬运箱子"}, + {"id": "drop-quality", "quality": {"occlusion": 0.2}, "action": "搬运箱子"}, + {"id": "drop-regex", "quality": {"occlusion": 0.05}, "action": "静止"}, + ]}} + artifacts = MODULE.builtin_json_filter("run-1", "filter", output_dir, node_data, [inputs]) + payload = MODULE.load_artifact_payload(artifacts[0]) + self.assertEqual([item["id"] for item in payload["items"]], ["keep"]) + self.assertEqual(len(payload["clauses"]), 2) + + def test_filter_dsl_rejects_unsupported_syntax(self): + with self.assertRaisesRegex(RuntimeError, "DSL clause 1 is invalid"): + MODULE.parse_filter_dsl("quality.score or true") + def test_json_map_rejects_non_object_field_map(self): with tempfile.TemporaryDirectory() as temp_dir: output_dir = pathlib.Path(temp_dir) diff --git a/backend/operators/builtin/json_filter/operator.py b/backend/operators/builtin/json_filter/operator.py index 8e1d2012..4a00c64b 100644 --- a/backend/operators/builtin/json_filter/operator.py +++ b/backend/operators/builtin/json_filter/operator.py @@ -15,19 +15,10 @@ def run( helpers: HelperMap, ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) - field_path = params.get("FIELD_PATH", "").strip() - operator = helpers["normalize_comparison_operator"](params.get("OPERATOR", "eq")) - expected = helpers["parse_json_param"](params, "EXPECTED_JSON", params.get("EXPECTED", "")) rows = helpers["collect_items"](helpers["payloads_from_inputs"](inputs)) - filtered = [ - row - for row in rows - if helpers["compare_values"](helpers["get_field_value"](row, field_path), operator, expected) - ] + filtered, filter_metadata = helpers["filter_rows"](rows, params) payload = { - "fieldPath": field_path, - "operator": operator, - "expected": expected, + **filter_metadata, "items": filtered, "count": len(filtered), } diff --git a/docs/operators/JSON-过滤.md b/docs/operators/JSON-过滤.md index 6850b398..20d923ec 100644 --- a/docs/operators/JSON-过滤.md +++ b/docs/operators/JSON-过滤.md @@ -48,8 +48,9 @@ flowchart LR ## 关键超参数 +- `DSL`:类型 `string`,默认值 空字符串或由工作流显式填写。可选安全 DSL,支持使用 & 连接条件,例如 quality.occlusion<0.1 & duration>360s;设置后优先于单字段条件 通常建议在工作流中显式配置。 - `FIELD_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。待判断字段路径 通常建议在工作流中显式配置。 -- `OPERATOR`:类型 `string`,默认值 `eq`。eq/ne/contains/gt/gte/lt/lte/exists/missing/in;显式覆盖时必须是受支持值 可直接使用默认值,也可在节点级覆盖。 +- `OPERATOR`:类型 `string`,默认值 `eq`。eq/ne/contains/gt/gte/lt/lte/exists/missing/in/regex;显式覆盖时必须是受支持值 可直接使用默认值,也可在节点级覆盖。 - `EXPECTED`:类型 `string`,默认值 空字符串或由工作流显式填写。字符串比较值 通常建议在工作流中显式配置。 - `EXPECTED_JSON`:类型 `string`,默认值 空字符串或由工作流显式填写。JSON 格式比较值,优先于 EXPECTED 通常建议在工作流中显式配置。 @@ -77,6 +78,7 @@ flowchart LR "type": "builtin", "category": "DATA", "hyperparams": { + "DSL": "<按节点填写>", "FIELD_PATH": "<按节点填写>", "OPERATOR": "eq", "EXPECTED": "<按节点填写>", diff --git a/docs/operators/抽检采样.md b/docs/operators/抽检采样.md index 845da495..c1bc4943 100644 --- a/docs/operators/抽检采样.md +++ b/docs/operators/抽检采样.md @@ -47,7 +47,7 @@ flowchart LR ## 关键超参数 -- `RULES_JSON`:类型 `string`,默认值 `[]`。抽检规则数组;显式规则必须包含 id/field/operator/value/samplePercent,operator 必须受支持,samplePercent 必须在 (0,100] 可直接使用默认值,也可在节点级覆盖。 +- `RULES_JSON`:类型 `string`,默认值 `[{"id":"low-clarity","field":"quality.clarity","operator":"lt","value":0.5,"samplePercent":10},{"id":"low-confidence","field":"annotation.confidence","operator":"lt","value":0.7,"samplePercent":10},{"id":"high-jitter","field":"quality.jitter","operator":"gt","value":0.15,"samplePercent":10},{"id":"high-no-hand","field":"quality.noHandRate","operator":"gt","value":0.15,"samplePercent":10}]`。抽检规则数组;显式规则必须包含 id/field/operator/value/samplePercent,operator 必须受支持,samplePercent 必须在 (0,100] 可直接使用默认值,也可在节点级覆盖。 ## 执行前准备 @@ -73,7 +73,7 @@ flowchart LR "type": "builtin", "category": "QA", "hyperparams": { - "RULES_JSON": "[]" + "RULES_JSON": "[{\"id\":\"low-clarity\",\"field\":\"quality.clarity\",\"operator\":\"lt\",\"value\":0.5,\"samplePercent\":10},{\"id\":\"low-confidence\",\"field\":\"annotation.confidence\",\"operator\":\"lt\",\"value\":0.7,\"samplePercent\":10},{\"id\":\"high-jitter\",\"field\":\"quality.jitter\",\"operator\":\"gt\",\"value\":0.15,\"samplePercent\":10},{\"id\":\"high-no-hand\",\"field\":\"quality.noHandRate\",\"operator\":\"gt\",\"value\":0.15,\"samplePercent\":10}]" } } ``` diff --git a/go.mod b/go.mod index 455275e3..bf53aaef 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,12 @@ module droboticflow go 1.24.0 require ( + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/baidubce/bce-sdk-go v0.9.264 github.com/go-chi/chi/v5 v5.2.3 go.mongodb.org/mongo-driver v1.17.6 golang.org/x/crypto v0.36.0 + golang.org/x/sync v0.12.0 k8s.io/api v0.34.1 k8s.io/apimachinery v0.34.1 k8s.io/client-go v0.34.1 @@ -46,7 +48,6 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/term v0.30.0 // indirect golang.org/x/text v0.23.0 // indirect diff --git a/go.sum b/go.sum index 36dc87d0..46c74588 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/baidubce/bce-sdk-go v0.9.264 h1:qqOW8OEjNIqFOzULGTum7m0y6NBebG9+2rJBdBgdQso= diff --git a/internal/api/annotation_operations.go b/internal/api/annotation_operations.go index 355b9291..257ad5b5 100644 --- a/internal/api/annotation_operations.go +++ b/internal/api/annotation_operations.go @@ -27,12 +27,7 @@ func annotationManager(principal *authenticatedPrincipal) bool { if principal == nil { return false } - switch principal.User.Role { - case domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleOperator: - return true - default: - return false - } + return principal.User.Role == domain.UserRoleAdmin } func annotationPrincipalUserID(principal *authenticatedPrincipal) string { @@ -529,12 +524,17 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques return } revisions, revisionsErr := server.store.ListAnnotationRevisions(request.Context(), item.OrganizationID, item.ID) + sliceRevisions, sliceRevisionsErr := server.store.ListSliceRevisions(request.Context(), item.OrganizationID, item.ID) reviews, reviewsErr := server.store.ListAnnotationReviews(request.Context(), item.OrganizationID, item.ID) - if revisionsErr != nil || reviewsErr != nil { - renderError(writer, http.StatusInternalServerError, fmt.Errorf("load annotation task history: revisions=%v reviews=%v", revisionsErr, reviewsErr)) + if revisionsErr != nil || sliceRevisionsErr != nil || reviewsErr != nil { + renderError(writer, http.StatusInternalServerError, fmt.Errorf("load annotation task history: slices=%v revisions=%v reviews=%v", sliceRevisionsErr, revisionsErr, reviewsErr)) return } - renderJSON(writer, http.StatusOK, map[string]any{"task": item, "revisions": revisions, "reviews": reviews}) + var humanItem *domain.HumanItemRecord + if linked, linkedErr := server.humanItemForLegacyTask(request.Context(), *item); linkedErr == nil { + humanItem = linked + } + renderJSON(writer, http.StatusOK, map[string]any{"task": item, "sliceRevisions": sliceRevisions, "revisions": revisions, "reviews": reviews, "humanItem": humanItem}) } func (server *server) handleAnnotationTaskCreate(writer http.ResponseWriter, request *http.Request) { @@ -616,10 +616,195 @@ func (server *server) handleAnnotationTaskCreate(writer http.ResponseWriter, req type annotationRevisionInput struct { Segments []domain.AnnotationSegment `json:"segments"` + ObjectHints []string `json:"objectHints,omitempty"` ChangeReason string `json:"changeReason,omitempty"` DurationSeconds int64 `json:"durationSeconds,omitempty"` } +type sliceRevisionInput struct { + Segments []domain.SliceSegment `json:"segments"` + Operations []domain.SliceOperation `json:"operations,omitempty"` + ChangeReason string `json:"changeReason,omitempty"` + DurationSeconds int64 `json:"durationSeconds,omitempty"` +} + +type annotationTaskRejectInput struct { + ReasonCodes []string `json:"reasonCodes"` + Comment string `json:"comment"` +} + +func (server *server) handleAnnotationTaskReject(writer http.ResponseWriter, request *http.Request) { + task, err := server.store.GetAnnotationTask(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if task == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Annotation task not found"}) + return + } + principal, memberships, ok := server.ensureAnnotationTaskAccess(writer, request, *task) + if !ok { + return + } + if !annotationManager(principal) && !membershipHasRole(memberships, *task, domain.AnnotationRoleAnnotator) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "annotator assignment is required"}) + return + } + var payload annotationTaskRejectInput + if !decodeJSON(writer, request, &payload) { + return + } + if len(payload.ReasonCodes) == 0 || strings.TrimSpace(payload.Comment) == "" { + renderValidationError(writer, validationIssue{Field: "rejection", Message: "reasonCodes and comment are required"}) + return + } + item, err := server.humanItemForLegacyTask(request.Context(), *task) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item is required for a workflow-visible rejection", "code": "human_item_required"}) + return + } + batch, err := server.store.GetHumanBatch(request.Context(), item.BatchID) + if err != nil || batch == nil { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Human batch not found"}) + } + return + } + previousVersion := item.Version + item.Status = domain.HumanItemRejected + item.RejectedAt = nowISO() + item.Version++ + updated, err := server.store.UpdateHumanItemVersioned(request.Context(), *item, previousVersion) + if err != nil || !updated { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"}) + } + return + } + allowed := allowedHumanReopenStages(batch, item.CurrentStageID) + reject := domain.HumanRejectRecord{OrganizationID: item.OrganizationID, BatchID: item.BatchID, ItemID: item.ID, ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.RunID, StageID: item.CurrentStageID, AttemptRound: item.ReviewRound, RejectedByUserID: principal.User.ID, ReasonCodes: payload.ReasonCodes, Comment: strings.TrimSpace(payload.Comment), AllowedReopenStages: allowed, Status: domain.HumanRejectOpen} + if err := server.store.UpsertHumanReject(request.Context(), reject); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + task.Status = domain.AnnotationTaskStatusRejected + if err := server.store.UpsertAnnotationTask(request.Context(), *task); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + _ = server.appendHumanEvent(request.Context(), *item, "rejected", principal.User.ID, map[string]any{"reasonCodes": payload.ReasonCodes, "comment": strings.TrimSpace(payload.Comment)}) + renderJSON(writer, http.StatusCreated, map[string]any{"task": task, "item": item}) +} + +func validateSliceSegments(segments []domain.SliceSegment, task domain.AnnotationTaskRecord) []validationIssue { + issues := make([]validationIssue, 0) + if len(segments) == 0 { + return append(issues, validationIssue{Field: "segments", Message: "at least one corrected slice is required"}) + } + sorted := append([]domain.SliceSegment(nil), segments...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start }) + for index, segment := range sorted { + if strings.TrimSpace(segment.ID) == "" { + issues = append(issues, validationIssue{Field: fmt.Sprintf("segments.%d.id", index), Message: "slice id is required"}) + } + if math.IsNaN(segment.Start) || math.IsNaN(segment.End) || segment.Start < task.ClipStart || segment.End > task.ClipEnd || segment.End <= segment.Start { + issues = append(issues, validationIssue{Field: fmt.Sprintf("segments.%d", index), Message: "slice must be finite and inside the episode range"}) + } + if index > 0 && segment.Start < sorted[index-1].End { + issues = append(issues, validationIssue{Field: fmt.Sprintf("segments.%d", index), Message: "corrected slices must not overlap"}) + } + } + return issues +} + +func (server *server) handleSliceRevisionCreate(writer http.ResponseWriter, request *http.Request) { + task, err := server.store.GetAnnotationTask(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if task == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Annotation task not found"}) + return + } + principal, memberships, ok := server.ensureAnnotationTaskAccess(writer, request, *task) + if !ok { + return + } + if !annotationManager(principal) && !membershipHasRole(memberships, *task, domain.AnnotationRoleAnnotator) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "annotator assignment is required"}) + return + } + if task.AssignedUserID != "" && task.AssignedUserID != principal.User.ID && !annotationManager(principal) { + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "task is assigned to another annotator"}) + return + } + var payload sliceRevisionInput + if !decodeJSON(writer, request, &payload) { + return + } + if issues := validateSliceSegments(payload.Segments, *task); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + existing, err := server.store.ListSliceRevisions(request.Context(), task.OrganizationID, task.ID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + revision := domain.SliceRevisionRecord{ + ID: domain.NewID("slice-revision"), OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: len(existing) + 1, + Segments: payload.Segments, Operations: payload.Operations, AuthorUserID: principal.User.ID, + ChangeReason: strings.TrimSpace(payload.ChangeReason), DurationSeconds: payload.DurationSeconds, CreatedAt: nowISO(), + } + if err := server.store.UpsertSliceRevision(request.Context(), revision); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + task.AssignedUserID = principal.User.ID + task.Status = domain.AnnotationTaskStatusPendingAnnotation + task.UpdatedAt = nowISO() + if err := server.store.UpsertAnnotationTask(request.Context(), *task); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + item, err := server.humanItemForLegacyTask(request.Context(), *task) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item != nil { + previousVersion := item.Version + item.CurrentStageID = "semantic_annotation" + item.Status = domain.HumanItemReady + item.AssignedUserID = principal.User.ID + item.LatestAuthorUserID = principal.User.ID + item.Version++ + updated, updateErr := server.store.UpdateHumanItemVersioned(request.Context(), *item, previousVersion) + if updateErr != nil || !updated { + if updateErr != nil { + renderError(writer, http.StatusInternalServerError, updateErr) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"}) + } + return + } + _ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: "split_annotation", Round: item.ReviewRound, Status: domain.HumanItemSubmitted, AssigneeUserID: principal.User.ID, SubmissionRef: revision.ID}) + _ = server.appendHumanEvent(request.Context(), *item, "submitted", principal.User.ID, map[string]any{"submissionRef": revision.ID, "sliceRevision": revision.Revision}) + } + _ = server.store.UpsertAnnotationWorklogEvent(request.Context(), domain.AnnotationWorklogEventRecord{ID: domain.NewID("annotation-worklog"), OrganizationID: task.OrganizationID, ProjectID: task.ProjectID, TaskID: task.ID, UserID: principal.User.ID, WorkgroupID: task.WorkgroupID, VendorID: task.VendorID, EventType: "slice_submitted", Revision: revision.Revision, DurationSeconds: payload.DurationSeconds, OutputDelta: 0, CreatedAt: nowISO()}) + renderJSON(writer, http.StatusCreated, map[string]any{"task": task, "sliceRevision": revision}) +} + func validateAnnotationSegments(segments []domain.AnnotationSegment, task domain.AnnotationTaskRecord) []validationIssue { issues := make([]validationIssue, 0) sorted := append([]domain.AnnotationSegment(nil), segments...) @@ -675,7 +860,7 @@ func (server *server) handleAnnotationRevisionCreate(writer http.ResponseWriter, revisionNumber := task.CurrentRevision + 1 revision := domain.AnnotationRevisionRecord{ ID: domain.NewID("annotation-revision"), OrganizationID: task.OrganizationID, TaskID: task.ID, - Revision: revisionNumber, TemplateVersion: task.TemplateVersion, Segments: payload.Segments, + Revision: revisionNumber, TemplateVersion: task.TemplateVersion, Segments: payload.Segments, ObjectHints: payload.ObjectHints, AuthorUserID: principal.User.ID, ChangeReason: strings.TrimSpace(payload.ChangeReason), IsRework: isRework, CountsAsOutput: !isRework, DurationSeconds: payload.DurationSeconds, CreatedAt: nowISO(), } @@ -712,12 +897,16 @@ func (server *server) handleAnnotationRevisionCreate(writer http.ResponseWriter, } type annotationReviewInput struct { - Stage domain.AnnotationReviewStage `json:"stage"` - Decision domain.AnnotationReviewDecision `json:"decision"` - Comment string `json:"comment,omitempty"` - DurationSeconds int64 `json:"durationSeconds,omitempty"` - RejectTargets []string `json:"rejectTargets,omitempty"` - ReasonCodes []string `json:"reasonCodes,omitempty"` + Stage domain.AnnotationReviewStage `json:"stage"` + Decision domain.AnnotationReviewDecision `json:"decision"` + Comment string `json:"comment,omitempty"` + DurationSeconds int64 `json:"durationSeconds,omitempty"` + RejectTargets []string `json:"rejectTargets,omitempty"` + ReasonCodes []string `json:"reasonCodes,omitempty"` + VideoQuality string `json:"videoQuality,omitempty"` + AnnotationAccuracy float64 `json:"annotationAccuracy,omitempty"` + AnnotationCompleteness float64 `json:"annotationCompleteness,omitempty"` + OverallGrade string `json:"overallGrade,omitempty"` } func (server *server) queueAcceptedAnnotationGateRun(ctx context.Context, task domain.AnnotationTaskRecord) error { @@ -855,12 +1044,15 @@ func (server *server) handleAnnotationReviewCreate(writer http.ResponseWriter, r ID: domain.NewID("annotation-review"), OrganizationID: task.OrganizationID, TaskID: task.ID, Revision: task.CurrentRevision, Stage: payload.Stage, Decision: payload.Decision, ReviewerUserID: principal.User.ID, Comment: strings.TrimSpace(payload.Comment), CreatedAt: nowISO(), + VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: payload.AnnotationAccuracy, + AnnotationCompleteness: payload.AnnotationCompleteness, OverallGrade: strings.TrimSpace(payload.OverallGrade), + ReasonCodes: payload.ReasonCodes, RejectTargets: payload.RejectTargets, } if payload.Decision == domain.AnnotationReviewRejected { review.NextRevision = task.CurrentRevision + 1 - task.Status = domain.AnnotationTaskStatusRework + task.Status = domain.AnnotationTaskStatusRejected } else if payload.Stage == domain.AnnotationReviewStageQuality { - task.Status = domain.AnnotationTaskStatusPendingAcceptance + task.Status = domain.AnnotationTaskStatusAccepted } else { task.Status = domain.AnnotationTaskStatusAccepted } diff --git a/internal/api/auth.go b/internal/api/auth.go index 95a157ad..fde567ad 100644 --- a/internal/api/auth.go +++ b/internal/api/auth.go @@ -822,7 +822,7 @@ func (server *server) handleAdminOrganizationUsersCreate(writer http.ResponseWri return } if !validUserRole(payload.Role) { - renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, pipeline_engineer, data_operator, qa_reviewer, compliance_auditor, auditor, or operator"}) + renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, annotator, quality_reviewer, or acceptance_reviewer"}) return } passwordHash, err := hashPassword(payload.Password) diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index 3a31c5c8..f00f7ed6 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -954,6 +954,12 @@ ], "description": "按字段条件过滤 JSON 列表。", "hyperparams": [ + { + "description": "可选安全 DSL,支持使用 & 连接条件,例如 quality.occlusion<0.1 & duration>360s;设置后优先于单字段条件", + "key": "DSL", + "type": "string", + "value": "" + }, { "description": "待判断字段路径", "key": "FIELD_PATH", @@ -961,7 +967,7 @@ "value": "" }, { - "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing/in;显式覆盖时必须是受支持值", + "description": "eq/ne/contains/gt/gte/lt/lte/exists/missing/in/regex;显式覆盖时必须是受支持值", "key": "OPERATOR", "type": "string", "value": "eq" @@ -2104,7 +2110,7 @@ "description": "抽检规则数组;显式规则必须包含 id/field/operator/value/samplePercent,operator 必须受支持,samplePercent 必须在 (0,100]", "key": "RULES_JSON", "type": "string", - "value": "[]" + "value": "[{\"id\":\"low-clarity\",\"field\":\"quality.clarity\",\"operator\":\"lt\",\"value\":0.5,\"samplePercent\":10},{\"id\":\"low-confidence\",\"field\":\"annotation.confidence\",\"operator\":\"lt\",\"value\":0.7,\"samplePercent\":10},{\"id\":\"high-jitter\",\"field\":\"quality.jitter\",\"operator\":\"gt\",\"value\":0.15,\"samplePercent\":10},{\"id\":\"high-no-hand\",\"field\":\"quality.noHandRate\",\"operator\":\"gt\",\"value\":0.15,\"samplePercent\":10}]" } ], "id": "operator-builtin-audit-sample", diff --git a/internal/api/builtin_operators_test.go b/internal/api/builtin_operators_test.go index c3b728a1..0a486e0b 100644 --- a/internal/api/builtin_operators_test.go +++ b/internal/api/builtin_operators_test.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/json" "testing" "droboticflow/internal/domain" @@ -35,6 +36,44 @@ func TestUpsertBuiltinOperatorPreservesSharedMemoryOverride(t *testing.T) { } } +func TestFanyuRequiredOperatorDefaultsMatchDeliveryContract(t *testing.T) { + items, err := builtinOperatorRecords("2026-08-19T00:00:00Z", "registry.example/runner:test") + if err != nil { + t.Fatal(err) + } + want := map[string][3]string{ + "operator-builtin-diffusion-policy-train": {"16", "96Gi", "1"}, + "operator-builtin-openvla-finetune": {"24", "128Gi", "1"}, + "operator-builtin-openvla-inference": {"12", "64Gi", "1"}, + "operator-builtin-yolo-embodied-detection-train": {"16", "64Gi", "1"}, + "operator-builtin-yolo-video-target-clip": {"12", "32Gi", "1"}, + } + for id, expected := range want { + item := findBuiltinOperator(items, id, "") + if item == nil || item.Resources == nil { + t.Fatalf("missing resources for %s", id) + } + if item.Resources.CPULimit != expected[0] || item.Resources.MemoryLimit != expected[1] || item.Resources.GPULimit != expected[2] || item.Resources.GPUType != "NVIDIA A800 80GB" { + t.Fatalf("resources for %s = %#v", id, item.Resources) + } + } + sampling := findBuiltinOperator(items, "operator-builtin-audit-sample", "") + if sampling == nil { + t.Fatal("missing audit sampling operator") + } + var rules []map[string]any + for _, hyperparam := range sampling.Hyperparams { + if hyperparam.Key == "RULES_JSON" { + if err := json.Unmarshal([]byte(hyperparam.Value), &rules); err != nil { + t.Fatal(err) + } + } + } + if len(rules) != 4 { + t.Fatalf("sampling rules=%#v", rules) + } +} + func TestBuiltinCatalogIncludesNativeAnnotationGateContract(t *testing.T) { items, err := builtinOperatorRecords("2026-08-18T00:00:00Z", "registry.example/droboticflow-operator-runner:test") if err != nil { diff --git a/internal/api/delivery_packages.go b/internal/api/delivery_packages.go new file mode 100644 index 00000000..7436aeff --- /dev/null +++ b/internal/api/delivery_packages.go @@ -0,0 +1,109 @@ +package api + +import ( + "net/http" + "strings" + + "droboticflow/internal/domain" + + "github.com/go-chi/chi/v5" +) + +type deliveryPackageDecisionRequest struct { + Version int `json:"version"` + Decision domain.HumanDecision `json:"decision"` + Comment string `json:"comment"` + VideoQuality string `json:"videoQuality,omitempty"` + AnnotationAccuracy float64 `json:"annotationAccuracy,omitempty"` + AnnotationCompleteness float64 `json:"annotationCompleteness,omitempty"` + OverallGrade string `json:"overallGrade,omitempty"` + ReasonCodes []string `json:"reasonCodes,omitempty"` +} + +func parseDeliveryPackageQuery(request *http.Request) domain.DeliveryPackageQuery { + query := request.URL.Query() + return domain.DeliveryPackageQuery{ProjectID: query.Get("projectId"), WorkflowID: query.Get("workflowId"), ReleasePartitionID: query.Get("releasePartitionId"), Status: domain.DeliveryPackageStatus(query.Get("status"))} +} + +func (server *server) handleDeliveryPackagesList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListDeliveryPackages(request.Context(), server.requestOrganizationID(request), parseDeliveryPackageQuery(request)) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleDeliveryPackageGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetDeliveryPackage(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Delivery package not found"}) + return + } + if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Delivery package") { + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleDeliveryPackageDecision(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetDeliveryPackage(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Delivery package not found"}) + return + } + if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Delivery package") { + return + } + var payload deliveryPackageDecisionRequest + if !decodeJSON(writer, request, &payload) { + return + } + if payload.Version != item.Version { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Delivery package version conflict", "code": "version_conflict"}) + return + } + if item.Status != domain.DeliveryPackagePendingAcceptance { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Delivery package is not pending acceptance", "code": "package_not_pending"}) + return + } + if payload.Decision != domain.HumanDecisionApproved && payload.Decision != domain.HumanDecisionRejected { + renderValidationError(writer, validationIssue{Field: "decision", Message: "decision must be approved or rejected"}) + return + } + if payload.Decision == domain.HumanDecisionRejected && (strings.TrimSpace(payload.Comment) == "" || len(payload.ReasonCodes) == 0) { + renderValidationError(writer, validationIssue{Field: "rejection", Message: "reasonCodes and comment are required"}) + return + } + principal, _ := currentPrincipal(request) + review := domain.DeliveryPackageAcceptanceReview{ID: domain.NewID("package-acceptance"), Decision: payload.Decision, ReviewerUserID: principal.User.ID, Comment: strings.TrimSpace(payload.Comment), VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: payload.AnnotationAccuracy, AnnotationCompleteness: payload.AnnotationCompleteness, OverallGrade: strings.TrimSpace(payload.OverallGrade), ReasonCodes: payload.ReasonCodes, CreatedAt: nowISO()} + previousVersion := item.Version + item.Version++ + item.AcceptanceReviews = append(item.AcceptanceReviews, review) + if payload.Decision == domain.HumanDecisionApproved { + item.Status = domain.DeliveryPackageAccepted + } else { + item.Status = domain.DeliveryPackageRejected + } + updated, err := server.store.UpdateDeliveryPackageVersioned(request.Context(), *item, previousVersion) + if err != nil || !updated { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Delivery package version conflict", "code": "version_conflict"}) + } + return + } + if payload.Decision == domain.HumanDecisionRejected { + _ = server.store.UpsertHumanReject(request.Context(), domain.HumanRejectRecord{OrganizationID: item.OrganizationID, BatchID: item.ReleasePartitionID, ItemID: item.ID, ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.ChildRunID, StageID: "acceptance", AttemptRound: len(item.AcceptanceReviews) - 1, ReviewedRevision: item.Version, RejectedByUserID: principal.User.ID, ReasonCodes: payload.ReasonCodes, Comment: strings.TrimSpace(payload.Comment), AllowedReopenStages: []string{"split_annotation", "semantic_annotation", "quality_review"}, Status: domain.HumanRejectOpen}) + } + renderJSON(writer, http.StatusCreated, map[string]any{"package": item, "review": review}) +} diff --git a/internal/api/fanyu_template.go b/internal/api/fanyu_template.go new file mode 100644 index 00000000..b9cacbac --- /dev/null +++ b/internal/api/fanyu_template.go @@ -0,0 +1,94 @@ +package api + +import ( + "context" + "fmt" + "strings" + + "droboticflow/internal/domain" + pluginregistry "droboticflow/internal/plugins" +) + +const fanyuDeliveryTemplateID = "template-fanyu-right-camera-delivery-v1" + +func fanyuNode(id string, label string, operator domain.OperatorRecord, x float64, y float64) domain.WorkflowNode { + cpu, memory, gpu, sharedMemory := "", "", "", "" + if operator.Resources != nil { + cpu = operator.Resources.CPULimit + memory = operator.Resources.MemoryLimit + gpu = operator.Resources.GPULimit + sharedMemory = operator.Resources.SharedMemorySize + } + return domain.WorkflowNode{ID: id, Type: "operator", Position: domain.Position{X: x, Y: y}, Data: domain.WorkflowNodeData{ + Label: label, Category: operator.Category, Description: operator.Description, Inputs: operator.Inputs, Outputs: operator.Outputs, + OperatorID: operator.ID, OperatorVersion: operator.Version, OperatorImageID: operator.ImageID, Image: operator.Image, + ContainerCommand: operator.ContainerCommand, ContainerArgs: operator.ContainerArgs, RuntimeContract: operator.RuntimeContract, + SkillDefinition: operator.SkillDefinition, PublicationStatus: operator.PublicationStatus, GenerationSource: operator.GenerationSource, + PublicationEvidence: operator.PublicationEvidence, Hyperparams: operator.Hyperparams, CPU: cpu, Memory: memory, GPU: gpu, + SharedMemorySize: sharedMemory, + }} +} + +func fanyuEdge(id string, source string, sourceHandle string, target string, targetHandle string) domain.WorkflowEdge { + return domain.WorkflowEdge{ID: id, Source: source, SourceHandle: sourceHandle, Target: target, TargetHandle: targetHandle, Animated: true} +} + +func (server *server) ensureFanyuDeliveryTemplate(ctx context.Context, organizationID string) error { + organizationID = normalizedOrganizationID(organizationID) + templateID := fanyuDeliveryTemplateID + "-" + organizationID + if existing, err := server.store.GetTemplate(ctx, templateID); err != nil || existing != nil { + return err + } + operatorIDs := []string{ + "operator-builtin-bos-incremental-scan", "operator-builtin-xplatform-dataset-index", "operator-builtin-json-filter", + "operator-builtin-video-slice", "operator-builtin-annotation-gate", "operator-builtin-wilor-hand-pose", + "operator-builtin-merge", "operator-builtin-package", + } + operators := map[string]domain.OperatorRecord{} + for _, operatorID := range operatorIDs { + operator, err := server.store.GetOperator(ctx, operatorID) + if err != nil { + return err + } + if operator == nil { + return fmt.Errorf("fanyu delivery template requires builtin operator %s", operatorID) + } + operators[operatorID] = *operator + } + filter := operators["operator-builtin-json-filter"] + for index := range filter.Hyperparams { + if filter.Hyperparams[index].Key == "DSL" { + filter.Hyperparams[index].Value = "camera~right|右" + } + } + nodes := []domain.WorkflowNode{ + fanyuNode("fanyu-scan", "BOS 增量扫描", operators[operatorIDs[0]], 0, 220), + fanyuNode("fanyu-index", "X 平台数据集索引", operators[operatorIDs[1]], 280, 220), + fanyuNode("fanyu-filter", "数据过滤(右相机)", filter, 560, 220), + fanyuNode("fanyu-slice", "视频切片", operators[operatorIDs[3]], 840, 80), + fanyuNode("fanyu-human", "切片微调 → 时间动作分割 → 质检复核", operators[operatorIDs[4]], 1120, 80), + fanyuNode("fanyu-wilor", "WiLoR 3D 手部姿态估计", operators[operatorIDs[5]], 840, 360), + fanyuNode("fanyu-join", "质检数据 + handpose 汇合", operators[operatorIDs[6]], 1400, 220), + fanyuNode("fanyu-package", "成品打包(进入最终验收)", operators[operatorIDs[7]], 1680, 220), + } + edges := []domain.WorkflowEdge{ + fanyuEdge("fanyu-e1", "fanyu-scan", "out-1", "fanyu-index", "in-1"), + fanyuEdge("fanyu-e2", "fanyu-index", "out-1", "fanyu-filter", "in-1"), + fanyuEdge("fanyu-e3", "fanyu-filter", "out-1", "fanyu-slice", "in-1"), + fanyuEdge("fanyu-e4", "fanyu-filter", "out-1", "fanyu-wilor", "in-1"), + fanyuEdge("fanyu-e5", "fanyu-slice", "out-1", "fanyu-human", "clips"), + fanyuEdge("fanyu-e6", "fanyu-human", "approved", "fanyu-join", "in-1"), + fanyuEdge("fanyu-e7", "fanyu-wilor", "out-1", "fanyu-join", "in-2"), + fanyuEdge("fanyu-e8", "fanyu-join", "out-1", "fanyu-package", "in-1"), + } + now := nowISO() + template := domain.TemplateRecord{ID: templateID, OrganizationID: organizationID, Name: "01-右相机切分标注质检-v1", Description: "BOS 增量扫描、X 平台索引、右相机过滤、视频切片、切片/语义/质检人类在环、WiLoR 并行推理、数据汇合、成品打包与包级最终验收。", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, ExecutionScope: domain.WorkflowExecutionScopeProduction, MaxConcurrentRuns: 1, Nodes: nodes, Edges: edges, CreatedAt: now, UpdatedAt: now} + if issues := validateWorkflowTopology(nodes, edges); len(issues) > 0 { + return fmt.Errorf("invalid fanyu delivery template: %s", strings.TrimSpace(issues[0].Message)) + } + if err := server.store.UpsertTemplate(ctx, template); err != nil { + return err + } + release := domain.WorkflowReleaseRecord{ID: "workflow-release-fanyu-v1-" + organizationID, OrganizationID: organizationID, TemplateID: template.ID, Version: 1, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, MaxConcurrentRuns: template.MaxConcurrentRuns, Nodes: nodes, Edges: edges, Capabilities: []string{"human_annotation", "human_quality_review", "human_acceptance"}, RequiredPluginIDs: []string{pluginregistry.HumanLoopCoreID, pluginregistry.AnnotationQAID}, ContentDigest: workflowReleaseDigest(template), ReleaseNotes: "System Fanyu delivery baseline", PublishedByUserID: "system", PublishedAt: now, CreatedAt: now} + return server.store.UpsertWorkflowRelease(ctx, release) +} diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index 46881fae..ca3ab43a 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -2,9 +2,6 @@ package api import ( "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "net/http" "sort" "strconv" @@ -36,6 +33,13 @@ type humanDecisionRequest struct { Comment string `json:"comment"` } +type humanReopenRequest struct { + ItemVersion int `json:"itemVersion"` + RejectVersion int `json:"rejectVersion"` + TargetStageID string `json:"targetStageId"` + Comment string `json:"comment"` +} + func parseHumanBatchQuery(request *http.Request) domain.HumanBatchQuery { query := request.URL.Query() return domain.HumanBatchQuery{ @@ -53,6 +57,14 @@ func parseHumanItemQuery(request *http.Request) domain.HumanItemQuery { } } +func parseHumanRejectQuery(request *http.Request) domain.HumanRejectQuery { + query := request.URL.Query() + return domain.HumanRejectQuery{ + BatchID: query.Get("batchId"), ItemID: query.Get("itemId"), ProjectID: query.Get("projectId"), + WorkflowID: query.Get("workflowId"), RunID: query.Get("runId"), Status: domain.HumanRejectStatus(query.Get("status")), + } +} + func (server *server) handleHumanBatchesList(writer http.ResponseWriter, request *http.Request) { items, err := server.store.ListHumanBatches(request.Context(), server.requestOrganizationID(request), parseHumanBatchQuery(request)) if err != nil { @@ -96,7 +108,11 @@ func (server *server) humanItemDetail(ctx context.Context, item domain.HumanItem if err != nil { return nil, err } - return map[string]any{"item": item, "batch": batch, "attempts": attempts, "events": events}, nil + rejects, err := server.store.ListHumanRejects(ctx, item.OrganizationID, domain.HumanRejectQuery{ItemID: item.ID}) + if err != nil { + return nil, err + } + return map[string]any{"item": item, "batch": batch, "attempts": attempts, "events": events, "rejects": rejects}, nil } func (server *server) handleHumanItemGet(writer http.ResponseWriter, request *http.Request) { @@ -161,7 +177,7 @@ func (server *server) canAccessHumanItem(ctx context.Context, request *http.Requ return false } if item.PluginID == "human-loop-core" { - return principal.User.Role == domain.UserRoleAdmin || principal.User.Role == domain.UserRoleQAReviewer || principal.User.Role == domain.UserRoleAuditor + return principal.User.Role == domain.UserRoleAdmin || principal.User.Role == domain.UserRoleQualityReviewer || principal.User.Role == domain.UserRoleAcceptanceReviewer || principal.User.Role == domain.UserRoleQAReviewer || principal.User.Role == domain.UserRoleAuditor } if annotationManager(principal) { return true @@ -212,6 +228,8 @@ func (server *server) syncLegacyTaskFromHumanItem(ctx context.Context, item doma switch { case item.Status == domain.HumanItemAccepted: task.Status = domain.AnnotationTaskStatusAccepted + case item.Status == domain.HumanItemRejected: + task.Status = domain.AnnotationTaskStatusRejected case item.Status == domain.HumanItemEscalated: task.Status = domain.AnnotationTaskStatusRejected case item.Status == domain.HumanItemRework: @@ -232,6 +250,33 @@ func (server *server) syncLegacyTaskFromHumanItem(ctx context.Context, item doma return nil } +func allowedHumanReopenStages(batch *domain.HumanBatchRecord, rejectedStageID string) []string { + _, rejectedIndex, found := humanStage(batch, rejectedStageID) + if !found { + return nil + } + items := make([]string, 0, rejectedIndex+1) + for index, stage := range batch.Stages { + if index > rejectedIndex { + break + } + if stage.Kind == domain.HumanStageAcceptance || stage.Kind == domain.HumanStageApproval { + continue + } + items = append(items, stage.ID) + } + return items +} + +func containsHumanStage(items []string, target string) bool { + for _, item := range items { + if item == target { + return true + } + } + return false +} + func (server *server) syncLegacyAuditTaskFromHumanItem(ctx context.Context, item domain.HumanItemRecord) error { if item.LegacyAuditTaskID == "" || (item.Status != domain.HumanItemAccepted && item.Status != domain.HumanItemRejected) { return nil @@ -280,7 +325,7 @@ func (server *server) syncHumanItemFromLegacyRevision(ctx context.Context, task Status: domain.HumanItemSubmitted, AssigneeUserID: revision.AuthorUserID, SubmissionRef: revision.ID, }) return server.appendHumanEvent(ctx, *item, "submitted", revision.AuthorUserID, map[string]any{ - "submissionRef": revision.ID, "revision": revision.Revision, "compatibilityMode": "combined_split_semantic", + "submissionRef": revision.ID, "revision": revision.Revision, "stage": "semantic_annotation", }) } @@ -301,35 +346,28 @@ func (server *server) syncHumanItemFromLegacyReview(ctx context.Context, task do item.CurrentStageID = stageID if review.Decision == domain.AnnotationReviewApproved { if review.Stage == domain.AnnotationReviewStageQuality { - item.CurrentStageID = "acceptance" - item.Status = domain.HumanItemSubmitted + item.Status = domain.HumanItemAccepted + item.AcceptedAt = nowISO() } else { item.Status = domain.HumanItemAccepted item.AcceptedAt = nowISO() } } else { - stage, _, _ := humanStage(batch, stageID) + allowed := allowedHumanReopenStages(batch, stageID) if len(rejectTargets) == 0 { - if review.Stage == domain.AnnotationReviewStageQuality { - rejectTargets = []string{"semantic_annotation"} - } else { - rejectTargets = []string{"quality_review"} + rejectTargets = allowed + } + for _, target := range rejectTargets { + if !containsHumanStage(allowed, target) { + return nil } } - target, valid := earliestRejectTarget(batch, stage.RejectTargets, rejectTargets) - if !valid { - return nil - } - item.ReviewRound++ - if item.ReviewRound > item.MaxReworkRounds { - item.Status = domain.HumanItemEscalated - batch.Status = domain.HumanBatchBlocked - } else { - item.CurrentStageID = target - item.Status = domain.HumanItemRework - } + item.Status = domain.HumanItemRejected + item.RejectedAt = nowISO() + } + if item.Status != domain.HumanItemRejected { + item.AssignedUserID = "" } - item.AssignedUserID = "" item.Version++ updated, err := server.store.UpdateHumanItemVersioned(ctx, *item, previousVersion) if err != nil || !updated { @@ -344,13 +382,18 @@ func (server *server) syncHumanItemFromLegacyReview(ctx context.Context, task do Status: item.Status, AssigneeUserID: review.ReviewerUserID, ReviewedRevision: review.Revision, Decision: domain.HumanDecision(review.Decision), RejectTargets: rejectTargets, ReasonCodes: reasonCodes, Comment: review.Comment, }) - eventType := string(review.Decision) - if item.Status == domain.HumanItemEscalated { - eventType = "escalated" + if review.Decision == domain.AnnotationReviewRejected { + _ = server.store.UpsertHumanReject(ctx, domain.HumanRejectRecord{ + OrganizationID: item.OrganizationID, BatchID: item.BatchID, ItemID: item.ID, ProjectID: item.ProjectID, + WorkflowID: item.WorkflowID, RunID: item.RunID, StageID: stageID, AttemptRound: item.ReviewRound, + ReviewedRevision: review.Revision, RejectedByUserID: review.ReviewerUserID, ReasonCodes: reasonCodes, + Comment: review.Comment, AllowedReopenStages: allowedHumanReopenStages(batch, stageID), Status: domain.HumanRejectOpen, + }) } + eventType := string(review.Decision) if err := server.appendHumanEvent(ctx, *item, eventType, review.ReviewerUserID, map[string]any{ "reviewedRevision": review.Revision, "rejectTargets": rejectTargets, "reasonCodes": reasonCodes, - "comment": review.Comment, "compatibilityMode": true, + "comment": review.Comment, "source": "annotation-qa", }); err != nil { return err } @@ -583,15 +626,6 @@ func (server *server) handleHumanItemDecision(writer http.ResponseWriter, reques item.Status = domain.HumanItemReady } } - } else if stage.Kind == domain.HumanStageApproval { - if len(payload.ReasonCodes) == 0 || strings.TrimSpace(payload.Comment) == "" { - renderValidationError(writer, - validationIssue{Field: "reasonCodes", Message: "at least one reason code is required for rejection"}, - validationIssue{Field: "comment", Message: "comment is required for rejection"}, - ) - return - } - item.Status = domain.HumanItemRejected } else { if len(payload.ReasonCodes) == 0 || strings.TrimSpace(payload.Comment) == "" { renderValidationError(writer, @@ -600,21 +634,15 @@ func (server *server) handleHumanItemDecision(writer http.ResponseWriter, reques ) return } - target, valid := earliestRejectTarget(batch, stage.RejectTargets, payload.RejectTargets) - if !valid { - renderValidationError(writer, validationIssue{Field: "rejectTargets", Message: "rejectTargets must contain an allowed earlier stage"}) - return - } - item.ReviewRound++ - if item.ReviewRound > item.MaxReworkRounds { - item.Status = domain.HumanItemEscalated - batch.Status = domain.HumanBatchBlocked - eventType = "escalated" - } else { - item.CurrentStageID = target - item.Status = domain.HumanItemRework - item.AssignedUserID = "" + allowedReopenStages := allowedHumanReopenStages(batch, previousStage) + for _, target := range payload.RejectTargets { + if !containsHumanStage(allowedReopenStages, target) { + renderValidationError(writer, validationIssue{Field: "rejectTargets", Message: "rejectTargets must contain only current or earlier reopen stages"}) + return + } } + item.Status = domain.HumanItemRejected + item.RejectedAt = nowISO() } item.Version++ updated, err := server.store.UpdateHumanItemVersioned(request.Context(), *item, previousVersion) @@ -632,6 +660,18 @@ func (server *server) handleHumanItemDecision(writer http.ResponseWriter, reques Status: item.Status, AssigneeUserID: principal.User.ID, ReviewedRevision: payload.ReviewedRevision, Decision: payload.Decision, RejectTargets: payload.RejectTargets, ReasonCodes: payload.ReasonCodes, Comment: strings.TrimSpace(payload.Comment), }) + if payload.Decision == domain.HumanDecisionRejected { + allowedReopenStages := allowedHumanReopenStages(batch, previousStage) + if err := server.store.UpsertHumanReject(request.Context(), domain.HumanRejectRecord{ + OrganizationID: item.OrganizationID, BatchID: item.BatchID, ItemID: item.ID, ProjectID: item.ProjectID, + WorkflowID: item.WorkflowID, RunID: item.RunID, StageID: previousStage, AttemptRound: item.ReviewRound, + ReviewedRevision: payload.ReviewedRevision, RejectedByUserID: principal.User.ID, ReasonCodes: payload.ReasonCodes, + Comment: strings.TrimSpace(payload.Comment), AllowedReopenStages: allowedReopenStages, Status: domain.HumanRejectOpen, + }); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } _ = server.appendHumanEvent(request.Context(), *item, eventType, principal.User.ID, map[string]any{ "decision": payload.Decision, "reviewedRevision": payload.ReviewedRevision, "rejectTargets": payload.RejectTargets, "reasonCodes": payload.ReasonCodes, "comment": strings.TrimSpace(payload.Comment), @@ -650,6 +690,215 @@ func (server *server) handleHumanItemDecision(writer http.ResponseWriter, reques renderJSON(writer, http.StatusOK, item) } +func (server *server) handleHumanRejectsList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListHumanRejects(request.Context(), server.requestOrganizationID(request), parseHumanRejectQuery(request)) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleHumanRejectGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetHumanReject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Human reject record not found"}) + return + } + if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Human reject record") { + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleHumanRejectReopen(writer http.ResponseWriter, request *http.Request) { + reject, err := server.store.GetHumanReject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if reject == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Human reject record not found"}) + return + } + if !server.ensureTenantAccess(writer, request, reject.OrganizationID, "Human reject record") { + return + } + var payload humanReopenRequest + if !decodeJSON(writer, request, &payload) { + return + } + if reject.Version != payload.RejectVersion { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human reject version conflict", "code": "version_conflict"}) + return + } + if reject.Status != domain.HumanRejectOpen { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human reject record is not open", "code": "reject_not_open"}) + return + } + if !containsHumanStage(reject.AllowedReopenStages, strings.TrimSpace(payload.TargetStageID)) { + renderValidationError(writer, validationIssue{Field: "targetStageId", Message: "targetStageId must be an allowed current or earlier stage"}) + return + } + if strings.TrimSpace(payload.Comment) == "" { + renderValidationError(writer, validationIssue{Field: "comment", Message: "comment is required when reopening rejected work"}) + return + } + item, err := server.store.GetHumanItem(request.Context(), reject.ItemID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + server.reopenRejectedDeliveryPackage(writer, request, reject, payload) + return + } + if item.Version != payload.ItemVersion { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"}) + return + } + if item.Status != domain.HumanItemRejected { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item is not in the reject zone", "code": "item_not_rejected"}) + return + } + principal, _ := currentPrincipal(request) + previousItemVersion := item.Version + item.ReviewRound++ + item.Version++ + item.RejectedAt = "" + eventType := "reopened" + if item.ReviewRound > item.MaxReworkRounds { + item.Status = domain.HumanItemEscalated + reject.Status = domain.HumanRejectEscalated + eventType = "escalated" + if batch, loadErr := server.store.GetHumanBatch(request.Context(), item.BatchID); loadErr == nil && batch != nil { + batch.Status = domain.HumanBatchBlocked + _ = server.store.UpsertHumanBatch(request.Context(), *batch) + } + } else { + item.CurrentStageID = strings.TrimSpace(payload.TargetStageID) + item.Status = domain.HumanItemRework + item.AssignedUserID = item.LatestAuthorUserID + reject.Status = domain.HumanRejectReopened + } + updated, err := server.store.UpdateHumanItemVersioned(request.Context(), *item, previousItemVersion) + if err != nil || !updated { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"}) + } + return + } + previousRejectVersion := reject.Version + reject.Version++ + reject.ReopenCount++ + reject.ReopenedByUserID = principal.User.ID + reject.ReopenTargetStageID = strings.TrimSpace(payload.TargetStageID) + reject.ReopenComment = strings.TrimSpace(payload.Comment) + reject.ReopenedAt = nowISO() + updated, err = server.store.UpdateHumanRejectVersioned(request.Context(), *reject, previousRejectVersion) + if err != nil || !updated { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human reject version conflict", "code": "version_conflict"}) + } + return + } + _ = server.store.UpsertHumanStageAttempt(request.Context(), domain.HumanStageAttemptRecord{ + OrganizationID: item.OrganizationID, ItemID: item.ID, StageID: item.CurrentStageID, Round: item.ReviewRound, + Status: item.Status, AssigneeUserID: item.AssignedUserID, + }) + _ = server.appendHumanEvent(request.Context(), *item, eventType, principal.User.ID, map[string]any{ + "rejectId": reject.ID, "targetStageId": payload.TargetStageID, "comment": strings.TrimSpace(payload.Comment), + }) + if err := server.syncLegacyTaskFromHumanItem(request.Context(), *item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"item": item, "reject": reject}) +} + +func (server *server) reopenRejectedDeliveryPackage(writer http.ResponseWriter, request *http.Request, reject *domain.HumanRejectRecord, payload humanReopenRequest) { + item, err := server.store.GetDeliveryPackage(request.Context(), reject.ItemID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Rejected human item or delivery package not found"}) + return + } + if item.Version != payload.ItemVersion { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Delivery package version conflict", "code": "version_conflict"}) + return + } + if item.Status != domain.DeliveryPackageRejected { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Delivery package is not rejected", "code": "package_not_rejected"}) + return + } + principal, _ := currentPrincipal(request) + reopened := make([]string, 0, len(item.ItemIDs)) + for _, itemID := range item.ItemIDs { + humanItem, loadErr := server.store.GetHumanItem(request.Context(), itemID) + if loadErr != nil { + renderError(writer, http.StatusInternalServerError, loadErr) + return + } + if humanItem == nil { + continue + } + previousVersion := humanItem.Version + humanItem.ReviewRound++ + humanItem.Version++ + humanItem.AcceptedAt = "" + humanItem.ReleasePartitionID = "" + if humanItem.ReviewRound > humanItem.MaxReworkRounds { + humanItem.Status = domain.HumanItemEscalated + reject.Status = domain.HumanRejectEscalated + } else { + humanItem.CurrentStageID = strings.TrimSpace(payload.TargetStageID) + humanItem.Status = domain.HumanItemRework + humanItem.AssignedUserID = humanItem.LatestAuthorUserID + reject.Status = domain.HumanRejectReopened + } + updated, updateErr := server.store.UpdateHumanItemVersioned(request.Context(), *humanItem, previousVersion) + if updateErr != nil || !updated { + if updateErr != nil { + renderError(writer, http.StatusInternalServerError, updateErr) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human item version conflict", "code": "version_conflict"}) + } + return + } + _ = server.appendHumanEvent(request.Context(), *humanItem, "reopened", principal.User.ID, map[string]any{"rejectId": reject.ID, "deliveryPackageId": item.ID, "targetStageId": payload.TargetStageID}) + _ = server.syncLegacyTaskFromHumanItem(request.Context(), *humanItem) + reopened = append(reopened, humanItem.ID) + } + previousRejectVersion := reject.Version + reject.Version++ + reject.ReopenCount++ + reject.ReopenedByUserID = principal.User.ID + reject.ReopenTargetStageID = strings.TrimSpace(payload.TargetStageID) + reject.ReopenComment = strings.TrimSpace(payload.Comment) + reject.ReopenedAt = nowISO() + updated, err := server.store.UpdateHumanRejectVersioned(request.Context(), *reject, previousRejectVersion) + if err != nil || !updated { + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + } else { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Human reject version conflict", "code": "version_conflict"}) + } + return + } + renderJSON(writer, http.StatusOK, map[string]any{"package": item, "reject": reject, "reopenedItemIds": reopened}) +} + func (server *server) handleHumanItemEvents(writer http.ResponseWriter, request *http.Request) { item, ok := server.loadHumanItemForRequest(writer, request) if !ok { @@ -679,7 +928,7 @@ func (server *server) flushHumanReleasePartitions(ctx context.Context, batch *do accepted := make([]domain.HumanItemRecord, 0) allTerminal := len(items) > 0 for _, item := range items { - if item.Status != domain.HumanItemAccepted && item.Status != domain.HumanItemEscalated { + if item.Status != domain.HumanItemAccepted && item.Status != domain.HumanItemRejected && item.Status != domain.HumanItemEscalated { allTerminal = false } if item.Status == domain.HumanItemAccepted && item.ReleasePartitionID == "" { @@ -714,9 +963,7 @@ func (server *server) flushHumanReleasePartitions(ctx context.Context, batch *do itemIDs = append(itemIDs, item.ID) } sort.Strings(itemIDs) - payload, _ := json.Marshal(itemIDs) - digestBytes := sha256.Sum256(payload) - digest := hex.EncodeToString(digestBytes[:]) + digest := domain.HumanReleaseManifestDigest(itemIDs) sequence := batch.NextPartitionSequence + 1 uniqueKey := strings.Join([]string{batch.GateNodeID, strconv.Itoa(sequence), digest}, ":") existing, err := server.store.GetReleasePartitionByUniqueKey(ctx, batch.OrganizationID, uniqueKey) @@ -756,7 +1003,7 @@ func (server *server) flushHumanReleasePartitions(ctx context.Context, batch *do batch.AcceptedCount++ } } - if allTerminal && batch.AcceptedCount == len(items) { + if allTerminal { batch.Status = domain.HumanBatchCompleted } return server.store.UpsertHumanBatch(ctx, *batch) diff --git a/internal/api/human_loop_test.go b/internal/api/human_loop_test.go index 2532dfeb..9a2b8a86 100644 --- a/internal/api/human_loop_test.go +++ b/internal/api/human_loop_test.go @@ -24,3 +24,16 @@ func TestEarliestRejectTargetRejectsUnknownStage(t *testing.T) { t.Fatalf("unexpected target=%q ok=%v", target, ok) } } + +func TestAllowedHumanReopenStagesKeepsRejectZoneSeparateFromRework(t *testing.T) { + batch := &domain.HumanBatchRecord{Stages: domain.DefaultAnnotationHumanStages()} + quality := allowedHumanReopenStages(batch, "quality_review") + if len(quality) != 3 || quality[0] != "split_annotation" || quality[2] != "quality_review" { + t.Fatalf("quality reopen stages=%#v", quality) + } + batch.Stages = append(batch.Stages, domain.HumanStageDefinition{ID: "acceptance", Label: "验收", Kind: domain.HumanStageAcceptance, RequiredRole: "acceptance_reviewer", RejectTargets: []string{"split_annotation", "semantic_annotation", "quality_review"}}) + acceptance := allowedHumanReopenStages(batch, "acceptance") + if len(acceptance) != 3 || containsHumanStage(acceptance, "acceptance") { + t.Fatalf("acceptance must reopen an earlier production stage: %#v", acceptance) + } +} diff --git a/internal/api/object_storage_browser.go b/internal/api/object_storage_browser.go index b0cb77cc..506ea102 100644 --- a/internal/api/object_storage_browser.go +++ b/internal/api/object_storage_browser.go @@ -98,6 +98,9 @@ var storageConnectionBrowserResolver = func(ctx context.Context, connection *dom return nil, nil, nil } endpoint := storageEndpointForProvider(connection.Provider, connection.Endpoint) + if normalizeStorageProvider(connection.Provider) == domain.StorageProviderOSS { + return newOSSObjectStorageBrowser(connection, endpoint) + } if normalizeStorageProvider(connection.Provider) == domain.StorageProviderBOS && endpoint == "" { return nil, nil, fmt.Errorf("BOS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name)) } @@ -1902,16 +1905,71 @@ func (server *server) handleObjectStorageObjectDelete(writer http.ResponseWriter ) return } + trashKey := ".droboticflow-trash/" + time.Now().UTC().Format("20060102T150405.000000000Z") + "/" + key if recursive { - err = browser.DeletePrefix(request.Context(), bucket, key) + _, err = browser.MovePrefix(request.Context(), bucket, key, trashKey) } else { - err = browser.DeleteObject(request.Context(), bucket, key) + _, err = browser.RenameObject(request.Context(), bucket, key, trashKey) + } + if err != nil { + if recursive && (os.IsNotExist(err) || isBOSNoSuchKey(err)) { + err = nil + } } if err != nil { renderObjectStorageWriteError(writer, err) return } - renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectDeleteResult{Bucket: bucket, Key: key, Deleted: true}) + renderJSON(writer, http.StatusOK, map[string]any{"bucket": bucket, "key": key, "originalKey": key, "trashKey": trashKey, "deleted": true}) +} + +type objectStorageRestoreInput struct { + Bucket string `json:"bucket"` + TrashKey string `json:"trashKey"` + OriginalKey string `json:"originalKey"` + Recursive bool `json:"recursive,omitempty"` +} + +func (server *server) handleObjectStorageObjectRestore(writer http.ResponseWriter, request *http.Request) { + connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) + browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if cleanup != nil { + defer cleanup() + } + if browser == nil || connection == nil { + renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) + return + } + var payload objectStorageRestoreInput + if !decodeJSON(writer, request, &payload) { + return + } + bucket, err := resolveObjectStorageBucket(connection, payload.Bucket) + if err != nil { + renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) + return + } + trashKey := strings.Trim(strings.TrimSpace(payload.TrashKey), "/") + originalKey := strings.Trim(strings.TrimSpace(payload.OriginalKey), "/") + if !strings.HasPrefix(trashKey, ".droboticflow-trash/") || originalKey == "" { + renderValidationError(writer, validationIssue{Field: "trashKey", Message: "a trash key and originalKey are required"}) + return + } + var restored domain.ObjectStorageObjectSummary + if payload.Recursive { + restored, err = browser.MovePrefix(request.Context(), bucket, trashKey, originalKey) + } else { + restored, err = browser.RenameObject(request.Context(), bucket, trashKey, originalKey) + } + if err != nil { + renderObjectStorageWriteError(writer, err) + return + } + renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: restored}) } func (server *server) handleObjectStorageContent(writer http.ResponseWriter, request *http.Request) { diff --git a/internal/api/oss_object_storage_browser.go b/internal/api/oss_object_storage_browser.go new file mode 100644 index 00000000..1f6efc34 --- /dev/null +++ b/internal/api/oss_object_storage_browser.go @@ -0,0 +1,364 @@ +package api + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "path" + "sort" + "strconv" + "strings" + "time" + + "droboticflow/internal/domain" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +type ossObjectStorageBrowser struct { + connection *domain.StorageConnectionRecord + client *oss.Client +} + +var _ objectStorageBrowser = (*ossObjectStorageBrowser)(nil) + +func newOSSObjectStorageBrowser(connection *domain.StorageConnectionRecord, endpoint string) (objectStorageBrowser, func(), error) { + if connection == nil { + return nil, nil, nil + } + endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/") + if endpoint == "" { + return nil, nil, fmt.Errorf("OSS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name)) + } + client, err := oss.New(endpoint, strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey)) + if err != nil { + return nil, nil, err + } + return &ossObjectStorageBrowser{connection: connection, client: client}, func() {}, nil +} + +func (browser *ossObjectStorageBrowser) bucket(name string) (*oss.Bucket, error) { + return browser.client.Bucket(strings.TrimSpace(name)) +} + +func (browser *ossObjectStorageBrowser) ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error) { + result, err := browser.client.ListBuckets() + if err != nil { + return nil, err + } + items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets)) + for _, item := range result.Buckets { + items = append(items, domain.ObjectStorageBucketSummary{Name: item.Name, CreatedAt: item.CreationDate.UTC().Format(time.RFC3339)}) + } + return items, nil +} + +func ossObjectSummary(bucketName string, item oss.ObjectProperties) domain.ObjectStorageObjectSummary { + contentType := detectObjectStorageContentType(item.Key, nil, "") + return domain.ObjectStorageObjectSummary{ + Bucket: bucketName, Key: item.Key, Name: path.Base(strings.TrimRight(item.Key, "/")), + Kind: objectStorageKindForObject(item.Key, contentType), SizeBytes: item.Size, ContentType: contentType, + ETag: strings.Trim(item.ETag, `"`), LastModified: item.LastModified.UTC().Format(time.RFC3339), StorageClass: item.StorageClass, + } +} + +func (browser *ossObjectStorageBrowser) ListObjects(ctx context.Context, bucketName string, prefix string) ([]domain.ObjectStorageObjectSummary, error) { + page, err := browser.ListObjectsPage(ctx, bucketName, prefix, "", 1000) + return page.Items, err +} + +func (browser *ossObjectStorageBrowser) ListObjectsPage(_ context.Context, bucketName string, prefix string, marker string, limit int) (objectStoragePage, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return objectStoragePage{}, err + } + if limit <= 0 || limit > 1000 { + limit = 1000 + } + result, err := bucket.ListObjects(oss.Prefix(strings.TrimSpace(prefix)), oss.Delimiter("/"), oss.Marker(strings.TrimSpace(marker)), oss.MaxKeys(limit)) + if err != nil { + return objectStoragePage{}, err + } + items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Objects)) + for _, commonPrefix := range result.CommonPrefixes { + items = append(items, objectStorageSummaryFromPrefix(bucketName, commonPrefix)) + } + for _, object := range result.Objects { + items = append(items, ossObjectSummary(bucketName, object)) + } + return objectStoragePage{Items: items, NextMarker: result.NextMarker, HasMore: result.IsTruncated}, nil +} + +func (browser *ossObjectStorageBrowser) CreateBucket(_ context.Context, bucketName string) (domain.ObjectStorageBucketSummary, error) { + bucketName = strings.TrimSpace(bucketName) + if err := browser.client.CreateBucket(bucketName); err != nil { + return domain.ObjectStorageBucketSummary{}, err + } + return domain.ObjectStorageBucketSummary{Name: bucketName, CreatedAt: nowISO()}, nil +} + +func (browser *ossObjectStorageBrowser) DeleteBucket(_ context.Context, bucketName string) error { + return browser.client.DeleteBucket(strings.TrimSpace(bucketName)) +} + +func (browser *ossObjectStorageBrowser) PutObject(ctx context.Context, bucketName string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) { + return browser.PutObjectStream(ctx, bucketName, key, bytes.NewReader(payload), contentType) +} + +func (browser *ossObjectStorageBrowser) PutObjectStream(ctx context.Context, bucketName string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + options := make([]oss.Option, 0, 1) + if strings.TrimSpace(contentType) != "" { + options = append(options, oss.ContentType(strings.TrimSpace(contentType))) + } + if err := bucket.PutObject(exactObjectStorageKey(key), reader, options...); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + return browser.GetObjectSummary(ctx, bucketName, key) +} + +func (browser *ossObjectStorageBrowser) PresignPutObject(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStoragePresignedUploadResult, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStoragePresignedUploadResult{}, err + } + expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) + options := make([]oss.Option, 0, 1) + headers := objectStorageUploadHeaders(contentType) + if strings.TrimSpace(contentType) != "" { + options = append(options, oss.ContentType(strings.TrimSpace(contentType))) + } + url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), options...) + if err != nil { + return domain.ObjectStoragePresignedUploadResult{}, err + } + return domain.ObjectStoragePresignedUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), Method: http.MethodPut, URL: url, Headers: headers, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil +} + +func (browser *ossObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStorageMultipartUploadCreateResult{}, err + } + options := make([]oss.Option, 0, 1) + if strings.TrimSpace(contentType) != "" { + options = append(options, oss.ContentType(strings.TrimSpace(contentType))) + } + result, err := bucket.InitiateMultipartUpload(exactObjectStorageKey(key), options...) + if err != nil { + return domain.ObjectStorageMultipartUploadCreateResult{}, err + } + expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) + return domain.ObjectStorageMultipartUploadCreateResult{Bucket: result.Bucket, Key: result.Key, UploadID: result.UploadID, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil +} + +func (browser *ossObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucketName string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStorageMultipartUploadPartURLResult{}, err + } + expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) + url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), oss.AddParam("uploadId", strings.TrimSpace(uploadID)), oss.AddParam("partNumber", partNumber)) + if err != nil { + return domain.ObjectStorageMultipartUploadPartURLResult{}, err + } + return domain.ObjectStorageMultipartUploadPartURLResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID), PartNumber: partNumber, Method: http.MethodPut, URL: url, Headers: map[string]string{}, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil +} + +func (browser *ossObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucketName string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + completed := make([]oss.UploadPart, 0, len(parts)) + for _, item := range parts { + completed = append(completed, oss.UploadPart{PartNumber: item.PartNumber, ETag: strings.Trim(item.ETag, `"`)}) + } + sort.Slice(completed, func(i, j int) bool { return completed[i].PartNumber < completed[j].PartNumber }) + _, err = bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)}, completed) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + return browser.GetObjectSummary(ctx, bucketName, key) +} + +func (browser *ossObjectStorageBrowser) AbortMultipartUpload(_ context.Context, bucketName string, key string, uploadID string) error { + bucket, err := browser.bucket(bucketName) + if err != nil { + return err + } + return bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)}) +} + +func (browser *ossObjectStorageBrowser) RenameObject(ctx context.Context, bucketName string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + key, newKey = exactObjectStorageKey(key), exactObjectStorageKey(newKey) + if key == "" || newKey == "" || key == newKey { + return domain.ObjectStorageObjectSummary{}, fmt.Errorf("distinct source and destination keys are required") + } + if _, err := bucket.CopyObject(key, newKey); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + if err := bucket.DeleteObject(key); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + return browser.GetObjectSummary(ctx, bucketName, newKey) +} + +func (browser *ossObjectStorageBrowser) MovePrefix(ctx context.Context, bucketName string, prefix string, newPrefix string) (domain.ObjectStorageObjectSummary, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + prefix, newPrefix = strings.Trim(strings.TrimSpace(prefix), "/"), strings.Trim(strings.TrimSpace(newPrefix), "/") + if prefix == "" || newPrefix == "" || prefix == newPrefix || strings.HasPrefix(newPrefix, prefix+"/") { + return domain.ObjectStorageObjectSummary{}, fmt.Errorf("valid distinct source and destination prefixes are required") + } + marker := "" + for { + result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000)) + if listErr != nil { + return domain.ObjectStorageObjectSummary{}, listErr + } + for _, item := range result.Objects { + destination := newPrefix + "/" + strings.TrimPrefix(item.Key, prefix+"/") + if _, err := bucket.CopyObject(item.Key, destination); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + if err := bucket.DeleteObject(item.Key); err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + } + if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker { + break + } + marker = result.NextMarker + } + return objectStorageSummaryFromPrefix(strings.TrimSpace(bucketName), newPrefix), nil +} + +func (browser *ossObjectStorageBrowser) DeleteObject(_ context.Context, bucketName string, key string) error { + bucket, err := browser.bucket(bucketName) + if err != nil { + return err + } + return bucket.DeleteObject(exactObjectStorageKey(key)) +} + +func (browser *ossObjectStorageBrowser) DeletePrefix(_ context.Context, bucketName string, prefix string) error { + bucket, err := browser.bucket(bucketName) + if err != nil { + return err + } + prefix = strings.Trim(strings.TrimSpace(prefix), "/") + marker := "" + for { + result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000)) + if listErr != nil { + return listErr + } + keys := make([]string, 0, len(result.Objects)) + for _, item := range result.Objects { + keys = append(keys, item.Key) + } + if len(keys) > 0 { + if _, deleteErr := bucket.DeleteObjects(keys); deleteErr != nil { + return deleteErr + } + } + if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker { + break + } + marker = result.NextMarker + } + _ = bucket.DeleteObject(prefix) + return bucket.DeleteObject(prefix + "/") +} + +func ossSummaryFromHeader(bucketName string, key string, header http.Header) domain.ObjectStorageObjectSummary { + size, _ := strconv.ParseInt(strings.TrimSpace(header.Get("Content-Length")), 10, 64) + contentType := strings.TrimSpace(header.Get("Content-Type")) + metadata := map[string]any{} + for name, values := range header { + if strings.HasPrefix(strings.ToLower(name), "x-oss-meta-") { + metadata[strings.TrimPrefix(strings.ToLower(name), "x-oss-meta-")] = strings.Join(values, ",") + } + } + if len(metadata) == 0 { + metadata = nil + } + return domain.ObjectStorageObjectSummary{Bucket: strings.TrimSpace(bucketName), Key: key, Name: path.Base(strings.TrimRight(key, "/")), Kind: objectStorageKindForObject(key, contentType), SizeBytes: size, ContentType: contentType, ETag: strings.Trim(header.Get("ETag"), `"`), LastModified: header.Get("Last-Modified"), StorageClass: header.Get("x-oss-storage-class"), VersionID: header.Get("x-oss-version-id"), Metadata: metadata} +} + +func (browser *ossObjectStorageBrowser) GetObjectSummary(_ context.Context, bucketName string, key string) (domain.ObjectStorageObjectSummary, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + key = exactObjectStorageKey(key) + header, err := bucket.GetObjectDetailedMeta(key) + if err != nil { + return domain.ObjectStorageObjectSummary{}, err + } + return ossSummaryFromHeader(bucketName, key, header), nil +} + +func (browser *ossObjectStorageBrowser) ReadObject(_ context.Context, bucketName string, key string) ([]byte, string, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return nil, "", err + } + body, err := bucket.GetObject(exactObjectStorageKey(key)) + if err != nil { + return nil, "", err + } + defer body.Close() + payload, err := io.ReadAll(body) + if err != nil { + return nil, "", err + } + summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key) + return payload, summary.ContentType, nil +} + +func (browser *ossObjectStorageBrowser) ReadObjectRange(_ context.Context, bucketName string, key string, start int64, end int64) (io.ReadCloser, string, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return nil, "", err + } + body, err := bucket.GetObject(exactObjectStorageKey(key), oss.Range(start, end)) + if err != nil { + return nil, "", err + } + summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key) + return body, summary.ContentType, nil +} + +func (browser *ossObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucketName string, key string, maxBytes int64) ([]byte, string, error) { + if maxBytes <= 0 { + return browser.ReadObject(ctx, bucketName, key) + } + body, contentType, err := browser.ReadObjectRange(ctx, bucketName, key, 0, maxBytes-1) + if err != nil { + return nil, "", err + } + defer body.Close() + payload, err := io.ReadAll(io.LimitReader(body, maxBytes)) + return payload, contentType, err +} + +func (browser *ossObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucketName string, key string) (string, error) { + bucket, err := browser.bucket(bucketName) + if err != nil { + return "", err + } + return bucket.SignURL(exactObjectStorageKey(key), oss.HTTPGet, 3600) +} diff --git a/internal/api/plugins.go b/internal/api/plugins.go index 40f7286e..a245e3a2 100644 --- a/internal/api/plugins.go +++ b/internal/api/plugins.go @@ -294,6 +294,23 @@ func (server *server) enrichWorkflowCapabilities(ctx context.Context, item *doma summary.Rework++ } } + humanItems, err := server.store.ListHumanItems(ctx, item.OrganizationID, domain.HumanItemQuery{ProjectID: item.ProjectID, WorkflowID: item.ID}) + if err != nil { + return err + } + for _, humanItem := range humanItems { + if humanItem.Status == domain.HumanItemRejected { + summary.Rejected++ + } + if humanItem.Status == domain.HumanItemEscalated { + summary.Escalated++ + } + } + packages, err := server.store.ListDeliveryPackages(ctx, item.OrganizationID, domain.DeliveryPackageQuery{ProjectID: item.ProjectID, WorkflowID: item.ID, Status: domain.DeliveryPackagePendingAcceptance}) + if err != nil { + return err + } + summary.PendingAcceptance = len(packages) item.HumanWorkSummary = summary for index := range item.Capabilities { switch item.Capabilities[index].ID { diff --git a/internal/api/robodata_100_item_test.go b/internal/api/robodata_100_item_test.go new file mode 100644 index 00000000..7de25ddc --- /dev/null +++ b/internal/api/robodata_100_item_test.go @@ -0,0 +1,57 @@ +package api + +import ( + "context" + "fmt" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func TestRobodataHundredItemsReleaseAcceptedDataWithoutRejectBlocking(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + server := &server{store: dataStore} + batch := domain.HumanBatchRecord{ID: "batch-100", OrganizationID: "org-a", PluginID: "annotation-qa", ProjectID: "project-a", WorkflowID: "workflow-a", RunID: "run-a", GateNodeID: "gate", ReleasePolicy: domain.HumanReleaseAcceptedPartition, PartitionMaxItems: 32, PartitionMaxWaitSeconds: 60, MaxReworkRounds: 5} + if err := dataStore.UpsertHumanBatch(ctx, batch); err != nil { + t.Fatal(err) + } + acceptedAt := time.Now().Add(-2 * time.Minute).UTC().Format(time.RFC3339Nano) + for index := 0; index < 100; index++ { + status := domain.HumanItemAccepted + if index == 17 { + status = domain.HumanItemRejected + } + item := domain.HumanItemRecord{ID: fmt.Sprintf("item-%03d", index), OrganizationID: "org-a", PluginID: "annotation-qa", BatchID: batch.ID, ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, GateNodeID: batch.GateNodeID, Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("clip-%03d", index)}, CurrentStageID: "quality_review", Status: status, Version: 1, AcceptedAt: acceptedAt} + if err := dataStore.UpsertHumanItem(ctx, item); err != nil { + t.Fatal(err) + } + } + storedBatch, _ := dataStore.GetHumanBatch(ctx, batch.ID) + if err := server.flushHumanReleasePartitions(ctx, storedBatch, true); err != nil { + t.Fatal(err) + } + partitions, err := dataStore.ListReleasePartitions(ctx, "org-a", batch.ID) + if err != nil { + t.Fatal(err) + } + if len(partitions) != 4 { + t.Fatalf("expected 4 partitions for 99 accepted items, got %d", len(partitions)) + } + released := 0 + for _, partition := range partitions { + released += len(partition.ItemIDs) + if partition.ManifestDigest == "" || partition.UniqueKey == "" { + t.Fatalf("invalid partition=%#v", partition) + } + } + if released != 99 { + t.Fatalf("released=%d want=99", released) + } + rejected, _ := dataStore.GetHumanItem(ctx, "item-017") + if rejected == nil || rejected.ReleasePartitionID != "" || rejected.Status != domain.HumanItemRejected { + t.Fatalf("rejected item was incorrectly released: %#v", rejected) + } +} diff --git a/internal/api/robodata_delivery_test.go b/internal/api/robodata_delivery_test.go new file mode 100644 index 00000000..060b2bb9 --- /dev/null +++ b/internal/api/robodata_delivery_test.go @@ -0,0 +1,42 @@ +package api + +import ( + "testing" + + "droboticflow/internal/domain" +) + +func TestStorageProviderSupportsBOSAndOSS(t *testing.T) { + if normalizeStorageProvider("bos") != domain.StorageProviderBOS { + t.Fatal("BOS provider was not normalized") + } + if normalizeStorageProvider("oss") != domain.StorageProviderOSS { + t.Fatal("OSS provider was not normalized") + } + server := &server{} + if issue := server.validateStorageConnectionEndpoint(domain.StorageProviderOSS, ""); issue == nil { + t.Fatal("OSS endpoint must be required") + } + if issue := server.validateStorageConnectionEndpoint(domain.StorageProviderOSS, "https://oss-cn-hangzhou.aliyuncs.com"); issue != nil { + t.Fatalf("valid OSS endpoint rejected: %#v", issue) + } +} + +func TestWorkflowReleaseDigestIgnoresMutableTimestamps(t *testing.T) { + item := domain.TemplateRecord{ID: "template-a", Name: "release", Nodes: []domain.WorkflowNode{{ID: "source"}}} + item.CreatedAt = "one" + item.UpdatedAt = "one" + left := workflowReleaseDigest(item) + item.CreatedAt = "two" + item.UpdatedAt = "two" + if right := workflowReleaseDigest(item); left == "" || left != right { + t.Fatalf("release digest drifted: left=%q right=%q", left, right) + } +} + +func TestDefaultAnnotationStagesEndAtQualityReview(t *testing.T) { + stages := domain.DefaultAnnotationHumanStages() + if len(stages) != 3 || stages[0].ID != "split_annotation" || stages[1].ID != "semantic_annotation" || stages[2].ID != "quality_review" { + t.Fatalf("stages=%#v", stages) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 41dd7c1b..e37baa5c 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1317,12 +1317,18 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/api/human-items", instance.handleHumanItemsList) router.Get("/api/human-items/{id}", instance.handleHumanItemGet) router.Get("/api/human-items/{id}/events", instance.handleHumanItemEvents) + router.Get("/api/human-rejects", instance.handleHumanRejectsList) + router.Get("/api/human-rejects/{id}", instance.handleHumanRejectGet) router.Group(func(router chi.Router) { - router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleQAReviewer, domain.UserRoleAuditor, domain.UserRoleOperator)) + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleAnnotator, domain.UserRoleQualityReviewer, domain.UserRoleAcceptanceReviewer, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleQAReviewer, domain.UserRoleAuditor, domain.UserRoleOperator)) router.Post("/api/human-items/{id}/claim", instance.handleHumanItemClaim) router.Post("/api/human-items/{id}/submissions", instance.handleHumanItemSubmission) router.Post("/api/human-items/{id}/decisions", instance.handleHumanItemDecision) }) + router.Group(func(router chi.Router) { + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin)) + router.Post("/api/human-rejects/{id}/reopen", instance.handleHumanRejectReopen) + }) router.Get("/api/data-collection-tasks", instance.handleCollectionTasksList) router.Get("/api/data-collection-tasks/{id}", instance.handleCollectionTaskGet) router.Get("/api/data-operations/summary", instance.handleDataOperationsSummary) @@ -1342,14 +1348,17 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/api/plugins/annotation-qa/tasks/{id}", instance.handleAnnotationTaskGet) router.Get("/api/plugins/annotation-qa/workload-summary", instance.handleAnnotationWorkloadSummary) router.Group(func(router chi.Router) { - router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleOperator)) + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleAnnotator, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleOperator)) router.Post("/api/data-collection-tasks", instance.handleCollectionTaskCreate) router.Patch("/api/data-collection-tasks/{id}", instance.handleCollectionTaskUpdate) router.Post("/api/data-collection-tasks/{id}/start", instance.handleCollectionTaskStart) router.Post("/api/data-collection-tasks/{id}/complete", instance.handleCollectionTaskComplete) router.Post("/api/annotation-tasks", deprecatedAnnotationEndpoint(instance.handleAnnotationTaskCreate)) router.Post("/api/annotation-tasks/{id}/revisions", deprecatedAnnotationEndpoint(instance.handleAnnotationRevisionCreate)) + router.Post("/api/annotation-tasks/{id}/slice-revisions", deprecatedAnnotationEndpoint(instance.handleSliceRevisionCreate)) router.Post("/api/plugins/annotation-qa/tasks", instance.handleAnnotationTaskCreate) + router.Post("/api/plugins/annotation-qa/tasks/{id}/slice-revisions", instance.handleSliceRevisionCreate) + router.Post("/api/plugins/annotation-qa/tasks/{id}/reject", instance.handleAnnotationTaskReject) router.Post("/api/plugins/annotation-qa/tasks/{id}/revisions", instance.handleAnnotationRevisionCreate) }) router.Group(func(router chi.Router) { @@ -1364,7 +1373,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Post("/api/plugins/annotation-qa/templates/{id}/publish", instance.handleAnnotationTemplatePublish) }) router.Group(func(router chi.Router) { - router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleQAReviewer, domain.UserRoleAuditor, domain.UserRoleOperator)) + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleQualityReviewer, domain.UserRoleAcceptanceReviewer, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleQAReviewer, domain.UserRoleAuditor, domain.UserRoleOperator)) router.Post("/api/annotation-tasks/{id}/reviews", deprecatedAnnotationEndpoint(instance.handleAnnotationReviewCreate)) router.Post("/api/plugins/annotation-qa/tasks/{id}/reviews", instance.handleAnnotationReviewCreate) }) @@ -1377,7 +1386,10 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/api/projects/{id}", instance.handleProjectsGet) router.Get("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesGet) router.Get("/api/projects/{id}/workflows", instance.handleProjectWorkflowsList) + router.Get("/api/projects/{id}/workflow-instances", instance.handleProjectWorkflowInstancesList) router.Get("/api/workflows/{id}", instance.handleWorkflowsGet) + router.Get("/api/templates/{id}/releases", instance.handleTemplateReleasesList) + router.Get("/api/workflow-releases/{id}", instance.handleWorkflowReleaseGet) router.Get("/api/workflows/{id}/runs", instance.handleWorkflowRunsList) router.Get("/api/workflows/{id}/nodes/{nodeId}/source-state", instance.handleWorkflowNodeSourceStateGet) router.Get("/api/scan-configs", instance.handleScanConfigsList) @@ -1390,6 +1402,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Delete("/api/projects/{id}", instance.handleProjectsDelete) router.Put("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesUpdate) router.Post("/api/projects/{id}/workflows", instance.handleProjectWorkflowsCreate) + router.Post("/api/projects/{id}/workflow-instances", instance.handleProjectWorkflowInstanceCreate) router.Put("/api/workflows/{id}", instance.handleWorkflowsUpdate) router.Delete("/api/workflows/{id}", instance.handleWorkflowsDelete) router.Delete("/api/workflows/{id}/nodes/{nodeId}/source-state", instance.handleWorkflowNodeSourceStateDelete) @@ -1408,6 +1421,12 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/api/packages", instance.handlePackagesList) router.Get("/api/packages/{runId}/{packageName}", instance.handlePackageDetail) router.Get("/api/packages/{runId}/{packageName}/content", instance.handlePackageContent) + router.Get("/api/delivery-packages", instance.handleDeliveryPackagesList) + router.Get("/api/delivery-packages/{id}", instance.handleDeliveryPackageGet) + router.Group(func(router chi.Router) { + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleAcceptanceReviewer)) + router.Post("/api/delivery-packages/{id}/decisions", instance.handleDeliveryPackageDecision) + }) router.Get("/api/sampling-tasks", instance.handleSamplingTasksList) router.Get("/api/sampling-tasks/{id}", instance.handleSamplingTaskGet) @@ -1419,12 +1438,13 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/api/operators/{id}", instance.handleOperatorsGet) router.Group(func(router chi.Router) { - router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleQAReviewer, domain.UserRoleOperator)) + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleQualityReviewer, domain.UserRoleQAReviewer, domain.UserRoleOperator)) router.Post("/api/sampling-items/{id}/review", instance.handleSamplingItemReview) }) router.Group(func(router chi.Router) { router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleOperator)) router.Post("/api/templates", instance.handleTemplatesCreate) + router.Post("/api/templates/{id}/publish", instance.handleTemplatePublish) router.Put("/api/templates/{id}", instance.handleTemplatesUpdate) router.Delete("/api/templates/{id}", instance.handleTemplatesDelete) router.Post("/api/operators", instance.handleOperatorsCreate) @@ -1497,6 +1517,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig }) router.Group(func(router chi.Router) { router.Use(instance.requireBusinessRole(domain.UserRoleAdmin)) + router.Post("/api/storage/connections/test", instance.handleStorageConnectionTest) router.Post("/api/storage/connections", instance.handleStorageConnectionsCreate) router.Put("/api/storage/connections/{id}", instance.handleStorageConnectionsUpdate) router.Delete("/api/storage/connections/{id}", instance.handleStorageConnectionsDelete) @@ -1552,6 +1573,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Post("/api/object-storage/uploads/multipart/abort", instance.handleObjectStorageMultipartAbort) router.Patch("/api/object-storage/objects", instance.handleObjectStorageObjectRename) router.Delete("/api/object-storage/objects", instance.handleObjectStorageObjectDelete) + router.Post("/api/object-storage/objects/restore", instance.handleObjectStorageObjectRestore) }) router.Route("/api/object-sync", func(router chi.Router) { @@ -1789,6 +1811,7 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http. } summary := store.BuildDashboardSummaryAt(projects, operators, runSummaries, auditTasks, settings.Timezone, time.Now()) enrichDashboardOperatorInventory(&summary, operators) + server.enrichDashboardHumanWork(request.Context(), request, &summary) server.enrichDashboardPOC(request.Context(), request, &summary) renderJSON(writer, http.StatusOK, summary) return @@ -1804,10 +1827,70 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http. return } enrichDashboardOperatorInventory(&summary, operators) + server.enrichDashboardHumanWork(request.Context(), request, &summary) server.enrichDashboardPOC(request.Context(), request, &summary) renderJSON(writer, http.StatusOK, summary) } +func (server *server) enrichDashboardHumanWork(ctx context.Context, request *http.Request, summary *domain.DashboardSummary) { + if summary == nil { + return + } + organizationID := server.requestOrganizationID(request) + items, err := server.store.ListHumanItems(ctx, organizationID, domain.HumanItemQuery{}) + if err == nil { + accepted := 0 + decided := 0 + seenVideos := map[string]struct{}{} + for _, item := range items { + if key := strings.TrimSpace(item.Subject.ItemKey); key != "" { + seenVideos[key] = struct{}{} + } + if duration, ok := item.Subject.Payload["durationSeconds"].(float64); ok && duration > 0 { + summary.Data.CaptureDurationSeconds += int64(duration) + } + switch item.Status { + case domain.HumanItemReady, domain.HumanItemRework: + if item.CurrentStageID == "quality_review" { + summary.Tasks.PendingQuality++ + } else { + summary.Tasks.PendingAnnotation++ + } + case domain.HumanItemClaimed, domain.HumanItemInProgress: + summary.Tasks.InAnnotation++ + case domain.HumanItemSubmitted: + if item.CurrentStageID == "quality_review" { + summary.Tasks.PendingQuality++ + } else if item.CurrentStageID == "acceptance" { + summary.Tasks.PendingAcceptance++ + } + case domain.HumanItemAccepted: + accepted++ + decided++ + case domain.HumanItemRejected: + summary.Tasks.Rejected++ + decided++ + case domain.HumanItemEscalated: + summary.Tasks.Escalated++ + } + } + summary.Data.VideoCount = len(seenVideos) + if decided > 0 { + summary.Tasks.QualityPassRate = float64(accepted) * 100 / float64(decided) + } + } + packages, err := server.store.ListDeliveryPackages(ctx, organizationID, domain.DeliveryPackageQuery{}) + if err == nil { + summary.Data.PackageCount = len(packages) + for _, item := range packages { + if item.Status == domain.DeliveryPackagePendingAcceptance { + summary.Tasks.PendingAcceptance++ + } + } + } + summary.Tasks.Waiting = summary.Tasks.PendingAnnotation + summary.Tasks.PendingQuality + summary.Tasks.PendingAcceptance +} + func enrichDashboardOperatorInventory(summary *domain.DashboardSummary, operators []domain.OperatorRecord) { if summary == nil { return @@ -2882,6 +2965,12 @@ func (server *server) handleProjectsList(writer http.ResponseWriter, request *ht return } items = server.filterProjectsForRequest(request, items) + for index := range items { + if err := server.migrateLegacyProjectWorkflowInstances(request.Context(), &items[index]); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } items, err = server.populateProjectWorkflowCounts(request.Context(), request, items) if err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -2922,6 +3011,13 @@ func (server *server) handleProjectsGet(writer http.ResponseWriter, request *htt if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Project") { return } + if !server.ensureProjectVisibility(writer, request, item.OrganizationID, item.ID) { + return + } + if err := server.migrateLegacyProjectWorkflowInstances(request.Context(), item); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } item, err = server.populateSingleProjectWorkflowCount(request.Context(), item) if err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -2931,9 +3027,19 @@ func (server *server) handleProjectsGet(writer http.ResponseWriter, request *htt } type projectPatch struct { - Name *string `json:"name"` - Description *string `json:"description"` - Status *domain.ProjectStatus `json:"status"` + Name *string `json:"name"` + Description *string `json:"description"` + Status *domain.ProjectStatus `json:"status"` + Scene *string `json:"scene"` + OwnerUserID *string `json:"ownerUserId"` + Customer *string `json:"customer"` + DataBatch *string `json:"dataBatch"` + StorageConnectionID *string `json:"storageConnectionId"` + StorageBucket *string `json:"storageBucket"` + StoragePrefix *string `json:"storagePrefix"` + AnnotationStandard *string `json:"annotationStandard"` + QualityPolicy *map[string]any `json:"qualityPolicy"` + DeliveryConfig *map[string]any `json:"deliveryConfig"` } func validProjectStatus(value domain.ProjectStatus) bool { @@ -2968,6 +3074,36 @@ func (server *server) handleProjectsUpdate(writer http.ResponseWriter, request * if payload.Description != nil { item.Description = *payload.Description } + if payload.Scene != nil { + item.Scene = strings.TrimSpace(*payload.Scene) + } + if payload.OwnerUserID != nil { + item.OwnerUserID = strings.TrimSpace(*payload.OwnerUserID) + } + if payload.Customer != nil { + item.Customer = strings.TrimSpace(*payload.Customer) + } + if payload.DataBatch != nil { + item.DataBatch = strings.TrimSpace(*payload.DataBatch) + } + if payload.StorageConnectionID != nil { + item.StorageConnectionID = strings.TrimSpace(*payload.StorageConnectionID) + } + if payload.StorageBucket != nil { + item.StorageBucket = strings.TrimSpace(*payload.StorageBucket) + } + if payload.StoragePrefix != nil { + item.StoragePrefix = strings.TrimSpace(*payload.StoragePrefix) + } + if payload.AnnotationStandard != nil { + item.AnnotationStandard = strings.TrimSpace(*payload.AnnotationStandard) + } + if payload.QualityPolicy != nil { + item.QualityPolicy = *payload.QualityPolicy + } + if payload.DeliveryConfig != nil { + item.DeliveryConfig = *payload.DeliveryConfig + } if payload.Status != nil { status := domain.ProjectStatus(strings.TrimSpace(string(*payload.Status))) if !validProjectStatus(status) { @@ -3341,6 +3477,9 @@ func (server *server) handleWorkflowsGet(writer http.ResponseWriter, request *ht if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Workflow") { return } + if !server.ensureProjectVisibility(writer, request, item.OrganizationID, item.ProjectID) { + return + } if err := server.enrichWorkflowCapabilities(request.Context(), item); err != nil { renderError(writer, http.StatusInternalServerError, err) return @@ -4297,6 +4436,10 @@ func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request } func (server *server) handleTemplatesList(writer http.ResponseWriter, request *http.Request) { + if err := server.ensureFanyuDeliveryTemplate(request.Context(), server.requestOrganizationID(request)); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } items, err := server.listTemplatesForRequest(request.Context(), request) if err != nil { renderError(writer, http.StatusInternalServerError, err) @@ -6256,6 +6399,9 @@ func (server *server) handleRunsGet(writer http.ResponseWriter, request *http.Re if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") { return } + if !server.ensureProjectVisibility(writer, request, item.OrganizationID, item.ProjectID) { + return + } renderJSON(writer, http.StatusOK, runForResponse(request, *item)) } @@ -6359,6 +6505,9 @@ func (server *server) handleRunsLogs(writer http.ResponseWriter, request *http.R if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") { return } + if !server.ensureProjectVisibility(writer, request, item.OrganizationID, item.ProjectID) { + return + } // Surface the running node's live stdout so its execution logs are visible DURING the // run — the worker persists a node's structured logs only at completion. No-op for // terminal runs or runs with no running node; failures are silent (stored logs intact). @@ -6457,6 +6606,9 @@ func (server *server) handleRunsResources(writer http.ResponseWriter, request *h if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") { return } + if !server.ensureProjectVisibility(writer, request, item.OrganizationID, item.ProjectID) { + return + } // Fold nodeExecutions (including live GPU samples the worker streams during execution via // AppendNodeExecutionResourceSample) into the top-level series the resource monitor reads. // Idempotent with the worker's own fold; it only adds value while a node is mid-execution. @@ -6482,6 +6634,9 @@ func (server *server) handleRunsArtifacts(writer http.ResponseWriter, request *h if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") { return } + if !server.ensureProjectVisibility(writer, request, item.OrganizationID, item.ProjectID) { + return + } response, issues := pagedResponse(request, runArtifactsForResponse(request, *item), 1000, 5000, false) if len(issues) > 0 { renderValidationError(writer, issues...) @@ -6834,6 +6989,9 @@ func (server *server) handleRunsLineage(writer http.ResponseWriter, request *htt if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") { return } + if !server.ensureProjectVisibility(writer, request, item.OrganizationID, item.ProjectID) { + return + } renderJSON(writer, http.StatusOK, runLineageForResponse(request, *item)) } @@ -6961,6 +7119,15 @@ func (server *server) handleUsersList(writer http.ResponseWriter, request *http. return } items = server.filterUsersForRequest(request, items) + if request.URL.Query().Get("includeDisabled") != "true" { + active := items[:0] + for _, item := range items { + if item.Status == "" || item.Status == domain.UserStatusActive { + active = append(active, item) + } + } + items = active + } renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeUsers(items)}) } @@ -7005,7 +7172,7 @@ func (server *server) handleUsersCreate(writer http.ResponseWriter, request *htt return } if !validUserRole(payload.Role) { - renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, pipeline_engineer, data_operator, qa_reviewer, compliance_auditor, auditor, or operator"}) + renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, annotator, quality_reviewer, or acceptance_reviewer"}) return } if payload.Status != "" && !validUserStatus(payload.Status) { @@ -7045,6 +7212,9 @@ type userPatch struct { func validUserRole(value domain.UserRole) bool { switch value { case domain.UserRoleAdmin, + domain.UserRoleAnnotator, + domain.UserRoleQualityReviewer, + domain.UserRoleAcceptanceReviewer, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleQAReviewer, @@ -7102,7 +7272,7 @@ func (server *server) handleUsersUpdate(writer http.ResponseWriter, request *htt } if payload.Role != nil { if !validUserRole(*payload.Role) { - renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, pipeline_engineer, data_operator, qa_reviewer, compliance_auditor, auditor, or operator"}) + renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, annotator, quality_reviewer, or acceptance_reviewer"}) return } item.Role = *payload.Role @@ -7147,13 +7317,14 @@ func (server *server) handleUsersDelete(writer http.ResponseWriter, request *htt if !server.ensureTenantAccess(writer, request, item.OrganizationID, "User") { return } - deleted, err := server.store.DeleteUser(request.Context(), item.ID) - if err != nil { + item.Status = domain.UserStatusDisabled + item.UpdatedAt = nowISO() + if err := server.store.UpsertUser(request.Context(), *item); err != nil { renderError(writer, http.StatusInternalServerError, err) return } - if !deleted { - writer.WriteHeader(http.StatusNotFound) + if err := server.store.DeleteSessionsByUser(request.Context(), item.ID); err != nil { + renderError(writer, http.StatusInternalServerError, err) return } writer.WriteHeader(http.StatusNoContent) @@ -7236,6 +7407,8 @@ func normalizeStorageProvider(value domain.StorageProvider) domain.StorageProvid switch domain.StorageProvider(strings.ToUpper(strings.TrimSpace(string(value)))) { case domain.StorageProviderBOS: return domain.StorageProviderBOS + case domain.StorageProviderOSS: + return domain.StorageProviderOSS default: return "" } @@ -7287,6 +7460,8 @@ func storageEndpointForProvider(provider domain.StorageProvider, endpoint string switch normalizeStorageProvider(provider) { case domain.StorageProviderBOS: return chooseBOSEndpoint(endpoint) + case domain.StorageProviderOSS: + return strings.TrimRight(strings.TrimSpace(endpoint), "/") default: return strings.TrimSpace(endpoint) } @@ -7296,19 +7471,22 @@ func (server *server) storageEndpointForProvider(provider domain.StorageProvider switch normalizeStorageProvider(provider) { case domain.StorageProviderBOS: return server.chooseBOSEndpoint(endpoint) + case domain.StorageProviderOSS: + return strings.TrimRight(strings.TrimSpace(endpoint), "/") default: return strings.TrimSpace(endpoint) } } func (server *server) validateStorageConnectionEndpoint(provider domain.StorageProvider, endpoint string) *validationIssue { - if normalizeStorageProvider(provider) != domain.StorageProviderBOS { - return nil - } + normalizedProvider := normalizeStorageProvider(provider) if issue := validateOptionalHTTPURL("endpoint", endpoint); issue != nil { return issue } - if strings.TrimSpace(endpoint) == "" && strings.TrimSpace(server.globalBOSEndpoint) == "" { + if normalizedProvider == domain.StorageProviderOSS && strings.TrimSpace(endpoint) == "" { + return &validationIssue{Field: "endpoint", Message: "endpoint is required for OSS"} + } + if normalizedProvider == domain.StorageProviderBOS && strings.TrimSpace(endpoint) == "" && strings.TrimSpace(server.globalBOSEndpoint) == "" { return &validationIssue{Field: "endpoint", Message: "endpoint is required when global BOS_ENDPOINT is not configured"} } return nil @@ -7350,7 +7528,7 @@ func (server *server) handleStorageConnectionsCreate(writer http.ResponseWriter, return } if !validStorageProvider(payload.Provider) { - renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS"}) + renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS or OSS"}) return } if issue := server.validateStorageConnectionEndpoint(payload.Provider, payload.Endpoint); issue != nil { @@ -7417,7 +7595,7 @@ func (server *server) handleStorageConnectionsUpdate(writer http.ResponseWriter, if payload.Provider != nil { provider := normalizeStorageProvider(*payload.Provider) if !validStorageProvider(provider) { - renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS"}) + renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS or OSS"}) return } item.Provider = provider diff --git a/internal/api/storage_connection_probe.go b/internal/api/storage_connection_probe.go new file mode 100644 index 00000000..cb64b7be --- /dev/null +++ b/internal/api/storage_connection_probe.go @@ -0,0 +1,53 @@ +package api + +import ( + "net/http" + "strings" + + "droboticflow/internal/domain" +) + +func (server *server) handleStorageConnectionTest(writer http.ResponseWriter, request *http.Request) { + var payload domain.CreateStorageConnectionInput + if !decodeJSON(writer, request, &payload) { + return + } + payload.Provider = normalizeStorageProvider(payload.Provider) + payload.Endpoint = strings.TrimSpace(payload.Endpoint) + payload.AccessKeyID = strings.TrimSpace(payload.AccessKeyID) + payload.SecretAccessKey = strings.TrimSpace(payload.SecretAccessKey) + if !validStorageProvider(payload.Provider) { + renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS or OSS"}) + return + } + if issue := server.validateStorageConnectionEndpoint(payload.Provider, payload.Endpoint); issue != nil { + renderValidationError(writer, *issue) + return + } + if payload.AccessKeyID == "" || payload.SecretAccessKey == "" { + renderValidationError(writer, validationIssue{Field: "credentials", Message: "accessKeyId and secretAccessKey are required for a connection test"}) + return + } + connection := &domain.StorageConnectionRecord{ + ID: "storage-connection-test", OrganizationID: server.requestOrganizationID(request), Name: "connection test", + Provider: payload.Provider, Endpoint: server.storageEndpointForProvider(payload.Provider, payload.Endpoint), + Bucket: strings.TrimSpace(payload.Bucket), AccessKeyID: payload.AccessKeyID, SecretAccessKey: payload.SecretAccessKey, + } + browser, cleanup, err := storageConnectionBrowserResolver(request.Context(), connection) + if cleanup != nil { + defer cleanup() + } + if err != nil { + renderJSON(writer, http.StatusBadGateway, map[string]any{"ok": false, "error": err.Error()}) + return + } + buckets, err := browser.ListBuckets(request.Context()) + if err != nil { + renderJSON(writer, http.StatusBadGateway, map[string]any{"ok": false, "error": err.Error()}) + return + } + if len(buckets) > 20 { + buckets = buckets[:20] + } + renderJSON(writer, http.StatusOK, map[string]any{"ok": true, "provider": payload.Provider, "endpoint": connection.Endpoint, "buckets": buckets}) +} diff --git a/internal/api/tenant.go b/internal/api/tenant.go index 8d80a86a..6810ee9a 100644 --- a/internal/api/tenant.go +++ b/internal/api/tenant.go @@ -231,10 +231,54 @@ func (server *server) listMongoConnectionsForRequest(ctx context.Context, reques func (server *server) filterProjectsForRequest(request *http.Request, items []domain.ProjectRecord) []domain.ProjectRecord { return filterTenantRecords(items, func(item domain.ProjectRecord) bool { - return server.filterByTenant(request, item.OrganizationID) + return server.filterByTenant(request, item.OrganizationID) && server.projectVisibleToRequest(request, item.OrganizationID, item.ID) }) } +func (server *server) projectVisibleToRequest(request *http.Request, organizationID string, projectID string) bool { + if !server.tenantEnforced(request) { + return true + } + principal, ok := currentPrincipal(request) + if !ok || principal == nil { + // Organization-scoped API tokens do not carry a human project role. + return true + } + switch principal.User.Role { + case domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleOperator: + return true + case domain.UserRoleAnnotator, domain.UserRoleQualityReviewer, domain.UserRoleAcceptanceReviewer: + // Standard RoboData roles are project scoped below. + default: + // Legacy roles remain organization-scoped for one compatibility cycle. + return true + } + memberships, err := server.store.ListProjectAnnotationMemberships(request.Context(), organizationID, projectID) + if err != nil { + return false + } + // Historical projects predate project membership records. Keep them readable + // during the compatibility window; once the first membership is assigned the + // project becomes strictly membership-scoped. + if len(memberships) == 0 { + return true + } + for _, membership := range memberships { + if membership.UserID == principal.User.ID { + return true + } + } + return false +} + +func (server *server) ensureProjectVisibility(writer http.ResponseWriter, request *http.Request, organizationID string, projectID string) bool { + if server.projectVisibleToRequest(request, organizationID, projectID) { + return true + } + renderJSON(writer, http.StatusForbidden, map[string]string{"error": "project membership is required", "code": "project_access_required"}) + return false +} + func (server *server) filterOperatorsForRequest(request *http.Request, items []domain.OperatorRecord) []domain.OperatorRecord { return filterTenantRecords(items, func(item domain.OperatorRecord) bool { return server.operatorVisibleToRequest(request, item) @@ -261,7 +305,7 @@ func (server *server) ensureOperatorAccess(writer http.ResponseWriter, request * func (server *server) filterWorkflowsForRequest(request *http.Request, items []domain.WorkflowRecord) []domain.WorkflowRecord { return filterTenantRecords(items, func(item domain.WorkflowRecord) bool { - return server.filterByTenant(request, item.OrganizationID) + return server.filterByTenant(request, item.OrganizationID) && server.projectVisibleToRequest(request, item.OrganizationID, item.ProjectID) }) } @@ -273,25 +317,25 @@ func (server *server) filterTemplatesForRequest(request *http.Request, items []d func (server *server) filterRunsForRequest(request *http.Request, items []domain.RunRecord) []domain.RunRecord { return filterTenantRecords(items, func(item domain.RunRecord) bool { - return server.filterByTenant(request, item.OrganizationID) + return server.filterByTenant(request, item.OrganizationID) && server.projectVisibleToRequest(request, item.OrganizationID, item.ProjectID) }) } func (server *server) filterRunSummariesForRequest(request *http.Request, items []domain.RunSummary) []domain.RunSummary { return filterTenantRecords(items, func(item domain.RunSummary) bool { - return server.filterByTenant(request, item.OrganizationID) + return server.filterByTenant(request, item.OrganizationID) && server.projectVisibleToRequest(request, item.OrganizationID, item.ProjectID) }) } func (server *server) filterAuditTasksForRequest(request *http.Request, items []domain.AuditTaskRecord) []domain.AuditTaskRecord { return filterTenantRecords(items, func(item domain.AuditTaskRecord) bool { - return server.filterByTenant(request, item.OrganizationID) + return server.filterByTenant(request, item.OrganizationID) && server.projectVisibleToRequest(request, item.OrganizationID, item.ProjectID) }) } func (server *server) filterSamplingTasksForRequest(request *http.Request, items []domain.SamplingTaskRecord) []domain.SamplingTaskRecord { return filterTenantRecords(items, func(item domain.SamplingTaskRecord) bool { - return server.filterByTenant(request, item.OrganizationID) + return server.filterByTenant(request, item.OrganizationID) && server.projectVisibleToRequest(request, item.OrganizationID, item.ProjectID) }) } diff --git a/internal/api/workflow_releases.go b/internal/api/workflow_releases.go new file mode 100644 index 00000000..3dfc1550 --- /dev/null +++ b/internal/api/workflow_releases.go @@ -0,0 +1,321 @@ +package api + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "strings" + + "droboticflow/internal/domain" + pluginregistry "droboticflow/internal/plugins" + + "github.com/go-chi/chi/v5" +) + +type workflowPublishRequest struct { + ReleaseNotes string `json:"releaseNotes"` +} + +type projectWorkflowInstanceRequest struct { + ReleaseID string `json:"releaseId"` +} + +func templateSnapshotFromWorkflow(item domain.WorkflowRecord) domain.TemplateRecord { + return domain.TemplateRecord{ID: "legacy-workflow:" + item.ID, OrganizationID: item.OrganizationID, Name: item.Name, Description: item.Description, Type: item.Type, Kind: item.Kind, ExecutionScope: item.ExecutionScope, ScanConfig: item.ScanConfig, Hyperparams: item.Hyperparams, MaxConcurrentRuns: item.MaxConcurrentRuns, NodeTimeoutSeconds: item.NodeTimeoutSeconds, RunTimeoutSeconds: item.RunTimeoutSeconds, MaxRetryAttempts: item.MaxRetryAttempts, RetryBackoffSeconds: item.RetryBackoffSeconds, Schedule: item.Schedule, Nodes: item.Nodes, Edges: item.Edges} +} + +func (server *server) migrateLegacyProjectWorkflowInstances(ctx context.Context, project *domain.ProjectRecord) error { + if project == nil { + return nil + } + workflows, err := server.store.ListWorkflowsByProject(ctx, project.ID) + if err != nil { + return err + } + instances, err := server.store.ListProjectWorkflowInstances(ctx, project.OrganizationID, domain.ProjectWorkflowInstanceQuery{ProjectID: project.ID}) + if err != nil { + return err + } + byWorkflow := map[string]struct{}{} + for _, instance := range instances { + byWorkflow[instance.WorkflowID] = struct{}{} + } + for _, workflow := range workflows { + if _, exists := byWorkflow[workflow.ID]; exists { + continue + } + template := templateSnapshotFromWorkflow(workflow) + capabilities, requiredPlugins := workflowReleaseRequirements(template) + now := nowISO() + release := domain.WorkflowReleaseRecord{ID: "workflow-release-legacy-" + workflow.ID, OrganizationID: project.OrganizationID, TemplateID: template.ID, Version: 1, Name: workflow.Name, Description: workflow.Description, Type: workflow.Type, Kind: workflow.Kind, ExecutionScope: workflow.ExecutionScope, ScanConfig: workflow.ScanConfig, Hyperparams: workflow.Hyperparams, MaxConcurrentRuns: workflow.MaxConcurrentRuns, NodeTimeoutSeconds: workflow.NodeTimeoutSeconds, RunTimeoutSeconds: workflow.RunTimeoutSeconds, MaxRetryAttempts: workflow.MaxRetryAttempts, RetryBackoffSeconds: workflow.RetryBackoffSeconds, Schedule: workflow.Schedule, Nodes: workflow.Nodes, Edges: workflow.Edges, Capabilities: capabilities, RequiredPluginIDs: requiredPlugins, ContentDigest: workflowReleaseDigest(template), ReleaseNotes: "Historical workflow migration", PublishedByUserID: "system-migration", PublishedAt: now, CreatedAt: now} + if err := server.store.UpsertWorkflowRelease(ctx, release); err != nil { + return err + } + instance := domain.ProjectWorkflowInstanceRecord{ID: "project-workflow-instance-legacy-" + workflow.ID, OrganizationID: project.OrganizationID, ProjectID: project.ID, ReleaseID: release.ID, ReleaseVersion: 1, SourceTemplateID: release.TemplateID, WorkflowID: workflow.ID, Status: domain.ProjectWorkflowInstanceActive, CreatedByUserID: "system-migration", CreatedAt: now, UpdatedAt: now} + if err := server.store.UpsertProjectWorkflowInstance(ctx, instance); err != nil { + return err + } + if project.WorkflowInstanceID == "" { + project.WorkflowInstanceID = instance.ID + project.UpdatedAt = now + } + } + if project.WorkflowInstanceID != "" { + return server.store.UpsertProject(ctx, *project) + } + return nil +} + +func workflowReleaseDigest(item domain.TemplateRecord) string { + item.CreatedAt = "" + item.UpdatedAt = "" + payload, _ := json.Marshal(item) + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]) +} + +func workflowReleaseRequirements(item domain.TemplateRecord) ([]string, []string) { + workflow := domain.WorkflowRecord{ID: item.ID, Nodes: item.Nodes, Edges: item.Edges} + if !pluginregistry.RequiredByWorkflow(workflow, pluginregistry.AnnotationQAID) { + return nil, nil + } + return []string{"human_annotation", "human_quality_review", "human_acceptance"}, []string{pluginregistry.HumanLoopCoreID, pluginregistry.AnnotationQAID} +} + +func (server *server) handleTemplateReleasesList(writer http.ResponseWriter, request *http.Request) { + template, err := server.store.GetTemplate(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if template == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Template not found"}) + return + } + if !server.ensureTenantAccess(writer, request, template.OrganizationID, "Template") { + return + } + items, err := server.store.ListWorkflowReleases(request.Context(), template.OrganizationID, domain.WorkflowReleaseQuery{TemplateID: template.ID}) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if len(items) == 0 && len(template.Nodes) > 0 && len(validateWorkflowTopology(template.Nodes, template.Edges)) == 0 { + capabilities, requiredPlugins := workflowReleaseRequirements(*template) + now := nowISO() + release := domain.WorkflowReleaseRecord{ID: "workflow-release-legacy-template-" + template.ID, OrganizationID: template.OrganizationID, TemplateID: template.ID, Version: 1, Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, ExecutionScope: template.ExecutionScope, ScanConfig: template.ScanConfig, Hyperparams: template.Hyperparams, MaxConcurrentRuns: template.MaxConcurrentRuns, NodeTimeoutSeconds: template.NodeTimeoutSeconds, RunTimeoutSeconds: template.RunTimeoutSeconds, MaxRetryAttempts: template.MaxRetryAttempts, RetryBackoffSeconds: template.RetryBackoffSeconds, Schedule: template.Schedule, Nodes: template.Nodes, Edges: template.Edges, Capabilities: capabilities, RequiredPluginIDs: requiredPlugins, ContentDigest: workflowReleaseDigest(*template), ReleaseNotes: "Historical template migration", PublishedByUserID: "system-migration", PublishedAt: now, CreatedAt: now} + if err := server.store.UpsertWorkflowRelease(request.Context(), release); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + items = []domain.WorkflowReleaseRecord{release} + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleTemplatePublish(writer http.ResponseWriter, request *http.Request) { + template, err := server.store.GetTemplate(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if template == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Template not found"}) + return + } + if !server.ensureTenantAccess(writer, request, template.OrganizationID, "Template") { + return + } + var payload workflowPublishRequest + if !decodeJSON(writer, request, &payload) { + return + } + if issues := validateWorkflowTopology(template.Nodes, template.Edges); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + if issues := server.validateWorkflowOperatorPublication(request.Context(), template.Nodes, template.ExecutionScope); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + capabilities, requiredPlugins := workflowReleaseRequirements(*template) + if workflowReleaseContainsString(requiredPlugins, pluginregistry.AnnotationQAID) { + enabled, pluginErr := server.annotationPluginEnabled(request.Context(), template.OrganizationID) + if pluginErr != nil { + renderError(writer, http.StatusInternalServerError, pluginErr) + return + } + if !enabled { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Annotation QA plugin is disabled", "code": "plugin_disabled"}) + return + } + } + digest := workflowReleaseDigest(*template) + existing, err := server.store.ListWorkflowReleases(request.Context(), template.OrganizationID, domain.WorkflowReleaseQuery{TemplateID: template.ID}) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if len(existing) > 0 && existing[0].ContentDigest == digest { + renderJSON(writer, http.StatusOK, existing[0]) + return + } + version := 1 + if len(existing) > 0 { + version = existing[0].Version + 1 + } + principal, _ := currentPrincipal(request) + now := nowISO() + release := domain.WorkflowReleaseRecord{ + ID: domain.NewID("workflow-release"), OrganizationID: template.OrganizationID, TemplateID: template.ID, Version: version, + Name: template.Name, Description: template.Description, Type: template.Type, Kind: template.Kind, + ExecutionScope: template.ExecutionScope, ScanConfig: template.ScanConfig, Hyperparams: template.Hyperparams, + MaxConcurrentRuns: template.MaxConcurrentRuns, NodeTimeoutSeconds: template.NodeTimeoutSeconds, + RunTimeoutSeconds: template.RunTimeoutSeconds, MaxRetryAttempts: template.MaxRetryAttempts, + RetryBackoffSeconds: template.RetryBackoffSeconds, Schedule: template.Schedule, Nodes: template.Nodes, Edges: template.Edges, + Capabilities: capabilities, RequiredPluginIDs: requiredPlugins, ContentDigest: digest, + ReleaseNotes: strings.TrimSpace(payload.ReleaseNotes), PublishedByUserID: principal.User.ID, PublishedAt: now, CreatedAt: now, + } + if err := server.store.UpsertWorkflowRelease(request.Context(), release); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, release) +} + +func workflowReleaseContainsString(items []string, target string) bool { + for _, item := range items { + if item == target { + return true + } + } + return false +} + +func (server *server) handleWorkflowReleaseGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetWorkflowRelease(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow release not found"}) + return + } + if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Workflow release") { + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) handleProjectWorkflowInstancesList(writer http.ResponseWriter, request *http.Request) { + project, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if project == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"}) + return + } + if !server.ensureTenantAccess(writer, request, project.OrganizationID, "Project") { + return + } + items, err := server.store.ListProjectWorkflowInstances(request.Context(), project.OrganizationID, domain.ProjectWorkflowInstanceQuery{ProjectID: project.ID}) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func (server *server) handleProjectWorkflowInstanceCreate(writer http.ResponseWriter, request *http.Request) { + project, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if project == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"}) + return + } + if !server.ensureTenantAccess(writer, request, project.OrganizationID, "Project") { + return + } + var payload projectWorkflowInstanceRequest + if !decodeJSON(writer, request, &payload) { + return + } + release, err := server.store.GetWorkflowRelease(request.Context(), strings.TrimSpace(payload.ReleaseID)) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if release == nil || normalizedOrganizationID(release.OrganizationID) != normalizedOrganizationID(project.OrganizationID) { + renderValidationError(writer, validationIssue{Field: "releaseId", Message: "published workflow release is required"}) + return + } + for _, pluginID := range release.RequiredPluginIDs { + if pluginID != pluginregistry.AnnotationQAID { + continue + } + enabled, pluginErr := server.annotationPluginEnabled(request.Context(), project.OrganizationID) + if pluginErr != nil { + renderError(writer, http.StatusInternalServerError, pluginErr) + return + } + if !enabled { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "Annotation QA plugin is disabled", "code": "plugin_disabled"}) + return + } + } + workflow, err := server.store.CreateWorkflow(request.Context(), domain.CreateWorkflowInput{ + OrganizationID: project.OrganizationID, ProjectID: project.ID, Name: release.Name, Description: release.Description, + Type: release.Type, Kind: release.Kind, ExecutionScope: release.ExecutionScope, ScanConfig: release.ScanConfig, + Hyperparams: release.Hyperparams, MaxConcurrentRuns: release.MaxConcurrentRuns, NodeTimeoutSeconds: release.NodeTimeoutSeconds, + RunTimeoutSeconds: release.RunTimeoutSeconds, MaxRetryAttempts: release.MaxRetryAttempts, + RetryBackoffSeconds: release.RetryBackoffSeconds, Schedule: release.Schedule, Nodes: release.Nodes, Edges: release.Edges, + }) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + existing, err := server.store.ListProjectWorkflowInstances(request.Context(), project.OrganizationID, domain.ProjectWorkflowInstanceQuery{ProjectID: project.ID, Status: domain.ProjectWorkflowInstanceActive}) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + var supersedes string + for index := range existing { + existing[index].Status = domain.ProjectWorkflowInstanceSuperseded + existing[index].UpdatedAt = nowISO() + if supersedes == "" { + supersedes = existing[index].ID + } + if err := server.store.UpsertProjectWorkflowInstance(request.Context(), existing[index]); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + } + principal, _ := currentPrincipal(request) + now := nowISO() + instance := domain.ProjectWorkflowInstanceRecord{ + ID: domain.NewID("project-workflow-instance"), OrganizationID: project.OrganizationID, ProjectID: project.ID, + ReleaseID: release.ID, ReleaseVersion: release.Version, SourceTemplateID: release.TemplateID, WorkflowID: workflow.ID, + Status: domain.ProjectWorkflowInstanceActive, SupersedesInstanceID: supersedes, CreatedByUserID: principal.User.ID, + CreatedAt: now, UpdatedAt: now, + } + if err := server.store.UpsertProjectWorkflowInstance(request.Context(), instance); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + project.WorkflowInstanceID = instance.ID + project.UpdatedAt = now + if err := server.store.UpsertProject(request.Context(), *project); err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, map[string]any{"instance": instance, "workflow": workflow}) +} diff --git a/internal/domain/annotation_operations.go b/internal/domain/annotation_operations.go index f249d6de..1d014740 100644 --- a/internal/domain/annotation_operations.go +++ b/internal/domain/annotation_operations.go @@ -108,6 +108,36 @@ type AnnotationSegment struct { Source string `json:"source,omitempty" bson:"source,omitempty"` } +type SliceSegment struct { + ID string `json:"id" bson:"id"` + Start float64 `json:"start" bson:"start"` + End float64 `json:"end" bson:"end"` + SourceIDs []string `json:"sourceIds,omitempty" bson:"sourceIds,omitempty"` +} + +type SliceOperation struct { + Type string `json:"type" bson:"type"` + SegmentIDs []string `json:"segmentIds,omitempty" bson:"segmentIds,omitempty"` + At float64 `json:"at,omitempty" bson:"at,omitempty"` + Before []SliceSegment `json:"before,omitempty" bson:"before,omitempty"` + After []SliceSegment `json:"after,omitempty" bson:"after,omitempty"` +} + +type SliceRevisionRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + TaskID string `json:"taskId" bson:"taskId"` + Revision int `json:"revision" bson:"revision"` + Segments []SliceSegment `json:"segments" bson:"segments"` + Operations []SliceOperation `json:"operations,omitempty" bson:"operations,omitempty"` + AuthorUserID string `json:"authorUserId" bson:"authorUserId"` + ChangeReason string `json:"changeReason,omitempty" bson:"changeReason,omitempty"` + DurationSeconds int64 `json:"durationSeconds,omitempty" bson:"durationSeconds,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` +} + +type SemanticAnnotationRevisionRecord = AnnotationRevisionRecord + type AnnotationTaskRecord struct { ID string `json:"id" bson:"id"` OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` @@ -138,6 +168,7 @@ type AnnotationRevisionRecord struct { Revision int `json:"revision" bson:"revision"` TemplateVersion int `json:"templateVersion" bson:"templateVersion"` Segments []AnnotationSegment `json:"segments" bson:"segments"` + ObjectHints []string `json:"objectHints,omitempty" bson:"objectHints,omitempty"` AuthorUserID string `json:"authorUserId" bson:"authorUserId"` ChangeReason string `json:"changeReason,omitempty" bson:"changeReason,omitempty"` IsRework bool `json:"isRework" bson:"isRework"` @@ -147,16 +178,22 @@ type AnnotationRevisionRecord struct { } type AnnotationReviewRecord struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - TaskID string `json:"taskId" bson:"taskId"` - Revision int `json:"revision" bson:"revision"` - Stage AnnotationReviewStage `json:"stage" bson:"stage"` - Decision AnnotationReviewDecision `json:"decision" bson:"decision"` - ReviewerUserID string `json:"reviewerUserId" bson:"reviewerUserId"` - Comment string `json:"comment,omitempty" bson:"comment,omitempty"` - NextRevision int `json:"nextRevision,omitempty" bson:"nextRevision,omitempty"` - CreatedAt string `json:"createdAt" bson:"createdAt"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + TaskID string `json:"taskId" bson:"taskId"` + Revision int `json:"revision" bson:"revision"` + Stage AnnotationReviewStage `json:"stage" bson:"stage"` + Decision AnnotationReviewDecision `json:"decision" bson:"decision"` + ReviewerUserID string `json:"reviewerUserId" bson:"reviewerUserId"` + Comment string `json:"comment,omitempty" bson:"comment,omitempty"` + VideoQuality string `json:"videoQuality,omitempty" bson:"videoQuality,omitempty"` + AnnotationAccuracy float64 `json:"annotationAccuracy,omitempty" bson:"annotationAccuracy,omitempty"` + AnnotationCompleteness float64 `json:"annotationCompleteness,omitempty" bson:"annotationCompleteness,omitempty"` + OverallGrade string `json:"overallGrade,omitempty" bson:"overallGrade,omitempty"` + ReasonCodes []string `json:"reasonCodes,omitempty" bson:"reasonCodes,omitempty"` + RejectTargets []string `json:"rejectTargets,omitempty" bson:"rejectTargets,omitempty"` + NextRevision int `json:"nextRevision,omitempty" bson:"nextRevision,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` } type AnnotationWorklogEventRecord struct { diff --git a/internal/domain/delivery_package.go b/internal/domain/delivery_package.go new file mode 100644 index 00000000..77f9c538 --- /dev/null +++ b/internal/domain/delivery_package.go @@ -0,0 +1,51 @@ +package domain + +type DeliveryPackageStatus string + +const ( + DeliveryPackageBuilding DeliveryPackageStatus = "building" + DeliveryPackagePendingAcceptance DeliveryPackageStatus = "pending_acceptance" + DeliveryPackageAccepted DeliveryPackageStatus = "accepted" + DeliveryPackageRejected DeliveryPackageStatus = "rejected" + DeliveryPackageFailed DeliveryPackageStatus = "failed" +) + +type DeliveryPackageAcceptanceReview struct { + ID string `json:"id" bson:"id"` + Decision HumanDecision `json:"decision" bson:"decision"` + ReviewerUserID string `json:"reviewerUserId" bson:"reviewerUserId"` + Comment string `json:"comment,omitempty" bson:"comment,omitempty"` + VideoQuality string `json:"videoQuality,omitempty" bson:"videoQuality,omitempty"` + AnnotationAccuracy float64 `json:"annotationAccuracy,omitempty" bson:"annotationAccuracy,omitempty"` + AnnotationCompleteness float64 `json:"annotationCompleteness,omitempty" bson:"annotationCompleteness,omitempty"` + OverallGrade string `json:"overallGrade,omitempty" bson:"overallGrade,omitempty"` + ReasonCodes []string `json:"reasonCodes,omitempty" bson:"reasonCodes,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` +} + +type DeliveryPackageRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + ProjectID string `json:"projectId" bson:"projectId"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + ParentRunID string `json:"parentRunId" bson:"parentRunId"` + ChildRunID string `json:"childRunId" bson:"childRunId"` + ReleasePartitionID string `json:"releasePartitionId" bson:"releasePartitionId"` + ManifestDigest string `json:"manifestDigest" bson:"manifestDigest"` + ItemIDs []string `json:"itemIds" bson:"itemIds"` + ArtifactIDs []string `json:"artifactIds,omitempty" bson:"artifactIds,omitempty"` + HandposeArtifactIDs []string `json:"handposeArtifactIds,omitempty" bson:"handposeArtifactIds,omitempty"` + QualitySummary map[string]any `json:"qualitySummary,omitempty" bson:"qualitySummary,omitempty"` + Status DeliveryPackageStatus `json:"status" bson:"status"` + Version int `json:"version" bson:"version"` + AcceptanceReviews []DeliveryPackageAcceptanceReview `json:"acceptanceReviews,omitempty" bson:"acceptanceReviews,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type DeliveryPackageQuery struct { + ProjectID string + WorkflowID string + ReleasePartitionID string + Status DeliveryPackageStatus +} diff --git a/internal/domain/human_loop.go b/internal/domain/human_loop.go index 60395b74..e8bcfc5c 100644 --- a/internal/domain/human_loop.go +++ b/internal/domain/human_loop.go @@ -108,10 +108,49 @@ type HumanItemRecord struct { ClaimedAt string `json:"claimedAt,omitempty" bson:"claimedAt,omitempty"` SubmittedAt string `json:"submittedAt,omitempty" bson:"submittedAt,omitempty"` AcceptedAt string `json:"acceptedAt,omitempty" bson:"acceptedAt,omitempty"` + RejectedAt string `json:"rejectedAt,omitempty" bson:"rejectedAt,omitempty"` CreatedAt string `json:"createdAt" bson:"createdAt"` UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } +type HumanRejectStatus string + +const ( + HumanRejectOpen HumanRejectStatus = "open" + HumanRejectReopened HumanRejectStatus = "reopened" + HumanRejectResolved HumanRejectStatus = "resolved" + HumanRejectEscalated HumanRejectStatus = "escalated" +) + +// HumanRejectRecord is the immutable workflow-visible rejection fact. Reopening +// an item updates only the reject lifecycle metadata and creates a new attempt; +// it never erases the original rejected decision, reason, or reviewer. +type HumanRejectRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + BatchID string `json:"batchId" bson:"batchId"` + ItemID string `json:"itemId" bson:"itemId"` + ProjectID string `json:"projectId" bson:"projectId"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + RunID string `json:"runId" bson:"runId"` + StageID string `json:"stageId" bson:"stageId"` + AttemptRound int `json:"attemptRound" bson:"attemptRound"` + ReviewedRevision int `json:"reviewedRevision,omitempty" bson:"reviewedRevision,omitempty"` + RejectedByUserID string `json:"rejectedByUserId" bson:"rejectedByUserId"` + ReasonCodes []string `json:"reasonCodes" bson:"reasonCodes"` + Comment string `json:"comment" bson:"comment"` + AllowedReopenStages []string `json:"allowedReopenStages" bson:"allowedReopenStages"` + Status HumanRejectStatus `json:"status" bson:"status"` + Version int `json:"version" bson:"version"` + ReopenCount int `json:"reopenCount" bson:"reopenCount"` + ReopenedByUserID string `json:"reopenedByUserId,omitempty" bson:"reopenedByUserId,omitempty"` + ReopenTargetStageID string `json:"reopenTargetStageId,omitempty" bson:"reopenTargetStageId,omitempty"` + ReopenComment string `json:"reopenComment,omitempty" bson:"reopenComment,omitempty"` + ReopenedAt string `json:"reopenedAt,omitempty" bson:"reopenedAt,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + type HumanStageAttemptRecord struct { ID string `json:"id" bson:"id"` OrganizationID string `json:"organizationId" bson:"organizationId"` @@ -187,11 +226,19 @@ type HumanItemQuery struct { Status HumanItemStatus } +type HumanRejectQuery struct { + BatchID string + ItemID string + ProjectID string + WorkflowID string + RunID string + Status HumanRejectStatus +} + func DefaultAnnotationHumanStages() []HumanStageDefinition { return []HumanStageDefinition{ {ID: "split_annotation", Label: "切分标注", Kind: HumanStageAnnotation, RequiredRole: "annotator", EditorSchema: "temporal-segments/v1"}, {ID: "semantic_annotation", Label: "语义标注", Kind: HumanStageAnnotation, RequiredRole: "annotator", EditorSchema: "temporal-segments/v1"}, {ID: "quality_review", Label: "人工质检", Kind: HumanStageReview, RequiredRole: "quality_reviewer", RejectTargets: []string{"split_annotation", "semantic_annotation"}}, - {ID: "acceptance", Label: "验收", Kind: HumanStageAcceptance, RequiredRole: "acceptance_reviewer", RejectTargets: []string{"split_annotation", "semantic_annotation", "quality_review"}}, } } diff --git a/internal/domain/plugins.go b/internal/domain/plugins.go index 99552646..5ba73535 100644 --- a/internal/domain/plugins.go +++ b/internal/domain/plugins.go @@ -60,4 +60,5 @@ type HumanWorkSummary struct { PendingAcceptance int `json:"pendingAcceptance" bson:"-"` Rework int `json:"rework" bson:"-"` Escalated int `json:"escalated" bson:"-"` + Rejected int `json:"rejected" bson:"-"` } diff --git a/internal/domain/release_partition.go b/internal/domain/release_partition.go new file mode 100644 index 00000000..86622e17 --- /dev/null +++ b/internal/domain/release_partition.go @@ -0,0 +1,16 @@ +package domain + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" +) + +func HumanReleaseManifestDigest(itemIDs []string) string { + canonical := append([]string(nil), itemIDs...) + sort.Strings(canonical) + payload, _ := json.Marshal(canonical) + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/domain/types.go b/internal/domain/types.go index a4c82f98..d86c55a0 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -11,13 +11,16 @@ import ( type UserRole string const ( - UserRoleAdmin UserRole = "admin" - UserRolePipelineEngineer UserRole = "pipeline_engineer" - UserRoleDataOperator UserRole = "data_operator" - UserRoleQAReviewer UserRole = "qa_reviewer" - UserRoleComplianceAuditor UserRole = "compliance_auditor" - UserRoleAuditor UserRole = "auditor" - UserRoleOperator UserRole = "operator" + UserRoleAdmin UserRole = "admin" + UserRolePipelineEngineer UserRole = "pipeline_engineer" + UserRoleDataOperator UserRole = "data_operator" + UserRoleQAReviewer UserRole = "qa_reviewer" + UserRoleComplianceAuditor UserRole = "compliance_auditor" + UserRoleAuditor UserRole = "auditor" + UserRoleOperator UserRole = "operator" + UserRoleAnnotator UserRole = "annotator" + UserRoleQualityReviewer UserRole = "quality_reviewer" + UserRoleAcceptanceReviewer UserRole = "acceptance_reviewer" ) type UserStatus string @@ -88,6 +91,7 @@ type StorageProvider string const ( StorageProviderBOS StorageProvider = "BOS" + StorageProviderOSS StorageProvider = "OSS" ) type RunTrigger string @@ -211,6 +215,20 @@ type ProjectRecord struct { OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` Name string `json:"name" bson:"name"` Description string `json:"description" bson:"description"` + Scene string `json:"scene,omitempty" bson:"scene,omitempty"` + OwnerUserID string `json:"ownerUserId,omitempty" bson:"ownerUserId,omitempty"` + Customer string `json:"customer,omitempty" bson:"customer,omitempty"` + DataBatch string `json:"dataBatch,omitempty" bson:"dataBatch,omitempty"` + StorageConnectionID string `json:"storageConnectionId,omitempty" bson:"storageConnectionId,omitempty"` + StorageBucket string `json:"storageBucket,omitempty" bson:"storageBucket,omitempty"` + StoragePrefix string `json:"storagePrefix,omitempty" bson:"storagePrefix,omitempty"` + WorkflowInstanceID string `json:"workflowInstanceId,omitempty" bson:"workflowInstanceId,omitempty"` + AnnotationStandard string `json:"annotationStandard,omitempty" bson:"annotationStandard,omitempty"` + QualityPolicy map[string]any `json:"qualityPolicy,omitempty" bson:"qualityPolicy,omitempty"` + DeliveryConfig map[string]any `json:"deliveryConfig,omitempty" bson:"deliveryConfig,omitempty"` + ProgressStatus string `json:"progressStatus,omitempty" bson:"progressStatus,omitempty"` + QualityStatus string `json:"qualityStatus,omitempty" bson:"qualityStatus,omitempty"` + DeliveryStatus string `json:"deliveryStatus,omitempty" bson:"deliveryStatus,omitempty"` Status ProjectStatus `json:"status" bson:"status"` StoragePathFavorites []StoragePathFavoriteRecord `json:"storagePathFavorites,omitempty" bson:"storagePathFavorites,omitempty"` WorkflowCount int `json:"workflowCount" bson:"-"` @@ -931,7 +949,34 @@ type DashboardSummary struct { Audits struct { Pending int `json:"pending"` } `json:"audits"` - POC DashboardPOCReadiness `json:"poc"` + Data struct { + CaptureDurationSeconds int64 `json:"captureDurationSeconds"` + VideoCount int `json:"videoCount"` + PackageCount int `json:"packageCount"` + } `json:"data"` + Tasks struct { + Running int `json:"running"` + Succeeded int `json:"succeeded"` + Failed int `json:"failed"` + Waiting int `json:"waiting"` + PendingAnnotation int `json:"pendingAnnotation"` + InAnnotation int `json:"inAnnotation"` + PendingQuality int `json:"pendingQuality"` + PendingAcceptance int `json:"pendingAcceptance"` + Rejected int `json:"rejected"` + Escalated int `json:"escalated"` + QualityPassRate float64 `json:"qualityPassRate"` + } `json:"tasks"` + Trends []DashboardTrendPoint `json:"trends"` + POC DashboardPOCReadiness `json:"poc"` +} + +type DashboardTrendPoint struct { + Date string `json:"date"` + RunCount int `json:"runCount"` + Succeeded int `json:"succeeded"` + Failed int `json:"failed"` + NewData int `json:"newData"` } type DashboardGate struct { @@ -1016,9 +1061,19 @@ type MongoConnectionRecord struct { } type CreateProjectInput struct { - OrganizationID string `json:"organizationId,omitempty"` - Name string `json:"name"` - Description string `json:"description"` + OrganizationID string `json:"organizationId,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + Scene string `json:"scene,omitempty"` + OwnerUserID string `json:"ownerUserId,omitempty"` + Customer string `json:"customer,omitempty"` + DataBatch string `json:"dataBatch,omitempty"` + StorageConnectionID string `json:"storageConnectionId,omitempty"` + StorageBucket string `json:"storageBucket,omitempty"` + StoragePrefix string `json:"storagePrefix,omitempty"` + AnnotationStandard string `json:"annotationStandard,omitempty"` + QualityPolicy map[string]any `json:"qualityPolicy,omitempty"` + DeliveryConfig map[string]any `json:"deliveryConfig,omitempty"` } type CreateStorageConnectionInput struct { diff --git a/internal/domain/workflow_release.go b/internal/domain/workflow_release.go new file mode 100644 index 00000000..d77a2dea --- /dev/null +++ b/internal/domain/workflow_release.go @@ -0,0 +1,61 @@ +package domain + +type WorkflowReleaseRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + TemplateID string `json:"templateId" bson:"templateId"` + Version int `json:"version" bson:"version"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Type WorkflowType `json:"type" bson:"type"` + Kind WorkflowKind `json:"kind" bson:"kind"` + ExecutionScope WorkflowExecutionScope `json:"executionScope" bson:"executionScope"` + ScanConfig *ScanConfig `json:"scanConfig,omitempty" bson:"scanConfig,omitempty"` + Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` + MaxConcurrentRuns int `json:"maxConcurrentRuns,omitempty" bson:"maxConcurrentRuns,omitempty"` + NodeTimeoutSeconds int `json:"nodeTimeoutSeconds,omitempty" bson:"nodeTimeoutSeconds,omitempty"` + RunTimeoutSeconds int `json:"runTimeoutSeconds,omitempty" bson:"runTimeoutSeconds,omitempty"` + MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"` + RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"` + Schedule *WorkflowSchedule `json:"schedule,omitempty" bson:"schedule,omitempty"` + Nodes []WorkflowNode `json:"nodes" bson:"nodes"` + Edges []WorkflowEdge `json:"edges" bson:"edges"` + Capabilities []string `json:"capabilities,omitempty" bson:"capabilities,omitempty"` + RequiredPluginIDs []string `json:"requiredPluginIds,omitempty" bson:"requiredPluginIds,omitempty"` + ContentDigest string `json:"contentDigest" bson:"contentDigest"` + ReleaseNotes string `json:"releaseNotes,omitempty" bson:"releaseNotes,omitempty"` + PublishedByUserID string `json:"publishedByUserId" bson:"publishedByUserId"` + PublishedAt string `json:"publishedAt" bson:"publishedAt"` + CreatedAt string `json:"createdAt" bson:"createdAt"` +} + +type ProjectWorkflowInstanceStatus string + +const ( + ProjectWorkflowInstanceActive ProjectWorkflowInstanceStatus = "active" + ProjectWorkflowInstanceSuperseded ProjectWorkflowInstanceStatus = "superseded" +) + +type ProjectWorkflowInstanceRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + ProjectID string `json:"projectId" bson:"projectId"` + ReleaseID string `json:"releaseId" bson:"releaseId"` + ReleaseVersion int `json:"releaseVersion" bson:"releaseVersion"` + SourceTemplateID string `json:"sourceTemplateId" bson:"sourceTemplateId"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + Status ProjectWorkflowInstanceStatus `json:"status" bson:"status"` + SupersedesInstanceID string `json:"supersedesInstanceId,omitempty" bson:"supersedesInstanceId,omitempty"` + CreatedByUserID string `json:"createdByUserId" bson:"createdByUserId"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type WorkflowReleaseQuery struct { + TemplateID string +} + +type ProjectWorkflowInstanceQuery struct { + ProjectID string + Status ProjectWorkflowInstanceStatus +} diff --git a/internal/plugins/registry.go b/internal/plugins/registry.go index 6c4cb600..c11c585a 100644 --- a/internal/plugins/registry.go +++ b/internal/plugins/registry.go @@ -21,12 +21,12 @@ var builtin = map[string]domain.PluginManifest{ AnnotationQAID: { ID: AnnotationQAID, Name: "Annotation QA", Version: "1.0.0", RequiredCoreVersion: ">=0.1.0", Routes: []domain.PluginRouteContribution{ - {ID: "annotate", Path: "/plugins/annotation-qa/workflows/:workflowId/annotate", Label: "人工标注", RequiredRoles: []string{"admin", "pipeline_engineer", "data_operator", "operator"}}, - {ID: "quality", Path: "/plugins/annotation-qa/workflows/:workflowId/quality", Label: "人工质检", RequiredRoles: []string{"admin", "qa_reviewer", "auditor"}}, - {ID: "acceptance", Path: "/plugins/annotation-qa/workflows/:workflowId/acceptance", Label: "验收", RequiredRoles: []string{"admin", "qa_reviewer", "auditor"}}, + {ID: "annotate", Path: "/plugins/annotation-qa/workflows/:workflowId/annotate", Label: "人工标注", RequiredRoles: []string{"admin", "annotator"}}, + {ID: "quality", Path: "/plugins/annotation-qa/workflows/:workflowId/quality", Label: "人工质检", RequiredRoles: []string{"admin", "quality_reviewer"}}, + {ID: "acceptance", Path: "/plugins/annotation-qa/workflows/:workflowId/acceptance", Label: "验收", RequiredRoles: []string{"admin", "acceptance_reviewer"}}, {ID: "item", Path: "/plugins/annotation-qa/items/:itemId", Label: "标注任务详情"}, }, - Navigation: []domain.PluginNavigationContribution{{ID: "annotation-operations", Label: "标注运营", Path: "/plugins/annotation-qa/workflows/all/annotate", RequiredRoles: []string{"admin", "pipeline_engineer", "data_operator", "qa_reviewer", "auditor", "operator"}}}, + Navigation: nil, WorkflowCapabilities: []string{"human_annotation", "human_quality_review", "human_acceptance"}, OperatorIDs: []string{"operator-builtin-annotation-gate"}, Permissions: []string{"annotation.tasks.read", "annotation.tasks.write", "annotation.reviews.write", "annotation.acceptance.write"}, diff --git a/internal/store/annotation_operations.go b/internal/store/annotation_operations.go index 2fedfaa5..91c87a41 100644 --- a/internal/store/annotation_operations.go +++ b/internal/store/annotation_operations.go @@ -213,6 +213,33 @@ func (store *MemoryStore) UpsertAnnotationRevision(_ context.Context, item domai return nil } +func (store *MemoryStore) ListSliceRevisions(_ context.Context, organizationID string, taskID string) ([]domain.SliceRevisionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.SliceRevisionRecord, 0) + for _, item := range store.sliceRevisions { + if annotationOrganizationMatches(item.OrganizationID, organizationID) && (taskID == "" || item.TaskID == taskID) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(i, j int) bool { return items[i].Revision < items[j].Revision }) + return items, nil +} + +func (store *MemoryStore) UpsertSliceRevision(_ context.Context, item domain.SliceRevisionRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if item.ID == "" { + item.ID = domain.NewID("slice-revision") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + store.sliceRevisions[item.ID] = clone(item) + return nil +} + func (store *MemoryStore) ListAnnotationReviews(_ context.Context, organizationID string, taskID string) ([]domain.AnnotationReviewRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -422,6 +449,25 @@ func (store *MongoStore) UpsertAnnotationRevision(ctx context.Context, item doma return mongoUpsert(ctx, store.annotationRevisions, item.ID, item) } +func (store *MongoStore) ListSliceRevisions(ctx context.Context, organizationID string, taskID string) ([]domain.SliceRevisionRecord, error) { + filter := bson.M{"organizationId": normalizedOrganizationID(organizationID)} + if taskID != "" { + filter["taskId"] = taskID + } + return mongoList[domain.SliceRevisionRecord](ctx, store.sliceRevisions, filter, bson.D{{Key: "revision", Value: 1}}) +} + +func (store *MongoStore) UpsertSliceRevision(ctx context.Context, item domain.SliceRevisionRecord) error { + if item.ID == "" { + item.ID = domain.NewID("slice-revision") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.CreatedAt == "" { + item.CreatedAt = nowISO() + } + return mongoUpsert(ctx, store.sliceRevisions, item.ID, item) +} + func (store *MongoStore) ListAnnotationReviews(ctx context.Context, organizationID string, taskID string) ([]domain.AnnotationReviewRecord, error) { extra := bson.M{} if taskID != "" { diff --git a/internal/store/delivery_packages.go b/internal/store/delivery_packages.go new file mode 100644 index 00000000..249063e8 --- /dev/null +++ b/internal/store/delivery_packages.go @@ -0,0 +1,117 @@ +package store + +import ( + "context" + "sort" + "strings" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" +) + +func deliveryPackageMatches(item domain.DeliveryPackageRecord, query domain.DeliveryPackageQuery) bool { + return (query.ProjectID == "" || item.ProjectID == query.ProjectID) && + (query.WorkflowID == "" || item.WorkflowID == query.WorkflowID) && + (query.ReleasePartitionID == "" || item.ReleasePartitionID == query.ReleasePartitionID) && + (query.Status == "" || item.Status == query.Status) +} + +func normalizeDeliveryPackage(item domain.DeliveryPackageRecord) domain.DeliveryPackageRecord { + now := nowISO() + if item.ID == "" { + item.ID = domain.NewID("delivery-package") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.Status == "" { + item.Status = domain.DeliveryPackageBuilding + } + if item.Version <= 0 { + item.Version = 1 + } + if item.CreatedAt == "" { + item.CreatedAt = now + } + item.UpdatedAt = now + return item +} + +func (store *MemoryStore) ListDeliveryPackages(_ context.Context, organizationID string, query domain.DeliveryPackageQuery) ([]domain.DeliveryPackageRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.DeliveryPackageRecord, 0) + for _, item := range store.deliveryPackages { + if organizationMatches(item.OrganizationID, organizationID) && deliveryPackageMatches(item, query) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(i, j int) bool { return items[i].CreatedAt > items[j].CreatedAt }) + return items, nil +} + +func (store *MemoryStore) GetDeliveryPackage(_ context.Context, id string) (*domain.DeliveryPackageRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.deliveryPackages[strings.TrimSpace(id)] + if !ok { + return nil, nil + } + copy := clone(item) + return ©, nil +} + +func (store *MemoryStore) UpsertDeliveryPackage(_ context.Context, item domain.DeliveryPackageRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + item = normalizeDeliveryPackage(item) + store.deliveryPackages[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) UpdateDeliveryPackageVersioned(_ context.Context, item domain.DeliveryPackageRecord, expectedVersion int) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + existing, ok := store.deliveryPackages[item.ID] + if !ok || existing.Version != expectedVersion { + return false, nil + } + item = normalizeDeliveryPackage(item) + store.deliveryPackages[item.ID] = clone(item) + return true, nil +} + +func deliveryPackageMongoFilter(organizationID string, query domain.DeliveryPackageQuery) bson.M { + extra := bson.M{} + if query.ProjectID != "" { + extra["projectId"] = query.ProjectID + } + if query.WorkflowID != "" { + extra["workflowId"] = query.WorkflowID + } + if query.ReleasePartitionID != "" { + extra["releasePartitionId"] = query.ReleasePartitionID + } + if query.Status != "" { + extra["status"] = query.Status + } + return organizationScopedFilter(organizationID, extra) +} + +func (store *MongoStore) ListDeliveryPackages(ctx context.Context, organizationID string, query domain.DeliveryPackageQuery) ([]domain.DeliveryPackageRecord, error) { + return mongoList[domain.DeliveryPackageRecord](ctx, store.deliveryPackages, deliveryPackageMongoFilter(organizationID, query), bson.D{{Key: "createdAt", Value: -1}}) +} +func (store *MongoStore) GetDeliveryPackage(ctx context.Context, id string) (*domain.DeliveryPackageRecord, error) { + return mongoGetByID[domain.DeliveryPackageRecord](ctx, store.deliveryPackages, id) +} +func (store *MongoStore) UpsertDeliveryPackage(ctx context.Context, item domain.DeliveryPackageRecord) error { + item = normalizeDeliveryPackage(item) + return mongoUpsert(ctx, store.deliveryPackages, item.ID, item) +} +func (store *MongoStore) UpdateDeliveryPackageVersioned(ctx context.Context, item domain.DeliveryPackageRecord, expectedVersion int) (bool, error) { + item = normalizeDeliveryPackage(item) + result, err := store.deliveryPackages.ReplaceOne(ctx, bson.M{"id": item.ID, "version": expectedVersion}, item) + if err != nil { + return false, err + } + return result.MatchedCount == 1, nil +} diff --git a/internal/store/human_loop.go b/internal/store/human_loop.go index 0bb61db7..24aafa8e 100644 --- a/internal/store/human_loop.go +++ b/internal/store/human_loop.go @@ -245,6 +245,78 @@ func (store *MemoryStore) AppendHumanEvent(_ context.Context, item domain.HumanE return nil } +func humanRejectMatches(item domain.HumanRejectRecord, query domain.HumanRejectQuery) bool { + return (query.BatchID == "" || item.BatchID == query.BatchID) && + (query.ItemID == "" || item.ItemID == query.ItemID) && + (query.ProjectID == "" || item.ProjectID == query.ProjectID) && + (query.WorkflowID == "" || item.WorkflowID == query.WorkflowID) && + (query.RunID == "" || item.RunID == query.RunID) && + (query.Status == "" || item.Status == query.Status) +} + +func normalizeHumanReject(item domain.HumanRejectRecord) domain.HumanRejectRecord { + now := nowISO() + if item.ID == "" { + item.ID = domain.NewID("human-reject") + } + item.OrganizationID = normalizedOrganizationID(item.OrganizationID) + if item.Status == "" { + item.Status = domain.HumanRejectOpen + } + if item.Version <= 0 { + item.Version = 1 + } + if item.CreatedAt == "" { + item.CreatedAt = now + } + item.UpdatedAt = now + return item +} + +func (store *MemoryStore) ListHumanRejects(_ context.Context, organizationID string, query domain.HumanRejectQuery) ([]domain.HumanRejectRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.HumanRejectRecord, 0) + for _, item := range store.humanRejects { + if humanOrganizationMatches(item.OrganizationID, organizationID) && humanRejectMatches(item, query) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(i, j int) bool { return items[i].CreatedAt > items[j].CreatedAt }) + return items, nil +} + +func (store *MemoryStore) GetHumanReject(_ context.Context, id string) (*domain.HumanRejectRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.humanRejects[strings.TrimSpace(id)] + if !ok { + return nil, nil + } + copy := clone(item) + return ©, nil +} + +func (store *MemoryStore) UpsertHumanReject(_ context.Context, item domain.HumanRejectRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + item = normalizeHumanReject(item) + store.humanRejects[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) UpdateHumanRejectVersioned(_ context.Context, item domain.HumanRejectRecord, expectedVersion int) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + existing, ok := store.humanRejects[item.ID] + if !ok || existing.Version != expectedVersion { + return false, nil + } + item = normalizeHumanReject(item) + store.humanRejects[item.ID] = clone(item) + return true, nil +} + func (store *MemoryStore) ListReleasePartitions(_ context.Context, organizationID string, batchID string) ([]domain.ReleasePartitionRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -402,6 +474,51 @@ func (store *MongoStore) AppendHumanEvent(ctx context.Context, item domain.Human return err } +func humanRejectMongoFilter(organizationID string, query domain.HumanRejectQuery) bson.M { + filter := bson.M{"organizationId": normalizedOrganizationID(organizationID)} + if query.BatchID != "" { + filter["batchId"] = query.BatchID + } + if query.ItemID != "" { + filter["itemId"] = query.ItemID + } + if query.ProjectID != "" { + filter["projectId"] = query.ProjectID + } + if query.WorkflowID != "" { + filter["workflowId"] = query.WorkflowID + } + if query.RunID != "" { + filter["runId"] = query.RunID + } + if query.Status != "" { + filter["status"] = query.Status + } + return filter +} + +func (store *MongoStore) ListHumanRejects(ctx context.Context, organizationID string, query domain.HumanRejectQuery) ([]domain.HumanRejectRecord, error) { + return mongoList[domain.HumanRejectRecord](ctx, store.humanRejects, humanRejectMongoFilter(organizationID, query), bson.D{{Key: "createdAt", Value: -1}}) +} + +func (store *MongoStore) GetHumanReject(ctx context.Context, id string) (*domain.HumanRejectRecord, error) { + return mongoGetByID[domain.HumanRejectRecord](ctx, store.humanRejects, id) +} + +func (store *MongoStore) UpsertHumanReject(ctx context.Context, item domain.HumanRejectRecord) error { + item = normalizeHumanReject(item) + return mongoUpsert(ctx, store.humanRejects, item.ID, item) +} + +func (store *MongoStore) UpdateHumanRejectVersioned(ctx context.Context, item domain.HumanRejectRecord, expectedVersion int) (bool, error) { + item = normalizeHumanReject(item) + result, err := store.humanRejects.ReplaceOne(ctx, bson.M{"id": item.ID, "version": expectedVersion}, item) + if err != nil { + return false, err + } + return result.MatchedCount == 1, nil +} + func (store *MongoStore) ListReleasePartitions(ctx context.Context, organizationID string, batchID string) ([]domain.ReleasePartitionRecord, error) { filter := bson.M{"organizationId": normalizedOrganizationID(organizationID)} if batchID != "" { diff --git a/internal/store/human_loop_test.go b/internal/store/human_loop_test.go index f5de6d75..1e1bf00b 100644 --- a/internal/store/human_loop_test.go +++ b/internal/store/human_loop_test.go @@ -43,3 +43,37 @@ func TestMemoryStoreHumanItemVersionedUpdateAndEventHistory(t *testing.T) { t.Fatalf("events=%#v err=%v", events, err) } } + +func TestMemoryStoreHumanRejectPreservesDecisionAndUsesOptimisticLocking(t *testing.T) { + ctx := context.Background() + dataStore := NewMemoryStore() + reject := domain.HumanRejectRecord{ + ID: "reject-a", OrganizationID: "org-a", ItemID: "item-a", StageID: "quality_review", + RejectedByUserID: "reviewer-a", ReasonCodes: []string{"bad_annotation"}, Comment: "动作边界错误", + AllowedReopenStages: []string{"split_annotation", "semantic_annotation", "quality_review"}, + } + if err := dataStore.UpsertHumanReject(ctx, reject); err != nil { + t.Fatalf("upsert reject: %v", err) + } + stored, err := dataStore.GetHumanReject(ctx, reject.ID) + if err != nil || stored == nil { + t.Fatalf("get reject: item=%#v err=%v", stored, err) + } + if stored.Status != domain.HumanRejectOpen || stored.Version != 1 || stored.Comment != reject.Comment { + t.Fatalf("normalized reject=%#v", stored) + } + stored.Status = domain.HumanRejectReopened + stored.ReopenTargetStageID = "semantic_annotation" + stored.Version = 2 + updated, err := dataStore.UpdateHumanRejectVersioned(ctx, *stored, 1) + if err != nil || !updated { + t.Fatalf("versioned reject update: updated=%v err=%v", updated, err) + } + if stale, err := dataStore.UpdateHumanRejectVersioned(ctx, *stored, 1); err != nil || stale { + t.Fatalf("stale reject update must fail: updated=%v err=%v", stale, err) + } + items, err := dataStore.ListHumanRejects(ctx, "org-a", domain.HumanRejectQuery{ItemID: "item-a", Status: domain.HumanRejectReopened}) + if err != nil || len(items) != 1 || items[0].RejectedByUserID != "reviewer-a" { + t.Fatalf("reject history=%#v err=%v", items, err) + } +} diff --git a/internal/store/memory.go b/internal/store/memory.go index 30619468..255dd216 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -19,6 +19,8 @@ type MemoryStore struct { builtinOperatorDeletions map[string]domain.BuiltinOperatorDeletionRecord workflows map[string]domain.WorkflowRecord templates map[string]domain.TemplateRecord + workflowReleases map[string]domain.WorkflowReleaseRecord + projectWorkflowInstances map[string]domain.ProjectWorkflowInstanceRecord runs map[string]domain.RunRecord sourceStates map[string]domain.SourceStateRecord collectionTasks map[string]domain.CollectionTaskRecord @@ -31,6 +33,7 @@ type MemoryStore struct { annotationTemplates map[string]domain.AnnotationTemplateRecord annotationTasks map[string]domain.AnnotationTaskRecord annotationRevisions map[string]domain.AnnotationRevisionRecord + sliceRevisions map[string]domain.SliceRevisionRecord annotationReviews map[string]domain.AnnotationReviewRecord annotationWorklogEvents map[string]domain.AnnotationWorklogEventRecord organizationPlugins map[string]domain.OrganizationPluginRecord @@ -38,7 +41,9 @@ type MemoryStore struct { humanItems map[string]domain.HumanItemRecord humanStageAttempts map[string]domain.HumanStageAttemptRecord humanEvents map[string]domain.HumanEventRecord + humanRejects map[string]domain.HumanRejectRecord releasePartitions map[string]domain.ReleasePartitionRecord + deliveryPackages map[string]domain.DeliveryPackageRecord users map[string]domain.UserRecord sessions map[string]domain.SessionRecord storageConnections map[string]domain.StorageConnectionRecord @@ -72,6 +77,8 @@ func NewMemoryStore() *MemoryStore { builtinOperatorDeletions: map[string]domain.BuiltinOperatorDeletionRecord{}, workflows: map[string]domain.WorkflowRecord{}, templates: map[string]domain.TemplateRecord{}, + workflowReleases: map[string]domain.WorkflowReleaseRecord{}, + projectWorkflowInstances: map[string]domain.ProjectWorkflowInstanceRecord{}, runs: map[string]domain.RunRecord{}, sourceStates: map[string]domain.SourceStateRecord{}, collectionTasks: map[string]domain.CollectionTaskRecord{}, @@ -84,6 +91,7 @@ func NewMemoryStore() *MemoryStore { annotationTemplates: map[string]domain.AnnotationTemplateRecord{}, annotationTasks: map[string]domain.AnnotationTaskRecord{}, annotationRevisions: map[string]domain.AnnotationRevisionRecord{}, + sliceRevisions: map[string]domain.SliceRevisionRecord{}, annotationReviews: map[string]domain.AnnotationReviewRecord{}, annotationWorklogEvents: map[string]domain.AnnotationWorklogEventRecord{}, organizationPlugins: map[string]domain.OrganizationPluginRecord{}, @@ -91,7 +99,9 @@ func NewMemoryStore() *MemoryStore { humanItems: map[string]domain.HumanItemRecord{}, humanStageAttempts: map[string]domain.HumanStageAttemptRecord{}, humanEvents: map[string]domain.HumanEventRecord{}, + humanRejects: map[string]domain.HumanRejectRecord{}, releasePartitions: map[string]domain.ReleasePartitionRecord{}, + deliveryPackages: map[string]domain.DeliveryPackageRecord{}, users: map[string]domain.UserRecord{}, sessions: map[string]domain.SessionRecord{}, storageConnections: map[string]domain.StorageConnectionRecord{}, @@ -490,13 +500,23 @@ func (store *MemoryStore) CreateProject(_ context.Context, input domain.CreatePr defer store.mu.Unlock() now := nowISO() project := domain.ProjectRecord{ - ID: domain.NewID("project"), - OrganizationID: input.OrganizationID, - Name: input.Name, - Description: input.Description, - Status: domain.ProjectStatusActive, - CreatedAt: now, - UpdatedAt: now, + ID: domain.NewID("project"), + OrganizationID: input.OrganizationID, + Name: input.Name, + Description: input.Description, + Scene: input.Scene, + OwnerUserID: input.OwnerUserID, + Customer: input.Customer, + DataBatch: input.DataBatch, + StorageConnectionID: input.StorageConnectionID, + StorageBucket: input.StorageBucket, + StoragePrefix: input.StoragePrefix, + AnnotationStandard: input.AnnotationStandard, + QualityPolicy: clone(input.QualityPolicy), + DeliveryConfig: clone(input.DeliveryConfig), + Status: domain.ProjectStatusActive, + CreatedAt: now, + UpdatedAt: now, } store.projects[project.ID] = clone(project) cloned := clone(project) @@ -2070,7 +2090,6 @@ func BuildDashboardSummaryAt( runningRuns = append(runningRuns, run) } } - var durations []int64 for _, run := range completedRuns { if run.DurationMS > 0 { @@ -2107,6 +2126,32 @@ func BuildDashboardSummaryAt( location, _ = time.LoadLocation(domain.DefaultWorkflowScheduleTimezone) } localNow := now.In(location) + summary.Tasks.Running = len(runningRuns) + summary.Tasks.Succeeded = len(completedRuns) + summary.Tasks.Failed = len(failedRuns) + trendByDate := map[string]*domain.DashboardTrendPoint{} + for dayOffset := 6; dayOffset >= 0; dayOffset-- { + date := localNow.AddDate(0, 0, -dayOffset).Format("2006-01-02") + summary.Trends = append(summary.Trends, domain.DashboardTrendPoint{Date: date}) + trendByDate[date] = &summary.Trends[len(summary.Trends)-1] + } + for _, run := range runs { + createdAt, parseErr := time.Parse(time.RFC3339Nano, strings.TrimSpace(run.CreatedAt)) + if parseErr != nil { + continue + } + point := trendByDate[createdAt.In(location).Format("2006-01-02")] + if point == nil { + continue + } + point.RunCount++ + if run.Status == domain.RunStatusCompleted { + point.Succeeded++ + } + if run.Status == domain.RunStatusFailed { + point.Failed++ + } + } for _, run := range runs { createdAt, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(run.CreatedAt)) if err != nil { diff --git a/internal/store/mongo.go b/internal/store/mongo.go index 211442b9..369eabcd 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -26,6 +26,8 @@ type MongoStore struct { builtinOperatorDeletions *mongo.Collection workflows *mongo.Collection templates *mongo.Collection + workflowReleases *mongo.Collection + projectWorkflowInstances *mongo.Collection runs *mongo.Collection sourceStates *mongo.Collection collectionTasks *mongo.Collection @@ -38,6 +40,7 @@ type MongoStore struct { annotationTemplates *mongo.Collection annotationTasks *mongo.Collection annotationRevisions *mongo.Collection + sliceRevisions *mongo.Collection annotationReviews *mongo.Collection annotationWorklogEvents *mongo.Collection organizationPlugins *mongo.Collection @@ -45,7 +48,9 @@ type MongoStore struct { humanItems *mongo.Collection humanStageAttempts *mongo.Collection humanEvents *mongo.Collection + humanRejects *mongo.Collection releasePartitions *mongo.Collection + deliveryPackages *mongo.Collection users *mongo.Collection sessions *mongo.Collection storageConnections *mongo.Collection @@ -79,6 +84,8 @@ func NewMongoStore(db *mongo.Database) *MongoStore { builtinOperatorDeletions: db.Collection("operatorTombstones"), workflows: db.Collection("workflows"), templates: db.Collection("templates"), + workflowReleases: db.Collection("workflowReleases"), + projectWorkflowInstances: db.Collection("projectWorkflowInstances"), runs: db.Collection("runs"), sourceStates: db.Collection("sourceStates"), collectionTasks: db.Collection("collectionTasks"), @@ -91,6 +98,7 @@ func NewMongoStore(db *mongo.Database) *MongoStore { annotationTemplates: db.Collection("annotationTemplates"), annotationTasks: db.Collection("annotationTasks"), annotationRevisions: db.Collection("annotationRevisions"), + sliceRevisions: db.Collection("sliceRevisions"), annotationReviews: db.Collection("annotationReviews"), annotationWorklogEvents: db.Collection("annotationWorklogEvents"), organizationPlugins: db.Collection("organizationPlugins"), @@ -98,7 +106,9 @@ func NewMongoStore(db *mongo.Database) *MongoStore { humanItems: db.Collection("humanItems"), humanStageAttempts: db.Collection("humanStageAttempts"), humanEvents: db.Collection("humanEvents"), + humanRejects: db.Collection("humanRejects"), releasePartitions: db.Collection("humanReleasePartitions"), + deliveryPackages: db.Collection("deliveryPackages"), users: db.Collection("users"), sessions: db.Collection("sessions"), storageConnections: db.Collection("storageConnections"), @@ -253,6 +263,8 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin store.builtinOperatorDeletions, store.workflows, store.templates, + store.workflowReleases, + store.projectWorkflowInstances, store.runs, store.sourceStates, store.auditTasks, @@ -262,6 +274,7 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin store.annotationTemplates, store.annotationTasks, store.annotationRevisions, + store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents, store.organizationPlugins, @@ -269,7 +282,9 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin store.humanItems, store.humanStageAttempts, store.humanEvents, + store.humanRejects, store.releasePartitions, + store.deliveryPackages, store.storageConnections, store.mongoConnections, store.objectSyncSpaces, @@ -325,13 +340,23 @@ func (store *MongoStore) GetProject(ctx context.Context, id string) (*domain.Pro func (store *MongoStore) CreateProject(ctx context.Context, input domain.CreateProjectInput) (*domain.ProjectRecord, error) { now := nowISO() item := domain.ProjectRecord{ - ID: domain.NewID("project"), - OrganizationID: input.OrganizationID, - Name: input.Name, - Description: input.Description, - Status: domain.ProjectStatusActive, - CreatedAt: now, - UpdatedAt: now, + ID: domain.NewID("project"), + OrganizationID: input.OrganizationID, + Name: input.Name, + Description: input.Description, + Scene: input.Scene, + OwnerUserID: input.OwnerUserID, + Customer: input.Customer, + DataBatch: input.DataBatch, + StorageConnectionID: input.StorageConnectionID, + StorageBucket: input.StorageBucket, + StoragePrefix: input.StoragePrefix, + AnnotationStandard: input.AnnotationStandard, + QualityPolicy: input.QualityPolicy, + DeliveryConfig: input.DeliveryConfig, + Status: domain.ProjectStatusActive, + CreatedAt: now, + UpdatedAt: now, } if err := store.UpsertProject(ctx, item); err != nil { return nil, err diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 281b4dd5..b1bb7338 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -74,6 +74,15 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { index("organizationId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "createdAt", Value: -1}}), index("createdAt", bson.D{{Key: "createdAt", Value: -1}}), }, + store.workflowReleases: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + uniqueIndex("organizationId_templateId_version_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "templateId", Value: 1}, {Key: "version", Value: 1}}), + index("organizationId_templateId_publishedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "templateId", Value: 1}, {Key: "publishedAt", Value: -1}}), + }, + store.projectWorkflowInstances: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + index("organizationId_projectId_status", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "status", Value: 1}}), + }, store.runs: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), @@ -147,6 +156,10 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), uniqueIndex("organizationId_taskId_revision_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "revision", Value: 1}}), }, + store.sliceRevisions: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + uniqueIndex("organizationId_taskId_revision_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "revision", Value: 1}}), + }, store.annotationReviews: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), uniqueIndex("organizationId_taskId_revision_stage_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "revision", Value: 1}, {Key: "stage", Value: 1}}), @@ -178,11 +191,21 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organizationId_itemId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "itemId", Value: 1}, {Key: "createdAt", Value: 1}}), }, + store.humanRejects: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + index("organizationId_projectId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), + index("organizationId_itemId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "itemId", Value: 1}, {Key: "createdAt", Value: -1}}), + }, store.releasePartitions: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), uniqueIndex("organizationId_uniqueKey_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "uniqueKey", Value: 1}}), index("organizationId_batchId_sequence", bson.D{{Key: "organizationId", Value: 1}, {Key: "batchId", Value: 1}, {Key: "sequence", Value: 1}}), }, + store.deliveryPackages: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + uniqueIndex("organizationId_releasePartitionId_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "releasePartitionId", Value: 1}}), + index("organizationId_projectId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), + }, store.users: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), uniqueIndex("email_unique", bson.D{{Key: "email", Value: 1}}), diff --git a/internal/store/store.go b/internal/store/store.go index 7389a38a..fcd5bd3f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -66,6 +66,12 @@ type Store interface { CreateTemplate(context.Context, domain.CreateTemplateInput) (*domain.TemplateRecord, error) DeleteTemplate(context.Context, string) (bool, error) UpsertTemplate(context.Context, domain.TemplateRecord) error + ListWorkflowReleases(context.Context, string, domain.WorkflowReleaseQuery) ([]domain.WorkflowReleaseRecord, error) + GetWorkflowRelease(context.Context, string) (*domain.WorkflowReleaseRecord, error) + UpsertWorkflowRelease(context.Context, domain.WorkflowReleaseRecord) error + ListProjectWorkflowInstances(context.Context, string, domain.ProjectWorkflowInstanceQuery) ([]domain.ProjectWorkflowInstanceRecord, error) + GetProjectWorkflowInstance(context.Context, string) (*domain.ProjectWorkflowInstanceRecord, error) + UpsertProjectWorkflowInstance(context.Context, domain.ProjectWorkflowInstanceRecord) error ListRuns(context.Context, string) ([]domain.RunRecord, error) ListRunsByOrganization(context.Context, string, string) ([]domain.RunRecord, error) @@ -139,6 +145,8 @@ type Store interface { UpsertAnnotationTask(context.Context, domain.AnnotationTaskRecord) error ListAnnotationRevisions(context.Context, string, string) ([]domain.AnnotationRevisionRecord, error) UpsertAnnotationRevision(context.Context, domain.AnnotationRevisionRecord) error + ListSliceRevisions(context.Context, string, string) ([]domain.SliceRevisionRecord, error) + UpsertSliceRevision(context.Context, domain.SliceRevisionRecord) error ListAnnotationReviews(context.Context, string, string) ([]domain.AnnotationReviewRecord, error) UpsertAnnotationReview(context.Context, domain.AnnotationReviewRecord) error ListAnnotationWorklogEvents(context.Context, string, string) ([]domain.AnnotationWorklogEventRecord, error) @@ -159,9 +167,17 @@ type Store interface { UpsertHumanStageAttempt(context.Context, domain.HumanStageAttemptRecord) error ListHumanEvents(context.Context, string, string) ([]domain.HumanEventRecord, error) AppendHumanEvent(context.Context, domain.HumanEventRecord) error + ListHumanRejects(context.Context, string, domain.HumanRejectQuery) ([]domain.HumanRejectRecord, error) + GetHumanReject(context.Context, string) (*domain.HumanRejectRecord, error) + UpsertHumanReject(context.Context, domain.HumanRejectRecord) error + UpdateHumanRejectVersioned(context.Context, domain.HumanRejectRecord, int) (bool, error) ListReleasePartitions(context.Context, string, string) ([]domain.ReleasePartitionRecord, error) GetReleasePartitionByUniqueKey(context.Context, string, string) (*domain.ReleasePartitionRecord, error) UpsertReleasePartition(context.Context, domain.ReleasePartitionRecord) error + ListDeliveryPackages(context.Context, string, domain.DeliveryPackageQuery) ([]domain.DeliveryPackageRecord, error) + GetDeliveryPackage(context.Context, string) (*domain.DeliveryPackageRecord, error) + UpsertDeliveryPackage(context.Context, domain.DeliveryPackageRecord) error + UpdateDeliveryPackageVersioned(context.Context, domain.DeliveryPackageRecord, int) (bool, error) ListUsers(context.Context) ([]domain.UserRecord, error) ListUsersByOrganization(context.Context, string) ([]domain.UserRecord, error) diff --git a/internal/store/workflow_releases.go b/internal/store/workflow_releases.go new file mode 100644 index 00000000..3f9d1c7d --- /dev/null +++ b/internal/store/workflow_releases.go @@ -0,0 +1,113 @@ +package store + +import ( + "context" + "sort" + "strings" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" +) + +func (store *MemoryStore) ListWorkflowReleases(_ context.Context, organizationID string, query domain.WorkflowReleaseQuery) ([]domain.WorkflowReleaseRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.WorkflowReleaseRecord, 0) + for _, item := range store.workflowReleases { + if organizationMatches(item.OrganizationID, organizationID) && (query.TemplateID == "" || item.TemplateID == query.TemplateID) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(i, j int) bool { + if items[i].TemplateID != items[j].TemplateID { + return items[i].TemplateID < items[j].TemplateID + } + return items[i].Version > items[j].Version + }) + return items, nil +} + +func (store *MemoryStore) GetWorkflowRelease(_ context.Context, id string) (*domain.WorkflowReleaseRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.workflowReleases[strings.TrimSpace(id)] + if !ok { + return nil, nil + } + copy := clone(item) + return ©, nil +} + +func (store *MemoryStore) UpsertWorkflowRelease(_ context.Context, item domain.WorkflowReleaseRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.workflowReleases[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) ListProjectWorkflowInstances(_ context.Context, organizationID string, query domain.ProjectWorkflowInstanceQuery) ([]domain.ProjectWorkflowInstanceRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.ProjectWorkflowInstanceRecord, 0) + for _, item := range store.projectWorkflowInstances { + if organizationMatches(item.OrganizationID, organizationID) && (query.ProjectID == "" || item.ProjectID == query.ProjectID) && (query.Status == "" || item.Status == query.Status) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(i, j int) bool { return items[i].CreatedAt > items[j].CreatedAt }) + return items, nil +} + +func (store *MemoryStore) GetProjectWorkflowInstance(_ context.Context, id string) (*domain.ProjectWorkflowInstanceRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.projectWorkflowInstances[strings.TrimSpace(id)] + if !ok { + return nil, nil + } + copy := clone(item) + return ©, nil +} + +func (store *MemoryStore) UpsertProjectWorkflowInstance(_ context.Context, item domain.ProjectWorkflowInstanceRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.projectWorkflowInstances[item.ID] = clone(item) + return nil +} + +func (store *MongoStore) ListWorkflowReleases(ctx context.Context, organizationID string, query domain.WorkflowReleaseQuery) ([]domain.WorkflowReleaseRecord, error) { + filter := orgCompatibilityFilter(organizationID) + if query.TemplateID != "" { + filter = organizationScopedFilter(organizationID, bson.M{"templateId": query.TemplateID}) + } + return mongoList[domain.WorkflowReleaseRecord](ctx, store.workflowReleases, filter, bson.D{{Key: "templateId", Value: 1}, {Key: "version", Value: -1}}) +} + +func (store *MongoStore) GetWorkflowRelease(ctx context.Context, id string) (*domain.WorkflowReleaseRecord, error) { + return mongoGetByID[domain.WorkflowReleaseRecord](ctx, store.workflowReleases, id) +} + +func (store *MongoStore) UpsertWorkflowRelease(ctx context.Context, item domain.WorkflowReleaseRecord) error { + return mongoUpsert(ctx, store.workflowReleases, item.ID, item) +} + +func (store *MongoStore) ListProjectWorkflowInstances(ctx context.Context, organizationID string, query domain.ProjectWorkflowInstanceQuery) ([]domain.ProjectWorkflowInstanceRecord, error) { + extra := bson.M{} + if query.ProjectID != "" { + extra["projectId"] = query.ProjectID + } + if query.Status != "" { + extra["status"] = query.Status + } + return mongoList[domain.ProjectWorkflowInstanceRecord](ctx, store.projectWorkflowInstances, organizationScopedFilter(organizationID, extra), bson.D{{Key: "createdAt", Value: -1}}) +} + +func (store *MongoStore) GetProjectWorkflowInstance(ctx context.Context, id string) (*domain.ProjectWorkflowInstanceRecord, error) { + return mongoGetByID[domain.ProjectWorkflowInstanceRecord](ctx, store.projectWorkflowInstances, id) +} + +func (store *MongoStore) UpsertProjectWorkflowInstance(ctx context.Context, item domain.ProjectWorkflowInstanceRecord) error { + return mongoUpsert(ctx, store.projectWorkflowInstances, item.ID, item) +} diff --git a/internal/store/workflow_releases_test.go b/internal/store/workflow_releases_test.go new file mode 100644 index 00000000..c6b5387d --- /dev/null +++ b/internal/store/workflow_releases_test.go @@ -0,0 +1,52 @@ +package store + +import ( + "context" + "testing" + + "droboticflow/internal/domain" +) + +func TestMemoryStoreKeepsWorkflowReleasesAndProjectInstancesImmutableByVersion(t *testing.T) { + ctx := context.Background() + dataStore := NewMemoryStore() + for version := 1; version <= 2; version++ { + if err := dataStore.UpsertWorkflowRelease(ctx, domain.WorkflowReleaseRecord{ID: domain.NewID("release"), OrganizationID: "org-a", TemplateID: "template-a", Version: version, ContentDigest: string(rune('a' + version))}); err != nil { + t.Fatalf("upsert release v%d: %v", version, err) + } + } + releases, err := dataStore.ListWorkflowReleases(ctx, "org-a", domain.WorkflowReleaseQuery{TemplateID: "template-a"}) + if err != nil || len(releases) != 2 || releases[0].Version != 2 || releases[1].Version != 1 { + t.Fatalf("releases=%#v err=%v", releases, err) + } + first := domain.ProjectWorkflowInstanceRecord{ID: "instance-v1", OrganizationID: "org-a", ProjectID: "project-a", ReleaseID: releases[1].ID, ReleaseVersion: 1, Status: domain.ProjectWorkflowInstanceSuperseded} + second := domain.ProjectWorkflowInstanceRecord{ID: "instance-v2", OrganizationID: "org-a", ProjectID: "project-a", ReleaseID: releases[0].ID, ReleaseVersion: 2, Status: domain.ProjectWorkflowInstanceActive, SupersedesInstanceID: first.ID} + if err := dataStore.UpsertProjectWorkflowInstance(ctx, first); err != nil { + t.Fatal(err) + } + if err := dataStore.UpsertProjectWorkflowInstance(ctx, second); err != nil { + t.Fatal(err) + } + instances, err := dataStore.ListProjectWorkflowInstances(ctx, "org-a", domain.ProjectWorkflowInstanceQuery{ProjectID: "project-a"}) + if err != nil || len(instances) != 2 { + t.Fatalf("instances=%#v err=%v", instances, err) + } +} + +func TestMemoryStoreDeliveryPackageVersionConflict(t *testing.T) { + ctx := context.Background() + dataStore := NewMemoryStore() + item := domain.DeliveryPackageRecord{ID: "package-a", OrganizationID: "org-a", ProjectID: "project-a", ReleasePartitionID: "partition-a", Status: domain.DeliveryPackagePendingAcceptance, Version: 1} + if err := dataStore.UpsertDeliveryPackage(ctx, item); err != nil { + t.Fatal(err) + } + item.Status = domain.DeliveryPackageAccepted + item.Version = 2 + updated, err := dataStore.UpdateDeliveryPackageVersioned(ctx, item, 1) + if err != nil || !updated { + t.Fatalf("updated=%v err=%v", updated, err) + } + if stale, err := dataStore.UpdateDeliveryPackageVersioned(ctx, item, 1); err != nil || stale { + t.Fatalf("stale=%v err=%v", stale, err) + } +} diff --git a/internal/worker/human_release.go b/internal/worker/human_release.go index 00c33229..5423b6d9 100644 --- a/internal/worker/human_release.go +++ b/internal/worker/human_release.go @@ -2,8 +2,6 @@ package worker import ( "context" - "crypto/sha256" - "encoding/hex" "fmt" "sort" "strings" @@ -56,6 +54,9 @@ func humanReleaseBaseline(workflow domain.WorkflowRecord, parent domain.RunRecor } func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch domain.HumanBatchRecord, partition domain.ReleasePartitionRecord) error { + if partition.Status == domain.ReleasePartitionDispatched { + return runtime.syncDeliveryPackage(ctx, batch, partition) + } if partition.Status != domain.ReleasePartitionPending && partition.Status != domain.ReleasePartitionFailed { return nil } @@ -74,7 +75,10 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch if existing := humanReleaseChildForKey(runs, partition.UniqueKey); existing != nil { partition.ChildRunID = existing.ID partition.Status = domain.ReleasePartitionDispatched - return runtime.store.UpsertReleasePartition(ctx, partition) + if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil { + return err + } + return runtime.syncDeliveryPackage(ctx, batch, partition) } target, err := humanReleaseTarget(workflow, batch.GateNodeID) if err != nil { @@ -138,7 +142,56 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch partition.ChildRunID = child.ID partition.Status = domain.ReleasePartitionDispatched partition.ManifestPath = "annotations-manifest.json" - return runtime.store.UpsertReleasePartition(ctx, partition) + if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil { + return err + } + return runtime.syncDeliveryPackage(ctx, batch, partition) +} + +func (runtime *Runtime) syncDeliveryPackage(ctx context.Context, batch domain.HumanBatchRecord, partition domain.ReleasePartitionRecord) error { + packages, err := runtime.store.ListDeliveryPackages(ctx, batch.OrganizationID, domain.DeliveryPackageQuery{ReleasePartitionID: partition.ID}) + if err != nil { + return err + } + var item domain.DeliveryPackageRecord + if len(packages) > 0 { + item = packages[0] + } else { + item = domain.DeliveryPackageRecord{ + ID: "delivery-package-" + partition.ID, OrganizationID: batch.OrganizationID, ProjectID: batch.ProjectID, + WorkflowID: batch.WorkflowID, ParentRunID: batch.RunID, ChildRunID: partition.ChildRunID, + ReleasePartitionID: partition.ID, ManifestDigest: partition.ManifestDigest, ItemIDs: partition.ItemIDs, + Status: domain.DeliveryPackageBuilding, Version: 1, CreatedAt: nowISO(), UpdatedAt: nowISO(), + } + } + item.ChildRunID = partition.ChildRunID + child, err := runtime.store.GetRun(ctx, partition.ChildRunID) + if err != nil { + return err + } + if child != nil { + artifactIDs := make([]string, 0, len(child.Artifacts)) + handposeIDs := make([]string, 0) + for _, artifact := range child.Artifacts { + artifactIDs = append(artifactIDs, artifact.ID) + identity := strings.ToLower(artifact.Name + " " + artifact.Path + " " + artifact.NodeID) + if strings.Contains(identity, "handpose") || strings.Contains(identity, "wilor") { + handposeIDs = append(handposeIDs, artifact.ID) + } + } + item.ArtifactIDs = artifactIDs + item.HandposeArtifactIDs = handposeIDs + switch child.Status { + case domain.RunStatusCompleted: + if item.Status == domain.DeliveryPackageBuilding || item.Status == domain.DeliveryPackageFailed { + item.Status = domain.DeliveryPackagePendingAcceptance + } + case domain.RunStatusFailed, domain.RunStatusCancelled: + item.Status = domain.DeliveryPackageFailed + } + } + item.UpdatedAt = nowISO() + return runtime.store.UpsertDeliveryPackage(ctx, item) } func humanReleaseBatchTerminal(items []domain.HumanItemRecord) bool { @@ -146,7 +199,7 @@ func humanReleaseBatchTerminal(items []domain.HumanItemRecord) bool { return false } for _, item := range items { - if item.Status != domain.HumanItemAccepted && item.Status != domain.HumanItemEscalated { + if item.Status != domain.HumanItemAccepted && item.Status != domain.HumanItemRejected && item.Status != domain.HumanItemEscalated { return false } } @@ -194,8 +247,7 @@ func (runtime *Runtime) ensureDueHumanReleasePartitions(ctx context.Context, bat } sort.Strings(itemIDs) sequence := batch.NextPartitionSequence + 1 - digestBytes := sha256.Sum256([]byte(strings.Join(itemIDs, "\x00"))) - digest := hex.EncodeToString(digestBytes[:]) + digest := domain.HumanReleaseManifestDigest(itemIDs) uniqueKey := strings.Join([]string{batch.GateNodeID, fmt.Sprint(sequence), digest}, ":") partition := domain.ReleasePartitionRecord{ OrganizationID: batch.OrganizationID, BatchID: batch.ID, GateNodeID: batch.GateNodeID, diff --git a/internal/worker/human_release_test.go b/internal/worker/human_release_test.go index d1605b20..95efa14c 100644 --- a/internal/worker/human_release_test.go +++ b/internal/worker/human_release_test.go @@ -41,3 +41,11 @@ func TestHumanReleaseBaselineCompletesGateWithPartitionArtifacts(t *testing.T) { } t.Fatal("gate baseline missing") } + +func TestHumanReleaseManifestDigestIsStableAcrossOrdering(t *testing.T) { + left := domain.HumanReleaseManifestDigest([]string{"item-b", "item-a"}) + right := domain.HumanReleaseManifestDigest([]string{"item-a", "item-b"}) + if left == "" || left != right { + t.Fatalf("digest must be stable: left=%q right=%q", left, right) + } +} diff --git a/lightweight-operators/ubt-mcap-pipeline/robo_orchard_lab/dataset/ubt_manipulation/tools/utils.py b/lightweight-operators/ubt-mcap-pipeline/robo_orchard_lab/dataset/ubt_manipulation/tools/utils.py index ed70a2b5..2fd25e74 100644 --- a/lightweight-operators/ubt-mcap-pipeline/robo_orchard_lab/dataset/ubt_manipulation/tools/utils.py +++ b/lightweight-operators/ubt-mcap-pipeline/robo_orchard_lab/dataset/ubt_manipulation/tools/utils.py @@ -38,7 +38,6 @@ from typing import Any, Iterator import cv2 import matplotlib.pyplot as plt import numpy as np -import torch from scipy.spatial.transform import Rotation logger = logging.getLogger(__name__) @@ -1631,10 +1630,7 @@ def get_frequency(timestamp, prefix="", window_size=3): + f"mean frequency: {freq.mean():.1f}Hz" ) return freq - time_diff = torch.from_numpy(time_diff)[None, None] - time_diff = torch.nn.functional.avg_pool1d( - time_diff, window_size, 1 - ).numpy()[0, 0] + time_diff = np.convolve(time_diff, np.ones(window_size, dtype=np.float64) / window_size, mode="valid") freq = 1 / time_diff logger.info( f"{prefix:<50} - " diff --git a/operator-repos/builtin/annotation_gate/requirements.txt b/operator-repos/builtin/annotation_gate/requirements.txt index 8b137891..e69de29b 100644 --- a/operator-repos/builtin/annotation_gate/requirements.txt +++ b/operator-repos/builtin/annotation_gate/requirements.txt @@ -1 +0,0 @@ - diff --git a/operator-repos/builtin/annotation_gate/src/__init__.py b/operator-repos/builtin/annotation_gate/src/__init__.py index 8b137891..e69de29b 100644 --- a/operator-repos/builtin/annotation_gate/src/__init__.py +++ b/operator-repos/builtin/annotation_gate/src/__init__.py @@ -1 +0,0 @@ - diff --git a/operator-repos/builtin/json_filter/src/operator.py b/operator-repos/builtin/json_filter/src/operator.py index 8e1d2012..4a00c64b 100644 --- a/operator-repos/builtin/json_filter/src/operator.py +++ b/operator-repos/builtin/json_filter/src/operator.py @@ -15,19 +15,10 @@ def run( helpers: HelperMap, ) -> list[dict[str, Any]]: params = helpers["hyperparams"](node_data) - field_path = params.get("FIELD_PATH", "").strip() - operator = helpers["normalize_comparison_operator"](params.get("OPERATOR", "eq")) - expected = helpers["parse_json_param"](params, "EXPECTED_JSON", params.get("EXPECTED", "")) rows = helpers["collect_items"](helpers["payloads_from_inputs"](inputs)) - filtered = [ - row - for row in rows - if helpers["compare_values"](helpers["get_field_value"](row, field_path), operator, expected) - ] + filtered, filter_metadata = helpers["filter_rows"](rows, params) payload = { - "fieldPath": field_path, - "operator": operator, - "expected": expected, + **filter_metadata, "items": filtered, "count": len(filtered), } diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..e9e3d9ab --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,9 @@ +# Reproducible local Worker/operator test environment. +pytest==9.1.1 +numpy==2.5.2 +PyYAML==6.0.3 +pyarrow==25.0.1 +opencv-python-headless==5.0.0.93 +h5py==3.16.0 +matplotlib==3.11.1 +scipy==1.18.0 diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index 31f387f2..18407a95 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -7,10 +7,7 @@ import { FolderKanban, FileCode, Cpu, - Activity, ClipboardCheck, - ClipboardList, - Settings, Users, ChevronLeft, ChevronRight, @@ -19,10 +16,6 @@ import { Mail, Shield, Database, - Cloudy, - CloudUpload, - DatabaseZap, - BarChart3, Tags, LogOut, Loader2, @@ -45,58 +38,37 @@ const navItems: readonly NavigationItem[] = [ icon: LayoutDashboard, label: "仪表盘", path: "/", - roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "compliance_auditor", "auditor", "operator"], + roles: ["admin", "annotator", "quality_reviewer", "acceptance_reviewer"], }, { icon: FolderKanban, label: "项目管理", path: "/projects", - roles: ["admin", "pipeline_engineer", "data_operator", "operator"], - }, - { - icon: BarChart3, - label: "数据运营", - path: "/data-operations", - roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "compliance_auditor", "auditor", "operator"], + roles: ["admin", "annotator", "quality_reviewer", "acceptance_reviewer"], }, { icon: FileCode, - label: "工作流模板", + label: "工作流", path: "/templates", - roles: ["admin", "pipeline_engineer", "operator"], + roles: ["admin"], }, - { icon: Cpu, label: "算子管理", path: "/operators", roles: ["admin", "pipeline_engineer", "operator"] }, - { icon: Database, label: "业务数据库", path: "/database", roles: ["admin"] }, - { icon: Cloudy, label: "对象存储", path: "/object-storage", roles: ["admin"] }, - { - icon: CloudUpload, - label: "drosync 同步", - path: "/object-sync", - roles: ["admin", "pipeline_engineer", "data_operator", "compliance_auditor", "auditor", "operator"], - }, - { - icon: DatabaseZap, - label: "数据资产", - path: "/data-assets", - roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "compliance_auditor", "auditor", "operator"], - }, - { - icon: Activity, - label: "执行监控", - path: "/monitoring", - roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "compliance_auditor", "auditor", "operator"], - }, - { icon: ClipboardCheck, label: "任务审核", path: "/audit", roles: ["admin", "qa_reviewer", "auditor"] }, - { icon: ClipboardList, label: "抽检任务", path: "/sampling", roles: ["admin", "qa_reviewer", "operator"] }, + { icon: Cpu, label: "算子库", path: "/operators", roles: ["admin"] }, + { icon: Database, label: "数据存储", path: "/object-storage", roles: ["admin"] }, { icon: Tags, - label: "标注运营", + label: "标注中心", path: "/plugins/annotation-qa/workflows/all/annotate", - roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "auditor", "operator"], + roles: ["admin", "annotator"], + pluginId: "annotation-qa", + }, + { + icon: ClipboardCheck, + label: "质检中心", + path: "/plugins/annotation-qa/workflows/all/quality", + roles: ["admin", "quality_reviewer", "acceptance_reviewer"], pluginId: "annotation-qa", }, { icon: Users, label: "用户管理", path: "/users", roles: ["admin"] }, - { icon: Settings, label: "系统配置", path: "/settings", roles: ["admin"] }, ] as const; interface SidebarProps { @@ -158,8 +130,16 @@ export function Sidebar({ isCollapsed, onToggle }: SidebarProps) { if (!role) { return []; } + const effectiveRole = + role === "data_operator" + ? "annotator" + : role === "qa_reviewer" || role === "auditor" || role === "compliance_auditor" + ? "quality_reviewer" + : role === "pipeline_engineer" || role === "operator" + ? "admin" + : role; return navItems.filter( - (item) => item.roles.includes(role) && (!item.pluginId || enabledPluginIds.has(item.pluginId)), + (item) => item.roles.includes(effectiveRole) && (!item.pluginId || enabledPluginIds.has(item.pluginId)), ); }, [enabledPluginIds, role]); diff --git a/src/components/SuperAdminConsole.tsx b/src/components/SuperAdminConsole.tsx index 17ee1acb..3eaaaee8 100644 --- a/src/components/SuperAdminConsole.tsx +++ b/src/components/SuperAdminConsole.tsx @@ -19,7 +19,7 @@ type UserDraft = { }; const emptyOrganizationDraft: OrganizationDraft = { name: "", slug: "" }; -const emptyUserDraft: UserDraft = { name: "", email: "", role: "data_operator", password: "" }; +const emptyUserDraft: UserDraft = { name: "", email: "", role: "annotator", password: "" }; function userRoleOptionsFor(role: UserRole) { if (["operator", "auditor"].includes(role)) { diff --git a/src/lib/storage-connections.js b/src/lib/storage-connections.js index 98f767db..5dc97afd 100644 --- a/src/lib/storage-connections.js +++ b/src/lib/storage-connections.js @@ -1,7 +1,7 @@ -export function validateStorageConnectionEndpoint(value, { field = "endpoint" } = {}) { +export function validateStorageConnectionEndpoint(value, { field = "endpoint", required = false } = {}) { const endpoint = String(value ?? "").trim(); if (!endpoint) { - return ""; + return required ? `${field} is required` : ""; } if (/[ \t\r\n]/.test(endpoint)) { return `${field} must be an absolute http(s) URL without whitespace`; diff --git a/src/lib/user-roles.ts b/src/lib/user-roles.ts index 8f2a7a95..9432892a 100644 --- a/src/lib/user-roles.ts +++ b/src/lib/user-roles.ts @@ -11,30 +11,29 @@ export const USER_ROLE_OPTIONS: Array<{ description: "账号、系统配置、数据库、对象存储与全局治理", }, { - value: "pipeline_engineer", - label: "流程工程师", - description: "算子、模板、工作流设计与调试", + value: "annotator", + label: "标注员", + description: "在授权项目内执行切片校正与语义标注", }, { - value: "data_operator", - label: "数据运营", - description: "项目运行、调度、重跑、停止与生产监控", + value: "quality_reviewer", + label: "质检员", + description: "在授权项目内复核视频质量与标注质量", }, { - value: "qa_reviewer", - label: "质检验收", - description: "任务审核、抽检评审与成品质量确认", - }, - { - value: "compliance_auditor", - label: "审计观察员", - description: "只读查看运行、资产与审计记录", + value: "acceptance_reviewer", + label: "验收员", + description: "在授权项目内执行成品数据最终验收", }, ]; const legacyLabels: Partial> = { operator: "旧版操作员", auditor: "旧版审核员", + pipeline_engineer: "旧版流程工程师", + data_operator: "旧版数据运营", + qa_reviewer: "旧版质检角色", + compliance_auditor: "旧版审计角色", }; export function getUserRoleLabel(role: UserRole) { @@ -46,5 +45,5 @@ export function getUserRoleDescription(role: UserRole) { } export function canManagePipelineDesign(role: UserRole | null | undefined) { - return role === "admin" || role === "pipeline_engineer" || role === "operator"; + return role === "admin"; } diff --git a/src/lib/workflow-capabilities.js b/src/lib/workflow-capabilities.js index 54649f80..07e9e1f6 100644 --- a/src/lib/workflow-capabilities.js +++ b/src/lib/workflow-capabilities.js @@ -19,9 +19,13 @@ export function workflowCapabilityTone(capabilityId) { export function canAccessWorkflowCapability(role, capabilityId) { if (role === "admin") return true; if (capabilityId === "human_annotation") { - return ["pipeline_engineer", "data_operator", "operator"].includes(role); + return ["annotator", "data_operator"].includes(role); } - if (capabilityId === "human_quality_review" || capabilityId === "human_acceptance") { + if (capabilityId === "human_quality_review") { + return ["quality_reviewer", "qa_reviewer", "auditor"].includes(role); + } + if (capabilityId === "human_acceptance") { + if (role === "acceptance_reviewer") return true; return ["qa_reviewer", "auditor"].includes(role); } return false; diff --git a/src/pages/AnnotationOperations.tsx b/src/pages/AnnotationOperations.tsx index 724f7820..c8bfe44d 100644 --- a/src/pages/AnnotationOperations.tsx +++ b/src/pages/AnnotationOperations.tsx @@ -10,6 +10,10 @@ import { RefreshCw, RotateCcw, Send, + Scissors, + Merge, + Trash2, + Undo2, Tags, UsersRound, XCircle, @@ -21,20 +25,25 @@ import type { AnnotationTemplate, AnnotationWorkgroup, AnnotationWorkloadSummary, + HumanReject, + DeliveryPackage, Project, ProjectAnnotationMembership, + SliceOperation, + SliceSegment, } from "../types"; import { useUser } from "../context/UserContext"; import { useTimeZone } from "../context/TimeZoneContext"; import { annotationOperationsApi } from "../services/api/annotation-operations"; import { humanLoopApi } from "../services/api/human-loop"; +import { deliveryPackagesApi } from "../services/api/delivery-packages"; import { projectsApi } from "../services/api/projects"; import { normalizeAnnotationSegments, summarizeAnnotationWorkload } from "../lib/annotation-operations.js"; import { formatCompactDateTime } from "../lib/datetime"; import { getActionErrorMessage } from "../lib/action-errors.js"; import AnnotationTimelineEditor from "../components/AnnotationTimelineEditor"; -type Tab = "queue" | "templates" | "groups" | "workload"; +type Tab = "queue" | "rejects" | "templates" | "groups" | "workload"; type WorkspaceMode = "annotate" | "quality" | "acceptance"; const statusLabels: Record = { @@ -101,11 +110,22 @@ export default function AnnotationOperations() { const [templates, setTemplates] = useState([]); const [tasks, setTasks] = useState([]); const [workload, setWorkload] = useState({ items: [], eventCount: 0 }); + const [rejects, setRejects] = useState([]); + const [deliveryPackages, setDeliveryPackages] = useState([]); const [selectedTaskId, setSelectedTaskId] = useState(""); const [detail, setDetail] = useState(null); const [segments, setSegments] = useState([]); + const [sliceSegments, setSliceSegments] = useState([]); + const [sliceOperations, setSliceOperations] = useState([]); + const [sliceHistory, setSliceHistory] = useState([]); const [segmentDraft, setSegmentDraft] = useState({ labelId: "", label: "", start: "0", end: "1" }); const [reviewComment, setReviewComment] = useState(""); + const [qualityDraft, setQualityDraft] = useState({ + videoQuality: "qualified", + annotationAccuracy: "100", + annotationCompleteness: "100", + overallGrade: "A", + }); const [groupDraft, setGroupDraft] = useState({ name: "", vendorId: "", description: "" }); const [membershipDraft, setMembershipDraft] = useState({ userId: "", @@ -118,7 +138,7 @@ export default function AnnotationOperations() { const [saving, setSaving] = useState(false); const [error, setError] = useState(""); - const isManager = role === "admin" || role === "pipeline_engineer" || role === "operator"; + const isManager = role === "admin"; const loadBase = useCallback(async () => { setLoading(true); @@ -152,7 +172,7 @@ export default function AnnotationOperations() { setLoading(true); setError(""); try { - const [nextMemberships, nextTemplates, nextTasks, nextWorkload] = await Promise.all([ + const [nextMemberships, nextTemplates, nextTasks, nextWorkload, nextRejects, nextPackages] = await Promise.all([ annotationOperationsApi.listMemberships(projectId), annotationOperationsApi.listTemplates(projectId), annotationOperationsApi.listTasks({ @@ -163,11 +183,15 @@ export default function AnnotationOperations() { artifactId: requestedArtifactId || undefined, }), annotationOperationsApi.workloadSummary(projectId), + humanLoopApi.listRejects({ projectId }), + deliveryPackagesApi.list({ projectId, workflowId: requestedWorkflowId || undefined }), ]); setMemberships(nextMemberships); setTemplates(nextTemplates); setTasks(nextTasks); setWorkload(nextWorkload); + setRejects(nextRejects); + setDeliveryPackages(nextPackages); setSelectedTaskId((current) => { const visible = nextTasks.filter((item) => taskVisibleInMode(item.status, workspaceMode)); if (visible.some((item) => item.id === current)) return current; @@ -228,6 +252,14 @@ export default function AnnotationOperations() { setDetail(next); const latest = next.revisions[next.revisions.length - 1]; setSegments(latest?.segments || []); + const latestSlice = next.sliceRevisions[next.sliceRevisions.length - 1]; + setSliceSegments( + latestSlice?.segments || [ + { id: `slice-${next.task.id}-1`, start: next.task.clipStart, end: next.task.clipEnd }, + ], + ); + setSliceOperations([]); + setSliceHistory([]); }) .catch((loadError) => setError(getActionErrorMessage(loadError, "加载标注任务详情失败"))); }, [selectedTaskId]); @@ -240,6 +272,8 @@ export default function AnnotationOperations() { [detail?.task.templateId, templates], ); const workloadTotals = useMemo(() => summarizeAnnotationWorkload(workload.items), [workload.items]); + const annotationStage = + detail?.humanItem?.currentStageId || (detail?.sliceRevisions.length ? "semantic_annotation" : "split_annotation"); const visibleTasks = useMemo( () => tasks.filter((task) => taskVisibleInMode(task.status, workspaceMode)), [tasks, workspaceMode], @@ -273,6 +307,88 @@ export default function AnnotationOperations() { setSegments(next); }; + const applySliceChange = (next: SliceSegment[], operation: Omit) => { + const sorted = [...next].sort((left, right) => left.start - right.start); + setSliceHistory((history) => [...history, sliceSegments]); + setSliceOperations((operations) => [...operations, { ...operation, before: sliceSegments, after: sorted }]); + setSliceSegments(sorted); + }; + + const adjustSlice = (id: string, field: "start" | "end", value: number) => { + const next = sliceSegments.map((item) => (item.id === id ? { ...item, [field]: value } : item)); + applySliceChange(next, { type: "adjust", segmentIds: [id] }); + }; + + const splitSlice = (index: number) => { + const current = sliceSegments[index]; + if (!current || current.end - current.start < 0.2) return; + const at = Number(((current.start + current.end) / 2).toFixed(3)); + const next = [...sliceSegments]; + next.splice( + index, + 1, + { id: `${current.id}-a-${Date.now()}`, start: current.start, end: at, sourceIds: [current.id] }, + { id: `${current.id}-b-${Date.now()}`, start: at, end: current.end, sourceIds: [current.id] }, + ); + applySliceChange(next, { type: "split", segmentIds: [current.id], at }); + }; + + const mergeSlice = (index: number) => { + const current = sliceSegments[index]; + const nextItem = sliceSegments[index + 1]; + if (!current || !nextItem) return; + const next = [...sliceSegments]; + next.splice(index, 2, { + id: `merged-${Date.now()}`, + start: current.start, + end: nextItem.end, + sourceIds: [current.id, nextItem.id], + }); + applySliceChange(next, { type: "merge", segmentIds: [current.id, nextItem.id] }); + }; + + const deleteSlice = (index: number) => { + if (sliceSegments.length <= 1) return; + const current = sliceSegments[index]; + applySliceChange( + sliceSegments.filter((_, itemIndex) => itemIndex !== index), + { + type: "delete", + segmentIds: current ? [current.id] : [], + }, + ); + }; + + const undoSlice = () => { + const previous = sliceHistory[sliceHistory.length - 1]; + if (!previous) return; + setSliceSegments(previous); + setSliceHistory((history) => history.slice(0, -1)); + setSliceOperations((operations) => operations.slice(0, -1)); + }; + + const submitSliceRevision = async () => { + if (!detail || sliceSegments.length === 0) return; + setSaving(true); + setError(""); + try { + await annotationOperationsApi.submitSliceRevision(detail.task.id, { + segments: sliceSegments, + operations: sliceOperations, + changeReason: detail.sliceRevisions.length ? "切片返工修订" : "首轮切片校正", + }); + const next = await annotationOperationsApi.getTask(detail.task.id); + setDetail(next); + setSliceOperations([]); + setSliceHistory([]); + await loadProject(); + } catch (saveError) { + setError(getActionErrorMessage(saveError, "提交切片校正失败")); + } finally { + setSaving(false); + } + }; + const submitRevision = async () => { if (!detail || segments.length === 0) return; setSaving(true); @@ -291,6 +407,28 @@ export default function AnnotationOperations() { } }; + const rejectAnnotationInput = async () => { + if (!detail || !reviewComment.trim()) { + setError("拒绝输入必须填写原因。"); + return; + } + setSaving(true); + setError(""); + try { + await annotationOperationsApi.rejectTask(detail.task.id, { + reasonCodes: ["annotation_input_rejected"], + comment: reviewComment.trim(), + }); + setReviewComment(""); + await loadProject(); + setDetail(await annotationOperationsApi.getTask(detail.task.id)); + } catch (saveError) { + setError(getActionErrorMessage(saveError, "拒绝标注输入失败")); + } finally { + setSaving(false); + } + }; + const review = async (stage: "quality" | "acceptance", decision: "approved" | "rejected") => { if (!detail) return; if (decision === "rejected" && !reviewComment.trim()) { @@ -307,6 +445,10 @@ export default function AnnotationOperations() { rejectTargets: decision === "rejected" ? (stage === "quality" ? ["semantic_annotation"] : ["quality_review"]) : undefined, reasonCodes: decision === "rejected" ? ["manual_review_reject"] : undefined, + videoQuality: qualityDraft.videoQuality, + annotationAccuracy: Number(qualityDraft.annotationAccuracy), + annotationCompleteness: Number(qualityDraft.annotationCompleteness), + overallGrade: qualityDraft.overallGrade, }); setReviewComment(""); await loadProject(); @@ -400,6 +542,58 @@ export default function AnnotationOperations() { } }; + const reopenReject = async (reject: HumanReject, targetStageId: string) => { + setSaving(true); + setError(""); + try { + let itemVersion: number; + try { + const { item } = await humanLoopApi.getItem(reject.itemId); + itemVersion = item.version; + } catch { + itemVersion = (await deliveryPackagesApi.get(reject.itemId)).version; + } + await humanLoopApi.reopen(reject.id, { + itemVersion, + rejectVersion: reject.version, + targetStageId, + comment: "管理员从拒绝区定向重开", + }); + await loadProject(); + } catch (saveError) { + setError(getActionErrorMessage(saveError, "从拒绝区重开任务失败")); + } finally { + setSaving(false); + } + }; + + const decideDeliveryPackage = async (item: DeliveryPackage, decision: "approved" | "rejected") => { + if (decision === "rejected" && !reviewComment.trim()) { + setError("验收拒绝必须填写原因。"); + return; + } + setSaving(true); + setError(""); + try { + await deliveryPackagesApi.decide(item.id, { + version: item.version, + decision, + comment: reviewComment.trim(), + videoQuality: qualityDraft.videoQuality, + annotationAccuracy: Number(qualityDraft.annotationAccuracy), + annotationCompleteness: Number(qualityDraft.annotationCompleteness), + overallGrade: qualityDraft.overallGrade, + reasonCodes: decision === "rejected" ? ["acceptance_rejected"] : undefined, + }); + setReviewComment(""); + await loadProject(); + } catch (saveError) { + setError(getActionErrorMessage(saveError, "提交成品包验收失败")); + } finally { + setSaving(false); + } + }; + const selectProject = (nextProjectId: string) => { setProjectId(nextProjectId); const next = new URLSearchParams(searchParams); @@ -451,6 +645,29 @@ export default function AnnotationOperations() {
{error}
) : null} + {workspaceMode !== "annotate" ? ( +
+ + 质检复核 + + + 最终验收 + + + 主动抽检 + +
+ ) : null} +
{( [ @@ -459,6 +676,7 @@ export default function AnnotationOperations() { workspaceMode === "quality" ? "质检队列" : workspaceMode === "acceptance" ? "验收队列" : "标注队列", Layers3, ], + ["rejects", "拒绝区", XCircle], ["templates", "模板版本", Tags], ["groups", "供应商与角色", UsersRound], ["workload", "工时统计", Clock3], @@ -476,7 +694,165 @@ export default function AnnotationOperations() { ))}
- {tab === "queue" ? ( + {tab === "rejects" ? ( +
+
+
+
拒绝区
+
+ 拒绝事实永久保留;重开会创建新 Attempt,不修改原审核结论。 +
+
+ + {rejects.filter((item) => item.status === "open").length} 待处置 + +
+
+ {rejects.map((reject) => ( +
+
+
+
{reject.itemId}
+
+ {reject.stageId} · {reject.comment} +
+
+ 原因:{reject.reasonCodes.join("、") || "未填写"} · round {reject.attemptRound} +
+
+ {reject.status} +
+ {isManager && reject.status === "open" ? ( +
+ {reject.allowedReopenStages.map((stage) => ( + + ))} +
+ ) : null} +
+ ))} + {rejects.length === 0 ? ( +
当前项目没有拒绝记录
+ ) : null} +
+
+ ) : null} + + {tab === "queue" && workspaceMode === "acceptance" ? ( +
+ {deliveryPackages.map((item) => ( +
+
+
+
{item.id}
+
成品包 · {item.itemIds.length} 条数据
+
+ Partition {item.releasePartitionId} · manifest {item.manifestDigest.slice(0, 12)} · handpose{" "} + {item.handposeArtifactIds?.length || 0} +
+ + 查看打包 Run 与 lineage + +
+ + {item.status} + +
+ {item.status === "pending_acceptance" ? ( +
+
+ + + setQualityDraft((current) => ({ ...current, annotationAccuracy: event.target.value })) + } + placeholder="准确性" + /> + + setQualityDraft((current) => ({ ...current, annotationCompleteness: event.target.value })) + } + placeholder="完整性" + /> + +
+