diff --git a/backend/operator-runner/main.py b/backend/operator-runner/main.py index 19d68e23..50db25d1 100644 --- a/backend/operator-runner/main.py +++ b/backend/operator-runner/main.py @@ -195,6 +195,151 @@ def load_input_artifacts() -> list[dict[str, Any]]: return [] +def _first_numeric(payload: dict[str, Any], keys: tuple[str, ...]) -> int | float | None: + for key in keys: + value = payload.get(key) + if isinstance(value, bool): + continue + if isinstance(value, (int, float)) and value >= 0: + return value + return None + + +def _metric_volume(payloads: list[dict[str, Any]]) -> dict[str, int | float]: + candidates: dict[str, list[int | float]] = { + "trajectoryCount": [], + "durationSeconds": [], + "bytes": [], + "fileCount": [], + } + for payload in payloads: + data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {} + trajectory_count = _first_numeric( + payload, + ("trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"), + ) + if trajectory_count is None and data_ref: + trajectory_count = _first_numeric(data_ref, ("itemCount",)) + duration_seconds = _first_numeric(payload, ("durationSeconds", "totalDurationSeconds", "totalDuration", "duration")) + total_bytes = _first_numeric(payload, ("totalBytes", "bytes", "sizeBytes")) + file_count = _first_numeric(payload, ("fileCount", "filesCount")) + if trajectory_count is not None: + candidates["trajectoryCount"].append(int(trajectory_count)) + if duration_seconds is not None: + candidates["durationSeconds"].append(float(duration_seconds)) + if total_bytes is not None: + candidates["bytes"].append(int(total_bytes)) + if file_count is not None: + candidates["fileCount"].append(int(file_count)) + return {key: max(values) for key, values in candidates.items() if values} + + +def _artifact_payloads(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [item["data"] for item in artifacts if isinstance(item.get("data"), dict)] + + +def _dataset_refs(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]: + refs: list[dict[str, Any]] = [] + seen: set[str] = set() + for artifact in artifacts: + payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {} + data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {} + if not data_ref: + continue + item = { + "name": text_value(artifact.get("name")), + "kind": text_value(data_ref.get("kind")), + "datasetType": text_value(data_ref.get("datasetType")), + "manifestPath": text_value(data_ref.get("manifestPath")), + "payloadPath": text_value(data_ref.get("payloadPath")), + "partitionDir": text_value(data_ref.get("partitionDir")), + "partitionCount": int(data_ref.get("partitionCount") or 0), + "itemCount": int(data_ref.get("itemCount") or 0), + "datasetId": text_value(data_ref.get("datasetId")), + "datasetVersionId": text_value(data_ref.get("datasetVersionId")), + "split": text_value(data_ref.get("split")), + "fingerprint": text_value(data_ref.get("fingerprint")), + } + key = "|".join( + text_value(item.get(name)) + for name in ("datasetVersionId", "datasetId", "manifestPath", "payloadPath", "partitionDir", "split") + ) + if key in seen: + continue + seen.add(key) + refs.append(item) + return refs + + +def infer_data_metrics( + node_data: dict[str, Any], + artifacts: list[dict[str, Any]], + inputs: list[dict[str, Any]], +) -> dict[str, Any] | None: + stage = text_value(node_data.get("businessStage")).strip().lower() + if stage not in {"collect", "process", "clean", "validate", "annotate", "audit", "train"}: + return None + if node_data.get("metricBoundary") is False: + return None + output_payloads = _artifact_payloads(artifacts) + for payload in output_payloads: + explicit = payload.get("dataMetrics") + if isinstance(explicit, dict): + normalized = json_compatible(explicit) + normalized.setdefault("schemaVersion", 1) + normalized.setdefault("stage", stage) + normalized.setdefault("quality", "exact") + normalized.setdefault("occurredAt", now_iso()) + return normalized + + input_payloads = _artifact_payloads(inputs) + input_volume = _metric_volume(input_payloads) + output_volume = _metric_volume(output_payloads) + metrics: dict[str, Any] = { + "schemaVersion": 1, + "stage": stage, + "input": input_volume, + "output": output_volume, + "datasetRefs": _dataset_refs(artifacts), + "quality": "derived", + "occurredAt": now_iso(), + } + if stage == "collect" and not input_volume: + metrics.pop("input") + if stage == "validate": + valid = any(payload.get("valid") is True for payload in output_payloads) + metrics["technicalValid"] = output_volume if valid else {} + if not valid and input_volume: + metrics["rejected"] = input_volume + if stage in {"annotate", "audit"}: + metrics["trainReady"] = output_volume + if stage == "clean" and input_volume and output_volume: + rejected: dict[str, int | float] = {} + for key in ("trajectoryCount", "durationSeconds", "bytes", "fileCount"): + if key in input_volume and key in output_volume: + rejected[key] = max(0, input_volume[key] - output_volume[key]) + if rejected: + metrics["rejected"] = rejected + if stage == "train": + training_inputs: list[dict[str, Any]] = [] + for ref in _dataset_refs(inputs): + training_inputs.append( + { + "datasetId": ref.get("datasetId", ""), + "datasetVersionId": ref.get("datasetVersionId", ""), + "name": ref.get("name", ""), + "split": ref.get("split", ""), + "volume": input_volume, + "manifestPath": ref.get("manifestPath", ""), + "partitionDir": ref.get("partitionDir", ""), + "external": not bool(ref.get("datasetVersionId")), + "quality": "derived", + } + ) + metrics["trainingInputs"] = training_inputs + return metrics + + def stage_manifest_payload( run_id: str, workflow_id: str, @@ -202,6 +347,7 @@ def stage_manifest_payload( node_label: str, output_dir: pathlib.Path, artifacts: list[dict[str, Any]], + data_metrics: dict[str, Any] | None = None, ) -> dict[str, Any]: manifest_artifacts: list[dict[str, Any]] = [] data_refs: list[dict[str, Any]] = [] @@ -232,7 +378,7 @@ def stage_manifest_payload( } ) manifest_artifacts.append(item) - return { + result = { "runId": run_id, "workflowId": workflow_id, "nodeId": node_id, @@ -244,6 +390,9 @@ def stage_manifest_payload( "primaryDataRefs": data_refs, "generatedAt": now_iso(), } + if data_metrics: + result["dataMetrics"] = json_compatible(data_metrics) + return result def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path: @@ -5295,16 +5444,25 @@ def main() -> int: resource_samples = sampler.stop(ended_at) append_log("INFO", f"Produced {len(artifacts)} artifact(s)") + data_metrics = infer_data_metrics(node_data, artifacts, inputs) result = { "logs": logs, "artifacts": artifacts, "resourceSeries": resource_samples, } + if data_metrics: + result["dataMetrics"] = data_metrics stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip() if stage_manifest_path: - write_json_file(stage_manifest_path, stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts)) + write_json_file( + stage_manifest_path, + stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics), + ) else: - write_json_file(default_stage_manifest_path(output_dir), stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts)) + write_json_file( + default_stage_manifest_path(output_dir), + stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics), + ) result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip() if result_output_path: diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py index 6e0e217f..4912b6ce 100644 --- a/backend/operator-runner/test_main.py +++ b/backend/operator-runner/test_main.py @@ -646,6 +646,41 @@ class RuntimeFileContractTests(unittest.TestCase): self.assertEqual(len(manifest["dataRefs"]), 1) self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages") + def test_infer_data_metrics_uses_bound_stage_and_preserves_unknowns(self): + metrics = MODULE.infer_data_metrics( + {"businessStage": "clean", "metricBoundary": True}, + [{"name": "filtered.json", "data": {"count": 8, "fileCount": 16}}], + [{"name": "source.json", "data": {"count": 10, "fileCount": 20}}], + ) + + self.assertEqual(metrics["stage"], "clean") + self.assertEqual(metrics["input"]["trajectoryCount"], 10) + self.assertEqual(metrics["output"]["trajectoryCount"], 8) + self.assertEqual(metrics["rejected"]["trajectoryCount"], 2) + self.assertEqual(metrics["quality"], "derived") + + def test_stage_manifest_includes_standard_data_metrics(self): + with tempfile.TemporaryDirectory() as temp_dir: + output_dir = pathlib.Path(temp_dir) + metrics = { + "schemaVersion": 1, + "stage": "collect", + "output": {"trajectoryCount": 12}, + "quality": "exact", + } + manifest = MODULE.stage_manifest_payload( + "run-1", + "wf-1", + "node-1", + "数据采集", + output_dir, + [], + metrics, + ) + + self.assertEqual(manifest["dataMetrics"]["stage"], "collect") + self.assertEqual(manifest["dataMetrics"]["output"]["trajectoryCount"], 12) + class XPlatformIndexingTests(unittest.TestCase): def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self): diff --git a/internal/api/data_operations.go b/internal/api/data_operations.go new file mode 100644 index 00000000..ce91fe8c --- /dev/null +++ b/internal/api/data_operations.go @@ -0,0 +1,781 @@ +package api + +import ( + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" + + "droboticflow/internal/dataops" + "droboticflow/internal/domain" + + "github.com/go-chi/chi/v5" +) + +func validCollectionTaskMode(value domain.CollectionTaskMode) bool { + return value == domain.CollectionTaskModeCollectionOnly || value == domain.CollectionTaskModeFullTraining +} + +func validCollectionTaskStatus(value domain.CollectionTaskStatus) bool { + switch value { + case domain.CollectionTaskStatusDraft, domain.CollectionTaskStatusActive, domain.CollectionTaskStatusPaused, + domain.CollectionTaskStatusCompleted, domain.CollectionTaskStatusCancelled: + return true + default: + return false + } +} + +func validDataMetricStage(value domain.DataMetricStage) bool { + for _, stage := range domain.DataMetricStages { + if value == stage { + return true + } + } + return false +} + +func normalizedStringList(items []string) []string { + result := make([]string, 0, len(items)) + seen := map[string]struct{}{} + for _, value := range items { + trimmed := strings.TrimSpace(value) + key := strings.ToLower(trimmed) + if trimmed == "" { + continue + } + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + result = append(result, trimmed) + } + return result +} + +func validateNonNegativeVolume(field string, value domain.DataVolume) []validationIssue { + issues := make([]validationIssue, 0) + if value.TrajectoryCount != nil && *value.TrajectoryCount < 0 { + issues = append(issues, validationIssue{Field: field + ".trajectoryCount", Message: "must be greater than or equal to 0"}) + } + if value.DurationSeconds != nil && (*value.DurationSeconds < 0 || math.IsNaN(*value.DurationSeconds) || math.IsInf(*value.DurationSeconds, 0)) { + issues = append(issues, validationIssue{Field: field + ".durationSeconds", Message: "must be a finite value greater than or equal to 0"}) + } + if value.Bytes != nil && *value.Bytes < 0 { + issues = append(issues, validationIssue{Field: field + ".bytes", Message: "must be greater than or equal to 0"}) + } + if value.FileCount != nil && *value.FileCount < 0 { + issues = append(issues, validationIssue{Field: field + ".fileCount", Message: "must be greater than or equal to 0"}) + } + return issues +} + +func collectionTaskHasTarget(value domain.DataVolume) bool { + return (value.TrajectoryCount != nil && *value.TrajectoryCount > 0) || + (value.DurationSeconds != nil && *value.DurationSeconds > 0) || + (value.Bytes != nil && *value.Bytes > 0) +} + +func (server *server) normalizeCollectionTaskInput(ctxRequest *http.Request, payload *domain.CollectionTaskInput, existing *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, []validationIssue, error) { + payload.ProjectID = strings.TrimSpace(payload.ProjectID) + payload.Name = strings.TrimSpace(payload.Name) + payload.Description = strings.TrimSpace(payload.Description) + payload.RobotModel = strings.TrimSpace(payload.RobotModel) + payload.RobotVersion = strings.TrimSpace(payload.RobotVersion) + payload.Scene = strings.TrimSpace(payload.Scene) + payload.Format = strings.TrimSpace(payload.Format) + payload.WorkflowID = strings.TrimSpace(payload.WorkflowID) + payload.Modalities = normalizedStringList(payload.Modalities) + if payload.Mode == "" { + payload.Mode = domain.CollectionTaskModeFullTraining + } + if payload.Status == "" { + payload.Status = domain.CollectionTaskStatusDraft + } + issues := make([]validationIssue, 0) + if payload.ProjectID == "" { + issues = append(issues, validationIssue{Field: "projectId", Message: "projectId is required"}) + } + if payload.Name == "" { + issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) + } + if payload.RobotModel == "" { + issues = append(issues, validationIssue{Field: "robotModel", Message: "robotModel is required"}) + } + if payload.Scene == "" { + issues = append(issues, validationIssue{Field: "scene", Message: "scene is required"}) + } + if !validCollectionTaskMode(payload.Mode) { + issues = append(issues, validationIssue{Field: "mode", Message: "mode must be collection_only or full_training"}) + } + if !validCollectionTaskStatus(payload.Status) { + issues = append(issues, validationIssue{Field: "status", Message: "invalid task status"}) + } + issues = append(issues, validateNonNegativeVolume("targets", payload.Targets)...) + if !collectionTaskHasTarget(payload.Targets) { + issues = append(issues, validationIssue{Field: "targets", Message: "at least one positive trajectory, duration, or bytes target is required"}) + } + splitTotal := 0 + for split, percent := range payload.SplitRules { + if split != "train" && split != "val" && split != "test" { + issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split must be train, val, or test"}) + } + if percent < 0 || percent > 100 { + issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split percent must be between 0 and 100"}) + } + splitTotal += percent + } + if len(payload.SplitRules) > 0 && splitTotal != 100 { + issues = append(issues, validationIssue{Field: "splitRules", Message: "split percentages must total 100"}) + } + seenNodes := map[string]domain.DataMetricStage{} + for stage, nodeID := range payload.StageBindings { + nodeID = strings.TrimSpace(nodeID) + if !validDataMetricStage(stage) { + issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "invalid business stage"}) + continue + } + if nodeID == "" { + delete(payload.StageBindings, stage) + continue + } + if previous, exists := seenNodes[nodeID]; exists { + issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: fmt.Sprintf("node is already the metric boundary for %s", previous)}) + } + seenNodes[nodeID] = stage + payload.StageBindings[stage] = nodeID + } + if len(issues) > 0 { + return nil, issues, nil + } + project, err := server.store.GetProject(ctxRequest.Context(), payload.ProjectID) + if err != nil { + return nil, nil, err + } + if project == nil || !server.filterByTenant(ctxRequest, project.OrganizationID) { + return nil, []validationIssue{{Field: "projectId", Message: "project not found"}}, nil + } + workflowName := "" + if payload.WorkflowID != "" { + workflow, err := server.store.GetWorkflow(ctxRequest.Context(), payload.WorkflowID) + if err != nil { + return nil, nil, err + } + if workflow == nil || workflow.ProjectID != project.ID || !server.filterByTenant(ctxRequest, workflow.OrganizationID) { + return nil, []validationIssue{{Field: "workflowId", Message: "workflow not found in selected project"}}, nil + } + workflowName = workflow.Name + for stage, nodeID := range payload.StageBindings { + if domain.GetNodeByID(workflow.Nodes, nodeID) == nil { + issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "workflow node not found"}) + } + } + } + if len(issues) > 0 { + return nil, issues, nil + } + item := &domain.CollectionTaskRecord{ + OrganizationID: project.OrganizationID, + ProjectID: project.ID, + ProjectName: project.Name, + Name: payload.Name, + Description: payload.Description, + RobotModel: payload.RobotModel, + RobotVersion: payload.RobotVersion, + Scene: payload.Scene, + Modalities: payload.Modalities, + Format: payload.Format, + Mode: payload.Mode, + Status: payload.Status, + WorkflowID: payload.WorkflowID, + WorkflowName: workflowName, + StageBindings: payload.StageBindings, + Targets: payload.Targets, + Requirements: payload.Requirements, + SplitRules: payload.SplitRules, + } + if existing != nil { + item.ID = existing.ID + item.CreatedAt = existing.CreatedAt + item.LastRunID = existing.LastRunID + } + return item, nil, nil +} + +func (server *server) handleCollectionTasksList(writer http.ResponseWriter, request *http.Request) { + items, err := server.store.ListCollectionTasks(request.Context(), server.requestOrganizationID(request)) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + projectID := strings.TrimSpace(request.URL.Query().Get("projectId")) + status := strings.TrimSpace(request.URL.Query().Get("status")) + filtered := make([]domain.CollectionTaskRecord, 0, len(items)) + for _, item := range items { + if projectID != "" && item.ProjectID != projectID { + continue + } + if status != "" && string(item.Status) != status { + continue + } + filtered = append(filtered, item) + } + renderJSON(writer, http.StatusOK, map[string]any{"items": filtered}) +} + +func (server *server) handleCollectionTaskGet(writer http.ResponseWriter, request *http.Request) { + item, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if item == nil || !server.ensureTenantAccess(writer, request, item.OrganizationID, "Data task") { + if item == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"}) + } + return + } + renderJSON(writer, http.StatusOK, item) +} + +func (server *server) persistCollectionTask(request *http.Request, item *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, error) { + if err := server.store.UpsertCollectionTask(request.Context(), *item); err != nil { + return nil, err + } + return server.store.GetCollectionTask(request.Context(), item.ID) +} + +func (server *server) handleCollectionTaskCreate(writer http.ResponseWriter, request *http.Request) { + var payload domain.CollectionTaskInput + if !decodeJSON(writer, request, &payload) { + return + } + item, issues, err := server.normalizeCollectionTaskInput(request, &payload, nil) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + item.ID = domain.NewID("data-task") + saved, err := server.persistCollectionTask(request, item) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusCreated, saved) +} + +func (server *server) handleCollectionTaskUpdate(writer http.ResponseWriter, request *http.Request) { + existing, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if existing == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"}) + return + } + if !server.ensureTenantAccess(writer, request, existing.OrganizationID, "Data task") { + return + } + var payload domain.CollectionTaskInput + if !decodeJSON(writer, request, &payload) { + return + } + item, issues, err := server.normalizeCollectionTaskInput(request, &payload, existing) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + saved, err := server.persistCollectionTask(request, item) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, saved) +} + +func (server *server) handleCollectionTaskStart(writer http.ResponseWriter, request *http.Request) { + task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if task == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"}) + return + } + if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") { + return + } + if task.WorkflowID == "" { + renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bind a workflow before starting the task"}) + return + } + requiredStages := []domain.DataMetricStage{domain.DataMetricStageCollect} + if task.Mode == domain.CollectionTaskModeFullTraining { + requiredStages = append( + requiredStages, + domain.DataMetricStageProcess, + domain.DataMetricStageClean, + domain.DataMetricStageValidate, + domain.DataMetricStageTrain, + ) + if task.Requirements.RequireAnnotation { + requiredStages = append(requiredStages, domain.DataMetricStageAnnotate) + } + if task.Requirements.RequireAudit { + requiredStages = append(requiredStages, domain.DataMetricStageAudit) + } + } + missingStageIssues := make([]validationIssue, 0) + for _, stage := range requiredStages { + if strings.TrimSpace(task.StageBindings[stage]) == "" { + missingStageIssues = append(missingStageIssues, validationIssue{ + Field: "stageBindings." + string(stage), + Message: "required stage is not bound to a workflow node", + }) + } + } + if len(missingStageIssues) > 0 { + renderValidationError(writer, missingStageIssues...) + return + } + workflow, err := server.store.GetWorkflow(request.Context(), task.WorkflowID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if workflow == nil || workflow.ProjectID != task.ProjectID { + renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bound workflow is unavailable"}) + return + } + active, err := server.store.HasActiveRun(request.Context(), workflow.ID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if active { + renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已有进行中的运行", "code": "workflow_run_active"}) + return + } + if issues := validateWorkflowTopology(workflow.Nodes, workflow.Edges); len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + run, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerManual, task.ID) + if err != nil { + if renderRunCreationError(writer, err) { + return + } + renderError(writer, http.StatusInternalServerError, err) + return + } + task.Status = domain.CollectionTaskStatusActive + task.LastRunID = run.ID + saved, err := server.persistCollectionTask(request, task) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusAccepted, map[string]any{"task": saved, "run": run}) +} + +func (server *server) handleCollectionTaskComplete(writer http.ResponseWriter, request *http.Request) { + task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if task == nil { + renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"}) + return + } + if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") { + return + } + task.Status = domain.CollectionTaskStatusCompleted + saved, err := server.persistCollectionTask(request, task) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, saved) +} + +func parseDataOperationsQuery(request *http.Request, settings *domain.SettingsRecord) (domain.DataOperationsQuery, []validationIssue) { + timezone := strings.TrimSpace(request.URL.Query().Get("timezone")) + if timezone == "" && settings != nil { + timezone = strings.TrimSpace(settings.Timezone) + } + if timezone == "" { + timezone = domain.DefaultWorkflowScheduleTimezone + } + location, err := time.LoadLocation(timezone) + if err != nil { + return domain.DataOperationsQuery{}, []validationIssue{{Field: "timezone", Message: "invalid IANA timezone"}} + } + query := domain.DataOperationsQuery{ + Timezone: timezone, + ProjectID: strings.TrimSpace(request.URL.Query().Get("projectId")), + TaskID: strings.TrimSpace(request.URL.Query().Get("taskId")), + RobotModel: strings.TrimSpace(request.URL.Query().Get("robotModel")), + Scene: strings.TrimSpace(request.URL.Query().Get("scene")), + Stage: domain.DataMetricStage(strings.TrimSpace(request.URL.Query().Get("stage"))), + Cursor: strings.TrimSpace(request.URL.Query().Get("cursor")), + } + if query.Stage != "" && !validDataMetricStage(query.Stage) { + return domain.DataOperationsQuery{}, []validationIssue{{Field: "stage", Message: "invalid business stage"}} + } + if raw := strings.TrimSpace(request.URL.Query().Get("limit")); raw != "" { + value, parseErr := strconv.Atoi(raw) + if parseErr != nil || value < 1 || value > 100 { + return domain.DataOperationsQuery{}, []validationIssue{{Field: "limit", Message: "limit must be between 1 and 100"}} + } + query.Limit = value + } + now := time.Now() + fromRaw := strings.TrimSpace(request.URL.Query().Get("from")) + toRaw := strings.TrimSpace(request.URL.Query().Get("to")) + if fromRaw == "" { + localNow := now.In(location) + fromRaw = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, location).UTC().Format(time.RFC3339Nano) + } + if toRaw == "" { + toRaw = now.UTC().Format(time.RFC3339Nano) + } + from, fromErr := time.Parse(time.RFC3339Nano, fromRaw) + to, toErr := time.Parse(time.RFC3339Nano, toRaw) + if fromErr != nil { + return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be RFC3339"}} + } + if toErr != nil { + return domain.DataOperationsQuery{}, []validationIssue{{Field: "to", Message: "to must be RFC3339"}} + } + if !from.Before(to) { + return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be before to"}} + } + query.From = from.UTC().Format(time.RFC3339Nano) + query.To = to.UTC().Format(time.RFC3339Nano) + return query, nil +} + +func (server *server) dataOperationsQuery(writer http.ResponseWriter, request *http.Request) (domain.DataOperationsQuery, bool) { + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return domain.DataOperationsQuery{}, false + } + query, issues := parseDataOperationsQuery(request, settings) + if len(issues) > 0 { + renderValidationError(writer, issues...) + return domain.DataOperationsQuery{}, false + } + return query, true +} + +func (server *server) handleDataOperationsSummary(writer http.ResponseWriter, request *http.Request) { + query, ok := server.dataOperationsQuery(writer, request) + if !ok { + return + } + summary, err := server.store.GetDataOperationsSummary(request.Context(), server.requestOrganizationID(request), query) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, summary) +} + +func (server *server) handleDataOperationsStageItems(writer http.ResponseWriter, request *http.Request) { + query, ok := server.dataOperationsQuery(writer, request) + if !ok { + return + } + page, err := server.store.ListDataMetricFactsPage(request.Context(), server.requestOrganizationID(request), query) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, page) +} + +func (server *server) handleDataOperationsTrainingUsage(writer http.ResponseWriter, request *http.Request) { + query, ok := server.dataOperationsQuery(writer, request) + if !ok { + return + } + page, err := server.store.ListTrainingUsagePage(request.Context(), server.requestOrganizationID(request), query) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, page) +} + +func (server *server) handleDataOperationsDatasets(writer http.ResponseWriter, request *http.Request) { + query, ok := server.dataOperationsQuery(writer, request) + if !ok { + return + } + items, err := server.store.ListDatasetVersions(request.Context(), server.requestOrganizationID(request), query) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{"items": items}) +} + +func legacyStageFromExecution(execution domain.NodeExecutionRecord) domain.DataMetricStage { + label := strings.ToLower(strings.TrimSpace(execution.NodeLabel + " " + execution.OperatorID)) + stageTokens := []struct { + stage domain.DataMetricStage + tokens []string + }{ + {domain.DataMetricStageTrain, []string{"train", "训练"}}, + {domain.DataMetricStageAudit, []string{"audit", "审核"}}, + {domain.DataMetricStageAnnotate, []string{"annotat", "标注"}}, + {domain.DataMetricStageValidate, []string{"validat", "quality", "校验", "质检"}}, + {domain.DataMetricStageClean, []string{"clean", "filter", "清洗", "过滤"}}, + {domain.DataMetricStageProcess, []string{"process", "convert", "package", "处理", "转换", "打包"}}, + {domain.DataMetricStageCollect, []string{"collect", "scan", "source", "采集", "扫描", "数据源"}}, + } + for _, item := range stageTokens { + for _, token := range item.tokens { + if strings.Contains(label, token) { + return item.stage + } + } + } + return "" +} + +func numericArtifactValue(payload map[string]any, keys ...string) (float64, bool) { + for _, key := range keys { + switch value := payload[key].(type) { + case int: + return float64(value), value >= 0 + case int32: + return float64(value), value >= 0 + case int64: + return float64(value), value >= 0 + case float64: + return value, value >= 0 && !math.IsNaN(value) && !math.IsInf(value, 0) + } + } + return 0, false +} + +func legacyVolumeFromArtifacts(items []domain.ExecutionArtifact) domain.DataVolume { + var trajectory, duration, bytes, files float64 + var hasTrajectory, hasDuration, hasBytes, hasFiles bool + for _, artifact := range items { + payload := artifact.Data + if value, ok := numericArtifactValue(payload, "trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"); ok && (!hasTrajectory || value > trajectory) { + trajectory, hasTrajectory = value, true + } + if value, ok := numericArtifactValue(payload, "durationSeconds", "totalDurationSeconds", "totalDuration", "duration"); ok && (!hasDuration || value > duration) { + duration, hasDuration = value, true + } + if value, ok := numericArtifactValue(payload, "totalBytes", "bytes", "sizeBytes"); ok && (!hasBytes || value > bytes) { + bytes, hasBytes = value, true + } + if value, ok := numericArtifactValue(payload, "fileCount", "filesCount"); ok && (!hasFiles || value > files) { + files, hasFiles = value, true + } + } + result := domain.DataVolume{} + if hasTrajectory { + value := int64(trajectory) + result.TrajectoryCount = &value + } + if hasDuration { + value := duration + result.DurationSeconds = &value + } + if hasBytes { + value := int64(bytes) + result.Bytes = &value + } + if hasFiles { + value := int64(files) + result.FileCount = &value + } + return result +} + +func legacySourceCollectionMetrics(items []domain.ExecutionArtifact) (domain.DataVolume, domain.DataVolume, []domain.DatasetMetricRef, bool) { + var trajectories int64 + var validTrajectories int64 + var files int64 + var bytes int64 + refs := make([]domain.DatasetMetricRef, 0) + for artifactIndex := range items { + artifact := &items[artifactIndex] + payload := artifactStructuredData(artifact) + if payload == nil { + continue + } + for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") { + collection := mapValue(raw) + if collection == nil { + continue + } + trajectories++ + fileCount := int64Value(collection["fileCount"]) + totalBytes := int64Value(collection["totalBytes"]) + files += fileCount + bytes += totalBytes + if boolValue(collection["valid"]) { + validTrajectories++ + } + name := firstNonEmptyText(stringValue(collection["name"]), stringValue(collection["id"]), stringValue(collection["relativePath"])) + refs = append(refs, domain.DatasetMetricRef{ + DatasetID: stringValue(collection["id"]), + Name: name, + Kind: "source-collection", + DatasetType: firstNonEmptyText(stringValue(collection["datasetType"]), "trajectory"), + PayloadPath: stringValue(collection["path"]), + ItemCount: 1, + }) + } + } + if trajectories == 0 { + return domain.DataVolume{}, domain.DataVolume{}, nil, false + } + output := domain.DataVolume{ + TrajectoryCount: &trajectories, + Bytes: &bytes, + FileCount: &files, + } + valid := domain.DataVolume{TrajectoryCount: &validTrajectories} + return output, valid, refs, true +} + +func legacyRejectedVolume(input domain.DataVolume, output domain.DataVolume) domain.DataVolume { + result := domain.DataVolume{} + if input.TrajectoryCount != nil && output.TrajectoryCount != nil && *input.TrajectoryCount >= *output.TrajectoryCount { + value := *input.TrajectoryCount - *output.TrajectoryCount + result.TrajectoryCount = &value + } + if input.DurationSeconds != nil && output.DurationSeconds != nil && *input.DurationSeconds >= *output.DurationSeconds { + value := *input.DurationSeconds - *output.DurationSeconds + result.DurationSeconds = &value + } + if input.Bytes != nil && output.Bytes != nil && *input.Bytes >= *output.Bytes { + value := *input.Bytes - *output.Bytes + result.Bytes = &value + } + if input.FileCount != nil && output.FileCount != nil && *input.FileCount >= *output.FileCount { + value := *input.FileCount - *output.FileCount + result.FileCount = &value + } + return result +} + +func legacyMetricsFromExecution(run domain.RunRecord, execution domain.NodeExecutionRecord) *domain.DataMetricsRecord { + stage := legacyStageFromExecution(execution) + if stage == "" { + return nil + } + input := legacyVolumeFromArtifacts(execution.InputArtifacts) + output := legacyVolumeFromArtifacts(execution.Artifacts) + technicalValid := domain.DataVolume{} + refs := []domain.DatasetMetricRef(nil) + if collectionOutput, valid, collectionRefs, ok := legacySourceCollectionMetrics(execution.Artifacts); ok { + output = collectionOutput + technicalValid = valid + refs = collectionRefs + } + metrics := &domain.DataMetricsRecord{ + SchemaVersion: 1, + Stage: stage, + Input: input, + Output: output, + DatasetRefs: refs, + Quality: domain.DataMetricQualityEstimated, + OccurredAt: firstNonEmptyText(execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt), + } + switch stage { + case domain.DataMetricStageClean: + metrics.Rejected = legacyRejectedVolume(input, output) + case domain.DataMetricStageValidate: + if technicalValid.TrajectoryCount != nil { + metrics.TechnicalValid = technicalValid + } else { + metrics.TechnicalValid = output + } + case domain.DataMetricStageAudit: + metrics.TrainReady = output + } + return metrics +} + +func (server *server) handleDataOperationsBackfill(writer http.ResponseWriter, request *http.Request) { + limit, issue := parseDatabaseBrowserPageQuery(request.URL.Query().Get("limit"), "limit", 100, 1, 500) + if issue != nil { + renderValidationError(writer, *issue) + return + } + cursor, err := decodeRunSummaryCursor(request.URL.Query().Get("cursor")) + if err != nil { + renderValidationError(writer, validationIssue{Field: "cursor", Message: err.Error()}) + return + } + cutoff := time.Now().UTC().Add(-30 * 24 * time.Hour) + page, err := server.store.ListDataOperationsBackfillRuns( + request.Context(), + server.requestOrganizationID(request), + cutoff.Format(time.RFC3339Nano), + cursor, + limit, + ) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + processed := 0 + skipped := 0 + failures := 0 + for runIndex := range page.Items { + run := &page.Items[runIndex] + for executionIndex := range run.NodeExecutions { + execution := &run.NodeExecutions[executionIndex] + if execution.Status != domain.NodeExecutionStatusCompleted { + continue + } + legacy := execution.DataMetrics == nil + if legacy { + execution.DataMetrics = legacyMetricsFromExecution(*run, *execution) + if execution.DataMetrics == nil { + skipped++ + continue + } + } + if err := dataops.MaterializeLegacyNodeMetrics(request.Context(), server.store, run, execution); err != nil { + failures++ + continue + } + processed++ + } + } + renderJSON(writer, http.StatusOK, map[string]any{ + "windowDays": 30, + "scannedRuns": len(page.Items), + "processed": processed, + "skipped": skipped, + "failures": failures, + "nextCursor": encodeRunSummaryCursor(page.NextCursor), + "hasMore": page.HasMore, + "completedAt": nowISO(), + }) +} diff --git a/internal/api/data_operations_test.go b/internal/api/data_operations_test.go new file mode 100644 index 00000000..8b485119 --- /dev/null +++ b/internal/api/data_operations_test.go @@ -0,0 +1,150 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "droboticflow/internal/config" + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func apiDataOperationsInt64(value int64) *int64 { + return &value +} + +func TestCollectionTaskCreateAndStartCarriesTaskIntoRun(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Y1 桌面整理"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "Y1 完整链路", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{{ + ID: "collect-node", + Type: "default", + Data: domain.WorkflowNodeData{ + Label: "数据采集", + Category: "DATA", + Description: "collect", + Inputs: []domain.PortDefinition{}, + Outputs: []domain.PortDefinition{}, + }, + Position: domain.Position{}, + }}, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + handler := NewHandlerWithConfig(dataStore, nil, config.AppConfig{WorkspaceMountPath: t.TempDir()}) + payload := domain.CollectionTaskInput{ + ProjectID: project.ID, + Name: "Y1 桌面整理数据闭环", + RobotModel: "Y1", + Scene: "桌面整理", + Modalities: []string{"RGB", "关节状态"}, + Mode: domain.CollectionTaskModeCollectionOnly, + Status: domain.CollectionTaskStatusDraft, + WorkflowID: workflow.ID, + StageBindings: map[domain.DataMetricStage]string{ + domain.DataMetricStageCollect: "collect-node", + }, + Targets: domain.DataVolume{TrajectoryCount: apiDataOperationsInt64(100)}, + Requirements: domain.CollectionTaskRequirements{ + RequireTechnicalValidation: true, + RequireQuality: true, + RequireAnnotation: true, + RequireAudit: true, + }, + SplitRules: map[string]int{"train": 80, "val": 10, "test": 10}, + } + body, _ := json.Marshal(payload) + createRecorder := httptest.NewRecorder() + handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/data-collection-tasks", bytesReader(body))) + if createRecorder.Code != http.StatusCreated { + t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String()) + } + var task domain.CollectionTaskRecord + if err := json.Unmarshal(createRecorder.Body.Bytes(), &task); err != nil { + t.Fatalf("decode task: %v", err) + } + if task.ID == "" || task.CreatedAt == "" || task.UpdatedAt == "" { + t.Fatalf("created task is missing persisted fields: %#v", task) + } + + startRecorder := httptest.NewRecorder() + handler.ServeHTTP(startRecorder, httptest.NewRequest(http.MethodPost, "/api/data-collection-tasks/"+task.ID+"/start", nil)) + if startRecorder.Code != http.StatusAccepted { + t.Fatalf("start status = %d body=%s", startRecorder.Code, startRecorder.Body.String()) + } + var response struct { + Run domain.RunRecord `json:"run"` + } + if err := json.Unmarshal(startRecorder.Body.Bytes(), &response); err != nil { + t.Fatalf("decode start response: %v", err) + } + if response.Run.TaskID != task.ID { + t.Fatalf("run task id = %q, want %q", response.Run.TaskID, task.ID) + } +} + +func TestWorkflowNodeBusinessStageValidation(t *testing.T) { + issues := validateWorkflowNodeResources([]domain.WorkflowNode{{ + Data: domain.WorkflowNodeData{BusinessStage: domain.DataMetricStage("publish")}, + }}) + if len(issues) != 1 || issues[0].Field != "nodes[0].data.businessStage" { + t.Fatalf("issues = %#v", issues) + } +} + +func TestLegacySourceCollectionsProduceEstimatedTrajectoryMetrics(t *testing.T) { + metrics := legacyMetricsFromExecution( + domain.RunRecord{CreatedAt: "2026-07-23T01:00:00Z"}, + domain.NodeExecutionRecord{ + NodeLabel: "BOS 增量扫描", + Artifacts: []domain.ExecutionArtifact{{ + Name: "dataset-index.json", + Data: map[string]any{ + "collections": []any{ + map[string]any{"id": "trajectory-1", "path": "y1/1", "fileCount": float64(5), "totalBytes": float64(100), "valid": true}, + map[string]any{"id": "trajectory-2", "path": "y1/2", "fileCount": float64(4), "totalBytes": float64(80), "valid": false}, + }, + }, + }}, + }, + ) + if metrics == nil || metrics.Output.TrajectoryCount == nil || *metrics.Output.TrajectoryCount != 2 { + t.Fatalf("metrics = %#v", metrics) + } + if metrics.Output.FileCount == nil || *metrics.Output.FileCount != 9 || + metrics.Output.Bytes == nil || *metrics.Output.Bytes != 180 { + t.Fatalf("source collection volumes = %#v", metrics) + } + if len(metrics.DatasetRefs) != 2 || metrics.Quality != domain.DataMetricQualityEstimated { + t.Fatalf("source collection lineage = %#v", metrics) + } +} + +func TestParseDataOperationsQueryUsesUTC8TodayByDefault(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/api/data-operations/summary", nil) + query, issues := parseDataOperationsQuery(request, &domain.SettingsRecord{Timezone: "Asia/Shanghai"}) + if len(issues) > 0 { + t.Fatalf("issues = %#v", issues) + } + if query.Timezone != "Asia/Shanghai" || query.From == "" || query.To == "" { + t.Fatalf("query = %#v", query) + } +} + +func bytesReader(value []byte) *bytes.Reader { + return bytes.NewReader(value) +} diff --git a/internal/api/server.go b/internal/api/server.go index ce7ccd56..395df070 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -409,6 +409,7 @@ func runSummaryFromRecord(run domain.RunRecord) domain.RunSummary { WorkflowName: run.WorkflowName, ProjectID: run.ProjectID, ProjectName: run.ProjectName, + TaskID: run.TaskID, Trigger: run.Trigger, Attempt: run.Attempt, ParentRunID: run.ParentRunID, @@ -1205,6 +1206,23 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig }) router.Get("/api/acceptance/poc-readiness", instance.handlePocAcceptanceReadiness) router.Get("/api/dashboard", instance.handleDashboard) + router.Get("/api/data-collection-tasks", instance.handleCollectionTasksList) + router.Get("/api/data-collection-tasks/{id}", instance.handleCollectionTaskGet) + router.Get("/api/data-operations/summary", instance.handleDataOperationsSummary) + router.Get("/api/data-operations/stage-items", instance.handleDataOperationsStageItems) + router.Get("/api/data-operations/training-usage", instance.handleDataOperationsTrainingUsage) + router.Get("/api/data-operations/datasets", instance.handleDataOperationsDatasets) + router.Group(func(router chi.Router) { + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleOperator)) + router.Post("/api/data-collection-tasks", instance.handleCollectionTaskCreate) + router.Patch("/api/data-collection-tasks/{id}", instance.handleCollectionTaskUpdate) + router.Post("/api/data-collection-tasks/{id}/start", instance.handleCollectionTaskStart) + router.Post("/api/data-collection-tasks/{id}/complete", instance.handleCollectionTaskComplete) + }) + router.Group(func(router chi.Router) { + router.Use(instance.requireBusinessRole(domain.UserRoleAdmin)) + router.Post("/api/data-operations/backfill", instance.handleDataOperationsBackfill) + }) router.Get("/api/projects", instance.handleProjectsList) router.Get("/api/projects/{id}", instance.handleProjectsGet) @@ -3481,6 +3499,12 @@ func validateWorkflowNodeResources(nodes []domain.WorkflowNode) []validationIssu if err := k8s.ValidateResourceQuantity(field, node.Data.SharedMemorySize); err != nil { issues = append(issues, validationIssue{Field: field, Message: err.Error()}) } + if node.Data.BusinessStage != "" && !validDataMetricStage(node.Data.BusinessStage) { + issues = append(issues, validationIssue{ + Field: "nodes[" + strconv.Itoa(index) + "].data.businessStage", + Message: "invalid business stage", + }) + } } return issues } @@ -3739,7 +3763,7 @@ func validRunTrigger(value domain.RunTrigger) bool { return value == domain.RunTriggerManual || value == domain.RunTriggerScheduled || value == domain.RunTriggerRetry } -func (server *server) createRunWithLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger) (*domain.RunRecord, error) { +func (server *server) createRunWithLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) { if workflow == nil { return nil, fmt.Errorf("workflow not found") } @@ -3747,7 +3771,7 @@ func (server *server) createRunWithLightweightManifests(ctx context.Context, wor if err != nil { return nil, err } - run, err := server.store.CreateRun(ctx, workflow.ID, trigger) + run, err := server.store.CreateRun(ctx, workflow.ID, trigger, taskIDs...) if err != nil { return nil, err } @@ -3980,7 +4004,7 @@ func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request } } - childRun, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree) + childRun, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree, sourceRun.TaskID) if err != nil { if renderRunCreationError(writer, err) { return @@ -4735,6 +4759,9 @@ func (server *server) handleOperatorsCreate(writer http.ResponseWriter, request normalizeLightweightOperatorContract(&payload.RuntimeContract, containerCommandProvided, containerArgsProvided) } issues := validateOperatorMetadata(payload.Name, payload.Type, payload.Category, payload.Description, payload.Version, payload.Inputs, payload.Outputs, payload.Resources, payload.ContainerCommand, payload.ContainerArgs) + if payload.BusinessStage != "" && !validDataMetricStage(payload.BusinessStage) { + issues = append(issues, validationIssue{Field: "businessStage", Message: "invalid business stage"}) + } if payload.Type == "lightweight" { issues = append(issues, validateLightweightOperatorMetadata(payload.Image, payload.RuntimeContract, payload.ContainerCommand, payload.ContainerArgs, server.workspaceMountPath)...) } @@ -4801,6 +4828,7 @@ type operatorPatch struct { SkillDefinition *string `json:"skillDefinition"` Hyperparams *[]domain.HyperParam `json:"hyperparams"` Resources *domain.OperatorResources `json:"resources"` + BusinessStage *domain.DataMetricStage `json:"businessStage"` PublicationStatus *domain.OperatorPublicationStatus `json:"publicationStatus"` GenerationSource *domain.OperatorGenerationSource `json:"generationSource"` PublicationEvidence *domain.OperatorPublicationEvidence `json:"publicationEvidence"` @@ -4883,6 +4911,13 @@ func (server *server) handleOperatorsUpdate(writer http.ResponseWriter, request if payload.Resources != nil { next.Resources = normalizeOperatorResources(payload.Resources) } + if payload.BusinessStage != nil { + if *payload.BusinessStage != "" && !validDataMetricStage(*payload.BusinessStage) { + renderValidationError(writer, validationIssue{Field: "businessStage", Message: "invalid business stage"}) + return + } + next.BusinessStage = *payload.BusinessStage + } if payload.GenerationSource != nil { next.GenerationSource = normalizeOperatorGenerationSource(*payload.GenerationSource) } diff --git a/internal/dataops/materializer.go b/internal/dataops/materializer.go new file mode 100644 index 00000000..45841264 --- /dev/null +++ b/internal/dataops/materializer.go @@ -0,0 +1,292 @@ +package dataops + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func normalizedQuality(value domain.DataMetricQuality) domain.DataMetricQuality { + switch value { + case domain.DataMetricQualityExact, domain.DataMetricQualityDerived, domain.DataMetricQualityEstimated: + return value + default: + return domain.DataMetricQualityDerived + } +} + +func stableID(prefix string, value string) string { + sum := sha256.Sum256([]byte(value)) + return prefix + "-" + hex.EncodeToString(sum[:10]) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + } + return "" +} + +func dataRefFingerprint(ref domain.DatasetMetricRef, runID string, nodeID string) string { + value := strings.Join([]string{ + ref.DatasetVersionID, + ref.DatasetID, + ref.Fingerprint, + ref.ManifestPath, + ref.PayloadPath, + ref.PartitionDir, + ref.Split, + ref.Name, + }, "|") + if strings.Trim(value, "|") == "" { + value = runID + "|" + nodeID + "|" + ref.Name + } + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func metricRefFromDataRef(item domain.DataRefRecord) domain.DatasetMetricRef { + return domain.DatasetMetricRef{ + DatasetID: item.DatasetID, + DatasetVersionID: item.DatasetVersionID, + Name: item.ArtifactName, + Split: item.Split, + Fingerprint: item.Fingerprint, + Kind: item.Kind, + DatasetType: item.DatasetType, + ManifestPath: item.ManifestPath, + PayloadPath: item.PayloadPath, + PartitionDir: item.PartitionDir, + PartitionCount: item.PartitionCount, + ItemCount: item.ItemCount, + } +} + +func int64Pointer(value int64) *int64 { + return &value +} + +func volumeWithItemCount(volume domain.DataVolume, itemCount int) domain.DataVolume { + if volume.TrajectoryCount == nil && itemCount > 0 { + volume.TrajectoryCount = int64Pointer(int64(itemCount)) + } + return volume +} + +func anyString(value any) string { + if value == nil { + return "" + } + return strings.TrimSpace(fmt.Sprint(value)) +} + +func anyInt(value any) int { + switch typed := value.(type) { + case int: + return typed + case int32: + return int(typed) + case int64: + return int(typed) + case float64: + return int(typed) + default: + return 0 + } +} + +func metricRefFromArtifact(artifact domain.ExecutionArtifact) (domain.DatasetMetricRef, bool) { + if artifact.Data == nil { + return domain.DatasetMetricRef{}, false + } + raw, ok := artifact.Data["dataRef"].(map[string]any) + if !ok || len(raw) == 0 { + return domain.DatasetMetricRef{}, false + } + return domain.DatasetMetricRef{ + Name: artifact.Name, + DatasetID: anyString(raw["datasetId"]), + DatasetVersionID: anyString(raw["datasetVersionId"]), + Split: anyString(raw["split"]), + Fingerprint: anyString(raw["fingerprint"]), + Kind: anyString(raw["kind"]), + DatasetType: anyString(raw["datasetType"]), + ManifestPath: anyString(raw["manifestPath"]), + PayloadPath: anyString(raw["payloadPath"]), + PartitionDir: anyString(raw["partitionDir"]), + PartitionCount: anyInt(raw["partitionCount"]), + ItemCount: anyInt(raw["itemCount"]), + }, true +} + +func trainingInputsFromExecution(execution domain.NodeExecutionRecord, inputVolume domain.DataVolume) []domain.TrainingInputRecord { + result := make([]domain.TrainingInputRecord, 0) + seen := map[string]struct{}{} + for _, artifact := range execution.InputArtifacts { + ref, ok := metricRefFromArtifact(artifact) + if !ok { + continue + } + key := strings.Join([]string{ref.DatasetVersionID, ref.DatasetID, ref.ManifestPath, ref.PartitionDir, ref.Split}, "|") + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + result = append(result, domain.TrainingInputRecord{ + DatasetID: ref.DatasetID, + DatasetVersionID: ref.DatasetVersionID, + Name: ref.Name, + Split: ref.Split, + Volume: volumeWithItemCount(inputVolume, ref.ItemCount), + ManifestPath: ref.ManifestPath, + PartitionDir: ref.PartitionDir, + External: ref.DatasetVersionID == "", + Quality: domain.DataMetricQualityDerived, + }) + } + return result +} + +func materializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord, markRun bool) error { + if run == nil || execution == nil || execution.DataMetrics == nil { + return nil + } + metrics := execution.DataMetrics + if metrics.Stage == "" { + return nil + } + task := (*domain.CollectionTaskRecord)(nil) + if strings.TrimSpace(run.TaskID) != "" { + item, err := dataStore.GetCollectionTask(ctx, run.TaskID) + if err != nil { + return err + } + task = item + } + refs := append([]domain.DatasetMetricRef(nil), metrics.DatasetRefs...) + if len(refs) == 0 { + for _, item := range execution.DataRefs { + refs = append(refs, metricRefFromDataRef(item)) + } + } + quality := normalizedQuality(metrics.Quality) + for index := range refs { + ref := &refs[index] + fingerprint := firstNonEmpty(ref.Fingerprint, dataRefFingerprint(*ref, run.ID, execution.NodeID)) + ref.Fingerprint = fingerprint + ref.DatasetID = firstNonEmpty(ref.DatasetID, stableID("dataset", firstNonEmpty(ref.Name, ref.DatasetType, fingerprint))) + ref.DatasetVersionID = firstNonEmpty(ref.DatasetVersionID, stableID("dataset-version", fingerprint)) + volume := volumeWithItemCount(metrics.Output, ref.ItemCount) + version := domain.DatasetVersionRecord{ + ID: ref.DatasetVersionID, + OrganizationID: run.OrganizationID, + DatasetID: ref.DatasetID, + TaskID: run.TaskID, + ProjectID: run.ProjectID, + Name: firstNonEmpty(ref.Name, ref.DatasetType, ref.DatasetID), + Split: ref.Split, + Fingerprint: fingerprint, + Volume: volume, + ManifestPath: ref.ManifestPath, + PartitionDir: ref.PartitionDir, + ProducerRunID: run.ID, + ProducerNodeID: execution.NodeID, + Quality: quality, + CreatedAt: firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.UpdatedAt), + } + if err := dataStore.UpsertDatasetVersion(ctx, version); err != nil { + return err + } + } + trainingInputs := append([]domain.TrainingInputRecord(nil), metrics.TrainingInputs...) + if metrics.Stage == domain.DataMetricStageTrain && len(trainingInputs) == 0 { + trainingInputs = trainingInputsFromExecution(*execution, metrics.Input) + } + occurredAt := firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt) + if parsed, err := time.Parse(time.RFC3339Nano, occurredAt); err != nil || parsed.IsZero() { + occurredAt = time.Now().UTC().Format(time.RFC3339Nano) + quality = domain.DataMetricQualityEstimated + } else { + occurredAt = parsed.UTC().Format(time.RFC3339Nano) + } + fact := domain.DataMetricFactRecord{ + ID: stableID("data-fact", fmt.Sprintf("%s|%s|%d|%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage)), + SourceKey: fmt.Sprintf("%s/%s/%d/%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage), + OrganizationID: run.OrganizationID, + ProjectID: run.ProjectID, + ProjectName: run.ProjectName, + TaskID: run.TaskID, + WorkflowID: run.WorkflowID, + WorkflowName: run.WorkflowName, + RunID: run.ID, + NodeID: execution.NodeID, + NodeLabel: execution.NodeLabel, + Attempt: execution.Attempt, + Stage: metrics.Stage, + Input: metrics.Input, + Output: metrics.Output, + Rejected: metrics.Rejected, + TechnicalValid: metrics.TechnicalValid, + TrainReady: metrics.TrainReady, + DatasetRefs: refs, + TrainingInputs: trainingInputs, + RejectionReasons: metrics.RejectionReasons, + Quality: quality, + OccurredAt: occurredAt, + RecordedAt: time.Now().UTC().Format(time.RFC3339Nano), + } + if task != nil { + fact.TaskName = task.Name + fact.RobotModel = task.RobotModel + fact.Scene = task.Scene + } + if err := dataStore.UpsertDataMetricFact(ctx, fact); err != nil { + return err + } + if !markRun { + return nil + } + return dataStore.MarkNodeMetricsMaterialized(ctx, run.ID, execution.NodeID, execution.Attempt, time.Now().UTC().Format(time.RFC3339Nano)) +} + +func MaterializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error { + return materializeNodeMetrics(ctx, dataStore, run, execution, true) +} + +func MaterializeLegacyNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error { + return materializeNodeMetrics(ctx, dataStore, run, execution, false) +} + +func ReconcileRunMetrics(ctx context.Context, dataStore store.Store, limit int) (int, error) { + runs, err := dataStore.ListRunsNeedingMetricMaterialization(ctx, limit) + if err != nil { + return 0, err + } + materialized := 0 + var firstErr error + for runIndex := range runs { + run := &runs[runIndex] + for executionIndex := range run.NodeExecutions { + execution := &run.NodeExecutions[executionIndex] + if execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" { + continue + } + if err := MaterializeNodeMetrics(ctx, dataStore, run, execution); err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + materialized++ + } + } + return materialized, firstErr +} diff --git a/internal/dataops/materializer_test.go b/internal/dataops/materializer_test.go new file mode 100644 index 00000000..27d86501 --- /dev/null +++ b/internal/dataops/materializer_test.go @@ -0,0 +1,89 @@ +package dataops + +import ( + "context" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +func materializerInt64(value int64) *int64 { + return &value +} + +func TestMaterializeNodeMetricsCreatesFactDatasetAndMarksExecution(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Y1 项目"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: "Y1 数据准备", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{}, + Edges: []domain.WorkflowEdge{}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual, "task-y1") + if err != nil { + t.Fatalf("create run: %v", err) + } + execution := domain.NodeExecutionRecord{ + NodeID: "collect", + NodeLabel: "采集索引", + Attempt: 1, + Status: domain.NodeExecutionStatusCompleted, + EndedAt: "2026-07-23T01:00:00Z", + DataMetrics: &domain.DataMetricsRecord{ + SchemaVersion: 1, + Stage: domain.DataMetricStageCollect, + Output: domain.DataVolume{TrajectoryCount: materializerInt64(8)}, + Quality: domain.DataMetricQualityExact, + DatasetRefs: []domain.DatasetMetricRef{{ + Name: "Y1 batch", + DatasetType: "x-platform", + ManifestPath: "/workspace/y1/manifest.json", + ItemCount: 8, + }}, + }, + } + run.NodeExecutions = []domain.NodeExecutionRecord{execution} + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert run: %v", err) + } + if err := MaterializeNodeMetrics(ctx, dataStore, run, &execution); err != nil { + t.Fatalf("materialize: %v", err) + } + page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{ + From: "2026-07-23T00:00:00Z", + To: "2026-07-24T00:00:00Z", + }) + if err != nil { + t.Fatalf("list facts: %v", err) + } + if len(page.Items) != 1 || len(page.Items[0].DatasetRefs) != 1 { + t.Fatalf("facts = %#v", page.Items) + } + if page.Items[0].DatasetRefs[0].DatasetVersionID == "" { + t.Fatal("dataset version id was not materialized") + } + versions, err := dataStore.ListDatasetVersions(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{Limit: 10}) + if err != nil { + t.Fatalf("list versions: %v", err) + } + if len(versions) != 1 || versions[0].Volume.TrajectoryCount == nil || *versions[0].Volume.TrajectoryCount != 8 { + t.Fatalf("versions = %#v", versions) + } + persisted, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if persisted.NodeExecutions[0].MetricsMaterializedAt == "" { + t.Fatal("node metrics were not marked materialized") + } +} diff --git a/internal/domain/data_operations.go b/internal/domain/data_operations.go new file mode 100644 index 00000000..ea1b8d24 --- /dev/null +++ b/internal/domain/data_operations.go @@ -0,0 +1,295 @@ +package domain + +type DataMetricStage string + +const ( + DataMetricStageCollect DataMetricStage = "collect" + DataMetricStageProcess DataMetricStage = "process" + DataMetricStageClean DataMetricStage = "clean" + DataMetricStageValidate DataMetricStage = "validate" + DataMetricStageAnnotate DataMetricStage = "annotate" + DataMetricStageAudit DataMetricStage = "audit" + DataMetricStageTrain DataMetricStage = "train" +) + +var DataMetricStages = []DataMetricStage{ + DataMetricStageCollect, + DataMetricStageProcess, + DataMetricStageClean, + DataMetricStageValidate, + DataMetricStageAnnotate, + DataMetricStageAudit, + DataMetricStageTrain, +} + +type DataMetricQuality string + +const ( + DataMetricQualityExact DataMetricQuality = "exact" + DataMetricQualityDerived DataMetricQuality = "derived" + DataMetricQualityEstimated DataMetricQuality = "estimated" +) + +type DataVolume struct { + TrajectoryCount *int64 `json:"trajectoryCount,omitempty" bson:"trajectoryCount,omitempty"` + DurationSeconds *float64 `json:"durationSeconds,omitempty" bson:"durationSeconds,omitempty"` + Bytes *int64 `json:"bytes,omitempty" bson:"bytes,omitempty"` + FileCount *int64 `json:"fileCount,omitempty" bson:"fileCount,omitempty"` +} + +type DatasetMetricRef struct { + DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"` + DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"` + Name string `json:"name,omitempty" bson:"name,omitempty"` + Split string `json:"split,omitempty" bson:"split,omitempty"` + Fingerprint string `json:"fingerprint,omitempty" bson:"fingerprint,omitempty"` + Kind string `json:"kind,omitempty" bson:"kind,omitempty"` + DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"` + ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"` + PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"` + PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"` + PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"` + ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"` +} + +type TrainingInputRecord struct { + DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"` + DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"` + Name string `json:"name,omitempty" bson:"name,omitempty"` + Split string `json:"split,omitempty" bson:"split,omitempty"` + Volume DataVolume `json:"volume,omitempty" bson:"volume,omitempty"` + ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"` + PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"` + External bool `json:"external,omitempty" bson:"external,omitempty"` + Quality DataMetricQuality `json:"quality,omitempty" bson:"quality,omitempty"` +} + +type DataMetricsRecord struct { + SchemaVersion int `json:"schemaVersion" bson:"schemaVersion"` + Stage DataMetricStage `json:"stage" bson:"stage"` + Input DataVolume `json:"input,omitempty" bson:"input,omitempty"` + Output DataVolume `json:"output,omitempty" bson:"output,omitempty"` + Rejected DataVolume `json:"rejected,omitempty" bson:"rejected,omitempty"` + TechnicalValid DataVolume `json:"technicalValid,omitempty" bson:"technicalValid,omitempty"` + TrainReady DataVolume `json:"trainReady,omitempty" bson:"trainReady,omitempty"` + DatasetRefs []DatasetMetricRef `json:"datasetRefs,omitempty" bson:"datasetRefs,omitempty"` + TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"` + RejectionReasons map[string]int64 `json:"rejectionReasons,omitempty" bson:"rejectionReasons,omitempty"` + Quality DataMetricQuality `json:"quality,omitempty" bson:"quality,omitempty"` + OccurredAt string `json:"occurredAt,omitempty" bson:"occurredAt,omitempty"` +} + +type CollectionTaskStatus string + +const ( + CollectionTaskStatusDraft CollectionTaskStatus = "draft" + CollectionTaskStatusActive CollectionTaskStatus = "active" + CollectionTaskStatusPaused CollectionTaskStatus = "paused" + CollectionTaskStatusCompleted CollectionTaskStatus = "completed" + CollectionTaskStatusCancelled CollectionTaskStatus = "cancelled" +) + +type CollectionTaskMode string + +const ( + CollectionTaskModeCollectionOnly CollectionTaskMode = "collection_only" + CollectionTaskModeFullTraining CollectionTaskMode = "full_training" +) + +type CollectionTaskRequirements struct { + RequireTechnicalValidation bool `json:"requireTechnicalValidation" bson:"requireTechnicalValidation"` + RequireQuality bool `json:"requireQuality" bson:"requireQuality"` + RequireAnnotation bool `json:"requireAnnotation" bson:"requireAnnotation"` + RequireAudit bool `json:"requireAudit" bson:"requireAudit"` +} + +type CollectionTaskRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + ProjectID string `json:"projectId" bson:"projectId"` + ProjectName string `json:"projectName,omitempty" bson:"projectName,omitempty"` + Name string `json:"name" bson:"name"` + Description string `json:"description,omitempty" bson:"description,omitempty"` + RobotModel string `json:"robotModel" bson:"robotModel"` + RobotVersion string `json:"robotVersion,omitempty" bson:"robotVersion,omitempty"` + Scene string `json:"scene" bson:"scene"` + Modalities []string `json:"modalities" bson:"modalities"` + Format string `json:"format,omitempty" bson:"format,omitempty"` + Mode CollectionTaskMode `json:"mode" bson:"mode"` + Status CollectionTaskStatus `json:"status" bson:"status"` + WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"` + WorkflowName string `json:"workflowName,omitempty" bson:"workflowName,omitempty"` + StageBindings map[DataMetricStage]string `json:"stageBindings,omitempty" bson:"stageBindings,omitempty"` + Targets DataVolume `json:"targets" bson:"targets"` + Requirements CollectionTaskRequirements `json:"requirements" bson:"requirements"` + SplitRules map[string]int `json:"splitRules,omitempty" bson:"splitRules,omitempty"` + LastRunID string `json:"lastRunId,omitempty" bson:"lastRunId,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type CollectionTaskInput struct { + ProjectID string `json:"projectId"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + RobotModel string `json:"robotModel"` + RobotVersion string `json:"robotVersion,omitempty"` + Scene string `json:"scene"` + Modalities []string `json:"modalities"` + Format string `json:"format,omitempty"` + Mode CollectionTaskMode `json:"mode"` + Status CollectionTaskStatus `json:"status,omitempty"` + WorkflowID string `json:"workflowId,omitempty"` + StageBindings map[DataMetricStage]string `json:"stageBindings,omitempty"` + Targets DataVolume `json:"targets"` + Requirements CollectionTaskRequirements `json:"requirements"` + SplitRules map[string]int `json:"splitRules,omitempty"` +} + +type DatasetVersionRecord struct { + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + DatasetID string `json:"datasetId" bson:"datasetId"` + TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"` + ProjectID string `json:"projectId,omitempty" bson:"projectId,omitempty"` + Name string `json:"name" bson:"name"` + Split string `json:"split,omitempty" bson:"split,omitempty"` + Fingerprint string `json:"fingerprint" bson:"fingerprint"` + Volume DataVolume `json:"volume,omitempty" bson:"volume,omitempty"` + Storage AssetStorageLocation `json:"storage,omitempty" bson:"storage,omitempty"` + ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"` + PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"` + ProducerRunID string `json:"producerRunId,omitempty" bson:"producerRunId,omitempty"` + ProducerNodeID string `json:"producerNodeId,omitempty" bson:"producerNodeId,omitempty"` + ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` + Quality DataMetricQuality `json:"quality" bson:"quality"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type DataMetricFactRecord struct { + ID string `json:"id" bson:"id"` + SourceKey string `json:"sourceKey" bson:"sourceKey"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + ProjectID string `json:"projectId,omitempty" bson:"projectId,omitempty"` + ProjectName string `json:"projectName,omitempty" bson:"projectName,omitempty"` + TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"` + TaskName string `json:"taskName,omitempty" bson:"taskName,omitempty"` + RobotModel string `json:"robotModel,omitempty" bson:"robotModel,omitempty"` + Scene string `json:"scene,omitempty" bson:"scene,omitempty"` + WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"` + WorkflowName string `json:"workflowName,omitempty" bson:"workflowName,omitempty"` + RunID string `json:"runId" bson:"runId"` + NodeID string `json:"nodeId" bson:"nodeId"` + NodeLabel string `json:"nodeLabel,omitempty" bson:"nodeLabel,omitempty"` + Attempt int `json:"attempt" bson:"attempt"` + Stage DataMetricStage `json:"stage" bson:"stage"` + Input DataVolume `json:"input,omitempty" bson:"input,omitempty"` + Output DataVolume `json:"output,omitempty" bson:"output,omitempty"` + Rejected DataVolume `json:"rejected,omitempty" bson:"rejected,omitempty"` + TechnicalValid DataVolume `json:"technicalValid,omitempty" bson:"technicalValid,omitempty"` + TrainReady DataVolume `json:"trainReady,omitempty" bson:"trainReady,omitempty"` + DatasetRefs []DatasetMetricRef `json:"datasetRefs,omitempty" bson:"datasetRefs,omitempty"` + TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"` + RejectionReasons map[string]int64 `json:"rejectionReasons,omitempty" bson:"rejectionReasons,omitempty"` + Quality DataMetricQuality `json:"quality" bson:"quality"` + OccurredAt string `json:"occurredAt" bson:"occurredAt"` + RecordedAt string `json:"recordedAt" bson:"recordedAt"` +} + +type DataOperationsQuery struct { + From string + To string + Timezone string + ProjectID string + TaskID string + RobotModel string + Scene string + Stage DataMetricStage + Limit int + Cursor string +} + +type DataStageSummary struct { + Stage DataMetricStage `json:"stage"` + Input DataVolume `json:"input,omitempty"` + Output DataVolume `json:"output,omitempty"` + Rejected DataVolume `json:"rejected,omitempty"` + TechnicalValid DataVolume `json:"technicalValid,omitempty"` + TrainReady DataVolume `json:"trainReady,omitempty"` + RunCount int64 `json:"runCount"` +} + +type DataOperationsHourlyPoint struct { + Hour string `json:"hour"` + Stages map[DataMetricStage]DataVolume `json:"stages"` +} + +type DataOperationsTrainingSummary struct { + Consumed DataVolume `json:"consumed"` + Unique DataVolume `json:"unique"` + RunCount int64 `json:"runCount"` +} + +type CollectionTaskProgress struct { + TaskID string `json:"taskId"` + TaskName string `json:"taskName"` + Status CollectionTaskStatus `json:"status"` + Targets DataVolume `json:"targets"` + Collected DataVolume `json:"collected"` + Percent float64 `json:"percent"` +} + +type DataOperationsSummary struct { + From string `json:"from"` + To string `json:"to"` + Timezone string `json:"timezone"` + UpdatedAt string `json:"updatedAt"` + Freshness string `json:"freshness"` + Stages []DataStageSummary `json:"stages"` + Training DataOperationsTrainingSummary `json:"training"` + Hourly []DataOperationsHourlyPoint `json:"hourly"` + RejectionReasons map[string]int64 `json:"rejectionReasons"` + Tasks []CollectionTaskProgress `json:"tasks"` + FactCount int64 `json:"factCount"` + ExactCount int64 `json:"exactCount"` + DerivedCount int64 `json:"derivedCount"` + EstimatedCount int64 `json:"estimatedCount"` + UnknownCount int64 `json:"unknownCount"` +} + +type TrainingUsageRecord struct { + ID string `json:"id"` + TaskID string `json:"taskId,omitempty"` + TaskName string `json:"taskName,omitempty"` + ProjectID string `json:"projectId,omitempty"` + ProjectName string `json:"projectName,omitempty"` + WorkflowID string `json:"workflowId,omitempty"` + WorkflowName string `json:"workflowName,omitempty"` + RunID string `json:"runId"` + NodeID string `json:"nodeId"` + NodeLabel string `json:"nodeLabel,omitempty"` + Model string `json:"model,omitempty"` + Inputs []TrainingInputRecord `json:"inputs"` + Consumed DataVolume `json:"consumed"` + Quality DataMetricQuality `json:"quality"` + OccurredAt string `json:"occurredAt"` +} + +type TrainingUsagePage struct { + Items []TrainingUsageRecord `json:"items"` + NextCursor string `json:"nextCursor,omitempty"` + HasMore bool `json:"hasMore"` +} + +type DataMetricFactPage struct { + Items []DataMetricFactRecord `json:"items"` + NextCursor string `json:"nextCursor,omitempty"` + HasMore bool `json:"hasMore"` +} + +type DataOperationsRunPage struct { + Items []RunRecord + NextCursor *RunSummaryCursor + HasMore bool +} diff --git a/internal/domain/types.go b/internal/domain/types.go index 5abdedf9..81aa4c3a 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -183,6 +183,8 @@ type WorkflowNodeData struct { Memory string `json:"memory,omitempty" bson:"memory,omitempty"` GPU string `json:"gpu,omitempty" bson:"gpu,omitempty"` SharedMemorySize string `json:"sharedMemorySize,omitempty" bson:"sharedMemorySize,omitempty"` + BusinessStage DataMetricStage `json:"businessStage,omitempty" bson:"businessStage,omitempty"` + MetricBoundary bool `json:"metricBoundary,omitempty" bson:"metricBoundary,omitempty"` } type WorkflowNode struct { @@ -281,6 +283,7 @@ type OperatorRecord struct { SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"` Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"` Resources *OperatorResources `json:"resources,omitempty" bson:"resources,omitempty"` + BusinessStage DataMetricStage `json:"businessStage,omitempty" bson:"businessStage,omitempty"` PublicationStatus OperatorPublicationStatus `json:"publicationStatus,omitempty" bson:"publicationStatus,omitempty"` GenerationSource OperatorGenerationSource `json:"generationSource,omitempty" bson:"generationSource,omitempty"` PublicationEvidence *OperatorPublicationEvidence `json:"publicationEvidence,omitempty" bson:"publicationEvidence,omitempty"` @@ -463,37 +466,43 @@ func formatArtifactSize(bytes int64) string { } type DataRefRecord struct { - ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` - ArtifactName string `json:"artifactName,omitempty" bson:"artifactName,omitempty"` - PortID string `json:"portId,omitempty" bson:"portId,omitempty"` - Kind string `json:"kind,omitempty" bson:"kind,omitempty"` - DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"` - ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"` - PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"` - PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"` - PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"` - ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"` + ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"` + ArtifactName string `json:"artifactName,omitempty" bson:"artifactName,omitempty"` + PortID string `json:"portId,omitempty" bson:"portId,omitempty"` + Kind string `json:"kind,omitempty" bson:"kind,omitempty"` + DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"` + ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"` + PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"` + PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"` + PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"` + ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"` + DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"` + DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"` + Split string `json:"split,omitempty" bson:"split,omitempty"` + Fingerprint string `json:"fingerprint,omitempty" bson:"fingerprint,omitempty"` } type NodeExecutionRecord struct { - NodeID string `json:"nodeId" bson:"nodeId"` - NodeLabel string `json:"nodeLabel" bson:"nodeLabel"` - OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"` - OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"` - OperatorImage string `json:"operatorImage,omitempty" bson:"operatorImage,omitempty"` - OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"` - Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` - Status NodeExecutionStatus `json:"status" bson:"status"` - StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` - EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` - StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"` - InputArtifacts []ExecutionArtifact `json:"inputArtifacts,omitempty" bson:"inputArtifacts,omitempty"` - InputLineage []InputLineageRecord `json:"inputLineage,omitempty" bson:"inputLineage,omitempty"` - Logs []LogEntryRecord `json:"logs" bson:"logs"` - Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` - DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"` - ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` - Error string `json:"error,omitempty" bson:"error,omitempty"` + NodeID string `json:"nodeId" bson:"nodeId"` + NodeLabel string `json:"nodeLabel" bson:"nodeLabel"` + OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"` + OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"` + OperatorImage string `json:"operatorImage,omitempty" bson:"operatorImage,omitempty"` + OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"` + Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` + Status NodeExecutionStatus `json:"status" bson:"status"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"` + InputArtifacts []ExecutionArtifact `json:"inputArtifacts,omitempty" bson:"inputArtifacts,omitempty"` + InputLineage []InputLineageRecord `json:"inputLineage,omitempty" bson:"inputLineage,omitempty"` + Logs []LogEntryRecord `json:"logs" bson:"logs"` + Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` + DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"` + DataMetrics *DataMetricsRecord `json:"dataMetrics,omitempty" bson:"dataMetrics,omitempty"` + MetricsMaterializedAt string `json:"metricsMaterializedAt,omitempty" bson:"metricsMaterializedAt,omitempty"` + ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"` + Error string `json:"error,omitempty" bson:"error,omitempty"` } type InputLineageRecord struct { @@ -597,6 +606,7 @@ type LightweightOperatorCodeManifest struct { type RunMetadata struct { LightweightOperatorCodeManifests map[string]LightweightOperatorCodeManifest `json:"lightweightOperatorCodeManifests,omitempty" bson:"lightweightOperatorCodeManifests,omitempty"` + TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"` } // RunFailure is the machine-readable counterpart to RunRecord.Error. Error is @@ -630,6 +640,7 @@ type RunRecord struct { WorkflowName string `json:"workflowName" bson:"workflowName"` ProjectID string `json:"projectId" bson:"projectId"` ProjectName string `json:"projectName" bson:"projectName"` + TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"` Trigger RunTrigger `json:"trigger" bson:"trigger"` Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` @@ -666,6 +677,7 @@ type RunSummary struct { WorkflowName string `json:"workflowName" bson:"workflowName"` ProjectID string `json:"projectId" bson:"projectId"` ProjectName string `json:"projectName" bson:"projectName"` + TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"` Trigger RunTrigger `json:"trigger" bson:"trigger"` Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` @@ -1076,6 +1088,7 @@ type CreateOperatorInput struct { SkillDefinition string `json:"skillDefinition,omitempty"` Hyperparams []HyperParam `json:"hyperparams,omitempty"` Resources *OperatorResources `json:"resources,omitempty"` + BusinessStage DataMetricStage `json:"businessStage,omitempty"` PublicationStatus OperatorPublicationStatus `json:"publicationStatus,omitempty"` GenerationSource OperatorGenerationSource `json:"generationSource,omitempty"` PublicationEvidence *OperatorPublicationEvidence `json:"publicationEvidence,omitempty"` diff --git a/internal/store/data_operations_common.go b/internal/store/data_operations_common.go new file mode 100644 index 00000000..9cc0c6ab --- /dev/null +++ b/internal/store/data_operations_common.go @@ -0,0 +1,408 @@ +package store + +import ( + "encoding/base64" + "fmt" + "sort" + "strings" + "time" + + "droboticflow/internal/domain" +) + +type volumeAccumulator struct { + trajectoryCount int64 + durationSeconds float64 + bytes int64 + fileCount int64 + hasTrajectory bool + hasDuration bool + hasBytes bool + hasFileCount bool +} + +func dataMetricQualityRank(value domain.DataMetricQuality) int { + switch value { + case domain.DataMetricQualityExact: + return 3 + case domain.DataMetricQualityDerived: + return 2 + case domain.DataMetricQualityEstimated: + return 1 + default: + return 0 + } +} + +func (item *volumeAccumulator) add(value domain.DataVolume) { + if value.TrajectoryCount != nil { + item.trajectoryCount += *value.TrajectoryCount + item.hasTrajectory = true + } + if value.DurationSeconds != nil { + item.durationSeconds += *value.DurationSeconds + item.hasDuration = true + } + if value.Bytes != nil { + item.bytes += *value.Bytes + item.hasBytes = true + } + if value.FileCount != nil { + item.fileCount += *value.FileCount + item.hasFileCount = true + } +} + +func (item volumeAccumulator) value() domain.DataVolume { + result := domain.DataVolume{} + if item.hasTrajectory { + value := item.trajectoryCount + result.TrajectoryCount = &value + } + if item.hasDuration { + value := item.durationSeconds + result.DurationSeconds = &value + } + if item.hasBytes { + value := item.bytes + result.Bytes = &value + } + if item.hasFileCount { + value := item.fileCount + result.FileCount = &value + } + return result +} + +type stageAccumulator struct { + input volumeAccumulator + output volumeAccumulator + rejected volumeAccumulator + technicalValid volumeAccumulator + trainReady volumeAccumulator + runs map[string]struct{} +} + +func normalizedDataOperationsTimezone(value string) *time.Location { + name := strings.TrimSpace(value) + if name == "" { + name = domain.DefaultWorkflowScheduleTimezone + } + location, err := time.LoadLocation(name) + if err != nil { + location, _ = time.LoadLocation(domain.DefaultWorkflowScheduleTimezone) + } + if location == nil { + location = time.FixedZone("UTC+8", 8*60*60) + } + return location +} + +func dataOperationsFactMatches(item domain.DataMetricFactRecord, organizationID string, query domain.DataOperationsQuery) bool { + if !organizationMatches(item.OrganizationID, organizationID) { + return false + } + if query.ProjectID != "" && item.ProjectID != query.ProjectID { + return false + } + if query.TaskID != "" && item.TaskID != query.TaskID { + return false + } + if query.RobotModel != "" && !strings.EqualFold(item.RobotModel, query.RobotModel) { + return false + } + if query.Scene != "" && !strings.EqualFold(item.Scene, query.Scene) { + return false + } + if query.Stage != "" && item.Stage != query.Stage { + return false + } + if query.From != "" && item.OccurredAt < query.From { + return false + } + if query.To != "" && item.OccurredAt >= query.To { + return false + } + return true +} + +func datasetVersionMatches(item domain.DatasetVersionRecord, organizationID string, query domain.DataOperationsQuery) bool { + if !organizationMatches(item.OrganizationID, organizationID) { + return false + } + if query.ProjectID != "" && item.ProjectID != query.ProjectID { + return false + } + if query.TaskID != "" && item.TaskID != query.TaskID { + return false + } + if query.From != "" && item.CreatedAt < query.From { + return false + } + if query.To != "" && item.CreatedAt >= query.To { + return false + } + return true +} + +func encodeDataOperationsCursor(occurredAt string, id string) string { + return base64.RawURLEncoding.EncodeToString([]byte(occurredAt + "\n" + id)) +} + +func decodeDataOperationsCursor(value string) (string, string, bool) { + if strings.TrimSpace(value) == "" { + return "", "", true + } + payload, err := base64.RawURLEncoding.DecodeString(value) + if err != nil { + return "", "", false + } + parts := strings.SplitN(string(payload), "\n", 2) + if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" { + return "", "", false + } + return parts[0], parts[1], true +} + +func factsPage(items []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.DataMetricFactPage { + sort.Slice(items, func(left, right int) bool { + if items[left].OccurredAt == items[right].OccurredAt { + return items[left].ID > items[right].ID + } + return items[left].OccurredAt > items[right].OccurredAt + }) + cursorTime, cursorID, validCursor := decodeDataOperationsCursor(query.Cursor) + if !validCursor { + return domain.DataMetricFactPage{Items: []domain.DataMetricFactRecord{}} + } + if cursorTime != "" { + filtered := items[:0] + for _, item := range items { + if item.OccurredAt < cursorTime || (item.OccurredAt == cursorTime && item.ID < cursorID) { + filtered = append(filtered, item) + } + } + items = filtered + } + limit := query.Limit + if limit <= 0 { + limit = 50 + } + if limit > 100 { + limit = 100 + } + page := domain.DataMetricFactPage{Items: items, HasMore: len(items) > limit} + if page.HasMore { + page.Items = items[:limit] + last := page.Items[len(page.Items)-1] + page.NextCursor = encodeDataOperationsCursor(last.OccurredAt, last.ID) + } + if page.Items == nil { + page.Items = []domain.DataMetricFactRecord{} + } + return page +} + +func trainingUsageFromFacts(facts []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.TrainingUsagePage { + filtered := make([]domain.DataMetricFactRecord, 0) + for _, item := range facts { + if item.Stage == domain.DataMetricStageTrain { + filtered = append(filtered, item) + } + } + page := factsPage(filtered, query) + items := make([]domain.TrainingUsageRecord, 0, len(page.Items)) + for _, fact := range page.Items { + items = append(items, domain.TrainingUsageRecord{ + ID: fact.ID, + TaskID: fact.TaskID, + TaskName: fact.TaskName, + ProjectID: fact.ProjectID, + ProjectName: fact.ProjectName, + WorkflowID: fact.WorkflowID, + WorkflowName: fact.WorkflowName, + RunID: fact.RunID, + NodeID: fact.NodeID, + NodeLabel: fact.NodeLabel, + Model: fact.NodeLabel, + Inputs: fact.TrainingInputs, + Consumed: fact.Input, + Quality: fact.Quality, + OccurredAt: fact.OccurredAt, + }) + } + return domain.TrainingUsagePage{Items: items, NextCursor: page.NextCursor, HasMore: page.HasMore} +} + +func taskProgressPercent(target domain.DataVolume, collected domain.DataVolume) float64 { + switch { + case target.TrajectoryCount != nil && *target.TrajectoryCount > 0 && collected.TrajectoryCount != nil: + return minFloat(100, float64(*collected.TrajectoryCount)*100/float64(*target.TrajectoryCount)) + case target.DurationSeconds != nil && *target.DurationSeconds > 0 && collected.DurationSeconds != nil: + return minFloat(100, *collected.DurationSeconds*100 / *target.DurationSeconds) + case target.Bytes != nil && *target.Bytes > 0 && collected.Bytes != nil: + return minFloat(100, float64(*collected.Bytes)*100/float64(*target.Bytes)) + default: + return 0 + } +} + +func minFloat(left float64, right float64) float64 { + if left < right { + return left + } + return right +} + +func BuildDataOperationsSummary( + facts []domain.DataMetricFactRecord, + tasks []domain.CollectionTaskRecord, + query domain.DataOperationsQuery, +) domain.DataOperationsSummary { + location := normalizedDataOperationsTimezone(query.Timezone) + accumulators := make(map[domain.DataMetricStage]*stageAccumulator, len(domain.DataMetricStages)) + for _, stage := range domain.DataMetricStages { + accumulators[stage] = &stageAccumulator{runs: map[string]struct{}{}} + } + hourly := map[string]map[domain.DataMetricStage]*volumeAccumulator{} + reasons := map[string]int64{} + uniqueTraining := map[string]domain.DataVolume{} + var trainingConsumed volumeAccumulator + trainingRuns := map[string]struct{}{} + taskCollected := map[string]*volumeAccumulator{} + summary := domain.DataOperationsSummary{ + From: query.From, + To: query.To, + Timezone: location.String(), + UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Freshness: "live", + RejectionReasons: reasons, + } + + for _, fact := range facts { + accumulator, ok := accumulators[fact.Stage] + if !ok { + summary.UnknownCount++ + continue + } + accumulator.input.add(fact.Input) + accumulator.output.add(fact.Output) + accumulator.rejected.add(fact.Rejected) + accumulator.technicalValid.add(fact.TechnicalValid) + accumulator.trainReady.add(fact.TrainReady) + accumulator.runs[fact.RunID] = struct{}{} + summary.FactCount++ + switch fact.Quality { + case domain.DataMetricQualityExact: + summary.ExactCount++ + case domain.DataMetricQualityEstimated: + summary.EstimatedCount++ + case domain.DataMetricQualityDerived: + summary.DerivedCount++ + default: + summary.UnknownCount++ + } + for reason, count := range fact.RejectionReasons { + reasons[reason] += count + } + if occurredAt, err := time.Parse(time.RFC3339Nano, fact.OccurredAt); err == nil { + hour := occurredAt.In(location).Format("2006-01-02T15:00:00-07:00") + if hourly[hour] == nil { + hourly[hour] = map[domain.DataMetricStage]*volumeAccumulator{} + } + if hourly[hour][fact.Stage] == nil { + hourly[hour][fact.Stage] = &volumeAccumulator{} + } + hourly[hour][fact.Stage].add(fact.Output) + } + if fact.Stage == domain.DataMetricStageCollect && fact.TaskID != "" { + if taskCollected[fact.TaskID] == nil { + taskCollected[fact.TaskID] = &volumeAccumulator{} + } + taskCollected[fact.TaskID].add(fact.Output) + } + if fact.Stage == domain.DataMetricStageTrain { + trainingConsumed.add(fact.Input) + trainingRuns[fact.RunID] = struct{}{} + for _, input := range fact.TrainingInputs { + key := strings.TrimSpace(input.DatasetVersionID) + if key == "" { + key = strings.TrimSpace(input.DatasetID) + "|" + strings.TrimSpace(input.ManifestPath) + "|" + strings.TrimSpace(input.Split) + } + if key == "||" || key == "" { + key = fmt.Sprintf("%s|%s", fact.RunID, fact.NodeID) + } + if _, exists := uniqueTraining[key]; !exists { + uniqueTraining[key] = input.Volume + } + } + } + } + + summary.Stages = make([]domain.DataStageSummary, 0, len(domain.DataMetricStages)) + for _, stage := range domain.DataMetricStages { + item := accumulators[stage] + summary.Stages = append(summary.Stages, domain.DataStageSummary{ + Stage: stage, + Input: item.input.value(), + Output: item.output.value(), + Rejected: item.rejected.value(), + TechnicalValid: item.technicalValid.value(), + TrainReady: item.trainReady.value(), + RunCount: int64(len(item.runs)), + }) + } + var uniqueAccumulator volumeAccumulator + for _, volume := range uniqueTraining { + uniqueAccumulator.add(volume) + } + summary.Training = domain.DataOperationsTrainingSummary{ + Consumed: trainingConsumed.value(), + Unique: uniqueAccumulator.value(), + RunCount: int64(len(trainingRuns)), + } + + hours := make([]string, 0, len(hourly)) + for hour := range hourly { + hours = append(hours, hour) + } + sort.Strings(hours) + summary.Hourly = make([]domain.DataOperationsHourlyPoint, 0, len(hours)) + for _, hour := range hours { + point := domain.DataOperationsHourlyPoint{Hour: hour, Stages: map[domain.DataMetricStage]domain.DataVolume{}} + for stage, value := range hourly[hour] { + point.Stages[stage] = value.value() + } + summary.Hourly = append(summary.Hourly, point) + } + + summary.Tasks = make([]domain.CollectionTaskProgress, 0) + for _, task := range tasks { + if query.ProjectID != "" && task.ProjectID != query.ProjectID { + continue + } + if query.TaskID != "" && task.ID != query.TaskID { + continue + } + if query.RobotModel != "" && !strings.EqualFold(task.RobotModel, query.RobotModel) { + continue + } + if query.Scene != "" && !strings.EqualFold(task.Scene, query.Scene) { + continue + } + collected := domain.DataVolume{} + if taskCollected[task.ID] != nil { + collected = taskCollected[task.ID].value() + } + summary.Tasks = append(summary.Tasks, domain.CollectionTaskProgress{ + TaskID: task.ID, + TaskName: task.Name, + Status: task.Status, + Targets: task.Targets, + Collected: collected, + Percent: taskProgressPercent(task.Targets, collected), + }) + } + sort.Slice(summary.Tasks, func(left, right int) bool { return summary.Tasks[left].TaskName < summary.Tasks[right].TaskName }) + return summary +} diff --git a/internal/store/data_operations_memory.go b/internal/store/data_operations_memory.go new file mode 100644 index 00000000..63063826 --- /dev/null +++ b/internal/store/data_operations_memory.go @@ -0,0 +1,240 @@ +package store + +import ( + "context" + "fmt" + "sort" + "strings" + + "droboticflow/internal/domain" +) + +func (store *MemoryStore) ListCollectionTasks(_ context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.CollectionTaskRecord, 0) + for _, item := range store.collectionTasks { + if organizationMatches(item.OrganizationID, organizationID) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(left, right int) bool { return items[left].UpdatedAt > items[right].UpdatedAt }) + return items, nil +} + +func (store *MemoryStore) GetCollectionTask(_ context.Context, id string) (*domain.CollectionTaskRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.collectionTasks[id] + if !ok { + return nil, nil + } + cloned := clone(item) + return &cloned, nil +} + +func (store *MemoryStore) UpsertCollectionTask(_ context.Context, item domain.CollectionTaskRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if strings.TrimSpace(item.ID) == "" { + item.ID = domain.NewID("data-task") + } + now := nowISO() + if item.CreatedAt == "" { + item.CreatedAt = now + } + item.UpdatedAt = now + store.collectionTasks[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) UpsertDatasetVersion(_ context.Context, item domain.DatasetVersionRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if strings.TrimSpace(item.Fingerprint) == "" { + return fmt.Errorf("dataset fingerprint is required") + } + for id, existing := range store.datasetVersions { + if organizationMatches(existing.OrganizationID, item.OrganizationID) && existing.Fingerprint == item.Fingerprint { + if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) { + return nil + } + item.ID = id + item.CreatedAt = existing.CreatedAt + break + } + } + if item.ID == "" { + item.ID = domain.NewID("dataset-version") + } + now := nowISO() + if item.CreatedAt == "" { + item.CreatedAt = now + } + item.UpdatedAt = now + store.datasetVersions[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) ListDatasetVersions(_ context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.DatasetVersionRecord, 0) + for _, item := range store.datasetVersions { + if datasetVersionMatches(item, organizationID, query) { + items = append(items, clone(item)) + } + } + sort.Slice(items, func(left, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) + limit := query.Limit + if limit <= 0 || limit > 100 { + limit = 100 + } + if len(items) > limit { + items = items[:limit] + } + return items, nil +} + +func (store *MemoryStore) UpsertDataMetricFact(_ context.Context, item domain.DataMetricFactRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + if strings.TrimSpace(item.SourceKey) == "" { + return fmt.Errorf("metric source key is required") + } + for id, existing := range store.dataMetricFacts { + if existing.SourceKey == item.SourceKey { + if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) { + return nil + } + item.ID = id + break + } + } + if item.ID == "" { + item.ID = domain.NewID("data-fact") + } + if item.RecordedAt == "" { + item.RecordedAt = nowISO() + } + store.dataMetricFacts[item.ID] = clone(item) + return nil +} + +func (store *MemoryStore) filteredDataMetricFactsLocked(organizationID string, query domain.DataOperationsQuery) []domain.DataMetricFactRecord { + items := make([]domain.DataMetricFactRecord, 0) + for _, item := range store.dataMetricFacts { + if dataOperationsFactMatches(item, organizationID, query) { + items = append(items, clone(item)) + } + } + return items +} + +func (store *MemoryStore) ListDataMetricFactsPage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) { + store.mu.Lock() + defer store.mu.Unlock() + return factsPage(store.filteredDataMetricFactsLocked(organizationID, query), query), nil +} + +func (store *MemoryStore) GetDataOperationsSummary(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) { + store.mu.Lock() + defer store.mu.Unlock() + facts := store.filteredDataMetricFactsLocked(organizationID, query) + tasks := make([]domain.CollectionTaskRecord, 0) + for _, item := range store.collectionTasks { + if organizationMatches(item.OrganizationID, organizationID) { + tasks = append(tasks, clone(item)) + } + } + return BuildDataOperationsSummary(facts, tasks, query), nil +} + +func (store *MemoryStore) ListTrainingUsagePage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) { + store.mu.Lock() + defer store.mu.Unlock() + return trainingUsageFromFacts(store.filteredDataMetricFactsLocked(organizationID, query), query), nil +} + +func (store *MemoryStore) ListDataOperationsBackfillRuns( + _ context.Context, + organizationID string, + from string, + cursor *domain.RunSummaryCursor, + limit int, +) (domain.DataOperationsRunPage, error) { + store.mu.Lock() + defer store.mu.Unlock() + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + items := make([]domain.RunRecord, 0) + for _, run := range store.runs { + if !organizationMatches(run.OrganizationID, organizationID) || run.CreatedAt < from { + continue + } + if cursor != nil && !(run.CreatedAt < cursor.CreatedAt || (run.CreatedAt == cursor.CreatedAt && run.ID < cursor.ID)) { + continue + } + items = append(items, clone(run)) + } + sort.Slice(items, func(left, right int) bool { + if items[left].CreatedAt == items[right].CreatedAt { + return items[left].ID > items[right].ID + } + return items[left].CreatedAt > items[right].CreatedAt + }) + page := domain.DataOperationsRunPage{HasMore: len(items) > limit} + if page.HasMore { + items = items[:limit] + } + page.Items = items + if page.HasMore && len(items) > 0 { + last := items[len(items)-1] + page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID} + } + return page, nil +} + +func (store *MemoryStore) ListRunsNeedingMetricMaterialization(_ context.Context, limit int) ([]domain.RunRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.RunRecord, 0) + for _, run := range store.runs { + for _, execution := range run.NodeExecutions { + if execution.DataMetrics != nil && execution.MetricsMaterializedAt == "" { + items = append(items, clone(run)) + break + } + } + } + sortRuns(items) + if limit <= 0 { + limit = 20 + } + if len(items) > limit { + items = items[:limit] + } + return items, nil +} + +func (store *MemoryStore) MarkNodeMetricsMaterialized(_ context.Context, runID string, nodeID string, attempt int, at string) error { + store.mu.Lock() + defer store.mu.Unlock() + run, ok := store.runs[runID] + if !ok { + return fmt.Errorf("run %s not found", runID) + } + for index := range run.NodeExecutions { + execution := &run.NodeExecutions[index] + if execution.NodeID == nodeID && execution.Attempt == attempt { + execution.MetricsMaterializedAt = at + store.runs[runID] = run + return nil + } + } + return fmt.Errorf("node execution %s attempt %d not found", nodeID, attempt) +} diff --git a/internal/store/data_operations_mongo.go b/internal/store/data_operations_mongo.go new file mode 100644 index 00000000..450f52cb --- /dev/null +++ b/internal/store/data_operations_mongo.go @@ -0,0 +1,300 @@ +package store + +import ( + "context" + "fmt" + "strings" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func dataOperationsMongoFilter(organizationID string, query domain.DataOperationsQuery) bson.M { + extra := bson.M{} + if query.ProjectID != "" { + extra["projectId"] = query.ProjectID + } + if query.TaskID != "" { + extra["taskId"] = query.TaskID + } + if query.RobotModel != "" { + extra["robotModel"] = query.RobotModel + } + if query.Scene != "" { + extra["scene"] = query.Scene + } + if query.Stage != "" { + extra["stage"] = query.Stage + } + timeRange := bson.M{} + if query.From != "" { + timeRange["$gte"] = query.From + } + if query.To != "" { + timeRange["$lt"] = query.To + } + if len(timeRange) > 0 { + extra["occurredAt"] = timeRange + } + return organizationScopedFilter(organizationID, extra) +} + +func (store *MongoStore) ListCollectionTasks(ctx context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) { + cursor, err := store.collectionTasks.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}})) + if err != nil { + return nil, err + } + return decodeMany[domain.CollectionTaskRecord](ctx, cursor) +} + +func (store *MongoStore) GetCollectionTask(ctx context.Context, id string) (*domain.CollectionTaskRecord, error) { + var item domain.CollectionTaskRecord + err := store.collectionTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err == mongo.ErrNoDocuments { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) UpsertCollectionTask(ctx context.Context, item domain.CollectionTaskRecord) error { + if item.ID == "" { + item.ID = domain.NewID("data-task") + } + now := nowISO() + if item.CreatedAt == "" { + item.CreatedAt = now + } + item.UpdatedAt = now + _, err := store.collectionTasks.ReplaceOne(ctx, bson.M{"id": item.ID}, item, options.Replace().SetUpsert(true)) + return err +} + +func (store *MongoStore) UpsertDatasetVersion(ctx context.Context, item domain.DatasetVersionRecord) error { + if strings.TrimSpace(item.Fingerprint) == "" { + return fmt.Errorf("dataset fingerprint is required") + } + var existing domain.DatasetVersionRecord + existingErr := store.datasetVersions.FindOne( + ctx, + organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint}), + options.FindOne().SetProjection(bson.M{"quality": 1}), + ).Decode(&existing) + if existingErr != nil && existingErr != mongo.ErrNoDocuments { + return existingErr + } + if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) { + return nil + } + now := nowISO() + if item.ID == "" { + item.ID = domain.NewID("dataset-version") + } + if item.CreatedAt == "" { + item.CreatedAt = now + } + item.UpdatedAt = now + filter := organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint}) + update := bson.M{ + "$set": bson.M{ + "datasetId": item.DatasetID, "taskId": item.TaskID, "projectId": item.ProjectID, + "name": item.Name, "split": item.Split, "fingerprint": item.Fingerprint, "volume": item.Volume, + "storage": item.Storage, "manifestPath": item.ManifestPath, "partitionDir": item.PartitionDir, + "producerRunId": item.ProducerRunID, "producerNodeId": item.ProducerNodeID, "artifactId": item.ArtifactID, + "quality": item.Quality, "updatedAt": now, "organizationId": normalizedOrganizationID(item.OrganizationID), + }, + "$setOnInsert": bson.M{"id": item.ID, "createdAt": item.CreatedAt}, + } + _, err := store.datasetVersions.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true)) + return err +} + +func (store *MongoStore) ListDatasetVersions(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) { + extra := bson.M{} + if query.ProjectID != "" { + extra["projectId"] = query.ProjectID + } + if query.TaskID != "" { + extra["taskId"] = query.TaskID + } + timeRange := bson.M{} + if query.From != "" { + timeRange["$gte"] = query.From + } + if query.To != "" { + timeRange["$lt"] = query.To + } + if len(timeRange) > 0 { + extra["createdAt"] = timeRange + } + limit := int64(query.Limit) + if limit <= 0 || limit > 100 { + limit = 100 + } + cursor, err := store.datasetVersions.Find( + ctx, + organizationScopedFilter(organizationID, extra), + options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(limit), + ) + if err != nil { + return nil, err + } + return decodeMany[domain.DatasetVersionRecord](ctx, cursor) +} + +func (store *MongoStore) UpsertDataMetricFact(ctx context.Context, item domain.DataMetricFactRecord) error { + if strings.TrimSpace(item.SourceKey) == "" { + return fmt.Errorf("metric source key is required") + } + if item.ID == "" { + item.ID = domain.NewID("data-fact") + } + if item.RecordedAt == "" { + item.RecordedAt = nowISO() + } + var existing domain.DataMetricFactRecord + existingErr := store.dataMetricFacts.FindOne( + ctx, + bson.M{"sourceKey": item.SourceKey}, + options.FindOne().SetProjection(bson.M{"quality": 1}), + ).Decode(&existing) + if existingErr != nil && existingErr != mongo.ErrNoDocuments { + return existingErr + } + if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) { + return nil + } + _, err := store.dataMetricFacts.ReplaceOne(ctx, bson.M{"sourceKey": item.SourceKey}, item, options.Replace().SetUpsert(true)) + return err +} + +func (store *MongoStore) findDataMetricFacts(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DataMetricFactRecord, error) { + cursor, err := store.dataMetricFacts.Find( + ctx, + dataOperationsMongoFilter(organizationID, query), + options.Find().SetSort(bson.D{{Key: "occurredAt", Value: -1}, {Key: "id", Value: -1}}), + ) + if err != nil { + return nil, err + } + return decodeMany[domain.DataMetricFactRecord](ctx, cursor) +} + +func (store *MongoStore) ListDataMetricFactsPage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) { + items, err := store.findDataMetricFacts(ctx, organizationID, query) + if err != nil { + return domain.DataMetricFactPage{}, err + } + return factsPage(items, query), nil +} + +func (store *MongoStore) GetDataOperationsSummary(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) { + facts, err := store.findDataMetricFacts(ctx, organizationID, query) + if err != nil { + return domain.DataOperationsSummary{}, err + } + tasks, err := store.ListCollectionTasks(ctx, organizationID) + if err != nil { + return domain.DataOperationsSummary{}, err + } + return BuildDataOperationsSummary(facts, tasks, query), nil +} + +func (store *MongoStore) ListTrainingUsagePage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) { + query.Stage = domain.DataMetricStageTrain + items, err := store.findDataMetricFacts(ctx, organizationID, query) + if err != nil { + return domain.TrainingUsagePage{}, err + } + return trainingUsageFromFacts(items, query), nil +} + +func (store *MongoStore) ListDataOperationsBackfillRuns( + ctx context.Context, + organizationID string, + from string, + pageCursor *domain.RunSummaryCursor, + limit int, +) (domain.DataOperationsRunPage, error) { + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + filter := organizationScopedFilter(organizationID, bson.M{"createdAt": bson.M{"$gte": from}}) + filter = combineRunSummaryFilters(filter, runSummaryCursorFilter(pageCursor)) + cursor, err := store.runs.Find( + ctx, + filter, + options.Find(). + SetProjection(bson.M{ + "id": 1, "organizationId": 1, "workflowId": 1, "workflowName": 1, + "projectId": 1, "projectName": 1, "taskId": 1, "status": 1, + "createdAt": 1, "updatedAt": 1, "endedAt": 1, "nodeExecutions": 1, + }). + SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}). + SetLimit(int64(limit+1)), + ) + if err != nil { + return domain.DataOperationsRunPage{}, err + } + items, err := decodeMany[domain.RunRecord](ctx, cursor) + if err != nil { + return domain.DataOperationsRunPage{}, err + } + page := domain.DataOperationsRunPage{HasMore: len(items) > limit} + if page.HasMore { + items = items[:limit] + } + page.Items = items + if page.HasMore && len(items) > 0 { + last := items[len(items)-1] + page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID} + } + return page, nil +} + +func (store *MongoStore) ListRunsNeedingMetricMaterialization(ctx context.Context, limit int) ([]domain.RunRecord, error) { + if limit <= 0 { + limit = 20 + } + cursor, err := store.runs.Find( + ctx, + bson.M{"nodeExecutions": bson.M{"$elemMatch": bson.M{ + "dataMetrics": bson.M{"$exists": true}, + "$or": bson.A{ + bson.M{"metricsMaterializedAt": bson.M{"$exists": false}}, + bson.M{"metricsMaterializedAt": ""}, + }, + }}}, + options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)), + ) + if err != nil { + return nil, err + } + return decodeMany[domain.RunRecord](ctx, cursor) +} + +func (store *MongoStore) MarkNodeMetricsMaterialized(ctx context.Context, runID string, nodeID string, attempt int, at string) error { + result, err := store.runs.UpdateOne( + ctx, + bson.M{"id": runID}, + bson.M{"$set": bson.M{"nodeExecutions.$[execution].metricsMaterializedAt": at}}, + options.Update().SetArrayFilters(options.ArrayFilters{Filters: []any{ + bson.M{"execution.nodeId": nodeID, "execution.attempt": attempt}, + }}), + ) + if err != nil { + return err + } + if result.MatchedCount == 0 { + return fmt.Errorf("run %s not found", runID) + } + return nil +} diff --git a/internal/store/data_operations_test.go b/internal/store/data_operations_test.go new file mode 100644 index 00000000..c3339fb5 --- /dev/null +++ b/internal/store/data_operations_test.go @@ -0,0 +1,192 @@ +package store + +import ( + "context" + "testing" + + "droboticflow/internal/domain" +) + +func dataOperationsInt64(value int64) *int64 { + return &value +} + +func dataOperationsFloat64(value float64) *float64 { + return &value +} + +func TestMemoryDataOperationsSummarySeparatesConsumptionAndUniqueData(t *testing.T) { + ctx := context.Background() + dataStore := NewMemoryStore() + task := domain.CollectionTaskRecord{ + ID: "task-y1", + OrganizationID: domain.DefaultOrganizationID, + ProjectID: "project-y1", + Name: "Y1 桌面整理", + RobotModel: "Y1", + Scene: "桌面整理", + Status: domain.CollectionTaskStatusActive, + Targets: domain.DataVolume{TrajectoryCount: dataOperationsInt64(100)}, + } + if err := dataStore.UpsertCollectionTask(ctx, task); err != nil { + t.Fatalf("upsert task: %v", err) + } + for _, item := range []domain.DataMetricFactRecord{ + { + ID: "fact-collect", + SourceKey: "run-collect/node/1/collect", + OrganizationID: domain.DefaultOrganizationID, + ProjectID: "project-y1", + TaskID: task.ID, + TaskName: task.Name, + RobotModel: "Y1", + Scene: "桌面整理", + RunID: "run-collect", + NodeID: "collect", + Attempt: 1, + Stage: domain.DataMetricStageCollect, + Output: domain.DataVolume{ + TrajectoryCount: dataOperationsInt64(40), + DurationSeconds: dataOperationsFloat64(1200), + }, + Quality: domain.DataMetricQualityExact, + OccurredAt: "2026-07-23T01:00:00Z", + }, + { + ID: "fact-train-1", + SourceKey: "run-train-1/node/1/train", + OrganizationID: domain.DefaultOrganizationID, + ProjectID: "project-y1", + TaskID: task.ID, + RunID: "run-train-1", + NodeID: "train", + Attempt: 1, + Stage: domain.DataMetricStageTrain, + Input: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)}, + TrainingInputs: []domain.TrainingInputRecord{{ + DatasetVersionID: "dataset-version-y1", + Split: "train", + Volume: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)}, + }}, + Quality: domain.DataMetricQualityExact, + OccurredAt: "2026-07-23T02:00:00Z", + }, + { + ID: "fact-train-2", + SourceKey: "run-train-2/node/1/train", + OrganizationID: domain.DefaultOrganizationID, + ProjectID: "project-y1", + TaskID: task.ID, + RunID: "run-train-2", + NodeID: "train", + Attempt: 1, + Stage: domain.DataMetricStageTrain, + Input: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)}, + TrainingInputs: []domain.TrainingInputRecord{{ + DatasetVersionID: "dataset-version-y1", + Split: "train", + Volume: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)}, + }}, + Quality: domain.DataMetricQualityDerived, + OccurredAt: "2026-07-23T03:00:00Z", + }, + } { + if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil { + t.Fatalf("upsert fact %s: %v", item.ID, err) + } + } + + summary, err := dataStore.GetDataOperationsSummary(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{ + From: "2026-07-22T16:00:00Z", + To: "2026-07-23T16:00:00Z", + Timezone: "Asia/Shanghai", + }) + if err != nil { + t.Fatalf("get summary: %v", err) + } + if summary.Training.Consumed.TrajectoryCount == nil || *summary.Training.Consumed.TrajectoryCount != 80 { + t.Fatalf("consumed trajectories = %#v, want 80", summary.Training.Consumed.TrajectoryCount) + } + if summary.Training.Unique.TrajectoryCount == nil || *summary.Training.Unique.TrajectoryCount != 40 { + t.Fatalf("unique trajectories = %#v, want 40", summary.Training.Unique.TrajectoryCount) + } + if summary.Training.RunCount != 2 { + t.Fatalf("training run count = %d, want 2", summary.Training.RunCount) + } + if len(summary.Tasks) != 1 || summary.Tasks[0].Percent != 40 { + t.Fatalf("task progress = %#v, want 40 percent", summary.Tasks) + } + if summary.ExactCount != 2 || summary.DerivedCount != 1 { + t.Fatalf("quality counts = exact %d derived %d", summary.ExactCount, summary.DerivedCount) + } +} + +func TestMemoryDataMetricUpsertIsIdempotent(t *testing.T) { + ctx := context.Background() + dataStore := NewMemoryStore() + item := domain.DataMetricFactRecord{ + ID: "fact-one", + SourceKey: "run/node/1/collect", + OrganizationID: domain.DefaultOrganizationID, + RunID: "run", + NodeID: "node", + Attempt: 1, + Stage: domain.DataMetricStageCollect, + Output: domain.DataVolume{TrajectoryCount: dataOperationsInt64(10)}, + Quality: domain.DataMetricQualityDerived, + OccurredAt: "2026-07-23T01:00:00Z", + } + if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil { + t.Fatalf("first upsert: %v", err) + } + item.ID = "fact-two" + item.Output = domain.DataVolume{TrajectoryCount: dataOperationsInt64(12)} + if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil { + t.Fatalf("second upsert: %v", err) + } + page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{ + From: "2026-07-23T00:00:00Z", + To: "2026-07-24T00:00:00Z", + }) + if err != nil { + t.Fatalf("list facts: %v", err) + } + if len(page.Items) != 1 || page.Items[0].Output.TrajectoryCount == nil || *page.Items[0].Output.TrajectoryCount != 12 { + t.Fatalf("idempotent items = %#v", page.Items) + } +} + +func TestMemoryExactDataMetricIsNotDowngradedByBackfill(t *testing.T) { + ctx := context.Background() + dataStore := NewMemoryStore() + exact := domain.DataMetricFactRecord{ + ID: "fact-exact", + SourceKey: "run/node/1/collect", + OrganizationID: domain.DefaultOrganizationID, + RunID: "run", + NodeID: "node", + Attempt: 1, + Stage: domain.DataMetricStageCollect, + Output: domain.DataVolume{TrajectoryCount: dataOperationsInt64(10)}, + Quality: domain.DataMetricQualityExact, + OccurredAt: "2026-07-23T01:00:00Z", + } + if err := dataStore.UpsertDataMetricFact(ctx, exact); err != nil { + t.Fatalf("upsert exact fact: %v", err) + } + estimated := exact + estimated.ID = "fact-estimated" + estimated.Output = domain.DataVolume{TrajectoryCount: dataOperationsInt64(3)} + estimated.Quality = domain.DataMetricQualityEstimated + if err := dataStore.UpsertDataMetricFact(ctx, estimated); err != nil { + t.Fatalf("upsert estimated fact: %v", err) + } + page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{}) + if err != nil { + t.Fatalf("list facts: %v", err) + } + if len(page.Items) != 1 || page.Items[0].Quality != domain.DataMetricQualityExact || + page.Items[0].Output.TrajectoryCount == nil || *page.Items[0].Output.TrajectoryCount != 10 { + t.Fatalf("facts = %#v", page.Items) + } +} diff --git a/internal/store/memory.go b/internal/store/memory.go index 3f28745a..64ae6e6b 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -21,6 +21,9 @@ type MemoryStore struct { templates map[string]domain.TemplateRecord runs map[string]domain.RunRecord sourceStates map[string]domain.SourceStateRecord + collectionTasks map[string]domain.CollectionTaskRecord + datasetVersions map[string]domain.DatasetVersionRecord + dataMetricFacts map[string]domain.DataMetricFactRecord auditTasks map[string]domain.AuditTaskRecord samplingTasks map[string]domain.SamplingTaskRecord users map[string]domain.UserRecord @@ -42,6 +45,9 @@ func NewMemoryStore() *MemoryStore { templates: map[string]domain.TemplateRecord{}, runs: map[string]domain.RunRecord{}, sourceStates: map[string]domain.SourceStateRecord{}, + collectionTasks: map[string]domain.CollectionTaskRecord{}, + datasetVersions: map[string]domain.DatasetVersionRecord{}, + dataMetricFacts: map[string]domain.DataMetricFactRecord{}, auditTasks: map[string]domain.AuditTaskRecord{}, samplingTasks: map[string]domain.SamplingTaskRecord{}, users: map[string]domain.UserRecord{}, @@ -276,6 +282,21 @@ func (store *MemoryStore) DeleteOrganizationCascade(_ context.Context, id string delete(store.sourceStates, key) } } + for key, item := range store.collectionTasks { + if organizationMatches(item.OrganizationID, organizationID) { + delete(store.collectionTasks, key) + } + } + for key, item := range store.datasetVersions { + if organizationMatches(item.OrganizationID, organizationID) { + delete(store.datasetVersions, key) + } + } + for key, item := range store.dataMetricFacts { + if organizationMatches(item.OrganizationID, organizationID) { + delete(store.dataMetricFacts, key) + } + } for key, item := range store.auditTasks { if organizationMatches(item.OrganizationID, organizationID) { delete(store.auditTasks, key) @@ -486,6 +507,7 @@ func (store *MemoryStore) CreateOperator(_ context.Context, input domain.CreateO SkillDefinition: input.SkillDefinition, Hyperparams: clone(input.Hyperparams), Resources: clone(input.Resources), + BusinessStage: input.BusinessStage, PublicationStatus: input.PublicationStatus, GenerationSource: input.GenerationSource, PublicationEvidence: clone(input.PublicationEvidence), @@ -842,6 +864,7 @@ func summarizeRun(run domain.RunRecord) domain.RunSummary { WorkflowName: run.WorkflowName, ProjectID: run.ProjectID, ProjectName: run.ProjectName, + TaskID: run.TaskID, Trigger: run.Trigger, Attempt: run.Attempt, ParentRunID: run.ParentRunID, @@ -1140,7 +1163,7 @@ func (store *MemoryStore) GetRun(_ context.Context, id string) (*domain.RunRecor return &cloned, nil } -func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) { +func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) { store.mu.Lock() defer store.mu.Unlock() workflow, ok := store.workflows[workflowID] @@ -1187,6 +1210,9 @@ func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigge CreatedAt: now, UpdatedAt: now, } + if len(taskIDs) > 0 { + run.TaskID = strings.TrimSpace(taskIDs[0]) + } store.runs[run.ID] = clone(run) workflow.LastRunID = run.ID diff --git a/internal/store/mongo.go b/internal/store/mongo.go index 67c8ccd6..80521835 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -28,6 +28,9 @@ type MongoStore struct { templates *mongo.Collection runs *mongo.Collection sourceStates *mongo.Collection + collectionTasks *mongo.Collection + datasetVersions *mongo.Collection + dataMetricFacts *mongo.Collection auditTasks *mongo.Collection samplingTasks *mongo.Collection users *mongo.Collection @@ -49,6 +52,9 @@ func NewMongoStore(db *mongo.Database) *MongoStore { templates: db.Collection("templates"), runs: db.Collection("runs"), sourceStates: db.Collection("sourceStates"), + collectionTasks: db.Collection("collectionTasks"), + datasetVersions: db.Collection("datasetVersions"), + dataMetricFacts: db.Collection("dataMetricFacts"), auditTasks: db.Collection("auditTasks"), samplingTasks: db.Collection("samplingTasks"), users: db.Collection("users"), @@ -350,6 +356,7 @@ func (store *MongoStore) CreateOperator(ctx context.Context, input domain.Create SkillDefinition: input.SkillDefinition, Hyperparams: input.Hyperparams, Resources: input.Resources, + BusinessStage: input.BusinessStage, PublicationStatus: input.PublicationStatus, GenerationSource: input.GenerationSource, PublicationEvidence: input.PublicationEvidence, @@ -725,6 +732,7 @@ func runSummaryProjection() bson.M { "workflowName": 1, "projectId": 1, "projectName": 1, + "taskId": 1, "trigger": 1, "attempt": 1, "parentRunId": 1, @@ -1104,7 +1112,7 @@ func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunReco return &item, nil } -func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) { +func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) { workflow, err := store.GetWorkflow(ctx, workflowID) if err != nil { return nil, err @@ -1152,6 +1160,9 @@ func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigg CreatedAt: now, UpdatedAt: now, } + if len(taskIDs) > 0 { + run.TaskID = strings.TrimSpace(taskIDs[0]) + } if err := store.UpsertRun(ctx, run); err != nil { return nil, err } diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 6e4dbcea..af3aa853 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -88,6 +88,23 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { index("organizationId_workflow_node", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}}), index("workflowId_nodeId", bson.D{{Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}}), }, + store.collectionTasks: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + index("organizationId_status_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}), + index("organizationId_projectId_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "updatedAt", Value: -1}}), + }, + store.datasetVersions: { + uniqueIndex("organizationId_fingerprint_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "fingerprint", Value: 1}}), + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + index("organizationId_projectId_taskId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "createdAt", Value: -1}}), + }, + store.dataMetricFacts: { + uniqueIndex("sourceKey_unique", bson.D{{Key: "sourceKey", Value: 1}}), + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + index("organizationId_projectId_taskId_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "occurredAt", Value: -1}}), + index("organizationId_stage_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "stage", Value: 1}, {Key: "occurredAt", Value: -1}}), + index("organizationId_robotModel_scene_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "robotModel", Value: 1}, {Key: "scene", Value: 1}, {Key: "occurredAt", Value: -1}}), + }, store.auditTasks: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), diff --git a/internal/store/mongo_migrations.go b/internal/store/mongo_migrations.go index 7eea19bb..de637f5b 100644 --- a/internal/store/mongo_migrations.go +++ b/internal/store/mongo_migrations.go @@ -160,6 +160,9 @@ func (store *MongoStore) normalizeLegacyDefaultOrganizationReferences(ctx contex store.templates, store.runs, store.sourceStates, + store.collectionTasks, + store.datasetVersions, + store.dataMetricFacts, store.auditTasks, store.samplingTasks, store.users, @@ -228,7 +231,9 @@ func (store *MongoStore) dropLegacyBuiltinOperatorDeletionIDIndex(ctx context.Co return nil } var commandError mongo.CommandError - if errors.As(err, &commandError) && (commandError.Name == "IndexNotFound" || commandError.HasErrorCode(27)) { + if errors.As(err, &commandError) && + (commandError.Name == "IndexNotFound" || commandError.Name == "NamespaceNotFound" || + commandError.HasErrorCode(27) || commandError.HasErrorCode(26)) { return nil } return err diff --git a/internal/store/store.go b/internal/store/store.go index 92b95acd..cfc0e08d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -80,7 +80,7 @@ type Store interface { CountWorkerRuns(context.Context) (WorkerRunCounts, error) HasActiveRun(context.Context, string) (bool, error) GetRun(context.Context, string) (*domain.RunRecord, error) - CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error) + CreateRun(context.Context, string, domain.RunTrigger, ...string) (*domain.RunRecord, error) ClaimNextQueuedRun(context.Context, string, time.Duration) (*domain.RunRecord, error) ClaimQueuedRun(context.Context, string, string, time.Duration) (*domain.RunRecord, error) AcquireRunLease(context.Context, string, string, time.Duration) (*domain.RunRecord, bool, error) @@ -101,6 +101,19 @@ type Store interface { DeleteSourceState(context.Context, string, string, string) error DeleteSourceStatesByWorkflow(context.Context, string) error + ListCollectionTasks(context.Context, string) ([]domain.CollectionTaskRecord, error) + GetCollectionTask(context.Context, string) (*domain.CollectionTaskRecord, error) + UpsertCollectionTask(context.Context, domain.CollectionTaskRecord) error + UpsertDatasetVersion(context.Context, domain.DatasetVersionRecord) error + ListDatasetVersions(context.Context, string, domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) + UpsertDataMetricFact(context.Context, domain.DataMetricFactRecord) error + ListDataMetricFactsPage(context.Context, string, domain.DataOperationsQuery) (domain.DataMetricFactPage, error) + GetDataOperationsSummary(context.Context, string, domain.DataOperationsQuery) (domain.DataOperationsSummary, error) + ListTrainingUsagePage(context.Context, string, domain.DataOperationsQuery) (domain.TrainingUsagePage, error) + ListDataOperationsBackfillRuns(context.Context, string, string, *domain.RunSummaryCursor, int) (domain.DataOperationsRunPage, error) + ListRunsNeedingMetricMaterialization(context.Context, int) ([]domain.RunRecord, error) + MarkNodeMetricsMaterialized(context.Context, string, string, int, string) error + ListAuditTasks(context.Context, string) ([]domain.AuditTaskRecord, error) ListAuditTasksByOrganization(context.Context, string, string) ([]domain.AuditTaskRecord, error) GetAuditTask(context.Context, string) (*domain.AuditTaskRecord, error) diff --git a/internal/worker/executor.go b/internal/worker/executor.go index 41be7f76..3f0d031e 100644 --- a/internal/worker/executor.go +++ b/internal/worker/executor.go @@ -33,12 +33,14 @@ type executionResult struct { Logs []domain.LogEntryRecord `json:"logs"` Artifacts []domain.ExecutionArtifact `json:"artifacts"` DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"` + DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"` StageManifestPath string `json:"stageManifestPath,omitempty"` ResourceSeries []domain.ResourceSample `json:"resourceSeries"` } type stageManifestRecord struct { - DataRefs []domain.DataRefRecord `json:"dataRefs"` + DataRefs []domain.DataRefRecord `json:"dataRefs"` + DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"` } var protectedSystemEnv = map[string]struct{}{ @@ -593,7 +595,49 @@ func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) { return manifest.DataRefs, nil } +func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return nil, nil + } + payload, err := os.ReadFile(trimmed) + if err != nil { + return nil, err + } + var manifest stageManifestRecord + if err := json.Unmarshal(payload, &manifest); err != nil { + return nil, err + } + return manifest.DataMetrics, nil +} + +func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode { + if run != nil && strings.TrimSpace(run.TaskID) != "" { + if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil { + for stage, nodeID := range task.StageBindings { + if strings.TrimSpace(nodeID) == node.ID { + node.Data.BusinessStage = stage + node.Data.MetricBoundary = true + return node + } + } + } + } + if node.Data.BusinessStage != "" { + node.Data.MetricBoundary = true + return node + } + if strings.TrimSpace(node.Data.OperatorID) != "" { + if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" { + node.Data.BusinessStage = operator.BusinessStage + node.Data.MetricBoundary = true + } + } + return node +} + func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) { + node = runtime.resolveNodeBusinessStage(ctx, run, node) workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID) if err != nil { return executionResult{}, nil, err @@ -675,6 +719,9 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, if err == nil { parsed.StageManifestPath = workspace.StageManifestPath parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath) + if err == nil && parsed.DataMetrics == nil { + parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath) + } } // Custom-container operators bypass operator-runner's ResourceSampler, so they // emit no resourceSeries in result.json. The launcher samples their GPU usage diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 111256a5..0061ad3b 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -15,6 +15,7 @@ import ( "sync/atomic" "time" + "droboticflow/internal/dataops" "droboticflow/internal/domain" "droboticflow/internal/k8s" "droboticflow/internal/lightweight" @@ -564,6 +565,12 @@ func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord) } else { run.NodeExecutions = append(run.NodeExecutions, execution) } + if execution.DataMetrics != nil && execution.DataMetrics.Stage == domain.DataMetricStageTrain && len(execution.DataMetrics.TrainingInputs) > 0 { + if run.Metadata == nil { + run.Metadata = &domain.RunMetadata{} + } + run.Metadata.TrainingInputs = append([]domain.TrainingInputRecord(nil), execution.DataMetrics.TrainingInputs...) + } run.Logs = []domain.LogEntryRecord{} run.ResourceSeries = []domain.ResourceSample{} run.Artifacts = []domain.ExecutionArtifact{} @@ -901,6 +908,7 @@ func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord { WorkflowName: summary.WorkflowName, ProjectID: summary.ProjectID, ProjectName: summary.ProjectName, + TaskID: summary.TaskID, Trigger: summary.Trigger, Attempt: summary.Attempt, ParentRunID: summary.ParentRunID, @@ -923,6 +931,18 @@ func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord { return runs } +func (runtime *Runtime) materializeNodeMetricsBestEffort(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) { + if run == nil || execution == nil || execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" { + return + } + if err := dataops.MaterializeNodeMetrics(ctx, runtime.store, run, execution); err != nil { + fmt.Printf("[worker] materialize data metrics for run %s node %s failed: %v\n", run.ID, execution.NodeID, err) + return + } + execution.MetricsMaterializedAt = nowISO() + applyExecution(run, *execution) +} + func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) { if claimed == nil { return @@ -1090,6 +1110,9 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { return processErr } runtime.updateEffectivePollInterval(settings) + if _, err := dataops.ReconcileRunMetrics(ctx, runtime.store, 20); err != nil { + fmt.Printf("[worker] reconcile data metrics failed: %v\n", err) + } reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit) if err != nil { processErr = err @@ -1391,10 +1414,16 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run execution.Logs = append(execution.Logs, parsed.Logs...) execution.Artifacts = parsed.Artifacts execution.DataRefs = parsed.DataRefs + execution.DataMetrics = parsed.DataMetrics execution.ResourceSeries = parsed.ResourceSeries execution.Error = "" executionMap[node.ID] = execution applyExecution(run, execution) + if err := runtime.store.UpsertRun(ctx, *run); err != nil { + return err + } + runtime.materializeNodeMetricsBestEffort(ctx, run, &execution) + executionMap[node.ID] = execution if run.Status == domain.RunStatusStopping { _, err := runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流") return err @@ -2124,7 +2153,7 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo if err != nil { return nil, err } - retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry) + retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.TaskID) if err != nil { return nil, err } @@ -2375,6 +2404,27 @@ func enqueueReadyNodes(workflow domain.WorkflowRecord, executionMap map[string]d } } +func upstreamBusinessVolume(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) domain.DataVolume { + for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) { + execution, ok := executionMap[edge.Source] + if !ok || execution.DataMetrics == nil { + continue + } + if execution.DataMetrics.TrainReady.TrajectoryCount != nil || + execution.DataMetrics.TrainReady.DurationSeconds != nil || + execution.DataMetrics.TrainReady.Bytes != nil { + return execution.DataMetrics.TrainReady + } + if execution.DataMetrics.TechnicalValid.TrajectoryCount != nil || + execution.DataMetrics.TechnicalValid.DurationSeconds != nil || + execution.DataMetrics.TechnicalValid.Bytes != nil { + return execution.DataMetrics.TechnicalValid + } + return execution.DataMetrics.Output + } + return domain.DataVolume{} +} + func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) { if _, ok, err := runtime.store.AcquireRunLease(ctx, claimed.ID, runtime.leaseOwner(), runtime.leaseTTL()); err != nil { return nil, err @@ -2469,6 +2519,25 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor execution.Status = domain.NodeExecutionStatusCompleted execution.EndedAt = nowISO() execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label)) + resolvedNode := runtime.resolveNodeBusinessStage(ctx, currentRun, *node) + if resolvedNode.Data.BusinessStage == domain.DataMetricStageAudit { + volume := upstreamBusinessVolume(*workflow, executionMap, node.ID) + execution.DataMetrics = &domain.DataMetricsRecord{ + SchemaVersion: 1, + Stage: domain.DataMetricStageAudit, + Input: volume, + Output: volume, + Quality: domain.DataMetricQualityExact, + OccurredAt: execution.EndedAt, + } + if task.Status == domain.AuditStatusApproved { + execution.DataMetrics.TrainReady = volume + } else { + execution.DataMetrics.Output = domain.DataVolume{} + execution.DataMetrics.Rejected = volume + execution.DataMetrics.RejectionReasons = map[string]int64{"人工审核拒绝": 1} + } + } executionMap[node.ID] = execution applyExecution(currentRun, execution) for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) { @@ -2480,6 +2549,8 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } + runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution) + executionMap[node.ID] = execution } break } @@ -2868,6 +2939,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor execution.Logs = append(execution.Logs, outcome.result.Logs...) execution.Artifacts = outcome.result.Artifacts execution.DataRefs = outcome.result.DataRefs + execution.DataMetrics = outcome.result.DataMetrics execution.StageManifestPath = outcome.result.StageManifestPath execution.ResourceSeries = outcome.result.ResourceSeries executionMap[outcome.nodeID] = execution @@ -2890,6 +2962,8 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err } + runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution) + executionMap[outcome.nodeID] = execution if forceStopRequested && len(runningNodes) == 0 { return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流") } diff --git a/src/App.tsx b/src/App.tsx index 1dcd36dd..3bc30a9c 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -86,6 +86,7 @@ const Templates = lazyRoute(() => import("./pages/Templates")); const Users = lazyRoute(() => import("./pages/Users")); const Audit = lazyRoute(() => import("./pages/Audit")); const DataAssets = lazyRoute(() => import("./pages/DataAssets")); +const DataOperations = lazyRoute(() => import("./pages/DataOperations")); const DatabaseBrowser = lazyRoute(() => import("./pages/DatabaseBrowser")); const ObjectStorageBrowser = lazyRoute(() => import("./pages/ObjectStorageBrowser")); const LegacyDataAssetsRedirect = lazyRoute(() => import("./pages/LegacyDataAssetsRedirect")); @@ -110,6 +111,7 @@ export default function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index 32026640..9984529c 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -21,6 +21,7 @@ import { Database, Cloudy, DatabaseZap, + BarChart3, LogOut, Loader2, } from "lucide-react"; @@ -41,6 +42,12 @@ const navItems = [ path: "/projects", roles: ["admin", "pipeline_engineer", "data_operator", "operator"], }, + { + icon: BarChart3, + label: "数据运营", + path: "/data-operations", + roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "compliance_auditor", "auditor", "operator"], + }, { icon: FileCode, label: "工作流模板", diff --git a/src/lib/operators.js b/src/lib/operators.js index 7e6139c7..ca979f0e 100644 --- a/src/lib/operators.js +++ b/src/lib/operators.js @@ -1,6 +1,7 @@ const operatorTypes = new Set(["builtin", "custom", "lightweight"]); const portTypes = new Set(["json", "image", "video", "tensor", "file"]); const portTransmissions = new Set(["", "bus", "oss", "shm", "stream"]); +const businessStages = new Set(["", "collect", "process", "clean", "validate", "annotate", "audit", "train"]); const positiveInteger = /^[1-9]\d*$/; const positiveResourceQuantity = /^(?:[1-9]\d*(?:\.\d+)?|0\.\d+)(?:(?:[eE][+-]?\d+)|(?:[numkKMGTP](?:i)?))?$/; @@ -73,6 +74,7 @@ export function buildOperatorPayload(source = {}, overrides = {}) { publicationEvidence: deepClone(overrides.publicationEvidence ?? source.publicationEvidence), hyperparams: deepClone(overrides.hyperparams ?? source.hyperparams ?? []), resources: normalizeResources(deepClone(overrides.resources ?? source.resources)), + businessStage: trimString(overrides.businessStage ?? source.businessStage).toLowerCase(), }; } @@ -133,6 +135,9 @@ export function getOperatorPayloadIssue(source = {}, overrides = {}) { if (payload.version && !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(payload.version)) { return "算子版本只能包含字母、数字、下划线、点和短横线,长度不超过 128"; } + if (!businessStages.has(payload.businessStage)) { + return "业务阶段必须是采集、处理、清洗、校验、标注、审核或训练"; + } return ( validatePorts("inputs", payload.inputs) || validatePorts("outputs", payload.outputs) || diff --git a/src/pages/DataOperations.tsx b/src/pages/DataOperations.tsx new file mode 100644 index 00000000..3173fa0c --- /dev/null +++ b/src/pages/DataOperations.tsx @@ -0,0 +1,1337 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { + Activity, + AlertCircle, + ArrowRight, + BarChart3, + CheckCircle2, + Clock3, + Database, + DatabaseZap, + Edit3, + FileCheck2, + Filter, + Loader2, + Play, + Plus, + RefreshCw, + Sparkles, + Target, + X, +} from "lucide-react"; +import { Link, useSearchParams } from "react-router-dom"; +import { useTimeZone } from "../context/TimeZoneContext"; +import { useUser } from "../context/UserContext"; +import { getActionErrorMessage } from "../lib/action-errors.js"; +import { formatCompactDateTime } from "../lib/datetime"; +import { cn } from "../lib/utils"; +import { dataOperationsApi, type DataOperationsFilters } from "../services/api/data-operations"; +import { projectsApi } from "../services/api/projects"; +import type { + CollectionTask, + CollectionTaskInput, + DataMetricFact, + DataMetricStage, + DataOperationsSummary, + DataVolume, + DatasetVersion, + Project, + TrainingUsage, + Workflow, +} from "../types"; + +type PageTab = "overview" | "tasks" | "datasets" | "training"; + +const tabs: Array<{ id: PageTab; label: string }> = [ + { id: "overview", label: "数据总览" }, + { id: "tasks", label: "完整任务" }, + { id: "datasets", label: "数据版本" }, + { id: "training", label: "训练使用" }, +]; + +const stageOrder: DataMetricStage[] = ["collect", "process", "clean", "validate", "annotate", "audit", "train"]; +const stageLabels: Record = { + collect: "采集", + process: "处理", + clean: "清洗", + validate: "技术有效", + annotate: "标注", + audit: "训练可用", + train: "训练消费", +}; + +const statusLabels: Record = { + draft: "草稿", + active: "进行中", + paused: "已暂停", + completed: "已完成", + cancelled: "已取消", +}; + +const defaultTaskInput = (): CollectionTaskInput => ({ + projectId: "", + name: "Y1 桌面整理数据闭环", + description: "Y1 机器人桌面整理场景的采集、处理、清洗、审核与训练闭环", + robotModel: "Y1", + robotVersion: "", + scene: "桌面整理", + modalities: ["RGB", "关节状态"], + format: "LeRobot / XPlatform", + mode: "full_training", + status: "draft", + workflowId: "", + stageBindings: {}, + targets: { trajectoryCount: 100, durationSeconds: 3600 }, + requirements: { + requireTechnicalValidation: true, + requireQuality: true, + requireAnnotation: true, + requireAudit: true, + }, + splitRules: { train: 80, val: 10, test: 10 }, +}); + +const taskToInput = (task: CollectionTask): CollectionTaskInput => ({ + projectId: task.projectId, + name: task.name, + description: task.description || "", + robotModel: task.robotModel, + robotVersion: task.robotVersion || "", + scene: task.scene, + modalities: task.modalities, + format: task.format || "", + mode: task.mode, + status: task.status, + workflowId: task.workflowId || "", + stageBindings: task.stageBindings || {}, + targets: task.targets, + requirements: task.requirements, + splitRules: task.splitRules || { train: 80, val: 10, test: 10 }, +}); + +const formatNumber = (value?: number) => (value === undefined ? "未知" : new Intl.NumberFormat("zh-CN").format(value)); + +const formatDuration = (value?: number) => { + if (value === undefined) return "未知"; + if (value < 60) return `${Math.round(value)} 秒`; + return `${(value / 3600).toFixed(value >= 36000 ? 0 : 1)} 小时`; +}; + +const formatBytes = (value?: number) => { + if (value === undefined) return "未知"; + const units = ["B", "KB", "MB", "GB", "TB"]; + let current = value; + let index = 0; + while (current >= 1024 && index < units.length - 1) { + current /= 1024; + index += 1; + } + return `${current.toFixed(index === 0 || current >= 10 ? 0 : 1)} ${units[index]}`; +}; + +const volumePrimary = (volume?: DataVolume) => { + if (!volume) return "未知"; + if (volume.trajectoryCount !== undefined) return `${formatNumber(volume.trajectoryCount)} 条轨迹`; + if (volume.durationSeconds !== undefined) return formatDuration(volume.durationSeconds); + if (volume.bytes !== undefined) return formatBytes(volume.bytes); + return "未知"; +}; + +const volumeSecondary = (volume?: DataVolume) => { + if (!volume) return "暂无精确辅助量"; + return ( + [ + volume.durationSeconds !== undefined ? formatDuration(volume.durationSeconds) : "", + volume.bytes !== undefined ? formatBytes(volume.bytes) : "", + ] + .filter(Boolean) + .join(" · ") || "暂无精确辅助量" + ); +}; + +function StatusBadge({ status }: { status: CollectionTask["status"] }) { + return ( + + {statusLabels[status]} + + ); +} + +function KpiCard({ + label, + volume, + icon: Icon, + tone, + onClick, +}: { + label: string; + volume?: DataVolume; + icon: React.ComponentType<{ className?: string }>; + tone: string; + onClick?: () => void; +}) { + return ( + + ); +} + +function TaskModal({ + task, + projects, + onClose, + onSaved, +}: { + task: CollectionTask | null; + projects: Project[]; + onClose: () => void; + onSaved: (task: CollectionTask) => void; +}) { + const [draft, setDraft] = useState(() => (task ? taskToInput(task) : defaultTaskInput())); + const [workflows, setWorkflows] = useState([]); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(""); + + useEffect(() => { + if (!draft.projectId) { + setWorkflows([]); + return; + } + let active = true; + projectsApi + .listWorkflows(draft.projectId) + .then((items) => { + if (active) setWorkflows(items); + }) + .catch((loadError) => { + if (active) setError(getActionErrorMessage(loadError, "加载项目工作流失败")); + }); + return () => { + active = false; + }; + }, [draft.projectId]); + + const selectedWorkflow = workflows.find((item) => item.id === draft.workflowId); + const setField = (key: K, value: CollectionTaskInput[K]) => + setDraft((current) => ({ ...current, [key]: value })); + const setTarget = (key: keyof DataVolume, value: string) => + setDraft((current) => ({ + ...current, + targets: { ...current.targets, [key]: value === "" ? undefined : Number(value) }, + })); + + const save = async (event: React.FormEvent) => { + event.preventDefault(); + setSaving(true); + setError(""); + try { + const saved = task + ? await dataOperationsApi.updateTask(task.id, draft) + : await dataOperationsApi.createTask(draft); + onSaved(saved); + } catch (saveError) { + setError(getActionErrorMessage(saveError, "保存数据任务失败")); + } finally { + setSaving(false); + } + }; + + return ( +
+
+
+
+

{task ? "编辑数据任务" : "新建完整数据任务"}

+

绑定一个真实工作流,并指定每个阶段唯一的统计边界。

+
+ +
+ +
+ {error && ( +
+ + {error} +
+ )} + +
+

任务定义

+
+ + + + + + +
+
+ +
+

目标与流程

+
+ + + + + +
+
+ + {selectedWorkflow && ( +
+

阶段统计边界

+

每个节点只能承担一个业务阶段,避免重复统计同一批数据。

+
+ {stageOrder.map((stage) => ( + + ))} +
+
+ )} + +
+

训练可用门槛

+
+ {[ + ["requireTechnicalValidation", "技术校验"], + ["requireQuality", "质量检查"], + ["requireAnnotation", "标注完成"], + ["requireAudit", "人工审核"], + ].map(([key, label]) => ( + + ))} +
+
+
+ +
+ + +
+
+
+ ); +} + +export default function DataOperations() { + const [searchParams, setSearchParams] = useSearchParams(); + const { timeZone } = useTimeZone(); + const { role } = useUser(); + const activeTab = ( + tabs.some((item) => item.id === searchParams.get("tab")) ? searchParams.get("tab") : "overview" + ) as PageTab; + const [summary, setSummary] = useState(null); + const [tasks, setTasks] = useState([]); + const [projects, setProjects] = useState([]); + const [datasets, setDatasets] = useState([]); + const [trainingUsage, setTrainingUsage] = useState([]); + const [stageItems, setStageItems] = useState([]); + const [loading, setLoading] = useState(true); + const [secondaryLoading, setSecondaryLoading] = useState(false); + const [error, setError] = useState(""); + const [lastUpdated, setLastUpdated] = useState(""); + const [taskModal, setTaskModal] = useState<{ open: boolean; task: CollectionTask | null }>({ + open: false, + task: null, + }); + const [actionTaskID, setActionTaskID] = useState(""); + + const projectId = searchParams.get("projectId") || ""; + const taskId = searchParams.get("taskId") || ""; + const robotModel = searchParams.get("robotModel") || ""; + const scene = searchParams.get("scene") || ""; + const selectedStage = (searchParams.get("stage") || "") as DataMetricStage | ""; + + const filters = useMemo( + () => ({ timezone: timeZone, projectId, taskId, robotModel, scene }), + [projectId, robotModel, scene, taskId, timeZone], + ); + + const updateParam = (key: string, value: string) => { + setSearchParams( + (current) => { + const next = new URLSearchParams(current); + if (value) next.set(key, value); + else next.delete(key); + return next; + }, + { replace: true }, + ); + }; + + const loadReferenceData = useCallback(async () => { + const [taskItems, projectItems] = await Promise.all([dataOperationsApi.listTasks(), projectsApi.list()]); + setTasks(taskItems); + setProjects(projectItems); + }, []); + + const refreshSummary = useCallback( + async (signal?: AbortSignal, silent = false) => { + if (!silent) setLoading(true); + try { + const result = await dataOperationsApi.getSummary(filters, signal); + setSummary(result); + setLastUpdated(result.updatedAt); + setError(""); + } catch (loadError) { + if (loadError instanceof Error && loadError.name === "AbortError") return; + setError(getActionErrorMessage(loadError, "加载数据运营指标失败")); + } finally { + if (!silent) setLoading(false); + } + }, + [filters], + ); + + useEffect(() => { + const controller = new AbortController(); + Promise.all([loadReferenceData(), refreshSummary(controller.signal)]).catch((loadError) => { + if (!(loadError instanceof Error && loadError.name === "AbortError")) { + setError(getActionErrorMessage(loadError, "加载数据运营页面失败")); + setLoading(false); + } + }); + return () => controller.abort(); + }, [loadReferenceData, refreshSummary]); + + useEffect(() => { + let controller: AbortController | null = null; + let timer: number | null = null; + const start = () => { + if (document.hidden) return; + timer = window.setInterval(() => { + controller?.abort(); + controller = new AbortController(); + void refreshSummary(controller.signal, true); + }, 30_000); + }; + const handleVisibility = () => { + if (document.hidden) { + if (timer !== null) window.clearInterval(timer); + timer = null; + controller?.abort(); + return; + } + controller = new AbortController(); + void refreshSummary(controller.signal, true); + start(); + }; + document.addEventListener("visibilitychange", handleVisibility); + start(); + return () => { + document.removeEventListener("visibilitychange", handleVisibility); + if (timer !== null) window.clearInterval(timer); + controller?.abort(); + }; + }, [refreshSummary]); + + useEffect(() => { + const controller = new AbortController(); + setSecondaryLoading(true); + const request = + activeTab === "datasets" + ? dataOperationsApi.listDatasets({ ...filters, limit: 100 }, controller.signal).then(setDatasets) + : activeTab === "training" + ? dataOperationsApi + .listTrainingUsage({ ...filters, limit: 50 }, controller.signal) + .then((page) => setTrainingUsage(page.items)) + : selectedStage + ? dataOperationsApi + .listStageItems({ ...filters, stage: selectedStage, limit: 50 }, controller.signal) + .then((page) => setStageItems(page.items)) + : Promise.resolve(); + request + .catch((loadError) => { + if (!(loadError instanceof Error && loadError.name === "AbortError")) { + setError(getActionErrorMessage(loadError, "加载明细失败")); + } + }) + .finally(() => setSecondaryLoading(false)); + return () => controller.abort(); + }, [activeTab, filters, selectedStage]); + + const stageMap = useMemo(() => new Map((summary?.stages || []).map((item) => [item.stage, item])), [summary?.stages]); + const uniqueRobotModels = useMemo( + () => [...new Set(tasks.map((item) => item.robotModel).filter(Boolean))].sort(), + [tasks], + ); + const uniqueScenes = useMemo(() => [...new Set(tasks.map((item) => item.scene).filter(Boolean))].sort(), [tasks]); + const filteredTasks = useMemo( + () => + tasks.filter( + (task) => + (!projectId || task.projectId === projectId) && + (!taskId || task.id === taskId) && + (!robotModel || task.robotModel === robotModel) && + (!scene || task.scene === scene), + ), + [projectId, robotModel, scene, taskId, tasks], + ); + + const saveTask = async () => { + setTaskModal({ open: false, task: null }); + await loadReferenceData(); + await refreshSummary(undefined, true); + }; + + const runTaskAction = async (task: CollectionTask, action: "start" | "complete") => { + setActionTaskID(task.id); + setError(""); + try { + if (action === "start") await dataOperationsApi.startTask(task.id); + else await dataOperationsApi.completeTask(task.id); + await Promise.all([loadReferenceData(), refreshSummary(undefined, true)]); + } catch (actionError) { + setError(getActionErrorMessage(actionError, action === "start" ? "启动数据任务失败" : "完成数据任务失败")); + } finally { + setActionTaskID(""); + } + }; + + const openStage = (stage: DataMetricStage) => { + updateParam("stage", selectedStage === stage ? "" : stage); + }; + + const maxHourly = Math.max( + 1, + ...(summary?.hourly || []).flatMap((point) => stageOrder.map((stage) => point.stages[stage]?.trajectoryCount || 0)), + ); + + return ( +
+
+
+
+
+ + 数据运营 +
+

从采集目标到训练数据的完整闭环

+

+ 默认统计 UTC+8 的今日数据。轨迹是主口径,时长和存储量作为辅助证据;未知、派生和估算值不会按零处理。 +

+
+
+
+ + {error ? "显示最后成功数据" : "30 秒自动刷新"} + {lastUpdated ? ` · ${formatCompactDateTime(lastUpdated, timeZone)}` : ""} +
+ + +
+
+ + {error && ( +
+ + {error} +
+ )} + +
+
+ + 当前统计范围 +
+
+ + + + +
+
+ +
+ {tabs.map((tab) => ( + + ))} +
+ + {loading && !summary ? ( +
+ + 正在汇总数据闭环... +
+ ) : ( + <> + {activeTab === "overview" && ( +
+
+ openStage("collect")} + /> + openStage("process")} + /> + openStage("clean")} + /> + openStage("validate")} + /> + openStage("annotate")} + /> + openStage("audit")} + /> + updateParam("tab", "training")} + /> +
+ +
+
+
+
+

今日数据链路

+

使用绝对量展示,不用面积漏斗误导转化关系。

+
+
+ {summary?.factCount || 0} 条指标事实 +
+ 精确 {summary?.exactCount || 0} · 派生 {summary?.derivedCount || 0} · 估算{" "} + {summary?.estimatedCount || 0} +
+
+
+
+ {stageOrder.map((stage, index) => { + const item = stageMap.get(stage); + const volume = + stage === "validate" + ? item?.technicalValid || item?.output + : stage === "audit" + ? item?.trainReady || item?.output + : stage === "train" + ? summary?.training.consumed + : item?.output; + return ( + + ); + })} +
+
+ +
+

训练数据使用

+
+
+
训练消费
+
+ {volumePrimary(summary?.training.consumed)} +
+
+
+
去重数据
+
+ {volumePrimary(summary?.training.unique)} +
+
+
+
+ 训练 Run + {summary?.training.runCount || 0} +
+ +
+
+ +
+
+

小时趋势

+

各阶段独立编码,不堆叠相加。

+
+ {(summary?.hourly || []).length === 0 ? ( +
今日尚无带时间口径的指标
+ ) : ( + summary?.hourly.map((point) => ( +
+ + {new Date(point.hour).toLocaleTimeString("zh-CN", { + hour: "2-digit", + minute: "2-digit", + hour12: false, + timeZone, + })} + +
+ {stageOrder.map((stage) => { + const value = point.stages[stage]?.trajectoryCount || 0; + return ( +
+
+
+
+
+ {stageLabels[stage]} {value || "-"} +
+
+ ); + })} +
+
+ )) + )} +
+
+ +
+

质量淘汰原因

+
+ {Object.keys(summary?.rejectionReasons || {}).length === 0 ? ( +
暂无结构化淘汰原因
+ ) : ( + (Object.entries(summary?.rejectionReasons || {}) as Array<[string, number]>) + .sort((left, right) => right[1] - left[1]) + .slice(0, 8) + .map(([reason, count]) => ( +
+
+ {reason} + {count} +
+
+
+
+
+ )) + )} +
+
+
+ + {selectedStage && ( +
+
+
+

{stageLabels[selectedStage]}明细

+

显示最近 50 条事实,点击 Run 进入执行监控。

+
+ +
+ {secondaryLoading ? ( +
正在加载明细...
+ ) : ( +
+ + + + + + + + + + + + + {stageItems.map((item) => ( + + + + + + + + + ))} + +
时间任务/项目节点输出可信度Run
+ {formatCompactDateTime(item.occurredAt, timeZone)} + + {item.taskName || item.projectName || "历史数据/未归属任务"} + {item.nodeLabel || item.nodeId}{volumePrimary(item.output)} + {item.quality === "exact" ? "精确" : item.quality === "derived" ? "派生" : "估算"} + + + {item.runId.slice(0, 14)}… + +
+ {stageItems.length === 0 && ( +
当前筛选下暂无明细
+ )} +
+ )} +
+ )} +
+ )} + + {activeTab === "tasks" && ( +
+ {filteredTasks.map((task) => { + const progress = summary?.tasks.find((item) => item.taskId === task.id); + return ( +
+
+
+
+

{task.name}

+ +
+

+ {task.robotModel} · {task.scene} +

+
+ +
+
+
+ 采集目标 + {(progress?.percent || 0).toFixed(1)}% +
+
+
+
+
+ {volumePrimary(progress?.collected)} + 目标 {volumePrimary(task.targets)} +
+
+
+
+
工作流
+
{task.workflowName || "未绑定"}
+
+
+
阶段覆盖
+
{Object.keys(task.stageBindings || {}).length} / 7
+
+
+
+ + {task.status === "active" && ( + + )} +
+
+ ); + })} + {filteredTasks.length === 0 && ( +
+ +

还没有符合筛选条件的数据任务

+ +
+ )} +
+ )} + + {activeTab === "datasets" && ( +
+
+

数据集版本

+

每个版本保留生产 Run、节点、split、指纹和存储路径。

+
+ {secondaryLoading ? ( +
正在加载数据版本...
+ ) : ( +
+ {datasets.map((item) => ( +
+
+
+

{item.name}

+

+ {item.split || "未指定 split"} ·{" "} + {item.quality === "exact" ? "精确" : item.quality === "derived" ? "派生" : "估算"} +

+
+ +
+
{volumePrimary(item.volume)}
+
{item.id}
+ {item.producerRunId && ( + + 查看生产 Run + + )} +
+ ))} + {datasets.length === 0 && ( +
暂无物化的数据集版本
+ )} +
+ )} +
+ )} + + {activeTab === "training" && ( +
+

训练使用的数据

+

+ 区分重复消费量和唯一数据版本,历史路径引用会明确标记为外部数据。 +

+ {secondaryLoading ? ( +
正在加载训练血缘...
+ ) : ( +
+ + + + + + + + + + + + + {trainingUsage.map((item) => ( + + + + + + + + + ))} + +
开始时间任务/模型训练消费数据版本与 split可信度Run
{formatCompactDateTime(item.occurredAt, timeZone)} +
{item.taskName || "历史数据/未归属任务"}
+
{item.model || item.nodeLabel || "-"}
+
{volumePrimary(item.consumed)} + {item.inputs.length === 0 ? ( + 未记录输入版本 + ) : ( +
+ {item.inputs.slice(0, 3).map((input, index) => ( +
+ + {input.datasetVersionId || input.name || "外部路径"} + + + {input.split || "unspecified"} + + {input.external && 外部} +
+ ))} +
+ )} +
+ {item.quality === "exact" ? "精确" : item.quality === "derived" ? "派生" : "估算"} + + + {item.runId.slice(0, 14)}… + +
+ {trainingUsage.length === 0 && ( +
暂无带数据版本引用的训练记录
+ )} +
+ )} +
+ )} + + )} + + {role === "admin" && ( +
+ 历史兼容:管理员可通过数据运营回填接口幂等补齐最近 30 天的派生指标。 + 旧 Run 和旧产物不会被重写 +
+ )} +
+ + {taskModal.open && ( + setTaskModal({ open: false, task: null })} + onSaved={() => void saveTask()} + /> + )} +
+ ); +} diff --git a/src/pages/Operators.tsx b/src/pages/Operators.tsx index ef2ae203..3a5ef1d5 100644 --- a/src/pages/Operators.tsx +++ b/src/pages/Operators.tsx @@ -39,7 +39,7 @@ import { import { buildOperatorPayload, getOperatorPayloadIssue } from "../lib/operators.js"; import { cn } from "../lib/utils"; import { canManagePipelineDesign } from "../lib/user-roles"; -import { Operator, PortDefinition, PortTransmission, PortType } from "../types"; +import { DataMetricStage, Operator, PortDefinition, PortTransmission, PortType } from "../types"; import { generateOperatorFromDescription } from "../services/aiService"; import type { GeneratedOperator } from "../services/api/ai"; import { @@ -76,6 +76,19 @@ const gpuTypeOptions = [ "NVIDIA T4 16GB", "NVIDIA GPU", ]; +const businessStageOptions: Array<{ value: DataMetricStage; label: string }> = [ + { value: "collect", label: "采集" }, + { value: "process", label: "处理 / 转换" }, + { value: "clean", label: "清洗" }, + { value: "validate", label: "技术校验" }, + { value: "annotate", label: "标注" }, + { value: "audit", label: "审核 / 训练可用" }, + { value: "train", label: "训练" }, +]; + +function businessStageLabel(stage?: DataMetricStage) { + return businessStageOptions.find((option) => option.value === stage)?.label || ""; +} type GeneratedImplementation = NonNullable; @@ -1399,6 +1412,11 @@ export default function Operators() { SHM {trimResource(op.resources?.sharedMemorySize)} )} + {op.businessStage && ( + + 阶段 {businessStageLabel(op.businessStage)} + + )} @@ -2010,6 +2028,34 @@ export default function Operators() { +
+
+ + +
+

+ 用于算子独立运行时的指标归类;完整任务绑定和工作流节点覆盖值优先。 +

+
+
)} +
+
+ + +
+

+ 完整任务绑定优先,其次使用当前节点;留空时继承算子默认阶段。 +

+
+
diff --git a/src/services/api/data-operations.ts b/src/services/api/data-operations.ts new file mode 100644 index 00000000..30338ebd --- /dev/null +++ b/src/services/api/data-operations.ts @@ -0,0 +1,78 @@ +import type { + CollectionTask, + CollectionTaskInput, + DataMetricFact, + DataMetricStage, + DataOperationsSummary, + DatasetVersion, + ExecutionRun, + TrainingUsage, +} from "../../types"; +import { apiRequest } from "./client"; + +export type DataOperationsFilters = { + from?: string; + to?: string; + timezone?: string; + projectId?: string; + taskId?: string; + robotModel?: string; + scene?: string; + stage?: DataMetricStage; + limit?: number; + cursor?: string; +}; + +const queryString = (filters: DataOperationsFilters = {}) => { + const params = new URLSearchParams(); + for (const [key, value] of Object.entries(filters)) { + if (value !== undefined && value !== "") { + params.set(key, String(value)); + } + } + const value = params.toString(); + return value ? `?${value}` : ""; +}; + +export const dataOperationsApi = { + listTasks: async (filters: Pick & { status?: string } = {}) => + ( + await apiRequest<{ items: CollectionTask[] }>( + `/api/data-collection-tasks${queryString(filters as DataOperationsFilters)}`, + ) + ).items, + getTask: (id: string) => apiRequest(`/api/data-collection-tasks/${id}`), + createTask: (payload: CollectionTaskInput) => + apiRequest("/api/data-collection-tasks", { method: "POST" }, payload), + updateTask: (id: string, payload: CollectionTaskInput) => + apiRequest(`/api/data-collection-tasks/${id}`, { method: "PATCH" }, payload), + startTask: (id: string) => + apiRequest<{ task: CollectionTask; run: ExecutionRun }>(`/api/data-collection-tasks/${id}/start`, { + method: "POST", + }), + completeTask: (id: string) => + apiRequest(`/api/data-collection-tasks/${id}/complete`, { method: "POST" }), + getSummary: (filters: DataOperationsFilters = {}, signal?: AbortSignal) => + apiRequest(`/api/data-operations/summary${queryString(filters)}`, { signal }), + listStageItems: (filters: DataOperationsFilters = {}, signal?: AbortSignal) => + apiRequest<{ items: DataMetricFact[]; nextCursor?: string; hasMore: boolean }>( + `/api/data-operations/stage-items${queryString(filters)}`, + { signal }, + ), + listTrainingUsage: (filters: DataOperationsFilters = {}, signal?: AbortSignal) => + apiRequest<{ items: TrainingUsage[]; nextCursor?: string; hasMore: boolean }>( + `/api/data-operations/training-usage${queryString(filters)}`, + { signal }, + ), + listDatasets: async (filters: DataOperationsFilters = {}, signal?: AbortSignal) => + ( + await apiRequest<{ items: DatasetVersion[] }>(`/api/data-operations/datasets${queryString(filters)}`, { + signal, + }) + ).items, + backfill: () => + apiRequest<{ windowDays: number; processed: number; skipped: number; failures: number; completedAt: string }>( + "/api/data-operations/backfill", + { method: "POST" }, + ), +}; diff --git a/src/types.ts b/src/types.ts index c5b80719..a8bf4149 100644 --- a/src/types.ts +++ b/src/types.ts @@ -261,6 +261,7 @@ export interface Operator { gpuLimit?: string; sharedMemorySize?: string; }; + businessStage?: DataMetricStage; } export interface HyperParam { @@ -609,6 +610,7 @@ export interface LightweightOperatorCodeManifest { export interface ExecutionRunMetadata { lightweightOperatorCodeManifests?: Record; + trainingInputs?: TrainingInput[]; } export interface ScanConfig { @@ -817,6 +819,8 @@ export interface NodeExecution { logs: LogEntry[]; artifacts: ExecutionData[]; dataRefs?: DataRef[]; + dataMetrics?: DataMetrics; + metricsMaterializedAt?: string; resourceSeries: ResourceUsage[]; error?: string; } @@ -839,6 +843,207 @@ export interface DataRef { partitionDir?: string; partitionCount?: number; itemCount?: number; + datasetId?: string; + datasetVersionId?: string; + split?: string; + fingerprint?: string; +} + +export type DataMetricStage = "collect" | "process" | "clean" | "validate" | "annotate" | "audit" | "train"; +export type DataMetricQuality = "exact" | "derived" | "estimated"; +export type CollectionTaskStatus = "draft" | "active" | "paused" | "completed" | "cancelled"; +export type CollectionTaskMode = "collection_only" | "full_training"; + +export interface DataVolume { + trajectoryCount?: number; + durationSeconds?: number; + bytes?: number; + fileCount?: number; +} + +export interface DatasetMetricRef { + datasetId?: string; + datasetVersionId?: string; + name?: string; + split?: string; + fingerprint?: string; + kind?: string; + datasetType?: string; + manifestPath?: string; + payloadPath?: string; + partitionDir?: string; + partitionCount?: number; + itemCount?: number; +} + +export interface TrainingInput { + datasetId?: string; + datasetVersionId?: string; + name?: string; + split?: string; + volume?: DataVolume; + manifestPath?: string; + partitionDir?: string; + external?: boolean; + quality?: DataMetricQuality; +} + +export interface DataMetrics { + schemaVersion: number; + stage: DataMetricStage; + input?: DataVolume; + output?: DataVolume; + rejected?: DataVolume; + technicalValid?: DataVolume; + trainReady?: DataVolume; + datasetRefs?: DatasetMetricRef[]; + trainingInputs?: TrainingInput[]; + rejectionReasons?: Record; + quality?: DataMetricQuality; + occurredAt?: string; +} + +export interface CollectionTaskRequirements { + requireTechnicalValidation: boolean; + requireQuality: boolean; + requireAnnotation: boolean; + requireAudit: boolean; +} + +export interface CollectionTask { + id: string; + organizationId?: string; + projectId: string; + projectName?: string; + name: string; + description?: string; + robotModel: string; + robotVersion?: string; + scene: string; + modalities: string[]; + format?: string; + mode: CollectionTaskMode; + status: CollectionTaskStatus; + workflowId?: string; + workflowName?: string; + stageBindings?: Partial>; + targets: DataVolume; + requirements: CollectionTaskRequirements; + splitRules?: Partial>; + lastRunId?: string; + createdAt?: string; + updatedAt?: string; +} + +export type CollectionTaskInput = Omit< + CollectionTask, + "id" | "organizationId" | "projectName" | "workflowName" | "lastRunId" | "createdAt" | "updatedAt" +>; + +export interface DatasetVersion { + id: string; + datasetId: string; + taskId?: string; + projectId?: string; + name: string; + split?: string; + fingerprint: string; + volume?: DataVolume; + storage?: AssetStorageLocation; + manifestPath?: string; + partitionDir?: string; + producerRunId?: string; + producerNodeId?: string; + artifactId?: string; + quality: DataMetricQuality; + createdAt: string; + updatedAt?: string; +} + +export interface DataMetricFact { + id: string; + sourceKey: string; + projectId?: string; + projectName?: string; + taskId?: string; + taskName?: string; + robotModel?: string; + scene?: string; + workflowId?: string; + workflowName?: string; + runId: string; + nodeId: string; + nodeLabel?: string; + attempt: number; + stage: DataMetricStage; + input?: DataVolume; + output?: DataVolume; + rejected?: DataVolume; + technicalValid?: DataVolume; + trainReady?: DataVolume; + datasetRefs?: DatasetMetricRef[]; + trainingInputs?: TrainingInput[]; + rejectionReasons?: Record; + quality: DataMetricQuality; + occurredAt: string; + recordedAt: string; +} + +export interface DataStageSummary { + stage: DataMetricStage; + input?: DataVolume; + output?: DataVolume; + rejected?: DataVolume; + technicalValid?: DataVolume; + trainReady?: DataVolume; + runCount: number; +} + +export interface DataOperationsSummary { + from: string; + to: string; + timezone: string; + updatedAt: string; + freshness: string; + stages: DataStageSummary[]; + training: { + consumed: DataVolume; + unique: DataVolume; + runCount: number; + }; + hourly: Array<{ hour: string; stages: Partial> }>; + rejectionReasons: Record; + tasks: Array<{ + taskId: string; + taskName: string; + status: CollectionTaskStatus; + targets: DataVolume; + collected: DataVolume; + percent: number; + }>; + factCount: number; + exactCount: number; + derivedCount: number; + estimatedCount: number; + unknownCount: number; +} + +export interface TrainingUsage { + id: string; + taskId?: string; + taskName?: string; + projectId?: string; + projectName?: string; + workflowId?: string; + workflowName?: string; + runId: string; + nodeId: string; + nodeLabel?: string; + model?: string; + inputs: TrainingInput[]; + consumed: DataVolume; + quality: DataMetricQuality; + occurredAt: string; } export interface InputLineage { @@ -915,6 +1120,7 @@ export interface ExecutionRun { workflowName: string; projectId: string; projectName: string; + taskId?: string; trigger: "manual" | "scheduled" | "retry" | "rerun_subtree"; attempt?: number; parentRunId?: string; @@ -968,6 +1174,7 @@ export interface ExecutionRunSummary { workflowName: string; projectId: string; projectName: string; + taskId?: string; trigger: "manual" | "scheduled" | "retry" | "rerun_subtree"; attempt?: number; parentRunId?: string; diff --git a/tests/frontend/data-operations.test.mjs b/tests/frontend/data-operations.test.mjs new file mode 100644 index 00000000..1408c18c --- /dev/null +++ b/tests/frontend/data-operations.test.mjs @@ -0,0 +1,37 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; + +const pageURL = new URL("../../src/pages/DataOperations.tsx", import.meta.url); +const serviceURL = new URL("../../src/services/api/data-operations.ts", import.meta.url); + +test("data operations page exposes complete task and business metric surfaces", async () => { + const source = await readFile(pageURL, "utf8"); + + assert.match(source, /数据总览/); + assert.match(source, /完整任务/); + assert.match(source, /数据版本/); + assert.match(source, /训练使用/); + assert.match(source, /技术有效/); + assert.match(source, /训练可用/); + assert.match(source, /训练消费/); +}); + +test("data operations polling pauses while the page is hidden", async () => { + const source = await readFile(pageURL, "utf8"); + + assert.match(source, /document\.hidden/); + assert.match(source, /visibilitychange/); + assert.match(source, /30_000/); + assert.match(source, /controller\?\.abort\(\)/); +}); + +test("data operations client uses additive compatibility endpoints", async () => { + const source = await readFile(serviceURL, "utf8"); + + assert.match(source, /\/api\/data-collection-tasks/); + assert.match(source, /\/api\/data-operations\/summary/); + assert.match(source, /\/api\/data-operations\/stage-items/); + assert.match(source, /\/api\/data-operations\/training-usage/); + assert.match(source, /\/api\/data-operations\/datasets/); +}); diff --git a/tests/frontend/operators.test.mjs b/tests/frontend/operators.test.mjs index cf1b1490..9282c084 100644 --- a/tests/frontend/operators.test.mjs +++ b/tests/frontend/operators.test.mjs @@ -55,6 +55,22 @@ test("buildOperatorPayload deep clones mutable operator fields", () => { assert.equal(source.hyperparams[0].value, "fast"); }); +test("buildOperatorPayload normalizes and validates business stage", () => { + const base = { + name: "cleaner", + type: "custom", + category: "DATA", + description: "clean data", + inputs: [], + outputs: [], + businessStage: " CLEAN ", + }; + const payload = buildOperatorPayload(base); + assert.equal(payload.businessStage, "clean"); + assert.equal(getOperatorPayloadIssue(base), ""); + assert.match(getOperatorPayloadIssue({ ...base, businessStage: "publish" }), /业务阶段/); +}); + test("operator payload validation rejects missing metadata and invalid ports", () => { assert.equal(getOperatorPayloadIssue({ name: " ", description: "ok" }), "算子名称不能为空"); assert.equal(