package api import ( "fmt" "math" "net/http" "strconv" "strings" "time" "droboticflow/internal/dataops" "droboticflow/internal/domain" "github.com/go-chi/chi/v5" ) func validCollectionTaskMode(value domain.CollectionTaskMode) bool { return value == domain.CollectionTaskModeCollectionOnly || value == domain.CollectionTaskModeFullTraining } func validCollectionTaskStatus(value domain.CollectionTaskStatus) bool { switch value { case domain.CollectionTaskStatusDraft, domain.CollectionTaskStatusActive, domain.CollectionTaskStatusPaused, domain.CollectionTaskStatusCompleted, domain.CollectionTaskStatusCancelled: return true default: return false } } func validDataMetricStage(value domain.DataMetricStage) bool { for _, stage := range domain.DataMetricStages { if value == stage { return true } } return false } func normalizedStringList(items []string) []string { result := make([]string, 0, len(items)) seen := map[string]struct{}{} for _, value := range items { trimmed := strings.TrimSpace(value) key := strings.ToLower(trimmed) if trimmed == "" { continue } if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} result = append(result, trimmed) } return result } func validateNonNegativeVolume(field string, value domain.DataVolume) []validationIssue { issues := make([]validationIssue, 0) if value.TrajectoryCount != nil && *value.TrajectoryCount < 0 { issues = append(issues, validationIssue{Field: field + ".trajectoryCount", Message: "must be greater than or equal to 0"}) } if value.DurationSeconds != nil && (*value.DurationSeconds < 0 || math.IsNaN(*value.DurationSeconds) || math.IsInf(*value.DurationSeconds, 0)) { issues = append(issues, validationIssue{Field: field + ".durationSeconds", Message: "must be a finite value greater than or equal to 0"}) } if value.Bytes != nil && *value.Bytes < 0 { issues = append(issues, validationIssue{Field: field + ".bytes", Message: "must be greater than or equal to 0"}) } if value.FileCount != nil && *value.FileCount < 0 { issues = append(issues, validationIssue{Field: field + ".fileCount", Message: "must be greater than or equal to 0"}) } return issues } func collectionTaskHasTarget(value domain.DataVolume) bool { return (value.TrajectoryCount != nil && *value.TrajectoryCount > 0) || (value.DurationSeconds != nil && *value.DurationSeconds > 0) || (value.Bytes != nil && *value.Bytes > 0) || (value.FileCount != nil && *value.FileCount > 0) } func (server *server) normalizeCollectionTaskInput(ctxRequest *http.Request, payload *domain.CollectionTaskInput, existing *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, []validationIssue, error) { payload.ProjectID = strings.TrimSpace(payload.ProjectID) payload.Name = strings.TrimSpace(payload.Name) payload.Description = strings.TrimSpace(payload.Description) payload.RobotModel = strings.TrimSpace(payload.RobotModel) payload.RobotVersion = strings.TrimSpace(payload.RobotVersion) payload.Scene = strings.TrimSpace(payload.Scene) payload.Format = strings.TrimSpace(payload.Format) payload.WorkflowID = strings.TrimSpace(payload.WorkflowID) payload.Modalities = normalizedStringList(payload.Modalities) if payload.Mode == "" { payload.Mode = domain.CollectionTaskModeFullTraining } if payload.Status == "" { payload.Status = domain.CollectionTaskStatusDraft } issues := make([]validationIssue, 0) if payload.ProjectID == "" { issues = append(issues, validationIssue{Field: "projectId", Message: "projectId is required"}) } if payload.Name == "" { issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) } if payload.RobotModel == "" { issues = append(issues, validationIssue{Field: "robotModel", Message: "robotModel is required"}) } if payload.Scene == "" { issues = append(issues, validationIssue{Field: "scene", Message: "scene is required"}) } if !validCollectionTaskMode(payload.Mode) { issues = append(issues, validationIssue{Field: "mode", Message: "mode must be collection_only or full_training"}) } if !validCollectionTaskStatus(payload.Status) { issues = append(issues, validationIssue{Field: "status", Message: "invalid task status"}) } issues = append(issues, validateNonNegativeVolume("targets", payload.Targets)...) if !collectionTaskHasTarget(payload.Targets) { issues = append(issues, validationIssue{Field: "targets", Message: "at least one positive trajectory, duration, or bytes target is required"}) } splitTotal := 0 for split, percent := range payload.SplitRules { if split != "train" && split != "val" && split != "test" { issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split must be train, val, or test"}) } if percent < 0 || percent > 100 { issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split percent must be between 0 and 100"}) } splitTotal += percent } if len(payload.SplitRules) > 0 && splitTotal != 100 { issues = append(issues, validationIssue{Field: "splitRules", Message: "split percentages must total 100"}) } seenNodes := map[string]domain.DataMetricStage{} for stage, nodeID := range payload.StageBindings { nodeID = strings.TrimSpace(nodeID) if !validDataMetricStage(stage) { issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "invalid business stage"}) continue } if nodeID == "" { delete(payload.StageBindings, stage) continue } if previous, exists := seenNodes[nodeID]; exists { issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: fmt.Sprintf("node is already the metric boundary for %s", previous)}) } seenNodes[nodeID] = stage payload.StageBindings[stage] = nodeID } if len(issues) > 0 { return nil, issues, nil } project, err := server.store.GetProject(ctxRequest.Context(), payload.ProjectID) if err != nil { return nil, nil, err } if project == nil || !server.filterByTenant(ctxRequest, project.OrganizationID) { return nil, []validationIssue{{Field: "projectId", Message: "project not found"}}, nil } workflowName := "" if payload.WorkflowID != "" { workflow, err := server.store.GetWorkflow(ctxRequest.Context(), payload.WorkflowID) if err != nil { return nil, nil, err } if workflow == nil || workflow.ProjectID != project.ID || !server.filterByTenant(ctxRequest, workflow.OrganizationID) { return nil, []validationIssue{{Field: "workflowId", Message: "workflow not found in selected project"}}, nil } workflowName = workflow.Name for stage, nodeID := range payload.StageBindings { if domain.GetNodeByID(workflow.Nodes, nodeID) == nil { issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "workflow node not found"}) } } } if len(issues) > 0 { return nil, issues, nil } item := &domain.CollectionTaskRecord{ OrganizationID: project.OrganizationID, ProjectID: project.ID, ProjectName: project.Name, Name: payload.Name, Description: payload.Description, RobotModel: payload.RobotModel, RobotVersion: payload.RobotVersion, Scene: payload.Scene, Modalities: payload.Modalities, Format: payload.Format, Mode: payload.Mode, Status: payload.Status, WorkflowID: payload.WorkflowID, WorkflowName: workflowName, StageBindings: payload.StageBindings, Targets: payload.Targets, Requirements: payload.Requirements, SplitRules: payload.SplitRules, } if existing != nil { item.ID = existing.ID item.CreatedAt = existing.CreatedAt item.LastRunID = existing.LastRunID } return item, nil, nil } func (server *server) handleCollectionTasksList(writer http.ResponseWriter, request *http.Request) { items, err := server.store.ListCollectionTasks(request.Context(), server.requestOrganizationID(request)) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } projectID := strings.TrimSpace(request.URL.Query().Get("projectId")) status := strings.TrimSpace(request.URL.Query().Get("status")) filtered := make([]domain.CollectionTaskRecord, 0, len(items)) for _, item := range items { if projectID != "" && item.ProjectID != projectID { continue } if status != "" && string(item.Status) != status { continue } filtered = append(filtered, item) } renderJSON(writer, http.StatusOK, map[string]any{"items": filtered}) } func (server *server) handleCollectionTaskGet(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if item == nil || !server.ensureTenantAccess(writer, request, item.OrganizationID, "Data task") { if item == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"}) } return } renderJSON(writer, http.StatusOK, item) } func (server *server) persistCollectionTask(request *http.Request, item *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, error) { if err := server.store.UpsertCollectionTask(request.Context(), *item); err != nil { return nil, err } return server.store.GetCollectionTask(request.Context(), item.ID) } func (server *server) handleCollectionTaskCreate(writer http.ResponseWriter, request *http.Request) { var payload domain.CollectionTaskInput if !decodeJSON(writer, request, &payload) { return } item, issues, err := server.normalizeCollectionTaskInput(request, &payload, nil) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if len(issues) > 0 { renderValidationError(writer, issues...) return } item.ID = domain.NewID("data-task") saved, err := server.persistCollectionTask(request, item) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusCreated, saved) } func (server *server) handleCollectionTaskUpdate(writer http.ResponseWriter, request *http.Request) { existing, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if existing == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"}) return } if !server.ensureTenantAccess(writer, request, existing.OrganizationID, "Data task") { return } var payload domain.CollectionTaskInput if !decodeJSON(writer, request, &payload) { return } item, issues, err := server.normalizeCollectionTaskInput(request, &payload, existing) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if len(issues) > 0 { renderValidationError(writer, issues...) return } saved, err := server.persistCollectionTask(request, item) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, saved) } func (server *server) handleCollectionTaskComplete(writer http.ResponseWriter, request *http.Request) { task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if task == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"}) return } if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") { return } task.Status = domain.CollectionTaskStatusCompleted saved, err := server.persistCollectionTask(request, task) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, saved) } func parseDataOperationsQuery(request *http.Request, settings *domain.SettingsRecord) (domain.DataOperationsQuery, []validationIssue) { timezone := strings.TrimSpace(request.URL.Query().Get("timezone")) if timezone == "" && settings != nil { timezone = strings.TrimSpace(settings.Timezone) } if timezone == "" { timezone = domain.DefaultWorkflowScheduleTimezone } location, err := time.LoadLocation(timezone) if err != nil { return domain.DataOperationsQuery{}, []validationIssue{{Field: "timezone", Message: "invalid IANA timezone"}} } query := domain.DataOperationsQuery{ Timezone: timezone, ProjectID: strings.TrimSpace(request.URL.Query().Get("projectId")), TaskID: strings.TrimSpace(request.URL.Query().Get("taskId")), RobotModel: strings.TrimSpace(request.URL.Query().Get("robotModel")), Scene: strings.TrimSpace(request.URL.Query().Get("scene")), Stage: domain.DataMetricStage(strings.TrimSpace(request.URL.Query().Get("stage"))), Cursor: strings.TrimSpace(request.URL.Query().Get("cursor")), } if query.Stage != "" && !validDataMetricStage(query.Stage) { return domain.DataOperationsQuery{}, []validationIssue{{Field: "stage", Message: "invalid business stage"}} } if raw := strings.TrimSpace(request.URL.Query().Get("limit")); raw != "" { value, parseErr := strconv.Atoi(raw) if parseErr != nil || value < 1 || value > 100 { return domain.DataOperationsQuery{}, []validationIssue{{Field: "limit", Message: "limit must be between 1 and 100"}} } query.Limit = value } now := time.Now() fromRaw := strings.TrimSpace(request.URL.Query().Get("from")) toRaw := strings.TrimSpace(request.URL.Query().Get("to")) if fromRaw == "" { localNow := now.In(location) fromRaw = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, location).UTC().Format(time.RFC3339Nano) } if toRaw == "" { toRaw = now.UTC().Format(time.RFC3339Nano) } from, fromErr := time.Parse(time.RFC3339Nano, fromRaw) to, toErr := time.Parse(time.RFC3339Nano, toRaw) if fromErr != nil { return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be RFC3339"}} } if toErr != nil { return domain.DataOperationsQuery{}, []validationIssue{{Field: "to", Message: "to must be RFC3339"}} } if !from.Before(to) { return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be before to"}} } query.From = from.UTC().Format(time.RFC3339Nano) query.To = to.UTC().Format(time.RFC3339Nano) return query, nil } func (server *server) dataOperationsQuery(writer http.ResponseWriter, request *http.Request) (domain.DataOperationsQuery, bool) { settings, err := server.store.GetSettings(request.Context()) if err != nil { renderError(writer, http.StatusInternalServerError, err) return domain.DataOperationsQuery{}, false } query, issues := parseDataOperationsQuery(request, settings) if len(issues) > 0 { renderValidationError(writer, issues...) return domain.DataOperationsQuery{}, false } return query, true } func (server *server) handleDataOperationsSummary(writer http.ResponseWriter, request *http.Request) { query, ok := server.dataOperationsQuery(writer, request) if !ok { return } summary, err := server.store.GetDataOperationsSummary(request.Context(), server.requestOrganizationID(request), query) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, summary) } func (server *server) handleDataOperationsStageItems(writer http.ResponseWriter, request *http.Request) { query, ok := server.dataOperationsQuery(writer, request) if !ok { return } page, err := server.store.ListDataMetricFactsPage(request.Context(), server.requestOrganizationID(request), query) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, page) } func (server *server) handleDataOperationsTrainingUsage(writer http.ResponseWriter, request *http.Request) { query, ok := server.dataOperationsQuery(writer, request) if !ok { return } page, err := server.store.ListTrainingUsagePage(request.Context(), server.requestOrganizationID(request), query) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, page) } func (server *server) handleDataOperationsDatasets(writer http.ResponseWriter, request *http.Request) { query, ok := server.dataOperationsQuery(writer, request) if !ok { return } items, err := server.store.ListDatasetVersions(request.Context(), server.requestOrganizationID(request), query) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, map[string]any{"items": items}) } func legacyStageFromExecution(execution domain.NodeExecutionRecord) domain.DataMetricStage { label := strings.ToLower(strings.TrimSpace(execution.NodeLabel + " " + execution.OperatorID)) stageTokens := []struct { stage domain.DataMetricStage tokens []string }{ {domain.DataMetricStageTrain, []string{"train", "训练"}}, {domain.DataMetricStageAudit, []string{"audit", "审核"}}, {domain.DataMetricStageAnnotate, []string{"annotat", "标注"}}, {domain.DataMetricStageValidate, []string{"validat", "quality", "校验", "质检"}}, {domain.DataMetricStageClean, []string{"clean", "filter", "清洗", "过滤"}}, {domain.DataMetricStageProcess, []string{"process", "convert", "package", "处理", "转换", "打包"}}, {domain.DataMetricStageCollect, []string{"collect", "scan", "source", "采集", "扫描", "数据源"}}, } for _, item := range stageTokens { for _, token := range item.tokens { if strings.Contains(label, token) { return item.stage } } } return "" } func numericArtifactValue(payload map[string]any, keys ...string) (float64, bool) { for _, key := range keys { switch value := payload[key].(type) { case int: return float64(value), value >= 0 case int32: return float64(value), value >= 0 case int64: return float64(value), value >= 0 case float64: return value, value >= 0 && !math.IsNaN(value) && !math.IsInf(value, 0) } } return 0, false } func legacyVolumeFromArtifacts(items []domain.ExecutionArtifact) domain.DataVolume { var trajectory, duration, bytes, files float64 var hasTrajectory, hasDuration, hasBytes, hasFiles bool for _, artifact := range items { payload := artifact.Data if value, ok := numericArtifactValue(payload, "trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"); ok && (!hasTrajectory || value > trajectory) { trajectory, hasTrajectory = value, true } if value, ok := numericArtifactValue(payload, "durationSeconds", "totalDurationSeconds", "totalDuration", "duration"); ok && (!hasDuration || value > duration) { duration, hasDuration = value, true } if value, ok := numericArtifactValue(payload, "totalBytes", "bytes", "sizeBytes"); ok && (!hasBytes || value > bytes) { bytes, hasBytes = value, true } if value, ok := numericArtifactValue(payload, "fileCount", "filesCount"); ok && (!hasFiles || value > files) { files, hasFiles = value, true } } result := domain.DataVolume{} if hasTrajectory { value := int64(trajectory) result.TrajectoryCount = &value } if hasDuration { value := duration result.DurationSeconds = &value } if hasBytes { value := int64(bytes) result.Bytes = &value } if hasFiles { value := int64(files) result.FileCount = &value } return result } func legacySourceCollectionMetrics(items []domain.ExecutionArtifact) (domain.DataVolume, domain.DataVolume, []domain.DatasetMetricRef, bool) { var trajectories int64 var validTrajectories int64 var files int64 var bytes int64 refs := make([]domain.DatasetMetricRef, 0) for artifactIndex := range items { artifact := &items[artifactIndex] payload := artifactStructuredData(artifact) if payload == nil { continue } for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") { collection := mapValue(raw) if collection == nil { continue } trajectories++ fileCount := int64Value(collection["fileCount"]) totalBytes := int64Value(collection["totalBytes"]) files += fileCount bytes += totalBytes if boolValue(collection["valid"]) { validTrajectories++ } name := firstNonEmptyText(stringValue(collection["name"]), stringValue(collection["id"]), stringValue(collection["relativePath"])) refs = append(refs, domain.DatasetMetricRef{ DatasetID: stringValue(collection["id"]), Name: name, Kind: "source-collection", DatasetType: firstNonEmptyText(stringValue(collection["datasetType"]), "trajectory"), PayloadPath: stringValue(collection["path"]), ItemCount: 1, }) } } if trajectories == 0 { return domain.DataVolume{}, domain.DataVolume{}, nil, false } output := domain.DataVolume{ TrajectoryCount: &trajectories, Bytes: &bytes, FileCount: &files, } valid := domain.DataVolume{TrajectoryCount: &validTrajectories} return output, valid, refs, true } func legacyRejectedVolume(input domain.DataVolume, output domain.DataVolume) domain.DataVolume { result := domain.DataVolume{} if input.TrajectoryCount != nil && output.TrajectoryCount != nil && *input.TrajectoryCount >= *output.TrajectoryCount { value := *input.TrajectoryCount - *output.TrajectoryCount result.TrajectoryCount = &value } if input.DurationSeconds != nil && output.DurationSeconds != nil && *input.DurationSeconds >= *output.DurationSeconds { value := *input.DurationSeconds - *output.DurationSeconds result.DurationSeconds = &value } if input.Bytes != nil && output.Bytes != nil && *input.Bytes >= *output.Bytes { value := *input.Bytes - *output.Bytes result.Bytes = &value } if input.FileCount != nil && output.FileCount != nil && *input.FileCount >= *output.FileCount { value := *input.FileCount - *output.FileCount result.FileCount = &value } return result } func legacyMetricsFromExecution(run domain.RunRecord, execution domain.NodeExecutionRecord) *domain.DataMetricsRecord { stage := legacyStageFromExecution(execution) if stage == "" { return nil } input := legacyVolumeFromArtifacts(execution.InputArtifacts) output := legacyVolumeFromArtifacts(execution.Artifacts) technicalValid := domain.DataVolume{} refs := []domain.DatasetMetricRef(nil) if collectionOutput, valid, collectionRefs, ok := legacySourceCollectionMetrics(execution.Artifacts); ok { output = collectionOutput technicalValid = valid refs = collectionRefs } metrics := &domain.DataMetricsRecord{ SchemaVersion: 1, Stage: stage, Input: input, Output: output, DatasetRefs: refs, Quality: domain.DataMetricQualityEstimated, OccurredAt: firstNonEmptyText(execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt), } switch stage { case domain.DataMetricStageClean: metrics.Rejected = legacyRejectedVolume(input, output) case domain.DataMetricStageValidate: if technicalValid.TrajectoryCount != nil { metrics.TechnicalValid = technicalValid } else { metrics.TechnicalValid = output } case domain.DataMetricStageAudit: metrics.TrainReady = output } return metrics } func (server *server) handleDataOperationsBackfill(writer http.ResponseWriter, request *http.Request) { limit, issue := parseDatabaseBrowserPageQuery(request.URL.Query().Get("limit"), "limit", 100, 1, 500) if issue != nil { renderValidationError(writer, *issue) return } cursor, err := decodeRunSummaryCursor(request.URL.Query().Get("cursor")) if err != nil { renderValidationError(writer, validationIssue{Field: "cursor", Message: err.Error()}) return } cutoff := time.Now().UTC().Add(-30 * 24 * time.Hour) page, err := server.store.ListDataOperationsBackfillRuns( request.Context(), server.requestOrganizationID(request), cutoff.Format(time.RFC3339Nano), cursor, limit, ) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } processed := 0 skipped := 0 failures := 0 for runIndex := range page.Items { run := &page.Items[runIndex] for executionIndex := range run.NodeExecutions { execution := &run.NodeExecutions[executionIndex] if execution.Status != domain.NodeExecutionStatusCompleted { continue } legacy := execution.DataMetrics == nil if legacy { execution.DataMetrics = legacyMetricsFromExecution(*run, *execution) if execution.DataMetrics == nil { skipped++ continue } } if err := dataops.MaterializeLegacyNodeMetrics(request.Context(), server.store, run, execution); err != nil { failures++ continue } processed++ } } renderJSON(writer, http.StatusOK, map[string]any{ "windowDays": 30, "scannedRuns": len(page.Items), "processed": processed, "skipped": skipped, "failures": failures, "nextCursor": encodeRunSummaryCursor(page.NextCursor), "hasMore": page.HasMore, "completedAt": nowISO(), }) }