package dataops import ( "context" "crypto/sha256" "encoding/hex" "fmt" "strings" "time" "droboticflow/internal/domain" "droboticflow/internal/store" ) func normalizedQuality(value domain.DataMetricQuality) domain.DataMetricQuality { switch value { case domain.DataMetricQualityExact, domain.DataMetricQualityDerived, domain.DataMetricQualityEstimated: return value default: return domain.DataMetricQualityDerived } } func stableID(prefix string, value string) string { sum := sha256.Sum256([]byte(value)) return prefix + "-" + hex.EncodeToString(sum[:10]) } func firstNonEmpty(values ...string) string { for _, value := range values { if trimmed := strings.TrimSpace(value); trimmed != "" { return trimmed } } return "" } func dataRefFingerprint(ref domain.DatasetMetricRef, runID string, nodeID string) string { value := strings.Join([]string{ ref.DatasetVersionID, ref.DatasetID, ref.Fingerprint, ref.ManifestPath, ref.PayloadPath, ref.PartitionDir, ref.Split, ref.Name, }, "|") if strings.Trim(value, "|") == "" { value = runID + "|" + nodeID + "|" + ref.Name } sum := sha256.Sum256([]byte(value)) return hex.EncodeToString(sum[:]) } func metricRefFromDataRef(item domain.DataRefRecord) domain.DatasetMetricRef { return domain.DatasetMetricRef{ DatasetID: item.DatasetID, DatasetVersionID: item.DatasetVersionID, Name: item.ArtifactName, Split: item.Split, Fingerprint: item.Fingerprint, Kind: item.Kind, DatasetType: item.DatasetType, ManifestPath: item.ManifestPath, PayloadPath: item.PayloadPath, PartitionDir: item.PartitionDir, PartitionCount: item.PartitionCount, ItemCount: item.ItemCount, } } func int64Pointer(value int64) *int64 { return &value } func volumeWithItemCount(volume domain.DataVolume, itemCount int) domain.DataVolume { if volume.TrajectoryCount == nil && itemCount > 0 { volume.TrajectoryCount = int64Pointer(int64(itemCount)) } return volume } func anyString(value any) string { if value == nil { return "" } return strings.TrimSpace(fmt.Sprint(value)) } func anyInt(value any) int { switch typed := value.(type) { case int: return typed case int32: return int(typed) case int64: return int(typed) case float64: return int(typed) default: return 0 } } func metricRefFromArtifact(artifact domain.ExecutionArtifact) (domain.DatasetMetricRef, bool) { if artifact.Data == nil { return domain.DatasetMetricRef{}, false } raw, ok := artifact.Data["dataRef"].(map[string]any) if !ok || len(raw) == 0 { return domain.DatasetMetricRef{}, false } return domain.DatasetMetricRef{ Name: artifact.Name, DatasetID: anyString(raw["datasetId"]), DatasetVersionID: anyString(raw["datasetVersionId"]), Split: anyString(raw["split"]), Fingerprint: anyString(raw["fingerprint"]), Kind: anyString(raw["kind"]), DatasetType: anyString(raw["datasetType"]), ManifestPath: anyString(raw["manifestPath"]), PayloadPath: anyString(raw["payloadPath"]), PartitionDir: anyString(raw["partitionDir"]), PartitionCount: anyInt(raw["partitionCount"]), ItemCount: anyInt(raw["itemCount"]), }, true } func trainingInputsFromExecution(execution domain.NodeExecutionRecord, inputVolume domain.DataVolume) []domain.TrainingInputRecord { result := make([]domain.TrainingInputRecord, 0) seen := map[string]struct{}{} for _, artifact := range execution.InputArtifacts { ref, ok := metricRefFromArtifact(artifact) if !ok { continue } key := strings.Join([]string{ref.DatasetVersionID, ref.DatasetID, ref.ManifestPath, ref.PartitionDir, ref.Split}, "|") if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} result = append(result, domain.TrainingInputRecord{ DatasetID: ref.DatasetID, DatasetVersionID: ref.DatasetVersionID, Name: ref.Name, Split: ref.Split, Volume: volumeWithItemCount(inputVolume, ref.ItemCount), ManifestPath: ref.ManifestPath, PartitionDir: ref.PartitionDir, External: ref.DatasetVersionID == "", Quality: domain.DataMetricQualityDerived, }) } return result } func materializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord, markRun bool) error { if run == nil || execution == nil || execution.DataMetrics == nil { return nil } metrics := execution.DataMetrics if metrics.Stage == "" { return nil } task := (*domain.CollectionTaskRecord)(nil) if strings.TrimSpace(run.TaskID) != "" { item, err := dataStore.GetCollectionTask(ctx, run.TaskID) if err != nil { return err } task = item } refs := append([]domain.DatasetMetricRef(nil), metrics.DatasetRefs...) if len(refs) == 0 { for _, item := range execution.DataRefs { refs = append(refs, metricRefFromDataRef(item)) } } quality := normalizedQuality(metrics.Quality) for index := range refs { ref := &refs[index] fingerprint := firstNonEmpty(ref.Fingerprint, dataRefFingerprint(*ref, run.ID, execution.NodeID)) ref.Fingerprint = fingerprint ref.DatasetID = firstNonEmpty(ref.DatasetID, stableID("dataset", firstNonEmpty(ref.Name, ref.DatasetType, fingerprint))) ref.DatasetVersionID = firstNonEmpty(ref.DatasetVersionID, stableID("dataset-version", fingerprint)) volume := volumeWithItemCount(metrics.Output, ref.ItemCount) version := domain.DatasetVersionRecord{ ID: ref.DatasetVersionID, OrganizationID: run.OrganizationID, DatasetID: ref.DatasetID, TaskID: run.TaskID, ProjectID: run.ProjectID, Name: firstNonEmpty(ref.Name, ref.DatasetType, ref.DatasetID), Split: ref.Split, Fingerprint: fingerprint, Volume: volume, ManifestPath: ref.ManifestPath, PartitionDir: ref.PartitionDir, ProducerRunID: run.ID, ProducerNodeID: execution.NodeID, Quality: quality, CreatedAt: firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.UpdatedAt), } if err := dataStore.UpsertDatasetVersion(ctx, version); err != nil { return err } } trainingInputs := append([]domain.TrainingInputRecord(nil), metrics.TrainingInputs...) if metrics.Stage == domain.DataMetricStageTrain && len(trainingInputs) == 0 { trainingInputs = trainingInputsFromExecution(*execution, metrics.Input) } occurredAt := firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt) if parsed, err := time.Parse(time.RFC3339Nano, occurredAt); err != nil || parsed.IsZero() { occurredAt = time.Now().UTC().Format(time.RFC3339Nano) quality = domain.DataMetricQualityEstimated } else { occurredAt = parsed.UTC().Format(time.RFC3339Nano) } fact := domain.DataMetricFactRecord{ ID: stableID("data-fact", fmt.Sprintf("%s|%s|%d|%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage)), SourceKey: fmt.Sprintf("%s/%s/%d/%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage), OrganizationID: run.OrganizationID, ProjectID: run.ProjectID, ProjectName: run.ProjectName, TaskID: run.TaskID, WorkflowID: run.WorkflowID, WorkflowName: run.WorkflowName, RunID: run.ID, NodeID: execution.NodeID, NodeLabel: execution.NodeLabel, Attempt: execution.Attempt, Stage: metrics.Stage, Input: metrics.Input, Output: metrics.Output, Rejected: metrics.Rejected, TechnicalValid: metrics.TechnicalValid, TrainReady: metrics.TrainReady, DatasetRefs: refs, TrainingInputs: trainingInputs, RejectionReasons: metrics.RejectionReasons, Quality: quality, OccurredAt: occurredAt, RecordedAt: time.Now().UTC().Format(time.RFC3339Nano), } if task != nil { fact.TaskName = task.Name fact.RobotModel = task.RobotModel fact.Scene = task.Scene } if err := dataStore.UpsertDataMetricFact(ctx, fact); err != nil { return err } if !markRun { return nil } return dataStore.MarkNodeMetricsMaterialized(ctx, run.ID, execution.NodeID, execution.Attempt, time.Now().UTC().Format(time.RFC3339Nano)) } func MaterializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error { return materializeNodeMetrics(ctx, dataStore, run, execution, true) } func MaterializeLegacyNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error { return materializeNodeMetrics(ctx, dataStore, run, execution, false) } func ReconcileRunMetrics(ctx context.Context, dataStore store.Store, limit int) (int, error) { runs, err := dataStore.ListRunsNeedingMetricMaterialization(ctx, limit) if err != nil { return 0, err } materialized := 0 var firstErr error for runIndex := range runs { run := &runs[runIndex] for executionIndex := range run.NodeExecutions { execution := &run.NodeExecutions[executionIndex] if execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" { continue } if err := MaterializeNodeMetrics(ctx, dataStore, run, execution); err != nil { if firstErr == nil { firstErr = err } continue } materialized++ } } return materialized, firstErr }