droboticflow/internal/dataops/materializer.go
2026-07-23 21:11:15 +08:00

293 lines
9.3 KiB
Go

package dataops
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func normalizedQuality(value domain.DataMetricQuality) domain.DataMetricQuality {
switch value {
case domain.DataMetricQualityExact, domain.DataMetricQualityDerived, domain.DataMetricQualityEstimated:
return value
default:
return domain.DataMetricQualityDerived
}
}
func stableID(prefix string, value string) string {
sum := sha256.Sum256([]byte(value))
return prefix + "-" + hex.EncodeToString(sum[:10])
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func dataRefFingerprint(ref domain.DatasetMetricRef, runID string, nodeID string) string {
value := strings.Join([]string{
ref.DatasetVersionID,
ref.DatasetID,
ref.Fingerprint,
ref.ManifestPath,
ref.PayloadPath,
ref.PartitionDir,
ref.Split,
ref.Name,
}, "|")
if strings.Trim(value, "|") == "" {
value = runID + "|" + nodeID + "|" + ref.Name
}
sum := sha256.Sum256([]byte(value))
return hex.EncodeToString(sum[:])
}
func metricRefFromDataRef(item domain.DataRefRecord) domain.DatasetMetricRef {
return domain.DatasetMetricRef{
DatasetID: item.DatasetID,
DatasetVersionID: item.DatasetVersionID,
Name: item.ArtifactName,
Split: item.Split,
Fingerprint: item.Fingerprint,
Kind: item.Kind,
DatasetType: item.DatasetType,
ManifestPath: item.ManifestPath,
PayloadPath: item.PayloadPath,
PartitionDir: item.PartitionDir,
PartitionCount: item.PartitionCount,
ItemCount: item.ItemCount,
}
}
func int64Pointer(value int64) *int64 {
return &value
}
func volumeWithItemCount(volume domain.DataVolume, itemCount int) domain.DataVolume {
if volume.TrajectoryCount == nil && itemCount > 0 {
volume.TrajectoryCount = int64Pointer(int64(itemCount))
}
return volume
}
func anyString(value any) string {
if value == nil {
return ""
}
return strings.TrimSpace(fmt.Sprint(value))
}
func anyInt(value any) int {
switch typed := value.(type) {
case int:
return typed
case int32:
return int(typed)
case int64:
return int(typed)
case float64:
return int(typed)
default:
return 0
}
}
func metricRefFromArtifact(artifact domain.ExecutionArtifact) (domain.DatasetMetricRef, bool) {
if artifact.Data == nil {
return domain.DatasetMetricRef{}, false
}
raw, ok := artifact.Data["dataRef"].(map[string]any)
if !ok || len(raw) == 0 {
return domain.DatasetMetricRef{}, false
}
return domain.DatasetMetricRef{
Name: artifact.Name,
DatasetID: anyString(raw["datasetId"]),
DatasetVersionID: anyString(raw["datasetVersionId"]),
Split: anyString(raw["split"]),
Fingerprint: anyString(raw["fingerprint"]),
Kind: anyString(raw["kind"]),
DatasetType: anyString(raw["datasetType"]),
ManifestPath: anyString(raw["manifestPath"]),
PayloadPath: anyString(raw["payloadPath"]),
PartitionDir: anyString(raw["partitionDir"]),
PartitionCount: anyInt(raw["partitionCount"]),
ItemCount: anyInt(raw["itemCount"]),
}, true
}
func trainingInputsFromExecution(execution domain.NodeExecutionRecord, inputVolume domain.DataVolume) []domain.TrainingInputRecord {
result := make([]domain.TrainingInputRecord, 0)
seen := map[string]struct{}{}
for _, artifact := range execution.InputArtifacts {
ref, ok := metricRefFromArtifact(artifact)
if !ok {
continue
}
key := strings.Join([]string{ref.DatasetVersionID, ref.DatasetID, ref.ManifestPath, ref.PartitionDir, ref.Split}, "|")
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
result = append(result, domain.TrainingInputRecord{
DatasetID: ref.DatasetID,
DatasetVersionID: ref.DatasetVersionID,
Name: ref.Name,
Split: ref.Split,
Volume: volumeWithItemCount(inputVolume, ref.ItemCount),
ManifestPath: ref.ManifestPath,
PartitionDir: ref.PartitionDir,
External: ref.DatasetVersionID == "",
Quality: domain.DataMetricQualityDerived,
})
}
return result
}
func materializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord, markRun bool) error {
if run == nil || execution == nil || execution.DataMetrics == nil {
return nil
}
metrics := execution.DataMetrics
if metrics.Stage == "" {
return nil
}
task := (*domain.CollectionTaskRecord)(nil)
if strings.TrimSpace(run.TaskID) != "" {
item, err := dataStore.GetCollectionTask(ctx, run.TaskID)
if err != nil {
return err
}
task = item
}
refs := append([]domain.DatasetMetricRef(nil), metrics.DatasetRefs...)
if len(refs) == 0 {
for _, item := range execution.DataRefs {
refs = append(refs, metricRefFromDataRef(item))
}
}
quality := normalizedQuality(metrics.Quality)
for index := range refs {
ref := &refs[index]
fingerprint := firstNonEmpty(ref.Fingerprint, dataRefFingerprint(*ref, run.ID, execution.NodeID))
ref.Fingerprint = fingerprint
ref.DatasetID = firstNonEmpty(ref.DatasetID, stableID("dataset", firstNonEmpty(ref.Name, ref.DatasetType, fingerprint)))
ref.DatasetVersionID = firstNonEmpty(ref.DatasetVersionID, stableID("dataset-version", fingerprint))
volume := volumeWithItemCount(metrics.Output, ref.ItemCount)
version := domain.DatasetVersionRecord{
ID: ref.DatasetVersionID,
OrganizationID: run.OrganizationID,
DatasetID: ref.DatasetID,
TaskID: run.TaskID,
ProjectID: run.ProjectID,
Name: firstNonEmpty(ref.Name, ref.DatasetType, ref.DatasetID),
Split: ref.Split,
Fingerprint: fingerprint,
Volume: volume,
ManifestPath: ref.ManifestPath,
PartitionDir: ref.PartitionDir,
ProducerRunID: run.ID,
ProducerNodeID: execution.NodeID,
Quality: quality,
CreatedAt: firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.UpdatedAt),
}
if err := dataStore.UpsertDatasetVersion(ctx, version); err != nil {
return err
}
}
trainingInputs := append([]domain.TrainingInputRecord(nil), metrics.TrainingInputs...)
if metrics.Stage == domain.DataMetricStageTrain && len(trainingInputs) == 0 {
trainingInputs = trainingInputsFromExecution(*execution, metrics.Input)
}
occurredAt := firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt)
if parsed, err := time.Parse(time.RFC3339Nano, occurredAt); err != nil || parsed.IsZero() {
occurredAt = time.Now().UTC().Format(time.RFC3339Nano)
quality = domain.DataMetricQualityEstimated
} else {
occurredAt = parsed.UTC().Format(time.RFC3339Nano)
}
fact := domain.DataMetricFactRecord{
ID: stableID("data-fact", fmt.Sprintf("%s|%s|%d|%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage)),
SourceKey: fmt.Sprintf("%s/%s/%d/%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage),
OrganizationID: run.OrganizationID,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
TaskID: run.TaskID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
RunID: run.ID,
NodeID: execution.NodeID,
NodeLabel: execution.NodeLabel,
Attempt: execution.Attempt,
Stage: metrics.Stage,
Input: metrics.Input,
Output: metrics.Output,
Rejected: metrics.Rejected,
TechnicalValid: metrics.TechnicalValid,
TrainReady: metrics.TrainReady,
DatasetRefs: refs,
TrainingInputs: trainingInputs,
RejectionReasons: metrics.RejectionReasons,
Quality: quality,
OccurredAt: occurredAt,
RecordedAt: time.Now().UTC().Format(time.RFC3339Nano),
}
if task != nil {
fact.TaskName = task.Name
fact.RobotModel = task.RobotModel
fact.Scene = task.Scene
}
if err := dataStore.UpsertDataMetricFact(ctx, fact); err != nil {
return err
}
if !markRun {
return nil
}
return dataStore.MarkNodeMetricsMaterialized(ctx, run.ID, execution.NodeID, execution.Attempt, time.Now().UTC().Format(time.RFC3339Nano))
}
func MaterializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
return materializeNodeMetrics(ctx, dataStore, run, execution, true)
}
func MaterializeLegacyNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
return materializeNodeMetrics(ctx, dataStore, run, execution, false)
}
func ReconcileRunMetrics(ctx context.Context, dataStore store.Store, limit int) (int, error) {
runs, err := dataStore.ListRunsNeedingMetricMaterialization(ctx, limit)
if err != nil {
return 0, err
}
materialized := 0
var firstErr error
for runIndex := range runs {
run := &runs[runIndex]
for executionIndex := range run.NodeExecutions {
execution := &run.NodeExecutions[executionIndex]
if execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" {
continue
}
if err := MaterializeNodeMetrics(ctx, dataStore, run, execution); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
materialized++
}
}
return materialized, firstErr
}