forked from longtao.wu/droboticflow
293 lines
9.3 KiB
Go
293 lines
9.3 KiB
Go
package dataops
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
func normalizedQuality(value domain.DataMetricQuality) domain.DataMetricQuality {
|
|
switch value {
|
|
case domain.DataMetricQualityExact, domain.DataMetricQualityDerived, domain.DataMetricQualityEstimated:
|
|
return value
|
|
default:
|
|
return domain.DataMetricQualityDerived
|
|
}
|
|
}
|
|
|
|
func stableID(prefix string, value string) string {
|
|
sum := sha256.Sum256([]byte(value))
|
|
return prefix + "-" + hex.EncodeToString(sum[:10])
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
|
return trimmed
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func dataRefFingerprint(ref domain.DatasetMetricRef, runID string, nodeID string) string {
|
|
value := strings.Join([]string{
|
|
ref.DatasetVersionID,
|
|
ref.DatasetID,
|
|
ref.Fingerprint,
|
|
ref.ManifestPath,
|
|
ref.PayloadPath,
|
|
ref.PartitionDir,
|
|
ref.Split,
|
|
ref.Name,
|
|
}, "|")
|
|
if strings.Trim(value, "|") == "" {
|
|
value = runID + "|" + nodeID + "|" + ref.Name
|
|
}
|
|
sum := sha256.Sum256([]byte(value))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func metricRefFromDataRef(item domain.DataRefRecord) domain.DatasetMetricRef {
|
|
return domain.DatasetMetricRef{
|
|
DatasetID: item.DatasetID,
|
|
DatasetVersionID: item.DatasetVersionID,
|
|
Name: item.ArtifactName,
|
|
Split: item.Split,
|
|
Fingerprint: item.Fingerprint,
|
|
Kind: item.Kind,
|
|
DatasetType: item.DatasetType,
|
|
ManifestPath: item.ManifestPath,
|
|
PayloadPath: item.PayloadPath,
|
|
PartitionDir: item.PartitionDir,
|
|
PartitionCount: item.PartitionCount,
|
|
ItemCount: item.ItemCount,
|
|
}
|
|
}
|
|
|
|
func int64Pointer(value int64) *int64 {
|
|
return &value
|
|
}
|
|
|
|
func volumeWithItemCount(volume domain.DataVolume, itemCount int) domain.DataVolume {
|
|
if volume.TrajectoryCount == nil && itemCount > 0 {
|
|
volume.TrajectoryCount = int64Pointer(int64(itemCount))
|
|
}
|
|
return volume
|
|
}
|
|
|
|
func anyString(value any) string {
|
|
if value == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(fmt.Sprint(value))
|
|
}
|
|
|
|
func anyInt(value any) int {
|
|
switch typed := value.(type) {
|
|
case int:
|
|
return typed
|
|
case int32:
|
|
return int(typed)
|
|
case int64:
|
|
return int(typed)
|
|
case float64:
|
|
return int(typed)
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func metricRefFromArtifact(artifact domain.ExecutionArtifact) (domain.DatasetMetricRef, bool) {
|
|
if artifact.Data == nil {
|
|
return domain.DatasetMetricRef{}, false
|
|
}
|
|
raw, ok := artifact.Data["dataRef"].(map[string]any)
|
|
if !ok || len(raw) == 0 {
|
|
return domain.DatasetMetricRef{}, false
|
|
}
|
|
return domain.DatasetMetricRef{
|
|
Name: artifact.Name,
|
|
DatasetID: anyString(raw["datasetId"]),
|
|
DatasetVersionID: anyString(raw["datasetVersionId"]),
|
|
Split: anyString(raw["split"]),
|
|
Fingerprint: anyString(raw["fingerprint"]),
|
|
Kind: anyString(raw["kind"]),
|
|
DatasetType: anyString(raw["datasetType"]),
|
|
ManifestPath: anyString(raw["manifestPath"]),
|
|
PayloadPath: anyString(raw["payloadPath"]),
|
|
PartitionDir: anyString(raw["partitionDir"]),
|
|
PartitionCount: anyInt(raw["partitionCount"]),
|
|
ItemCount: anyInt(raw["itemCount"]),
|
|
}, true
|
|
}
|
|
|
|
func trainingInputsFromExecution(execution domain.NodeExecutionRecord, inputVolume domain.DataVolume) []domain.TrainingInputRecord {
|
|
result := make([]domain.TrainingInputRecord, 0)
|
|
seen := map[string]struct{}{}
|
|
for _, artifact := range execution.InputArtifacts {
|
|
ref, ok := metricRefFromArtifact(artifact)
|
|
if !ok {
|
|
continue
|
|
}
|
|
key := strings.Join([]string{ref.DatasetVersionID, ref.DatasetID, ref.ManifestPath, ref.PartitionDir, ref.Split}, "|")
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
result = append(result, domain.TrainingInputRecord{
|
|
DatasetID: ref.DatasetID,
|
|
DatasetVersionID: ref.DatasetVersionID,
|
|
Name: ref.Name,
|
|
Split: ref.Split,
|
|
Volume: volumeWithItemCount(inputVolume, ref.ItemCount),
|
|
ManifestPath: ref.ManifestPath,
|
|
PartitionDir: ref.PartitionDir,
|
|
External: ref.DatasetVersionID == "",
|
|
Quality: domain.DataMetricQualityDerived,
|
|
})
|
|
}
|
|
return result
|
|
}
|
|
|
|
func materializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord, markRun bool) error {
|
|
if run == nil || execution == nil || execution.DataMetrics == nil {
|
|
return nil
|
|
}
|
|
metrics := execution.DataMetrics
|
|
if metrics.Stage == "" {
|
|
return nil
|
|
}
|
|
task := (*domain.CollectionTaskRecord)(nil)
|
|
if strings.TrimSpace(run.TaskID) != "" {
|
|
item, err := dataStore.GetCollectionTask(ctx, run.TaskID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
task = item
|
|
}
|
|
refs := append([]domain.DatasetMetricRef(nil), metrics.DatasetRefs...)
|
|
if len(refs) == 0 {
|
|
for _, item := range execution.DataRefs {
|
|
refs = append(refs, metricRefFromDataRef(item))
|
|
}
|
|
}
|
|
quality := normalizedQuality(metrics.Quality)
|
|
for index := range refs {
|
|
ref := &refs[index]
|
|
fingerprint := firstNonEmpty(ref.Fingerprint, dataRefFingerprint(*ref, run.ID, execution.NodeID))
|
|
ref.Fingerprint = fingerprint
|
|
ref.DatasetID = firstNonEmpty(ref.DatasetID, stableID("dataset", firstNonEmpty(ref.Name, ref.DatasetType, fingerprint)))
|
|
ref.DatasetVersionID = firstNonEmpty(ref.DatasetVersionID, stableID("dataset-version", fingerprint))
|
|
volume := volumeWithItemCount(metrics.Output, ref.ItemCount)
|
|
version := domain.DatasetVersionRecord{
|
|
ID: ref.DatasetVersionID,
|
|
OrganizationID: run.OrganizationID,
|
|
DatasetID: ref.DatasetID,
|
|
TaskID: run.TaskID,
|
|
ProjectID: run.ProjectID,
|
|
Name: firstNonEmpty(ref.Name, ref.DatasetType, ref.DatasetID),
|
|
Split: ref.Split,
|
|
Fingerprint: fingerprint,
|
|
Volume: volume,
|
|
ManifestPath: ref.ManifestPath,
|
|
PartitionDir: ref.PartitionDir,
|
|
ProducerRunID: run.ID,
|
|
ProducerNodeID: execution.NodeID,
|
|
Quality: quality,
|
|
CreatedAt: firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.UpdatedAt),
|
|
}
|
|
if err := dataStore.UpsertDatasetVersion(ctx, version); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
trainingInputs := append([]domain.TrainingInputRecord(nil), metrics.TrainingInputs...)
|
|
if metrics.Stage == domain.DataMetricStageTrain && len(trainingInputs) == 0 {
|
|
trainingInputs = trainingInputsFromExecution(*execution, metrics.Input)
|
|
}
|
|
occurredAt := firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt)
|
|
if parsed, err := time.Parse(time.RFC3339Nano, occurredAt); err != nil || parsed.IsZero() {
|
|
occurredAt = time.Now().UTC().Format(time.RFC3339Nano)
|
|
quality = domain.DataMetricQualityEstimated
|
|
} else {
|
|
occurredAt = parsed.UTC().Format(time.RFC3339Nano)
|
|
}
|
|
fact := domain.DataMetricFactRecord{
|
|
ID: stableID("data-fact", fmt.Sprintf("%s|%s|%d|%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage)),
|
|
SourceKey: fmt.Sprintf("%s/%s/%d/%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage),
|
|
OrganizationID: run.OrganizationID,
|
|
ProjectID: run.ProjectID,
|
|
ProjectName: run.ProjectName,
|
|
TaskID: run.TaskID,
|
|
WorkflowID: run.WorkflowID,
|
|
WorkflowName: run.WorkflowName,
|
|
RunID: run.ID,
|
|
NodeID: execution.NodeID,
|
|
NodeLabel: execution.NodeLabel,
|
|
Attempt: execution.Attempt,
|
|
Stage: metrics.Stage,
|
|
Input: metrics.Input,
|
|
Output: metrics.Output,
|
|
Rejected: metrics.Rejected,
|
|
TechnicalValid: metrics.TechnicalValid,
|
|
TrainReady: metrics.TrainReady,
|
|
DatasetRefs: refs,
|
|
TrainingInputs: trainingInputs,
|
|
RejectionReasons: metrics.RejectionReasons,
|
|
Quality: quality,
|
|
OccurredAt: occurredAt,
|
|
RecordedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
|
}
|
|
if task != nil {
|
|
fact.TaskName = task.Name
|
|
fact.RobotModel = task.RobotModel
|
|
fact.Scene = task.Scene
|
|
}
|
|
if err := dataStore.UpsertDataMetricFact(ctx, fact); err != nil {
|
|
return err
|
|
}
|
|
if !markRun {
|
|
return nil
|
|
}
|
|
return dataStore.MarkNodeMetricsMaterialized(ctx, run.ID, execution.NodeID, execution.Attempt, time.Now().UTC().Format(time.RFC3339Nano))
|
|
}
|
|
|
|
func MaterializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
|
|
return materializeNodeMetrics(ctx, dataStore, run, execution, true)
|
|
}
|
|
|
|
func MaterializeLegacyNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
|
|
return materializeNodeMetrics(ctx, dataStore, run, execution, false)
|
|
}
|
|
|
|
func ReconcileRunMetrics(ctx context.Context, dataStore store.Store, limit int) (int, error) {
|
|
runs, err := dataStore.ListRunsNeedingMetricMaterialization(ctx, limit)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
materialized := 0
|
|
var firstErr error
|
|
for runIndex := range runs {
|
|
run := &runs[runIndex]
|
|
for executionIndex := range run.NodeExecutions {
|
|
execution := &run.NodeExecutions[executionIndex]
|
|
if execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" {
|
|
continue
|
|
}
|
|
if err := MaterializeNodeMetrics(ctx, dataStore, run, execution); err != nil {
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
continue
|
|
}
|
|
materialized++
|
|
}
|
|
}
|
|
return materialized, firstErr
|
|
}
|