droboticflow/internal/store/data_operations_common.go
2026-08-18 11:35:44 +08:00

411 lines
12 KiB
Go

package store
import (
"encoding/base64"
"fmt"
"sort"
"strings"
"time"
"droboticflow/internal/domain"
)
type volumeAccumulator struct {
trajectoryCount int64
durationSeconds float64
bytes int64
fileCount int64
hasTrajectory bool
hasDuration bool
hasBytes bool
hasFileCount bool
}
func dataMetricQualityRank(value domain.DataMetricQuality) int {
switch value {
case domain.DataMetricQualityExact:
return 3
case domain.DataMetricQualityDerived:
return 2
case domain.DataMetricQualityEstimated:
return 1
default:
return 0
}
}
func (item *volumeAccumulator) add(value domain.DataVolume) {
if value.TrajectoryCount != nil {
item.trajectoryCount += *value.TrajectoryCount
item.hasTrajectory = true
}
if value.DurationSeconds != nil {
item.durationSeconds += *value.DurationSeconds
item.hasDuration = true
}
if value.Bytes != nil {
item.bytes += *value.Bytes
item.hasBytes = true
}
if value.FileCount != nil {
item.fileCount += *value.FileCount
item.hasFileCount = true
}
}
func (item volumeAccumulator) value() domain.DataVolume {
result := domain.DataVolume{}
if item.hasTrajectory {
value := item.trajectoryCount
result.TrajectoryCount = &value
}
if item.hasDuration {
value := item.durationSeconds
result.DurationSeconds = &value
}
if item.hasBytes {
value := item.bytes
result.Bytes = &value
}
if item.hasFileCount {
value := item.fileCount
result.FileCount = &value
}
return result
}
type stageAccumulator struct {
input volumeAccumulator
output volumeAccumulator
rejected volumeAccumulator
technicalValid volumeAccumulator
trainReady volumeAccumulator
runs map[string]struct{}
}
func normalizedDataOperationsTimezone(value string) *time.Location {
name := strings.TrimSpace(value)
if name == "" {
name = domain.DefaultWorkflowScheduleTimezone
}
location, err := time.LoadLocation(name)
if err != nil {
location, _ = time.LoadLocation(domain.DefaultWorkflowScheduleTimezone)
}
if location == nil {
location = time.FixedZone("UTC+8", 8*60*60)
}
return location
}
func dataOperationsFactMatches(item domain.DataMetricFactRecord, organizationID string, query domain.DataOperationsQuery) bool {
if !organizationMatches(item.OrganizationID, organizationID) {
return false
}
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
return false
}
if query.TaskID != "" && item.TaskID != query.TaskID {
return false
}
if query.RobotModel != "" && !strings.EqualFold(item.RobotModel, query.RobotModel) {
return false
}
if query.Scene != "" && !strings.EqualFold(item.Scene, query.Scene) {
return false
}
if query.Stage != "" && item.Stage != query.Stage {
return false
}
if query.From != "" && item.OccurredAt < query.From {
return false
}
if query.To != "" && item.OccurredAt >= query.To {
return false
}
return true
}
func datasetVersionMatches(item domain.DatasetVersionRecord, organizationID string, query domain.DataOperationsQuery) bool {
if !organizationMatches(item.OrganizationID, organizationID) {
return false
}
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
return false
}
if query.TaskID != "" && item.TaskID != query.TaskID {
return false
}
if query.From != "" && item.CreatedAt < query.From {
return false
}
if query.To != "" && item.CreatedAt >= query.To {
return false
}
return true
}
func encodeDataOperationsCursor(occurredAt string, id string) string {
return base64.RawURLEncoding.EncodeToString([]byte(occurredAt + "\n" + id))
}
func decodeDataOperationsCursor(value string) (string, string, bool) {
if strings.TrimSpace(value) == "" {
return "", "", true
}
payload, err := base64.RawURLEncoding.DecodeString(value)
if err != nil {
return "", "", false
}
parts := strings.SplitN(string(payload), "\n", 2)
if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" {
return "", "", false
}
return parts[0], parts[1], true
}
func factsPage(items []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.DataMetricFactPage {
sort.Slice(items, func(left, right int) bool {
if items[left].OccurredAt == items[right].OccurredAt {
return items[left].ID > items[right].ID
}
return items[left].OccurredAt > items[right].OccurredAt
})
cursorTime, cursorID, validCursor := decodeDataOperationsCursor(query.Cursor)
if !validCursor {
return domain.DataMetricFactPage{Items: []domain.DataMetricFactRecord{}}
}
if cursorTime != "" {
filtered := items[:0]
for _, item := range items {
if item.OccurredAt < cursorTime || (item.OccurredAt == cursorTime && item.ID < cursorID) {
filtered = append(filtered, item)
}
}
items = filtered
}
limit := query.Limit
if limit <= 0 {
limit = 50
}
if limit > 100 {
limit = 100
}
page := domain.DataMetricFactPage{Items: items, HasMore: len(items) > limit}
if page.HasMore {
page.Items = items[:limit]
last := page.Items[len(page.Items)-1]
page.NextCursor = encodeDataOperationsCursor(last.OccurredAt, last.ID)
}
if page.Items == nil {
page.Items = []domain.DataMetricFactRecord{}
}
return page
}
func trainingUsageFromFacts(facts []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.TrainingUsagePage {
filtered := make([]domain.DataMetricFactRecord, 0)
for _, item := range facts {
if item.Stage == domain.DataMetricStageTrain {
filtered = append(filtered, item)
}
}
page := factsPage(filtered, query)
items := make([]domain.TrainingUsageRecord, 0, len(page.Items))
for _, fact := range page.Items {
items = append(items, domain.TrainingUsageRecord{
ID: fact.ID,
TaskID: fact.TaskID,
TaskName: fact.TaskName,
ProjectID: fact.ProjectID,
ProjectName: fact.ProjectName,
WorkflowID: fact.WorkflowID,
WorkflowName: fact.WorkflowName,
RunID: fact.RunID,
NodeID: fact.NodeID,
NodeLabel: fact.NodeLabel,
Model: fact.NodeLabel,
Inputs: fact.TrainingInputs,
Consumed: fact.Input,
Quality: fact.Quality,
OccurredAt: fact.OccurredAt,
})
}
return domain.TrainingUsagePage{Items: items, NextCursor: page.NextCursor, HasMore: page.HasMore}
}
func taskProgressPercent(target domain.DataVolume, collected domain.DataVolume) float64 {
switch {
case target.TrajectoryCount != nil && *target.TrajectoryCount > 0 && collected.TrajectoryCount != nil:
return minFloat(100, float64(*collected.TrajectoryCount)*100/float64(*target.TrajectoryCount))
case target.DurationSeconds != nil && *target.DurationSeconds > 0 && collected.DurationSeconds != nil:
return minFloat(100, *collected.DurationSeconds*100 / *target.DurationSeconds)
case target.Bytes != nil && *target.Bytes > 0 && collected.Bytes != nil:
return minFloat(100, float64(*collected.Bytes)*100/float64(*target.Bytes))
case target.FileCount != nil && *target.FileCount > 0 && collected.FileCount != nil:
return minFloat(100, float64(*collected.FileCount)*100/float64(*target.FileCount))
default:
return 0
}
}
func minFloat(left float64, right float64) float64 {
if left < right {
return left
}
return right
}
func BuildDataOperationsSummary(
facts []domain.DataMetricFactRecord,
tasks []domain.CollectionTaskRecord,
query domain.DataOperationsQuery,
) domain.DataOperationsSummary {
location := normalizedDataOperationsTimezone(query.Timezone)
accumulators := make(map[domain.DataMetricStage]*stageAccumulator, len(domain.DataMetricStages))
for _, stage := range domain.DataMetricStages {
accumulators[stage] = &stageAccumulator{runs: map[string]struct{}{}}
}
hourly := map[string]map[domain.DataMetricStage]*volumeAccumulator{}
reasons := map[string]int64{}
uniqueTraining := map[string]domain.DataVolume{}
var trainingConsumed volumeAccumulator
trainingRuns := map[string]struct{}{}
taskCollected := map[string]*volumeAccumulator{}
summary := domain.DataOperationsSummary{
From: query.From,
To: query.To,
Timezone: location.String(),
UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano),
Freshness: "live",
RejectionReasons: reasons,
}
for _, fact := range facts {
accumulator, ok := accumulators[fact.Stage]
if !ok {
summary.UnknownCount++
continue
}
accumulator.input.add(fact.Input)
accumulator.output.add(fact.Output)
accumulator.rejected.add(fact.Rejected)
accumulator.technicalValid.add(fact.TechnicalValid)
accumulator.trainReady.add(fact.TrainReady)
accumulator.runs[fact.RunID] = struct{}{}
summary.FactCount++
switch fact.Quality {
case domain.DataMetricQualityExact:
summary.ExactCount++
case domain.DataMetricQualityEstimated:
summary.EstimatedCount++
case domain.DataMetricQualityDerived:
summary.DerivedCount++
default:
summary.UnknownCount++
}
for reason, count := range fact.RejectionReasons {
reasons[reason] += count
}
if occurredAt, err := time.Parse(time.RFC3339Nano, fact.OccurredAt); err == nil {
hour := occurredAt.In(location).Format("2006-01-02T15:00:00-07:00")
if hourly[hour] == nil {
hourly[hour] = map[domain.DataMetricStage]*volumeAccumulator{}
}
if hourly[hour][fact.Stage] == nil {
hourly[hour][fact.Stage] = &volumeAccumulator{}
}
hourly[hour][fact.Stage].add(fact.Output)
}
if fact.Stage == domain.DataMetricStageCollect && fact.TaskID != "" {
if taskCollected[fact.TaskID] == nil {
taskCollected[fact.TaskID] = &volumeAccumulator{}
}
taskCollected[fact.TaskID].add(fact.Output)
}
if fact.Stage == domain.DataMetricStageTrain {
trainingConsumed.add(fact.Input)
trainingRuns[fact.RunID] = struct{}{}
for _, input := range fact.TrainingInputs {
key := strings.TrimSpace(input.DatasetVersionID)
if key == "" {
key = strings.TrimSpace(input.DatasetID) + "|" + strings.TrimSpace(input.ManifestPath) + "|" + strings.TrimSpace(input.Split)
}
if key == "||" || key == "" {
key = fmt.Sprintf("%s|%s", fact.RunID, fact.NodeID)
}
if _, exists := uniqueTraining[key]; !exists {
uniqueTraining[key] = input.Volume
}
}
}
}
summary.Stages = make([]domain.DataStageSummary, 0, len(domain.DataMetricStages))
for _, stage := range domain.DataMetricStages {
item := accumulators[stage]
summary.Stages = append(summary.Stages, domain.DataStageSummary{
Stage: stage,
Input: item.input.value(),
Output: item.output.value(),
Rejected: item.rejected.value(),
TechnicalValid: item.technicalValid.value(),
TrainReady: item.trainReady.value(),
RunCount: int64(len(item.runs)),
})
}
var uniqueAccumulator volumeAccumulator
for _, volume := range uniqueTraining {
uniqueAccumulator.add(volume)
}
summary.Training = domain.DataOperationsTrainingSummary{
Consumed: trainingConsumed.value(),
Unique: uniqueAccumulator.value(),
RunCount: int64(len(trainingRuns)),
}
hours := make([]string, 0, len(hourly))
for hour := range hourly {
hours = append(hours, hour)
}
sort.Strings(hours)
summary.Hourly = make([]domain.DataOperationsHourlyPoint, 0, len(hours))
for _, hour := range hours {
point := domain.DataOperationsHourlyPoint{Hour: hour, Stages: map[domain.DataMetricStage]domain.DataVolume{}}
for stage, value := range hourly[hour] {
point.Stages[stage] = value.value()
}
summary.Hourly = append(summary.Hourly, point)
}
summary.Tasks = make([]domain.CollectionTaskProgress, 0)
for _, task := range tasks {
if query.ProjectID != "" && task.ProjectID != query.ProjectID {
continue
}
if query.TaskID != "" && task.ID != query.TaskID {
continue
}
if query.RobotModel != "" && !strings.EqualFold(task.RobotModel, query.RobotModel) {
continue
}
if query.Scene != "" && !strings.EqualFold(task.Scene, query.Scene) {
continue
}
collected := domain.DataVolume{}
if taskCollected[task.ID] != nil {
collected = taskCollected[task.ID].value()
}
summary.Tasks = append(summary.Tasks, domain.CollectionTaskProgress{
TaskID: task.ID,
TaskName: task.Name,
Status: task.Status,
Targets: task.Targets,
Collected: collected,
Percent: taskProgressPercent(task.Targets, collected),
})
}
sort.Slice(summary.Tasks, func(left, right int) bool { return summary.Tasks[left].TaskName < summary.Tasks[right].TaskName })
return summary
}