forked from longtao.wu/droboticflow
411 lines
12 KiB
Go
411 lines
12 KiB
Go
package store
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
type volumeAccumulator struct {
|
|
trajectoryCount int64
|
|
durationSeconds float64
|
|
bytes int64
|
|
fileCount int64
|
|
hasTrajectory bool
|
|
hasDuration bool
|
|
hasBytes bool
|
|
hasFileCount bool
|
|
}
|
|
|
|
func dataMetricQualityRank(value domain.DataMetricQuality) int {
|
|
switch value {
|
|
case domain.DataMetricQualityExact:
|
|
return 3
|
|
case domain.DataMetricQualityDerived:
|
|
return 2
|
|
case domain.DataMetricQualityEstimated:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func (item *volumeAccumulator) add(value domain.DataVolume) {
|
|
if value.TrajectoryCount != nil {
|
|
item.trajectoryCount += *value.TrajectoryCount
|
|
item.hasTrajectory = true
|
|
}
|
|
if value.DurationSeconds != nil {
|
|
item.durationSeconds += *value.DurationSeconds
|
|
item.hasDuration = true
|
|
}
|
|
if value.Bytes != nil {
|
|
item.bytes += *value.Bytes
|
|
item.hasBytes = true
|
|
}
|
|
if value.FileCount != nil {
|
|
item.fileCount += *value.FileCount
|
|
item.hasFileCount = true
|
|
}
|
|
}
|
|
|
|
func (item volumeAccumulator) value() domain.DataVolume {
|
|
result := domain.DataVolume{}
|
|
if item.hasTrajectory {
|
|
value := item.trajectoryCount
|
|
result.TrajectoryCount = &value
|
|
}
|
|
if item.hasDuration {
|
|
value := item.durationSeconds
|
|
result.DurationSeconds = &value
|
|
}
|
|
if item.hasBytes {
|
|
value := item.bytes
|
|
result.Bytes = &value
|
|
}
|
|
if item.hasFileCount {
|
|
value := item.fileCount
|
|
result.FileCount = &value
|
|
}
|
|
return result
|
|
}
|
|
|
|
type stageAccumulator struct {
|
|
input volumeAccumulator
|
|
output volumeAccumulator
|
|
rejected volumeAccumulator
|
|
technicalValid volumeAccumulator
|
|
trainReady volumeAccumulator
|
|
runs map[string]struct{}
|
|
}
|
|
|
|
func normalizedDataOperationsTimezone(value string) *time.Location {
|
|
name := strings.TrimSpace(value)
|
|
if name == "" {
|
|
name = domain.DefaultWorkflowScheduleTimezone
|
|
}
|
|
location, err := time.LoadLocation(name)
|
|
if err != nil {
|
|
location, _ = time.LoadLocation(domain.DefaultWorkflowScheduleTimezone)
|
|
}
|
|
if location == nil {
|
|
location = time.FixedZone("UTC+8", 8*60*60)
|
|
}
|
|
return location
|
|
}
|
|
|
|
func dataOperationsFactMatches(item domain.DataMetricFactRecord, organizationID string, query domain.DataOperationsQuery) bool {
|
|
if !organizationMatches(item.OrganizationID, organizationID) {
|
|
return false
|
|
}
|
|
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
|
|
return false
|
|
}
|
|
if query.TaskID != "" && item.TaskID != query.TaskID {
|
|
return false
|
|
}
|
|
if query.RobotModel != "" && !strings.EqualFold(item.RobotModel, query.RobotModel) {
|
|
return false
|
|
}
|
|
if query.Scene != "" && !strings.EqualFold(item.Scene, query.Scene) {
|
|
return false
|
|
}
|
|
if query.Stage != "" && item.Stage != query.Stage {
|
|
return false
|
|
}
|
|
if query.From != "" && item.OccurredAt < query.From {
|
|
return false
|
|
}
|
|
if query.To != "" && item.OccurredAt >= query.To {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func datasetVersionMatches(item domain.DatasetVersionRecord, organizationID string, query domain.DataOperationsQuery) bool {
|
|
if !organizationMatches(item.OrganizationID, organizationID) {
|
|
return false
|
|
}
|
|
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
|
|
return false
|
|
}
|
|
if query.TaskID != "" && item.TaskID != query.TaskID {
|
|
return false
|
|
}
|
|
if query.From != "" && item.CreatedAt < query.From {
|
|
return false
|
|
}
|
|
if query.To != "" && item.CreatedAt >= query.To {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func encodeDataOperationsCursor(occurredAt string, id string) string {
|
|
return base64.RawURLEncoding.EncodeToString([]byte(occurredAt + "\n" + id))
|
|
}
|
|
|
|
func decodeDataOperationsCursor(value string) (string, string, bool) {
|
|
if strings.TrimSpace(value) == "" {
|
|
return "", "", true
|
|
}
|
|
payload, err := base64.RawURLEncoding.DecodeString(value)
|
|
if err != nil {
|
|
return "", "", false
|
|
}
|
|
parts := strings.SplitN(string(payload), "\n", 2)
|
|
if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" {
|
|
return "", "", false
|
|
}
|
|
return parts[0], parts[1], true
|
|
}
|
|
|
|
func factsPage(items []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.DataMetricFactPage {
|
|
sort.Slice(items, func(left, right int) bool {
|
|
if items[left].OccurredAt == items[right].OccurredAt {
|
|
return items[left].ID > items[right].ID
|
|
}
|
|
return items[left].OccurredAt > items[right].OccurredAt
|
|
})
|
|
cursorTime, cursorID, validCursor := decodeDataOperationsCursor(query.Cursor)
|
|
if !validCursor {
|
|
return domain.DataMetricFactPage{Items: []domain.DataMetricFactRecord{}}
|
|
}
|
|
if cursorTime != "" {
|
|
filtered := items[:0]
|
|
for _, item := range items {
|
|
if item.OccurredAt < cursorTime || (item.OccurredAt == cursorTime && item.ID < cursorID) {
|
|
filtered = append(filtered, item)
|
|
}
|
|
}
|
|
items = filtered
|
|
}
|
|
limit := query.Limit
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
if limit > 100 {
|
|
limit = 100
|
|
}
|
|
page := domain.DataMetricFactPage{Items: items, HasMore: len(items) > limit}
|
|
if page.HasMore {
|
|
page.Items = items[:limit]
|
|
last := page.Items[len(page.Items)-1]
|
|
page.NextCursor = encodeDataOperationsCursor(last.OccurredAt, last.ID)
|
|
}
|
|
if page.Items == nil {
|
|
page.Items = []domain.DataMetricFactRecord{}
|
|
}
|
|
return page
|
|
}
|
|
|
|
func trainingUsageFromFacts(facts []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.TrainingUsagePage {
|
|
filtered := make([]domain.DataMetricFactRecord, 0)
|
|
for _, item := range facts {
|
|
if item.Stage == domain.DataMetricStageTrain {
|
|
filtered = append(filtered, item)
|
|
}
|
|
}
|
|
page := factsPage(filtered, query)
|
|
items := make([]domain.TrainingUsageRecord, 0, len(page.Items))
|
|
for _, fact := range page.Items {
|
|
items = append(items, domain.TrainingUsageRecord{
|
|
ID: fact.ID,
|
|
TaskID: fact.TaskID,
|
|
TaskName: fact.TaskName,
|
|
ProjectID: fact.ProjectID,
|
|
ProjectName: fact.ProjectName,
|
|
WorkflowID: fact.WorkflowID,
|
|
WorkflowName: fact.WorkflowName,
|
|
RunID: fact.RunID,
|
|
NodeID: fact.NodeID,
|
|
NodeLabel: fact.NodeLabel,
|
|
Model: fact.NodeLabel,
|
|
Inputs: fact.TrainingInputs,
|
|
Consumed: fact.Input,
|
|
Quality: fact.Quality,
|
|
OccurredAt: fact.OccurredAt,
|
|
})
|
|
}
|
|
return domain.TrainingUsagePage{Items: items, NextCursor: page.NextCursor, HasMore: page.HasMore}
|
|
}
|
|
|
|
func taskProgressPercent(target domain.DataVolume, collected domain.DataVolume) float64 {
|
|
switch {
|
|
case target.TrajectoryCount != nil && *target.TrajectoryCount > 0 && collected.TrajectoryCount != nil:
|
|
return minFloat(100, float64(*collected.TrajectoryCount)*100/float64(*target.TrajectoryCount))
|
|
case target.DurationSeconds != nil && *target.DurationSeconds > 0 && collected.DurationSeconds != nil:
|
|
return minFloat(100, *collected.DurationSeconds*100 / *target.DurationSeconds)
|
|
case target.Bytes != nil && *target.Bytes > 0 && collected.Bytes != nil:
|
|
return minFloat(100, float64(*collected.Bytes)*100/float64(*target.Bytes))
|
|
case target.FileCount != nil && *target.FileCount > 0 && collected.FileCount != nil:
|
|
return minFloat(100, float64(*collected.FileCount)*100/float64(*target.FileCount))
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func minFloat(left float64, right float64) float64 {
|
|
if left < right {
|
|
return left
|
|
}
|
|
return right
|
|
}
|
|
|
|
func BuildDataOperationsSummary(
|
|
facts []domain.DataMetricFactRecord,
|
|
tasks []domain.CollectionTaskRecord,
|
|
query domain.DataOperationsQuery,
|
|
) domain.DataOperationsSummary {
|
|
location := normalizedDataOperationsTimezone(query.Timezone)
|
|
accumulators := make(map[domain.DataMetricStage]*stageAccumulator, len(domain.DataMetricStages))
|
|
for _, stage := range domain.DataMetricStages {
|
|
accumulators[stage] = &stageAccumulator{runs: map[string]struct{}{}}
|
|
}
|
|
hourly := map[string]map[domain.DataMetricStage]*volumeAccumulator{}
|
|
reasons := map[string]int64{}
|
|
uniqueTraining := map[string]domain.DataVolume{}
|
|
var trainingConsumed volumeAccumulator
|
|
trainingRuns := map[string]struct{}{}
|
|
taskCollected := map[string]*volumeAccumulator{}
|
|
summary := domain.DataOperationsSummary{
|
|
From: query.From,
|
|
To: query.To,
|
|
Timezone: location.String(),
|
|
UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
|
Freshness: "live",
|
|
RejectionReasons: reasons,
|
|
}
|
|
|
|
for _, fact := range facts {
|
|
accumulator, ok := accumulators[fact.Stage]
|
|
if !ok {
|
|
summary.UnknownCount++
|
|
continue
|
|
}
|
|
accumulator.input.add(fact.Input)
|
|
accumulator.output.add(fact.Output)
|
|
accumulator.rejected.add(fact.Rejected)
|
|
accumulator.technicalValid.add(fact.TechnicalValid)
|
|
accumulator.trainReady.add(fact.TrainReady)
|
|
accumulator.runs[fact.RunID] = struct{}{}
|
|
summary.FactCount++
|
|
switch fact.Quality {
|
|
case domain.DataMetricQualityExact:
|
|
summary.ExactCount++
|
|
case domain.DataMetricQualityEstimated:
|
|
summary.EstimatedCount++
|
|
case domain.DataMetricQualityDerived:
|
|
summary.DerivedCount++
|
|
default:
|
|
summary.UnknownCount++
|
|
}
|
|
for reason, count := range fact.RejectionReasons {
|
|
reasons[reason] += count
|
|
}
|
|
if occurredAt, err := time.Parse(time.RFC3339Nano, fact.OccurredAt); err == nil {
|
|
hour := occurredAt.In(location).Format("2006-01-02T15:00:00-07:00")
|
|
if hourly[hour] == nil {
|
|
hourly[hour] = map[domain.DataMetricStage]*volumeAccumulator{}
|
|
}
|
|
if hourly[hour][fact.Stage] == nil {
|
|
hourly[hour][fact.Stage] = &volumeAccumulator{}
|
|
}
|
|
hourly[hour][fact.Stage].add(fact.Output)
|
|
}
|
|
if fact.Stage == domain.DataMetricStageCollect && fact.TaskID != "" {
|
|
if taskCollected[fact.TaskID] == nil {
|
|
taskCollected[fact.TaskID] = &volumeAccumulator{}
|
|
}
|
|
taskCollected[fact.TaskID].add(fact.Output)
|
|
}
|
|
if fact.Stage == domain.DataMetricStageTrain {
|
|
trainingConsumed.add(fact.Input)
|
|
trainingRuns[fact.RunID] = struct{}{}
|
|
for _, input := range fact.TrainingInputs {
|
|
key := strings.TrimSpace(input.DatasetVersionID)
|
|
if key == "" {
|
|
key = strings.TrimSpace(input.DatasetID) + "|" + strings.TrimSpace(input.ManifestPath) + "|" + strings.TrimSpace(input.Split)
|
|
}
|
|
if key == "||" || key == "" {
|
|
key = fmt.Sprintf("%s|%s", fact.RunID, fact.NodeID)
|
|
}
|
|
if _, exists := uniqueTraining[key]; !exists {
|
|
uniqueTraining[key] = input.Volume
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
summary.Stages = make([]domain.DataStageSummary, 0, len(domain.DataMetricStages))
|
|
for _, stage := range domain.DataMetricStages {
|
|
item := accumulators[stage]
|
|
summary.Stages = append(summary.Stages, domain.DataStageSummary{
|
|
Stage: stage,
|
|
Input: item.input.value(),
|
|
Output: item.output.value(),
|
|
Rejected: item.rejected.value(),
|
|
TechnicalValid: item.technicalValid.value(),
|
|
TrainReady: item.trainReady.value(),
|
|
RunCount: int64(len(item.runs)),
|
|
})
|
|
}
|
|
var uniqueAccumulator volumeAccumulator
|
|
for _, volume := range uniqueTraining {
|
|
uniqueAccumulator.add(volume)
|
|
}
|
|
summary.Training = domain.DataOperationsTrainingSummary{
|
|
Consumed: trainingConsumed.value(),
|
|
Unique: uniqueAccumulator.value(),
|
|
RunCount: int64(len(trainingRuns)),
|
|
}
|
|
|
|
hours := make([]string, 0, len(hourly))
|
|
for hour := range hourly {
|
|
hours = append(hours, hour)
|
|
}
|
|
sort.Strings(hours)
|
|
summary.Hourly = make([]domain.DataOperationsHourlyPoint, 0, len(hours))
|
|
for _, hour := range hours {
|
|
point := domain.DataOperationsHourlyPoint{Hour: hour, Stages: map[domain.DataMetricStage]domain.DataVolume{}}
|
|
for stage, value := range hourly[hour] {
|
|
point.Stages[stage] = value.value()
|
|
}
|
|
summary.Hourly = append(summary.Hourly, point)
|
|
}
|
|
|
|
summary.Tasks = make([]domain.CollectionTaskProgress, 0)
|
|
for _, task := range tasks {
|
|
if query.ProjectID != "" && task.ProjectID != query.ProjectID {
|
|
continue
|
|
}
|
|
if query.TaskID != "" && task.ID != query.TaskID {
|
|
continue
|
|
}
|
|
if query.RobotModel != "" && !strings.EqualFold(task.RobotModel, query.RobotModel) {
|
|
continue
|
|
}
|
|
if query.Scene != "" && !strings.EqualFold(task.Scene, query.Scene) {
|
|
continue
|
|
}
|
|
collected := domain.DataVolume{}
|
|
if taskCollected[task.ID] != nil {
|
|
collected = taskCollected[task.ID].value()
|
|
}
|
|
summary.Tasks = append(summary.Tasks, domain.CollectionTaskProgress{
|
|
TaskID: task.ID,
|
|
TaskName: task.Name,
|
|
Status: task.Status,
|
|
Targets: task.Targets,
|
|
Collected: collected,
|
|
Percent: taskProgressPercent(task.Targets, collected),
|
|
})
|
|
}
|
|
sort.Slice(summary.Tasks, func(left, right int) bool { return summary.Tasks[left].TaskName < summary.Tasks[right].TaskName })
|
|
return summary
|
|
}
|