forked from longtao.wu/droboticflow
241 lines
7.1 KiB
Go
241 lines
7.1 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
func (store *MemoryStore) ListCollectionTasks(_ context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
items := make([]domain.CollectionTaskRecord, 0)
|
|
for _, item := range store.collectionTasks {
|
|
if organizationMatches(item.OrganizationID, organizationID) {
|
|
items = append(items, clone(item))
|
|
}
|
|
}
|
|
sort.Slice(items, func(left, right int) bool { return items[left].UpdatedAt > items[right].UpdatedAt })
|
|
return items, nil
|
|
}
|
|
|
|
func (store *MemoryStore) GetCollectionTask(_ context.Context, id string) (*domain.CollectionTaskRecord, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
item, ok := store.collectionTasks[id]
|
|
if !ok {
|
|
return nil, nil
|
|
}
|
|
cloned := clone(item)
|
|
return &cloned, nil
|
|
}
|
|
|
|
func (store *MemoryStore) UpsertCollectionTask(_ context.Context, item domain.CollectionTaskRecord) error {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
if strings.TrimSpace(item.ID) == "" {
|
|
item.ID = domain.NewID("data-task")
|
|
}
|
|
now := nowISO()
|
|
if item.CreatedAt == "" {
|
|
item.CreatedAt = now
|
|
}
|
|
item.UpdatedAt = now
|
|
store.collectionTasks[item.ID] = clone(item)
|
|
return nil
|
|
}
|
|
|
|
func (store *MemoryStore) UpsertDatasetVersion(_ context.Context, item domain.DatasetVersionRecord) error {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
if strings.TrimSpace(item.Fingerprint) == "" {
|
|
return fmt.Errorf("dataset fingerprint is required")
|
|
}
|
|
for id, existing := range store.datasetVersions {
|
|
if organizationMatches(existing.OrganizationID, item.OrganizationID) && existing.Fingerprint == item.Fingerprint {
|
|
if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
|
return nil
|
|
}
|
|
item.ID = id
|
|
item.CreatedAt = existing.CreatedAt
|
|
break
|
|
}
|
|
}
|
|
if item.ID == "" {
|
|
item.ID = domain.NewID("dataset-version")
|
|
}
|
|
now := nowISO()
|
|
if item.CreatedAt == "" {
|
|
item.CreatedAt = now
|
|
}
|
|
item.UpdatedAt = now
|
|
store.datasetVersions[item.ID] = clone(item)
|
|
return nil
|
|
}
|
|
|
|
func (store *MemoryStore) ListDatasetVersions(_ context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
items := make([]domain.DatasetVersionRecord, 0)
|
|
for _, item := range store.datasetVersions {
|
|
if datasetVersionMatches(item, organizationID, query) {
|
|
items = append(items, clone(item))
|
|
}
|
|
}
|
|
sort.Slice(items, func(left, right int) bool { return items[left].CreatedAt > items[right].CreatedAt })
|
|
limit := query.Limit
|
|
if limit <= 0 || limit > 100 {
|
|
limit = 100
|
|
}
|
|
if len(items) > limit {
|
|
items = items[:limit]
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (store *MemoryStore) UpsertDataMetricFact(_ context.Context, item domain.DataMetricFactRecord) error {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
if strings.TrimSpace(item.SourceKey) == "" {
|
|
return fmt.Errorf("metric source key is required")
|
|
}
|
|
for id, existing := range store.dataMetricFacts {
|
|
if existing.SourceKey == item.SourceKey {
|
|
if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
|
return nil
|
|
}
|
|
item.ID = id
|
|
break
|
|
}
|
|
}
|
|
if item.ID == "" {
|
|
item.ID = domain.NewID("data-fact")
|
|
}
|
|
if item.RecordedAt == "" {
|
|
item.RecordedAt = nowISO()
|
|
}
|
|
store.dataMetricFacts[item.ID] = clone(item)
|
|
return nil
|
|
}
|
|
|
|
func (store *MemoryStore) filteredDataMetricFactsLocked(organizationID string, query domain.DataOperationsQuery) []domain.DataMetricFactRecord {
|
|
items := make([]domain.DataMetricFactRecord, 0)
|
|
for _, item := range store.dataMetricFacts {
|
|
if dataOperationsFactMatches(item, organizationID, query) {
|
|
items = append(items, clone(item))
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func (store *MemoryStore) ListDataMetricFactsPage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
return factsPage(store.filteredDataMetricFactsLocked(organizationID, query), query), nil
|
|
}
|
|
|
|
func (store *MemoryStore) GetDataOperationsSummary(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
facts := store.filteredDataMetricFactsLocked(organizationID, query)
|
|
tasks := make([]domain.CollectionTaskRecord, 0)
|
|
for _, item := range store.collectionTasks {
|
|
if organizationMatches(item.OrganizationID, organizationID) {
|
|
tasks = append(tasks, clone(item))
|
|
}
|
|
}
|
|
return BuildDataOperationsSummary(facts, tasks, query), nil
|
|
}
|
|
|
|
func (store *MemoryStore) ListTrainingUsagePage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
return trainingUsageFromFacts(store.filteredDataMetricFactsLocked(organizationID, query), query), nil
|
|
}
|
|
|
|
func (store *MemoryStore) ListDataOperationsBackfillRuns(
|
|
_ context.Context,
|
|
organizationID string,
|
|
from string,
|
|
cursor *domain.RunSummaryCursor,
|
|
limit int,
|
|
) (domain.DataOperationsRunPage, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
if limit <= 0 {
|
|
limit = 100
|
|
}
|
|
if limit > 500 {
|
|
limit = 500
|
|
}
|
|
items := make([]domain.RunRecord, 0)
|
|
for _, run := range store.runs {
|
|
if !organizationMatches(run.OrganizationID, organizationID) || run.CreatedAt < from {
|
|
continue
|
|
}
|
|
if cursor != nil && !(run.CreatedAt < cursor.CreatedAt || (run.CreatedAt == cursor.CreatedAt && run.ID < cursor.ID)) {
|
|
continue
|
|
}
|
|
items = append(items, clone(run))
|
|
}
|
|
sort.Slice(items, func(left, right int) bool {
|
|
if items[left].CreatedAt == items[right].CreatedAt {
|
|
return items[left].ID > items[right].ID
|
|
}
|
|
return items[left].CreatedAt > items[right].CreatedAt
|
|
})
|
|
page := domain.DataOperationsRunPage{HasMore: len(items) > limit}
|
|
if page.HasMore {
|
|
items = items[:limit]
|
|
}
|
|
page.Items = items
|
|
if page.HasMore && len(items) > 0 {
|
|
last := items[len(items)-1]
|
|
page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
func (store *MemoryStore) ListRunsNeedingMetricMaterialization(_ context.Context, limit int) ([]domain.RunRecord, error) {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
items := make([]domain.RunRecord, 0)
|
|
for _, run := range store.runs {
|
|
for _, execution := range run.NodeExecutions {
|
|
if execution.DataMetrics != nil && execution.MetricsMaterializedAt == "" {
|
|
items = append(items, clone(run))
|
|
break
|
|
}
|
|
}
|
|
}
|
|
sortRuns(items)
|
|
if limit <= 0 {
|
|
limit = 20
|
|
}
|
|
if len(items) > limit {
|
|
items = items[:limit]
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (store *MemoryStore) MarkNodeMetricsMaterialized(_ context.Context, runID string, nodeID string, attempt int, at string) error {
|
|
store.mu.Lock()
|
|
defer store.mu.Unlock()
|
|
run, ok := store.runs[runID]
|
|
if !ok {
|
|
return fmt.Errorf("run %s not found", runID)
|
|
}
|
|
for index := range run.NodeExecutions {
|
|
execution := &run.NodeExecutions[index]
|
|
if execution.NodeID == nodeID && execution.Attempt == attempt {
|
|
execution.MetricsMaterializedAt = at
|
|
store.runs[runID] = run
|
|
return nil
|
|
}
|
|
}
|
|
return fmt.Errorf("node execution %s attempt %d not found", nodeID, attempt)
|
|
}
|