droboticflow/internal/store/data_operations_mongo.go

327 lines
10 KiB
Go

package store
import (
"context"
"fmt"
"strings"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func dataOperationsMongoFilter(organizationID string, query domain.DataOperationsQuery) bson.M {
extra := bson.M{}
if query.ProjectID != "" {
extra["projectId"] = query.ProjectID
}
if query.TaskID != "" {
extra["taskId"] = query.TaskID
}
if query.RobotModel != "" {
extra["robotModel"] = query.RobotModel
}
if query.Scene != "" {
extra["scene"] = query.Scene
}
if query.Stage != "" {
extra["stage"] = query.Stage
}
timeRange := bson.M{}
if query.From != "" {
timeRange["$gte"] = query.From
}
if query.To != "" {
timeRange["$lt"] = query.To
}
if len(timeRange) > 0 {
extra["occurredAt"] = timeRange
}
return organizationScopedFilter(organizationID, extra)
}
func (store *MongoStore) ListCollectionTasks(ctx context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) {
cursor, err := store.collectionTasks.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.CollectionTaskRecord](ctx, cursor)
}
func (store *MongoStore) GetCollectionTask(ctx context.Context, id string) (*domain.CollectionTaskRecord, error) {
var item domain.CollectionTaskRecord
err := store.collectionTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) UpsertCollectionTask(ctx context.Context, item domain.CollectionTaskRecord) error {
if item.ID == "" {
item.ID = domain.NewID("data-task")
}
now := nowISO()
if item.CreatedAt == "" {
item.CreatedAt = now
}
item.UpdatedAt = now
_, err := store.collectionTasks.ReplaceOne(ctx, bson.M{"id": item.ID}, item, options.Replace().SetUpsert(true))
return err
}
func (store *MongoStore) UpsertDatasetVersion(ctx context.Context, item domain.DatasetVersionRecord) error {
return supplierGuardedMongoWrite(ctx, store.datasetVersions, item.ID, item, func(ctx context.Context, d bson.M) error {
raw, e := bson.Marshal(d)
if e != nil {
return e
}
var next domain.DatasetVersionRecord
if e = bson.Unmarshal(raw, &next); e != nil {
return e
}
return store.upsertDatasetVersionSupplierGuarded(ctx, next)
})
}
func (store *MongoStore) upsertDatasetVersionSupplierGuarded(ctx context.Context, item domain.DatasetVersionRecord) error {
if strings.TrimSpace(item.Fingerprint) == "" {
return fmt.Errorf("dataset fingerprint is required")
}
var existing domain.DatasetVersionRecord
existingErr := store.datasetVersions.FindOne(
ctx,
organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint}),
options.FindOne().SetProjection(bson.M{"quality": 1}),
).Decode(&existing)
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
return existingErr
}
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
return nil
}
now := nowISO()
if item.ID == "" {
item.ID = domain.NewID("dataset-version")
}
if item.CreatedAt == "" {
item.CreatedAt = now
}
item.UpdatedAt = now
filter := organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint})
update := bson.M{
"$set": bson.M{
"datasetId": item.DatasetID, "taskId": item.TaskID, "projectId": item.ProjectID,
"name": item.Name, "split": item.Split, "fingerprint": item.Fingerprint, "volume": item.Volume,
"storage": item.Storage, "manifestPath": item.ManifestPath, "partitionDir": item.PartitionDir,
"producerRunId": item.ProducerRunID, "producerNodeId": item.ProducerNodeID, "artifactId": item.ArtifactID,
"quality": item.Quality, "updatedAt": now, "organizationId": normalizedOrganizationID(item.OrganizationID),
},
"$setOnInsert": bson.M{"id": item.ID, "createdAt": item.CreatedAt},
}
_, err := store.datasetVersions.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) ListDatasetVersions(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) {
extra := bson.M{}
if query.ProjectID != "" {
extra["projectId"] = query.ProjectID
}
if query.TaskID != "" {
extra["taskId"] = query.TaskID
}
timeRange := bson.M{}
if query.From != "" {
timeRange["$gte"] = query.From
}
if query.To != "" {
timeRange["$lt"] = query.To
}
if len(timeRange) > 0 {
extra["createdAt"] = timeRange
}
limit := int64(query.Limit)
if limit <= 0 || limit > 100 {
limit = 100
}
cursor, err := store.datasetVersions.Find(
ctx,
organizationScopedFilter(organizationID, extra),
options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(limit),
)
if err != nil {
return nil, err
}
return decodeMany[domain.DatasetVersionRecord](ctx, cursor)
}
func (store *MongoStore) UpsertDataMetricFact(ctx context.Context, item domain.DataMetricFactRecord) error {
return supplierGuardedMongoWrite(ctx, store.dataMetricFacts, item.ID, item, func(ctx context.Context, d bson.M) error {
raw, e := bson.Marshal(d)
if e != nil {
return e
}
var next domain.DataMetricFactRecord
if e = bson.Unmarshal(raw, &next); e != nil {
return e
}
return store.upsertDataMetricFactSupplierGuarded(ctx, next)
})
}
func (store *MongoStore) upsertDataMetricFactSupplierGuarded(ctx context.Context, item domain.DataMetricFactRecord) error {
if strings.TrimSpace(item.SourceKey) == "" {
return fmt.Errorf("metric source key is required")
}
if item.ID == "" {
item.ID = domain.NewID("data-fact")
}
if item.RecordedAt == "" {
item.RecordedAt = nowISO()
}
var existing domain.DataMetricFactRecord
existingErr := store.dataMetricFacts.FindOne(
ctx,
bson.M{"sourceKey": item.SourceKey},
options.FindOne().SetProjection(bson.M{"quality": 1}),
).Decode(&existing)
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
return existingErr
}
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
return nil
}
_, err := store.dataMetricFacts.ReplaceOne(ctx, bson.M{"sourceKey": item.SourceKey}, item, options.Replace().SetUpsert(true))
return err
}
func (store *MongoStore) findDataMetricFacts(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DataMetricFactRecord, error) {
cursor, err := store.dataMetricFacts.Find(
ctx,
dataOperationsMongoFilter(organizationID, query),
options.Find().SetSort(bson.D{{Key: "occurredAt", Value: -1}, {Key: "id", Value: -1}}),
)
if err != nil {
return nil, err
}
return decodeMany[domain.DataMetricFactRecord](ctx, cursor)
}
func (store *MongoStore) ListDataMetricFactsPage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) {
items, err := store.findDataMetricFacts(ctx, organizationID, query)
if err != nil {
return domain.DataMetricFactPage{}, err
}
return factsPage(items, query), nil
}
func (store *MongoStore) GetDataOperationsSummary(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) {
facts, err := store.findDataMetricFacts(ctx, organizationID, query)
if err != nil {
return domain.DataOperationsSummary{}, err
}
tasks, err := store.ListCollectionTasks(ctx, organizationID)
if err != nil {
return domain.DataOperationsSummary{}, err
}
return BuildDataOperationsSummary(facts, tasks, query), nil
}
func (store *MongoStore) ListTrainingUsagePage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) {
query.Stage = domain.DataMetricStageTrain
items, err := store.findDataMetricFacts(ctx, organizationID, query)
if err != nil {
return domain.TrainingUsagePage{}, err
}
return trainingUsageFromFacts(items, query), nil
}
func (store *MongoStore) ListDataOperationsBackfillRuns(
ctx context.Context,
organizationID string,
from string,
pageCursor *domain.RunSummaryCursor,
limit int,
) (domain.DataOperationsRunPage, error) {
if limit <= 0 {
limit = 100
}
if limit > 500 {
limit = 500
}
filter := organizationScopedFilter(organizationID, bson.M{"createdAt": bson.M{"$gte": from}})
filter = combineRunSummaryFilters(filter, runSummaryCursorFilter(pageCursor))
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(bson.M{
"id": 1, "organizationId": 1, "workflowId": 1, "workflowName": 1,
"projectId": 1, "projectName": 1, "taskId": 1, "status": 1,
"createdAt": 1, "updatedAt": 1, "endedAt": 1, "nodeExecutions": 1,
}).
SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}).
SetLimit(int64(limit+1)),
)
if err != nil {
return domain.DataOperationsRunPage{}, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return domain.DataOperationsRunPage{}, err
}
page := domain.DataOperationsRunPage{HasMore: len(items) > limit}
if page.HasMore {
items = items[:limit]
}
page.Items = items
if page.HasMore && len(items) > 0 {
last := items[len(items)-1]
page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
return page, nil
}
func (store *MongoStore) ListRunsNeedingMetricMaterialization(ctx context.Context, limit int) ([]domain.RunRecord, error) {
if limit <= 0 {
limit = 20
}
cursor, err := store.runs.Find(
ctx,
bson.M{"nodeExecutions": bson.M{"$elemMatch": bson.M{
"dataMetrics": bson.M{"$exists": true},
"$or": bson.A{
bson.M{"metricsMaterializedAt": bson.M{"$exists": false}},
bson.M{"metricsMaterializedAt": ""},
},
}}},
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
return decodeMany[domain.RunRecord](ctx, cursor)
}
func (store *MongoStore) MarkNodeMetricsMaterialized(ctx context.Context, runID string, nodeID string, attempt int, at string) error {
result, err := store.runs.UpdateOne(
ctx,
bson.M{"id": runID},
bson.M{"$set": bson.M{"nodeExecutions.$[execution].metricsMaterializedAt": at}},
options.Update().SetArrayFilters(options.ArrayFilters{Filters: []any{
bson.M{"execution.nodeId": nodeID, "execution.attempt": attempt},
}}),
)
if err != nil {
return err
}
if result.MatchedCount == 0 {
return fmt.Errorf("run %s not found", runID)
}
return nil
}