forked from longtao.wu/droboticflow
327 lines
10 KiB
Go
327 lines
10 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
func dataOperationsMongoFilter(organizationID string, query domain.DataOperationsQuery) bson.M {
|
|
extra := bson.M{}
|
|
if query.ProjectID != "" {
|
|
extra["projectId"] = query.ProjectID
|
|
}
|
|
if query.TaskID != "" {
|
|
extra["taskId"] = query.TaskID
|
|
}
|
|
if query.RobotModel != "" {
|
|
extra["robotModel"] = query.RobotModel
|
|
}
|
|
if query.Scene != "" {
|
|
extra["scene"] = query.Scene
|
|
}
|
|
if query.Stage != "" {
|
|
extra["stage"] = query.Stage
|
|
}
|
|
timeRange := bson.M{}
|
|
if query.From != "" {
|
|
timeRange["$gte"] = query.From
|
|
}
|
|
if query.To != "" {
|
|
timeRange["$lt"] = query.To
|
|
}
|
|
if len(timeRange) > 0 {
|
|
extra["occurredAt"] = timeRange
|
|
}
|
|
return organizationScopedFilter(organizationID, extra)
|
|
}
|
|
|
|
func (store *MongoStore) ListCollectionTasks(ctx context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) {
|
|
cursor, err := store.collectionTasks.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.CollectionTaskRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetCollectionTask(ctx context.Context, id string) (*domain.CollectionTaskRecord, error) {
|
|
var item domain.CollectionTaskRecord
|
|
err := store.collectionTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertCollectionTask(ctx context.Context, item domain.CollectionTaskRecord) error {
|
|
if item.ID == "" {
|
|
item.ID = domain.NewID("data-task")
|
|
}
|
|
now := nowISO()
|
|
if item.CreatedAt == "" {
|
|
item.CreatedAt = now
|
|
}
|
|
item.UpdatedAt = now
|
|
_, err := store.collectionTasks.ReplaceOne(ctx, bson.M{"id": item.ID}, item, options.Replace().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) UpsertDatasetVersion(ctx context.Context, item domain.DatasetVersionRecord) error {
|
|
return supplierGuardedMongoWrite(ctx, store.datasetVersions, item.ID, item, func(ctx context.Context, d bson.M) error {
|
|
raw, e := bson.Marshal(d)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var next domain.DatasetVersionRecord
|
|
if e = bson.Unmarshal(raw, &next); e != nil {
|
|
return e
|
|
}
|
|
return store.upsertDatasetVersionSupplierGuarded(ctx, next)
|
|
})
|
|
}
|
|
func (store *MongoStore) upsertDatasetVersionSupplierGuarded(ctx context.Context, item domain.DatasetVersionRecord) error {
|
|
if strings.TrimSpace(item.Fingerprint) == "" {
|
|
return fmt.Errorf("dataset fingerprint is required")
|
|
}
|
|
var existing domain.DatasetVersionRecord
|
|
existingErr := store.datasetVersions.FindOne(
|
|
ctx,
|
|
organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint}),
|
|
options.FindOne().SetProjection(bson.M{"quality": 1}),
|
|
).Decode(&existing)
|
|
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
|
|
return existingErr
|
|
}
|
|
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
|
return nil
|
|
}
|
|
now := nowISO()
|
|
if item.ID == "" {
|
|
item.ID = domain.NewID("dataset-version")
|
|
}
|
|
if item.CreatedAt == "" {
|
|
item.CreatedAt = now
|
|
}
|
|
item.UpdatedAt = now
|
|
filter := organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint})
|
|
update := bson.M{
|
|
"$set": bson.M{
|
|
"datasetId": item.DatasetID, "taskId": item.TaskID, "projectId": item.ProjectID,
|
|
"name": item.Name, "split": item.Split, "fingerprint": item.Fingerprint, "volume": item.Volume,
|
|
"storage": item.Storage, "manifestPath": item.ManifestPath, "partitionDir": item.PartitionDir,
|
|
"producerRunId": item.ProducerRunID, "producerNodeId": item.ProducerNodeID, "artifactId": item.ArtifactID,
|
|
"quality": item.Quality, "updatedAt": now, "organizationId": normalizedOrganizationID(item.OrganizationID),
|
|
},
|
|
"$setOnInsert": bson.M{"id": item.ID, "createdAt": item.CreatedAt},
|
|
}
|
|
_, err := store.datasetVersions.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListDatasetVersions(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) {
|
|
extra := bson.M{}
|
|
if query.ProjectID != "" {
|
|
extra["projectId"] = query.ProjectID
|
|
}
|
|
if query.TaskID != "" {
|
|
extra["taskId"] = query.TaskID
|
|
}
|
|
timeRange := bson.M{}
|
|
if query.From != "" {
|
|
timeRange["$gte"] = query.From
|
|
}
|
|
if query.To != "" {
|
|
timeRange["$lt"] = query.To
|
|
}
|
|
if len(timeRange) > 0 {
|
|
extra["createdAt"] = timeRange
|
|
}
|
|
limit := int64(query.Limit)
|
|
if limit <= 0 || limit > 100 {
|
|
limit = 100
|
|
}
|
|
cursor, err := store.datasetVersions.Find(
|
|
ctx,
|
|
organizationScopedFilter(organizationID, extra),
|
|
options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(limit),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.DatasetVersionRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) UpsertDataMetricFact(ctx context.Context, item domain.DataMetricFactRecord) error {
|
|
return supplierGuardedMongoWrite(ctx, store.dataMetricFacts, item.ID, item, func(ctx context.Context, d bson.M) error {
|
|
raw, e := bson.Marshal(d)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var next domain.DataMetricFactRecord
|
|
if e = bson.Unmarshal(raw, &next); e != nil {
|
|
return e
|
|
}
|
|
return store.upsertDataMetricFactSupplierGuarded(ctx, next)
|
|
})
|
|
}
|
|
func (store *MongoStore) upsertDataMetricFactSupplierGuarded(ctx context.Context, item domain.DataMetricFactRecord) error {
|
|
if strings.TrimSpace(item.SourceKey) == "" {
|
|
return fmt.Errorf("metric source key is required")
|
|
}
|
|
if item.ID == "" {
|
|
item.ID = domain.NewID("data-fact")
|
|
}
|
|
if item.RecordedAt == "" {
|
|
item.RecordedAt = nowISO()
|
|
}
|
|
var existing domain.DataMetricFactRecord
|
|
existingErr := store.dataMetricFacts.FindOne(
|
|
ctx,
|
|
bson.M{"sourceKey": item.SourceKey},
|
|
options.FindOne().SetProjection(bson.M{"quality": 1}),
|
|
).Decode(&existing)
|
|
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
|
|
return existingErr
|
|
}
|
|
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
|
return nil
|
|
}
|
|
_, err := store.dataMetricFacts.ReplaceOne(ctx, bson.M{"sourceKey": item.SourceKey}, item, options.Replace().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) findDataMetricFacts(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DataMetricFactRecord, error) {
|
|
cursor, err := store.dataMetricFacts.Find(
|
|
ctx,
|
|
dataOperationsMongoFilter(organizationID, query),
|
|
options.Find().SetSort(bson.D{{Key: "occurredAt", Value: -1}, {Key: "id", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.DataMetricFactRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListDataMetricFactsPage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) {
|
|
items, err := store.findDataMetricFacts(ctx, organizationID, query)
|
|
if err != nil {
|
|
return domain.DataMetricFactPage{}, err
|
|
}
|
|
return factsPage(items, query), nil
|
|
}
|
|
|
|
func (store *MongoStore) GetDataOperationsSummary(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) {
|
|
facts, err := store.findDataMetricFacts(ctx, organizationID, query)
|
|
if err != nil {
|
|
return domain.DataOperationsSummary{}, err
|
|
}
|
|
tasks, err := store.ListCollectionTasks(ctx, organizationID)
|
|
if err != nil {
|
|
return domain.DataOperationsSummary{}, err
|
|
}
|
|
return BuildDataOperationsSummary(facts, tasks, query), nil
|
|
}
|
|
|
|
func (store *MongoStore) ListTrainingUsagePage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) {
|
|
query.Stage = domain.DataMetricStageTrain
|
|
items, err := store.findDataMetricFacts(ctx, organizationID, query)
|
|
if err != nil {
|
|
return domain.TrainingUsagePage{}, err
|
|
}
|
|
return trainingUsageFromFacts(items, query), nil
|
|
}
|
|
|
|
func (store *MongoStore) ListDataOperationsBackfillRuns(
|
|
ctx context.Context,
|
|
organizationID string,
|
|
from string,
|
|
pageCursor *domain.RunSummaryCursor,
|
|
limit int,
|
|
) (domain.DataOperationsRunPage, error) {
|
|
if limit <= 0 {
|
|
limit = 100
|
|
}
|
|
if limit > 500 {
|
|
limit = 500
|
|
}
|
|
filter := organizationScopedFilter(organizationID, bson.M{"createdAt": bson.M{"$gte": from}})
|
|
filter = combineRunSummaryFilters(filter, runSummaryCursorFilter(pageCursor))
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetProjection(bson.M{
|
|
"id": 1, "organizationId": 1, "workflowId": 1, "workflowName": 1,
|
|
"projectId": 1, "projectName": 1, "taskId": 1, "status": 1,
|
|
"createdAt": 1, "updatedAt": 1, "endedAt": 1, "nodeExecutions": 1,
|
|
}).
|
|
SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}).
|
|
SetLimit(int64(limit+1)),
|
|
)
|
|
if err != nil {
|
|
return domain.DataOperationsRunPage{}, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return domain.DataOperationsRunPage{}, err
|
|
}
|
|
page := domain.DataOperationsRunPage{HasMore: len(items) > limit}
|
|
if page.HasMore {
|
|
items = items[:limit]
|
|
}
|
|
page.Items = items
|
|
if page.HasMore && len(items) > 0 {
|
|
last := items[len(items)-1]
|
|
page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
|
}
|
|
return page, nil
|
|
}
|
|
|
|
func (store *MongoStore) ListRunsNeedingMetricMaterialization(ctx context.Context, limit int) ([]domain.RunRecord, error) {
|
|
if limit <= 0 {
|
|
limit = 20
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
bson.M{"nodeExecutions": bson.M{"$elemMatch": bson.M{
|
|
"dataMetrics": bson.M{"$exists": true},
|
|
"$or": bson.A{
|
|
bson.M{"metricsMaterializedAt": bson.M{"$exists": false}},
|
|
bson.M{"metricsMaterializedAt": ""},
|
|
},
|
|
}}},
|
|
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.RunRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) MarkNodeMetricsMaterialized(ctx context.Context, runID string, nodeID string, attempt int, at string) error {
|
|
result, err := store.runs.UpdateOne(
|
|
ctx,
|
|
bson.M{"id": runID},
|
|
bson.M{"$set": bson.M{"nodeExecutions.$[execution].metricsMaterializedAt": at}},
|
|
options.Update().SetArrayFilters(options.ArrayFilters{Filters: []any{
|
|
bson.M{"execution.nodeId": nodeID, "execution.attempt": attempt},
|
|
}}),
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if result.MatchedCount == 0 {
|
|
return fmt.Errorf("run %s not found", runID)
|
|
}
|
|
return nil
|
|
}
|