forked from longtao.wu/droboticflow
3127 lines
104 KiB
Go
3127 lines
104 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"golang.org/x/sync/singleflight"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
type MongoStore struct {
|
|
querySyncEnabled atomic.Bool
|
|
queryReplicaReads *queryReplicaReads
|
|
queryReplicaFactStore bool
|
|
queryPeriodic bool
|
|
queryPublishedResults bool
|
|
workforcePublicationTouches workforcePublicationTouches
|
|
workforcePublicationBuilds workforcePublicationBuildMemo
|
|
queryBuildWrites *queryBuildWriteGate
|
|
queryReplicaCounts queryCountCache
|
|
queryReplicaPageFlights singleflight.Group
|
|
queryReplicaBatchFacts queryBatchFacts
|
|
queryServing *queryServingOptions
|
|
productionReadFlights singleflight.Group
|
|
claimReadFlights singleflight.Group
|
|
poolSearchReadFlights singleflight.Group
|
|
claimReadOnce sync.Once
|
|
claimReadSlots chan struct{}
|
|
db *mongo.Database
|
|
organizations mongoCollection
|
|
projects mongoCollection
|
|
operators mongoCollection
|
|
builtinOperatorDeletions mongoCollection
|
|
workflows mongoCollection
|
|
workflowRuntimePolicies mongoCollection
|
|
templates mongoCollection
|
|
workflowReleases mongoCollection
|
|
projectWorkflowInstances mongoCollection
|
|
runs mongoCollection
|
|
sourceStates mongoCollection
|
|
captureRecords mongoCollection
|
|
collectionTasks mongoCollection
|
|
datasetVersions mongoCollection
|
|
dataMetricFacts mongoCollection
|
|
auditTasks mongoCollection
|
|
samplingTasks mongoCollection
|
|
annotationWorkgroups mongoCollection
|
|
annotationMemberships mongoCollection
|
|
suppliers mongoCollection
|
|
projectMembers mongoCollection
|
|
supplierProjectAuthorizations mongoCollection
|
|
supplierProjectAuthorizationInit mongoCollection
|
|
annotationLabels mongoCollection
|
|
projectAnnotationLabels mongoCollection
|
|
annotationTemplates mongoCollection
|
|
annotationTasks mongoCollection
|
|
annotationRevisions mongoCollection
|
|
sliceRevisions mongoCollection
|
|
annotationReviews mongoCollection
|
|
annotationWorklogEvents mongoCollection
|
|
organizationPlugins mongoCollection
|
|
humanBatches mongoCollection
|
|
humanItems mongoCollection
|
|
humanStageAttempts mongoCollection
|
|
humanEvents mongoCollection
|
|
humanRejects mongoCollection
|
|
releasePartitions mongoCollection
|
|
deliveryPackages mongoCollection
|
|
users mongoCollection
|
|
sessions mongoCollection
|
|
storageConnections mongoCollection
|
|
uploadSessions mongoCollection
|
|
mongoConnections mongoCollection
|
|
objectSyncSpaces mongoCollection
|
|
objectSyncDevices mongoCollection
|
|
objectSyncAuthorizations mongoCollection
|
|
objectSyncCredentials mongoCollection
|
|
objectSyncSessions mongoCollection
|
|
objectSyncItems mongoCollection
|
|
objectSyncLeases mongoCollection
|
|
objectSyncHeads mongoCollection
|
|
objectSyncVersions mongoCollection
|
|
objectSyncAuditEvents mongoCollection
|
|
objectSyncAuditSeals mongoCollection
|
|
objectSyncReleases mongoCollection
|
|
objectSyncClientReleases mongoCollection
|
|
objectSyncReconciles mongoCollection
|
|
objectSyncIdempotency mongoCollection
|
|
objectSyncRateLimits mongoCollection
|
|
settings mongoCollection
|
|
runSummaryAggregates *runSummaryAggregateCache
|
|
}
|
|
|
|
func NewMongoStore(db *mongo.Database) *MongoStore {
|
|
out := &MongoStore{db: db, runSummaryAggregates: newRunSummaryAggregateCache(runSummaryAggregateCacheTTL, runSummaryAggregateCacheMaxEntries)}
|
|
out.organizations = out.queryCollection("organizations")
|
|
out.projects = out.queryCollection("projects")
|
|
out.operators = out.queryCollection("operators")
|
|
out.builtinOperatorDeletions = out.queryCollection("operatorTombstones")
|
|
out.workflows = out.queryCollection("workflows")
|
|
out.workflowRuntimePolicies = out.queryCollection("workflowRuntimePolicies")
|
|
out.templates = out.queryCollection("templates")
|
|
out.workflowReleases = out.queryCollection("workflowReleases")
|
|
out.projectWorkflowInstances = out.queryCollection("projectWorkflowInstances")
|
|
out.runs = out.queryCollection("runs")
|
|
out.sourceStates = out.queryCollection("sourceStates")
|
|
out.captureRecords = out.queryCollection("captureRecords")
|
|
out.collectionTasks = out.queryCollection("collectionTasks")
|
|
out.datasetVersions = out.queryCollection("datasetVersions")
|
|
out.dataMetricFacts = out.queryCollection("dataMetricFacts")
|
|
out.auditTasks = out.queryCollection("auditTasks")
|
|
out.samplingTasks = out.queryCollection("samplingTasks")
|
|
out.annotationWorkgroups = out.queryCollection("annotationWorkgroups")
|
|
out.annotationMemberships = out.queryCollection("annotationMemberships")
|
|
out.suppliers = out.queryCollection("suppliers")
|
|
out.projectMembers = out.queryCollection("projectMembers")
|
|
out.supplierProjectAuthorizations = out.queryCollection("supplierProjectAuthorizations")
|
|
out.supplierProjectAuthorizationInit = out.queryCollection("supplierProjectAuthorizationInit")
|
|
out.annotationLabels = out.queryCollection("annotationLabels")
|
|
out.projectAnnotationLabels = out.queryCollection("projectAnnotationLabels")
|
|
out.annotationTemplates = out.queryCollection("annotationTemplates")
|
|
out.annotationTasks = out.queryCollection("annotationTasks")
|
|
out.annotationRevisions = out.queryCollection("annotationRevisions")
|
|
out.sliceRevisions = out.queryCollection("sliceRevisions")
|
|
out.annotationReviews = out.queryCollection("annotationReviews")
|
|
out.annotationWorklogEvents = out.queryCollection("annotationWorklogEvents")
|
|
out.organizationPlugins = out.queryCollection("organizationPlugins")
|
|
out.humanBatches = out.queryCollection("humanBatches")
|
|
out.humanItems = out.queryCollection("humanItems")
|
|
out.humanStageAttempts = out.queryCollection("humanStageAttempts")
|
|
out.humanEvents = out.queryCollection("humanEvents")
|
|
out.humanRejects = out.queryCollection("humanRejects")
|
|
out.releasePartitions = out.queryCollection("humanReleasePartitions")
|
|
out.deliveryPackages = out.queryCollection("deliveryPackages")
|
|
out.users = out.queryCollection("users")
|
|
out.sessions = out.queryCollection("sessions")
|
|
out.storageConnections = out.queryCollection("storageConnections")
|
|
out.uploadSessions = out.queryCollection("objectStorageUploadSessions")
|
|
out.mongoConnections = out.queryCollection("mongoConnections")
|
|
out.objectSyncSpaces = out.queryCollection("objectSyncSpaces")
|
|
out.objectSyncDevices = out.queryCollection("objectSyncDevices")
|
|
out.objectSyncAuthorizations = out.queryCollection("objectSyncAuthorizations")
|
|
out.objectSyncCredentials = out.queryCollection("objectSyncCredentials")
|
|
out.objectSyncSessions = out.queryCollection("objectSyncSessions")
|
|
out.objectSyncItems = out.queryCollection("objectSyncItems")
|
|
out.objectSyncLeases = out.queryCollection("objectSyncLeases")
|
|
out.objectSyncHeads = out.queryCollection("objectSyncHeads")
|
|
out.objectSyncVersions = out.queryCollection("objectSyncVersions")
|
|
out.objectSyncAuditEvents = out.queryCollection("objectSyncAuditEvents")
|
|
out.objectSyncAuditSeals = out.queryCollection("objectSyncAuditSeals")
|
|
out.objectSyncReleases = out.queryCollection("objectSyncReleases")
|
|
out.objectSyncClientReleases = out.queryCollection("objectSyncClientReleases")
|
|
out.objectSyncReconciles = out.queryCollection("objectSyncReconciles")
|
|
out.objectSyncIdempotency = out.queryCollection("objectSyncIdempotency")
|
|
out.objectSyncRateLimits = out.queryCollection("objectSyncRateLimits")
|
|
out.settings = out.queryCollection("settings")
|
|
return out
|
|
}
|
|
|
|
func (store *MongoStore) Ready(ctx context.Context) error {
|
|
return store.db.Client().Ping(ctx, nil)
|
|
}
|
|
|
|
func decodeMany[T any](ctx context.Context, cursor *mongo.Cursor) ([]T, error) {
|
|
defer cursor.Close(ctx)
|
|
items := make([]T, 0)
|
|
for cursor.Next(ctx) {
|
|
var item T
|
|
if err := cursor.Decode(&item); err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
return items, cursor.Err()
|
|
}
|
|
|
|
func orgCompatibilityFilter(organizationID string) bson.M {
|
|
normalized := normalizedOrganizationID(organizationID)
|
|
if normalized == domain.DefaultOrganizationID {
|
|
return bson.M{"$or": bson.A{
|
|
bson.M{"organizationId": normalized},
|
|
bson.M{"organizationId": bson.M{"$exists": false}},
|
|
bson.M{"organizationId": ""},
|
|
}}
|
|
}
|
|
return bson.M{"organizationId": normalized}
|
|
}
|
|
|
|
func operatorOrgCompatibilityFilter(organizationID string) bson.M {
|
|
return bson.M{"$or": bson.A{
|
|
orgCompatibilityFilter(organizationID),
|
|
bson.M{"type": "builtin"},
|
|
}}
|
|
}
|
|
|
|
func organizationScopedFilter(organizationID string, extra bson.M) bson.M {
|
|
orgFilter := orgCompatibilityFilter(organizationID)
|
|
if len(extra) == 0 {
|
|
return orgFilter
|
|
}
|
|
return bson.M{"$and": bson.A{orgFilter, extra}}
|
|
}
|
|
|
|
func (store *MongoStore) EnsureDefaultOrganization(ctx context.Context) (*domain.OrganizationRecord, error) {
|
|
existing, err := store.GetOrganization(ctx, domain.DefaultOrganizationID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if existing != nil {
|
|
return existing, nil
|
|
}
|
|
now := nowISO()
|
|
item := domain.OrganizationRecord{
|
|
ID: domain.DefaultOrganizationID,
|
|
Name: "Default Organization",
|
|
Slug: "default",
|
|
Status: domain.OrganizationStatusActive,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
if err := store.UpsertOrganization(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) ListOrganizations(ctx context.Context) ([]domain.OrganizationRecord, error) {
|
|
if _, err := store.EnsureDefaultOrganization(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
cursor, err := store.organizations.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.OrganizationRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetOrganization(ctx context.Context, id string) (*domain.OrganizationRecord, error) {
|
|
var item domain.OrganizationRecord
|
|
err := store.organizations.FindOne(ctx, bson.M{"id": normalizedOrganizationID(id)}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertOrganization(ctx context.Context, item domain.OrganizationRecord) error {
|
|
if strings.TrimSpace(item.ID) == "" {
|
|
item.ID = domain.NewID("org")
|
|
}
|
|
item.Slug = strings.TrimSpace(strings.ToLower(item.Slug))
|
|
now := nowISO()
|
|
if item.CreatedAt == "" {
|
|
item.CreatedAt = now
|
|
}
|
|
if item.Status == "" {
|
|
item.Status = domain.OrganizationStatusActive
|
|
}
|
|
item.UpdatedAt = now
|
|
_, err := store.organizations.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
if mongo.IsDuplicateKeyError(err) {
|
|
return ErrDuplicateOrganizationSlug
|
|
}
|
|
if err == nil {
|
|
_, err = store.EnsureDefaultSupplier(ctx, item.ID)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id string) (bool, error) {
|
|
organizationID := normalizedOrganizationID(id)
|
|
if organizationID == domain.DefaultOrganizationID {
|
|
return false, fmt.Errorf("default organization cannot be deleted")
|
|
}
|
|
if existing, err := store.GetOrganization(ctx, organizationID); err != nil {
|
|
return false, err
|
|
} else if existing == nil {
|
|
return false, nil
|
|
}
|
|
filter := orgCompatibilityFilter(organizationID)
|
|
for _, collection := range []mongoCollection{
|
|
store.users,
|
|
store.sessions,
|
|
store.projects,
|
|
store.operators,
|
|
store.builtinOperatorDeletions,
|
|
store.workflows,
|
|
store.templates,
|
|
store.workflowReleases,
|
|
store.projectWorkflowInstances,
|
|
store.runs,
|
|
store.sourceStates,
|
|
store.captureRecords,
|
|
store.auditTasks,
|
|
store.samplingTasks,
|
|
store.annotationWorkgroups,
|
|
store.annotationMemberships,
|
|
store.suppliers,
|
|
store.projectMembers,
|
|
store.annotationTemplates,
|
|
store.annotationTasks,
|
|
store.annotationRevisions,
|
|
store.sliceRevisions,
|
|
store.annotationReviews,
|
|
store.annotationWorklogEvents,
|
|
store.organizationPlugins,
|
|
store.humanBatches,
|
|
store.humanItems,
|
|
store.humanStageAttempts,
|
|
store.humanEvents,
|
|
store.humanRejects,
|
|
store.releasePartitions,
|
|
store.deliveryPackages,
|
|
store.storageConnections,
|
|
store.uploadSessions,
|
|
store.mongoConnections,
|
|
store.objectSyncSpaces,
|
|
store.objectSyncDevices,
|
|
store.objectSyncAuthorizations,
|
|
store.objectSyncCredentials,
|
|
store.objectSyncSessions,
|
|
store.objectSyncItems,
|
|
store.objectSyncLeases,
|
|
store.objectSyncHeads,
|
|
store.objectSyncVersions,
|
|
store.objectSyncAuditEvents,
|
|
store.objectSyncReleases,
|
|
store.objectSyncClientReleases,
|
|
store.annotationLabels,
|
|
store.settings,
|
|
} {
|
|
if _, err := collection.DeleteMany(ctx, filter); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
_, err := store.organizations.DeleteOne(ctx, bson.M{"id": organizationID})
|
|
return err == nil, err
|
|
}
|
|
|
|
func (store *MongoStore) ListProjects(ctx context.Context) ([]domain.ProjectRecord, error) {
|
|
cursor, err := store.projects.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.ProjectRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListProjectsByOrganization(ctx context.Context, organizationID string) ([]domain.ProjectRecord, error) {
|
|
cursor, err := store.projects.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.ProjectRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetProject(ctx context.Context, id string) (*domain.ProjectRecord, error) {
|
|
var item domain.ProjectRecord
|
|
err := store.projects.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateProject(ctx context.Context, input domain.CreateProjectInput) (*domain.ProjectRecord, error) {
|
|
now := nowISO()
|
|
item := domain.ProjectRecord{
|
|
ID: domain.NewID("project"),
|
|
OrganizationID: input.OrganizationID,
|
|
Name: input.Name,
|
|
Description: input.Description,
|
|
Scene: input.Scene,
|
|
OwnerUserID: input.OwnerUserID,
|
|
Customer: input.Customer,
|
|
DataBatch: input.DataBatch,
|
|
StorageConnectionID: input.StorageConnectionID,
|
|
StorageBucket: input.StorageBucket,
|
|
StoragePrefix: input.StoragePrefix,
|
|
AnnotationStandard: input.AnnotationStandard,
|
|
QualityPolicy: input.QualityPolicy,
|
|
DeliveryConfig: input.DeliveryConfig,
|
|
Status: domain.ProjectStatusActive,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
// The marker is created before any default-member compatibility sync. A
|
|
// crash may leave an orphan marker for a never-created ID, which is safe.
|
|
_, err := store.supplierProjectAuthorizationInit.UpdateOne(ctx, bson.M{"id": supplierAuthorizationInitID(item.OrganizationID, item.ID)}, bson.M{"$setOnInsert": bson.M{"id": supplierAuthorizationInitID(item.OrganizationID, item.ID), "organizationId": normalizedOrganizationID(item.OrganizationID), "projectId": item.ID, "initializedAt": now}}, options.Update().SetUpsert(true))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := store.UpsertProject(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, error) {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
return false, nil
|
|
}
|
|
workflowCursor, err := store.workflows.Find(ctx, bson.M{"projectId": id}, options.Find().SetProjection(bson.M{"id": 1}))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
workflowDocs, err := decodeMany[struct {
|
|
ID string `bson:"id"`
|
|
}](ctx, workflowCursor)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, item := range workflowDocs {
|
|
if _, err := store.DeleteWorkflow(ctx, item.ID); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
|
|
taskIDs, err := store.annotationTasks.Distinct(ctx, "id", bson.M{"projectId": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
itemIDs, err := store.humanItems.Distinct(ctx, "id", bson.M{"projectId": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
batchIDs, err := store.humanBatches.Distinct(ctx, "id", bson.M{"projectId": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if len(taskIDs) > 0 {
|
|
taskFilter := bson.M{"taskId": bson.M{"$in": taskIDs}}
|
|
for _, collection := range []mongoCollection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} {
|
|
if _, err := collection.DeleteMany(ctx, taskFilter); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
}
|
|
if len(itemIDs) > 0 {
|
|
itemFilter := bson.M{"itemId": bson.M{"$in": itemIDs}}
|
|
for _, collection := range []mongoCollection{store.humanStageAttempts, store.humanEvents} {
|
|
if _, err := collection.DeleteMany(ctx, itemFilter); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
}
|
|
if len(batchIDs) > 0 {
|
|
if _, err := store.releasePartitions.DeleteMany(ctx, bson.M{"batchId": bson.M{"$in": batchIDs}}); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
|
|
for _, operation := range []struct {
|
|
collection mongoCollection
|
|
filter bson.M
|
|
}{
|
|
{store.annotationTasks, bson.M{"projectId": id}},
|
|
{store.queryCollection("annotationTimelineIndexes"), bson.M{"projectId": id}},
|
|
{store.queryCollection("annotationIndexJobs"), bson.M{"projectId": id}},
|
|
{store.humanRejects, bson.M{"projectId": id}},
|
|
{store.humanItems, bson.M{"projectId": id}},
|
|
{store.humanBatches, bson.M{"projectId": id}},
|
|
{store.deliveryPackages, bson.M{"projectId": id}},
|
|
{store.annotationMemberships, bson.M{"projectId": id}},
|
|
{store.annotationTemplates, bson.M{"projectId": id}},
|
|
{store.projectMembers, bson.M{"projectId": id}},
|
|
{store.workflows, bson.M{"projectId": id}},
|
|
{store.projectWorkflowInstances, bson.M{"projectId": id}},
|
|
{store.runs, bson.M{"projectId": id}},
|
|
{store.captureRecords, bson.M{"projectId": id}},
|
|
{store.auditTasks, bson.M{"projectId": id}},
|
|
{store.samplingTasks, bson.M{"projectId": id}},
|
|
} {
|
|
if _, err := operation.collection.DeleteMany(ctx, operation.filter); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
|
|
result, err := store.projects.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount == 1, nil
|
|
}
|
|
|
|
func projectPersistenceDocument(item domain.ProjectRecord) bson.M {
|
|
// ProjectRecord has bson,omitempty fields for compact reads. Passing the
|
|
// struct directly to $set therefore skips empty strings and makes it
|
|
// impossible to clear optional project fields through the edit API. Build
|
|
// the persisted document explicitly so empty values replace stale values.
|
|
return bson.M{
|
|
"id": item.ID,
|
|
"organizationId": item.OrganizationID,
|
|
"name": item.Name,
|
|
"description": item.Description,
|
|
"scene": item.Scene,
|
|
"ownerUserId": item.OwnerUserID,
|
|
"customer": item.Customer,
|
|
"dataBatch": item.DataBatch,
|
|
"storageConnectionId": item.StorageConnectionID,
|
|
"storageBucket": item.StorageBucket,
|
|
"storagePrefix": item.StoragePrefix,
|
|
"workflowInstanceId": item.WorkflowInstanceID,
|
|
"annotationStandard": item.AnnotationStandard,
|
|
"qualityPolicy": item.QualityPolicy,
|
|
"deliveryConfig": item.DeliveryConfig,
|
|
"progressStatus": item.ProgressStatus,
|
|
"qualityStatus": item.QualityStatus,
|
|
"deliveryStatus": item.DeliveryStatus,
|
|
"status": item.Status,
|
|
"storagePathFavorites": item.StoragePathFavorites,
|
|
"createdAt": item.CreatedAt,
|
|
"updatedAt": item.UpdatedAt,
|
|
}
|
|
|
|
}
|
|
|
|
func (store *MongoStore) UpsertProject(ctx context.Context, item domain.ProjectRecord) error {
|
|
allow, err := projectAllowsUpsert(ctx, store.db, item.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
persisted := projectPersistenceDocument(item)
|
|
stampPoolSearchMetadata("projects", persisted)
|
|
_, err = store.projects.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": persisted}, options.Update().SetUpsert(allow))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListOperators(ctx context.Context) ([]domain.OperatorRecord, error) {
|
|
cursor, err := store.operators.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.OperatorRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListOperatorsByOrganization(ctx context.Context, organizationID string) ([]domain.OperatorRecord, error) {
|
|
cursor, err := store.operators.Find(ctx, operatorOrgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.OperatorRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetOperator(ctx context.Context, id string) (*domain.OperatorRecord, error) {
|
|
var item domain.OperatorRecord
|
|
err := store.operators.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateOperator(ctx context.Context, input domain.CreateOperatorInput) (*domain.OperatorRecord, error) {
|
|
now := nowISO()
|
|
item := domain.OperatorRecord{
|
|
ID: domain.NewID("operator"),
|
|
OrganizationID: input.OrganizationID,
|
|
Name: input.Name,
|
|
Type: input.Type,
|
|
Category: input.Category,
|
|
Description: input.Description,
|
|
Inputs: input.Inputs,
|
|
Outputs: input.Outputs,
|
|
ConfigSchema: input.ConfigSchema,
|
|
Version: input.Version,
|
|
Image: input.Image,
|
|
ImageID: input.ImageID,
|
|
ContainerCommand: input.ContainerCommand,
|
|
ContainerArgs: input.ContainerArgs,
|
|
RuntimeContract: input.RuntimeContract,
|
|
SkillDefinition: input.SkillDefinition,
|
|
Hyperparams: input.Hyperparams,
|
|
Resources: input.Resources,
|
|
BusinessStage: input.BusinessStage,
|
|
PublicationStatus: input.PublicationStatus,
|
|
GenerationSource: input.GenerationSource,
|
|
PublicationEvidence: input.PublicationEvidence,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
if err := store.UpsertOperator(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteOperator(ctx context.Context, id string) (bool, error) {
|
|
result, err := store.operators.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount == 1, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertOperator(ctx context.Context, item domain.OperatorRecord) error {
|
|
_, err := store.operators.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListBuiltinOperatorDeletions(ctx context.Context) ([]domain.BuiltinOperatorDeletionRecord, error) {
|
|
cursor, err := store.builtinOperatorDeletions.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "deletedAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.BuiltinOperatorDeletionRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) UpsertBuiltinOperatorDeletion(ctx context.Context, item domain.BuiltinOperatorDeletionRecord) error {
|
|
_, err := store.builtinOperatorDeletions.UpdateOne(ctx, bson.M{"key": item.Key}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListWorkflowsByProject(ctx context.Context, projectID string) ([]domain.WorkflowRecord, error) {
|
|
cursor, err := store.workflows.Find(ctx, bson.M{"projectId": projectID}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.WorkflowRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListAllWorkflows(ctx context.Context) ([]domain.WorkflowRecord, error) {
|
|
cursor, err := store.workflows.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.WorkflowRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListAllWorkflowsByOrganization(ctx context.Context, organizationID string) ([]domain.WorkflowRecord, error) {
|
|
cursor, err := store.workflows.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.WorkflowRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListWorkerWorkflows(ctx context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) {
|
|
filters := bson.A{bson.M{
|
|
"type": domain.WorkflowTypeStreaming,
|
|
"schedule": bson.M{"$exists": true, "$ne": nil},
|
|
}, bson.M{"captureAutomation.enabled": true}}
|
|
if len(workflowIDs) > 0 {
|
|
filters = append(filters, bson.M{"id": bson.M{"$in": workflowIDs}})
|
|
}
|
|
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
|
|
cursor, err := store.workflows.Find(
|
|
ctx,
|
|
bson.M{"$and": bson.A{
|
|
bson.M{"$or": filters},
|
|
bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}},
|
|
}},
|
|
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.WorkflowRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListEnabledCaptureAutomations(ctx context.Context, organizationID string) ([]domain.WorkflowRecord, error) {
|
|
filter := organizationScopedFilter(organizationID, bson.M{
|
|
"captureAutomation.enabled": true,
|
|
"$or": bson.A{
|
|
bson.M{"archivedAt": bson.M{"$exists": false}},
|
|
bson.M{"archivedAt": ""},
|
|
},
|
|
})
|
|
cursor, err := store.workflows.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().SetProjection(bson.M{"id": 1, "name": 1, "organizationId": 1, "captureAutomation": 1}).SetSort(bson.D{{Key: "updatedAt", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.WorkflowRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetWorkflow(ctx context.Context, id string) (*domain.WorkflowRecord, error) {
|
|
var item domain.WorkflowRecord
|
|
err := store.workflows.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
domain.NormalizeWorkflowDefinitionMetadata(&item)
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateWorkflow(ctx context.Context, input domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) {
|
|
if input.ProjectID != "" {
|
|
p, err := store.GetProject(ctx, input.ProjectID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if p == nil || p.DeletionTaskID != "" {
|
|
return nil, fmt.Errorf("project missing or deletion in progress")
|
|
}
|
|
}
|
|
now := nowISO()
|
|
item := domain.WorkflowRecord{
|
|
ID: domain.NewID("workflow"),
|
|
OrganizationID: input.OrganizationID,
|
|
ProjectID: input.ProjectID,
|
|
Name: input.Name,
|
|
Description: input.Description,
|
|
Type: input.Type,
|
|
Kind: input.Kind,
|
|
Hidden: input.Hidden,
|
|
ExecutionScope: input.ExecutionScope,
|
|
ScanConfig: input.ScanConfig,
|
|
CaptureAutomation: input.CaptureAutomation,
|
|
Hyperparams: input.Hyperparams,
|
|
MaxConcurrentRuns: input.MaxConcurrentRuns,
|
|
NodeTimeoutSeconds: input.NodeTimeoutSeconds,
|
|
RunTimeoutSeconds: input.RunTimeoutSeconds,
|
|
MaxRetryAttempts: input.MaxRetryAttempts,
|
|
RetryBackoffSeconds: input.RetryBackoffSeconds,
|
|
Schedule: input.Schedule,
|
|
DefinitionState: domain.WorkflowDefinitionStateDraft,
|
|
DefinitionVersion: 1,
|
|
WorkflowInputs: input.WorkflowInputs,
|
|
ArtifactStorage: input.ArtifactStorage,
|
|
Nodes: input.Nodes,
|
|
Edges: input.Edges,
|
|
Status: domain.WorkflowStatusDraft,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
item.DefinitionFamilyID = item.ID
|
|
if err := store.UpsertWorkflow(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, error) {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
return false, nil
|
|
}
|
|
count, err := store.workflows.CountDocuments(ctx, bson.M{"id": id}, options.Count().SetLimit(1))
|
|
if err != nil || count == 0 {
|
|
return false, err
|
|
}
|
|
taskIDs, err := store.annotationTasks.Distinct(ctx, "id", bson.M{"workflowId": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
itemIDs, err := store.humanItems.Distinct(ctx, "id", bson.M{"workflowId": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
batchIDs, err := store.humanBatches.Distinct(ctx, "id", bson.M{"workflowId": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if len(taskIDs) > 0 {
|
|
taskFilter := bson.M{"taskId": bson.M{"$in": taskIDs}}
|
|
for _, collection := range []mongoCollection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} {
|
|
if _, err := collection.DeleteMany(ctx, taskFilter); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
}
|
|
if len(itemIDs) > 0 {
|
|
itemFilter := bson.M{"itemId": bson.M{"$in": itemIDs}}
|
|
for _, collection := range []mongoCollection{store.humanStageAttempts, store.humanEvents} {
|
|
if _, err := collection.DeleteMany(ctx, itemFilter); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
}
|
|
if len(batchIDs) > 0 {
|
|
if _, err := store.releasePartitions.DeleteMany(ctx, bson.M{"batchId": bson.M{"$in": batchIDs}}); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
for _, operation := range []struct {
|
|
collection mongoCollection
|
|
filter bson.M
|
|
}{
|
|
{store.projectWorkflowInstances, bson.M{"workflowId": id}},
|
|
{store.annotationTasks, bson.M{"workflowId": id}},
|
|
{store.queryCollection("annotationTimelineIndexes"), bson.M{"workflowId": id}},
|
|
{store.queryCollection("annotationIndexJobs"), bson.M{"workflowId": id}},
|
|
{store.humanRejects, bson.M{"workflowId": id}},
|
|
{store.humanItems, bson.M{"workflowId": id}},
|
|
{store.humanBatches, bson.M{"workflowId": id}},
|
|
{store.deliveryPackages, bson.M{"workflowId": id}},
|
|
} {
|
|
if _, err := operation.collection.DeleteMany(ctx, operation.filter); err != nil {
|
|
return false, err
|
|
}
|
|
}
|
|
if _, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := store.captureRecords.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := store.runs.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := store.auditTasks.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
|
|
return false, err
|
|
}
|
|
if _, err := store.samplingTasks.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
|
|
return false, err
|
|
}
|
|
result, err := store.workflows.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount == 1, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertWorkflow(ctx context.Context, item domain.WorkflowRecord) error {
|
|
return supplierGuardedMongoWrite(ctx, store.workflows, item.ID, item, func(ctx context.Context, d bson.M) error {
|
|
raw, e := bson.Marshal(d)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var next domain.WorkflowRecord
|
|
if e = bson.Unmarshal(raw, &next); e != nil {
|
|
return e
|
|
}
|
|
return store.upsertWorkflowSupplierGuarded(ctx, next)
|
|
})
|
|
}
|
|
func (store *MongoStore) upsertWorkflowSupplierGuarded(ctx context.Context, item domain.WorkflowRecord) error {
|
|
allow, err := projectAllowsUpsert(ctx, store.db, item.ProjectID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !allow && item.CaptureAutomation != nil {
|
|
item.CaptureAutomation.Enabled = false
|
|
}
|
|
_, err = store.workflows.UpdateOne(ctx, bson.M{"id": item.ID}, workflowUpsertUpdate(item), options.Update().SetUpsert(allow))
|
|
return err
|
|
}
|
|
|
|
func workflowUpsertUpdate(item domain.WorkflowRecord) bson.M {
|
|
update := bson.M{"$set": item}
|
|
// WorkflowInputs is bson:"omitempty", so a normal $set silently leaves the
|
|
// previous array behind when a workflow version intentionally removes all
|
|
// root inputs. Explicitly unset it to keep the stored definition identical
|
|
// to the validated API payload.
|
|
if len(item.WorkflowInputs) == 0 {
|
|
update["$unset"] = bson.M{"workflowInputs": ""}
|
|
}
|
|
return update
|
|
}
|
|
|
|
// UpdateWorkflowRunState does a field-level $set of only status/lastRunId/updatedAt
|
|
// so the worker can bump a workflow's run state without overwriting concurrent PUT
|
|
// edits to Nodes/Edges (lost-update fix; see store.Store interface comment).
|
|
func (store *MongoStore) UpdateWorkflowRunState(ctx context.Context, workflowID string, status domain.WorkflowStatus, lastRunID string, updatedAt string) error {
|
|
_, err := store.workflows.UpdateOne(ctx, bson.M{"id": workflowID}, bson.M{"$set": bson.M{
|
|
"status": status,
|
|
"lastRunId": lastRunID,
|
|
"updatedAt": updatedAt,
|
|
}})
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListTemplates(ctx context.Context) ([]domain.TemplateRecord, error) {
|
|
cursor, err := store.templates.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.TemplateRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListTemplatesByOrganization(ctx context.Context, organizationID string) ([]domain.TemplateRecord, error) {
|
|
cursor, err := store.templates.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.TemplateRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetTemplate(ctx context.Context, id string) (*domain.TemplateRecord, error) {
|
|
var item domain.TemplateRecord
|
|
err := store.templates.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateTemplate(ctx context.Context, input domain.CreateTemplateInput) (*domain.TemplateRecord, error) {
|
|
now := nowISO()
|
|
templateType := input.Type
|
|
if templateType == "" {
|
|
templateType = domain.WorkflowTypeBatch
|
|
}
|
|
item := domain.TemplateRecord{
|
|
ID: domain.NewID("template"),
|
|
OrganizationID: input.OrganizationID,
|
|
Name: input.Name,
|
|
Description: input.Description,
|
|
Type: templateType,
|
|
Kind: input.Kind,
|
|
ExecutionScope: input.ExecutionScope,
|
|
ScanConfig: input.ScanConfig,
|
|
Hyperparams: input.Hyperparams,
|
|
MaxConcurrentRuns: input.MaxConcurrentRuns,
|
|
NodeTimeoutSeconds: input.NodeTimeoutSeconds,
|
|
RunTimeoutSeconds: input.RunTimeoutSeconds,
|
|
MaxRetryAttempts: input.MaxRetryAttempts,
|
|
RetryBackoffSeconds: input.RetryBackoffSeconds,
|
|
Schedule: input.Schedule,
|
|
DefinitionState: domain.WorkflowDefinitionStateDraft,
|
|
DefinitionVersion: 1,
|
|
WorkflowInputs: input.WorkflowInputs,
|
|
ArtifactStorage: input.ArtifactStorage,
|
|
Nodes: input.Nodes,
|
|
Edges: input.Edges,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
item.DefinitionFamilyID = item.ID
|
|
if err := store.UpsertTemplate(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteTemplate(ctx context.Context, id string) (bool, error) {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
return false, nil
|
|
}
|
|
// Removing the template entry must not invalidate immutable snapshots
|
|
// still used by project instances or historical/active Runs.
|
|
referenced := []any{}
|
|
for _, collection := range []mongoCollection{store.projectWorkflowInstances, store.runs} {
|
|
ids, err := collection.Distinct(ctx, "releaseId", bson.M{"releaseId": bson.M{"$exists": true, "$ne": ""}})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
referenced = append(referenced, ids...)
|
|
}
|
|
if _, err := store.workflowReleases.DeleteMany(ctx, bson.M{"templateId": id, "id": bson.M{"$nin": referenced}}); err != nil {
|
|
return false, err
|
|
}
|
|
result, err := store.templates.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount == 1, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertTemplate(ctx context.Context, item domain.TemplateRecord) error {
|
|
_, err := store.templates.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListRuns(ctx context.Context, status string) ([]domain.RunRecord, error) {
|
|
filter := bson.M{}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.runs.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
hydrateRunDerivedFields(&items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (store *MongoStore) ListRunsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
|
|
filter := bson.M{}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.runs.Find(ctx, organizationScopedFilter(organizationID, filter), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
hydrateRunDerivedFields(&items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func runArtifactProjection() bson.M {
|
|
return bson.M{
|
|
"id": 1,
|
|
"organizationId": 1,
|
|
"workflowId": 1,
|
|
"workflowName": 1,
|
|
"projectId": 1,
|
|
"projectName": 1,
|
|
"status": 1,
|
|
"artifacts": 1,
|
|
"nodeExecutions.nodeId": 1,
|
|
"nodeExecutions.artifacts": 1,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
}
|
|
|
|
func (store *MongoStore) ListRunArtifacts(ctx context.Context, status string) ([]domain.RunRecord, error) {
|
|
filter := bson.M{}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetProjection(runArtifactProjection()).
|
|
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
hydrateRunDerivedFields(&items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// ListRunArtifactsForProject scopes the query to runs whose projectId
|
|
// matches projectID directly, or whose workflowId is in workflowIDs, using
|
|
// the existing organizationId_projectId_createdAt_id and
|
|
// organizationId_workflowId_createdAt_id indexes instead of scanning every
|
|
// run in the organization.
|
|
func (store *MongoStore) ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) {
|
|
or := bson.A{}
|
|
if strings.TrimSpace(projectID) != "" {
|
|
or = append(or, bson.M{"projectId": projectID})
|
|
}
|
|
if len(workflowIDs) > 0 {
|
|
or = append(or, bson.M{"workflowId": bson.M{"$in": workflowIDs}})
|
|
}
|
|
if len(or) == 0 {
|
|
return []domain.RunRecord{}, nil
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
organizationScopedFilter(organizationID, bson.M{"$or": or}),
|
|
options.Find().
|
|
SetProjection(runArtifactProjection()).
|
|
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
hydrateRunDerivedFields(&items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// Excludes archived runs: this is the query behind validateDeletionReferences'
|
|
// org-wide safety scan, and an organization's run history is unbounded and
|
|
// never otherwise shrinks, so a run old enough to be archived is excluded
|
|
// here to keep that scan's cost from growing forever (see ArchiveTerminalRuns).
|
|
func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
|
|
filter := bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
organizationScopedFilter(organizationID, filter),
|
|
options.Find().
|
|
SetProjection(runArtifactProjection()).
|
|
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
hydrateRunDerivedFields(&items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// runRecencySort orders candidates exactly like the in-memory selector's
|
|
// freshness key (dashboardLatestAcceptancePayload / latestPocGPUWorkflowRun:
|
|
// UpdatedAt when present, CreatedAt otherwise), so even runs that never got
|
|
// an UpdatedAt written sort the same on both sides of the store boundary.
|
|
func runRecencySort() bson.D {
|
|
return bson.D{
|
|
{Key: "updatedAt", Value: bson.M{"$ifNull": bson.A{"$updatedAt", "$createdAt"}}},
|
|
{Key: "createdAt", Value: -1},
|
|
}
|
|
}
|
|
|
|
// ListRunsAcceptanceCandidates implements Store.ListRunsAcceptanceCandidates.
|
|
// The $or clauses mirror dashboardAcceptancePayloadFromArtifact's four
|
|
// detection paths (artifact name + data/preview.value/value.acceptanceScope),
|
|
// so the latest acceptance run is always inside the bounded result set while
|
|
// the in-memory selector keeps its exact picking semantics.
|
|
func (store *MongoStore) ListRunsAcceptanceCandidates(ctx context.Context, organizationID string, limit int) ([]domain.RunRecord, error) {
|
|
filter := bson.M{"$or": bson.A{
|
|
bson.M{"artifacts.name": "poc-operator-full-acceptance-result.json"},
|
|
bson.M{"artifacts.data.acceptanceScope": bson.M{"$exists": true}},
|
|
bson.M{"artifacts.data.value.acceptanceScope": bson.M{"$exists": true}},
|
|
bson.M{"artifacts.data.preview.value.acceptanceScope": bson.M{"$exists": true}},
|
|
}}
|
|
if organizationID != "" {
|
|
filter = organizationScopedFilter(organizationID, filter)
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetProjection(runArtifactProjection()).
|
|
SetSort(runRecencySort()).
|
|
SetLimit(int64(limit)),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
hydrateRunDerivedFields(&items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// ListRunsWorkflowCandidates implements Store.ListRunsWorkflowCandidates.
|
|
// Candidates need full documents (logs + node executions feed the preflight
|
|
// gate's log scan), but the name filters keep the result set tiny.
|
|
func (store *MongoStore) ListRunsWorkflowCandidates(ctx context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error) {
|
|
or := bson.A{}
|
|
if len(workflowNames) > 0 {
|
|
or = append(or, bson.M{"workflowName": bson.M{"$in": workflowNames}})
|
|
}
|
|
if len(projectNames) > 0 {
|
|
or = append(or, bson.M{"projectName": bson.M{"$in": projectNames}})
|
|
}
|
|
if len(or) == 0 {
|
|
return []domain.RunRecord{}, nil
|
|
}
|
|
filter := bson.M{"$or": or}
|
|
if organizationID != "" {
|
|
filter = organizationScopedFilter(organizationID, filter)
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetSort(runRecencySort()).
|
|
SetLimit(int64(limit)),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
hydrateRunDerivedFields(&items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
|
|
func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) {
|
|
filter := bson.M{
|
|
"status": bson.M{"$in": bson.A{string(domain.RunStatusCompleted), string(domain.RunStatusFailed), string(domain.RunStatusCancelled)}},
|
|
"endedAt": bson.M{"$ne": "", "$lt": olderThan.UTC().Format(time.RFC3339Nano)},
|
|
"archivedAt": bson.M{"$in": bson.A{nil, ""}},
|
|
}
|
|
result, err := store.runs.UpdateMany(ctx, filter, bson.M{"$set": bson.M{"archivedAt": nowISO()}})
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return int(result.ModifiedCount), nil
|
|
}
|
|
|
|
func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) {
|
|
cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{
|
|
{{Key: "$group", Value: bson.M{
|
|
"_id": "$status",
|
|
"count": bson.M{"$sum": 1},
|
|
}}},
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
counts := map[domain.RunStatus]int{}
|
|
for cursor.Next(ctx) {
|
|
var item struct {
|
|
Status domain.RunStatus `bson:"_id"`
|
|
Count int `bson:"count"`
|
|
}
|
|
if err := cursor.Decode(&item); err != nil {
|
|
return nil, err
|
|
}
|
|
counts[item.Status] = item.Count
|
|
}
|
|
return counts, cursor.Err()
|
|
}
|
|
|
|
func runSummaryProjection() bson.M {
|
|
return bson.M{
|
|
"id": 1,
|
|
"organizationId": 1,
|
|
"workflowId": 1,
|
|
"workflowName": 1,
|
|
"projectId": 1,
|
|
"projectName": 1,
|
|
"captureDirectories": 1,
|
|
"taskId": 1,
|
|
"trigger": 1,
|
|
"attempt": 1,
|
|
"mode": 1,
|
|
"releaseId": 1,
|
|
"workflowVersion": 1,
|
|
"gitCommitSha": 1,
|
|
"imageDigest": 1,
|
|
"parentRunId": 1,
|
|
"restartFromNodeId": 1,
|
|
"availableAt": 1,
|
|
"status": 1,
|
|
"stopMode": 1,
|
|
"stopRequestedAt": 1,
|
|
"startedAt": 1,
|
|
"endedAt": 1,
|
|
"durationMs": 1,
|
|
"currentNodeId": 1,
|
|
"failure": 1,
|
|
"progress": 1,
|
|
"error": 1,
|
|
"createdAt": 1,
|
|
"updatedAt": 1,
|
|
}
|
|
}
|
|
|
|
func runSummaryPageProjection() bson.M {
|
|
projection := runSummaryProjection()
|
|
projection["error"] = bson.M{"$substrCP": bson.A{
|
|
bson.M{"$ifNull": bson.A{"$error", ""}},
|
|
0,
|
|
runSummaryTextLimit,
|
|
}}
|
|
return projection
|
|
}
|
|
|
|
func (store *MongoStore) ListRunSummaries(ctx context.Context, status string) ([]domain.RunSummary, error) {
|
|
filter := bson.M{}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetProjection(runSummaryProjection()).
|
|
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.RunSummary](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListRunSummariesByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunSummary, error) {
|
|
filter := bson.M{}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
organizationScopedFilter(organizationID, filter),
|
|
options.Find().
|
|
SetProjection(runSummaryProjection()).
|
|
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.RunSummary](ctx, cursor)
|
|
}
|
|
|
|
func combineRunSummaryFilters(filters ...bson.M) bson.M {
|
|
items := make(bson.A, 0, len(filters))
|
|
for _, filter := range filters {
|
|
if len(filter) > 0 {
|
|
items = append(items, filter)
|
|
}
|
|
}
|
|
if len(items) == 0 {
|
|
return bson.M{}
|
|
}
|
|
if len(items) == 1 {
|
|
return items[0].(bson.M)
|
|
}
|
|
return bson.M{"$and": items}
|
|
}
|
|
|
|
func runSummaryBaseFilter(query domain.RunSummaryQuery) bson.M {
|
|
filters := make([]bson.M, 0, 4)
|
|
if query.ProjectID != "" {
|
|
filters = append(filters, bson.M{"projectId": query.ProjectID})
|
|
}
|
|
if query.WorkflowID != "" {
|
|
filters = append(filters, bson.M{"workflowId": query.WorkflowID})
|
|
}
|
|
if query.Mode == domain.RunModeProduction {
|
|
filters = append(filters, bson.M{"$or": bson.A{
|
|
bson.M{"mode": string(domain.RunModeProduction)},
|
|
bson.M{"mode": ""},
|
|
bson.M{"mode": bson.M{"$exists": false}},
|
|
bson.M{"mode": nil},
|
|
}})
|
|
} else if query.Mode != "" && query.Mode != domain.RunModeAll && query.Mode != "all" {
|
|
filters = append(filters, bson.M{"mode": string(query.Mode)})
|
|
}
|
|
if search := strings.TrimSpace(query.Search); search != "" {
|
|
pattern := primitive.Regex{Pattern: regexp.QuoteMeta(search), Options: "i"}
|
|
filters = append(filters, bson.M{"$or": bson.A{
|
|
bson.M{"id": pattern},
|
|
bson.M{"workflowId": pattern},
|
|
bson.M{"workflowName": pattern},
|
|
bson.M{"projectId": pattern},
|
|
bson.M{"projectName": pattern},
|
|
bson.M{"captureDirectories": pattern},
|
|
bson.M{"currentNodeId": pattern},
|
|
bson.M{"error": pattern},
|
|
}})
|
|
}
|
|
return combineRunSummaryFilters(filters...)
|
|
}
|
|
|
|
func runSummaryCursorFilter(cursor *domain.RunSummaryCursor) bson.M {
|
|
if cursor == nil {
|
|
return bson.M{}
|
|
}
|
|
return bson.M{"$or": bson.A{
|
|
bson.M{"createdAt": bson.M{"$lt": cursor.CreatedAt}},
|
|
bson.M{"createdAt": cursor.CreatedAt, "id": bson.M{"$lt": cursor.ID}},
|
|
}}
|
|
}
|
|
|
|
func (store *MongoStore) loadRunSummaryAggregates(ctx context.Context, filter bson.M) (runSummaryAggregates, error) {
|
|
pipeline := mongo.Pipeline{}
|
|
if len(filter) > 0 {
|
|
pipeline = append(pipeline, bson.D{{Key: "$match", Value: filter}})
|
|
}
|
|
pipeline = append(pipeline, bson.D{{Key: "$group", Value: bson.M{
|
|
"_id": "$status",
|
|
"count": bson.M{"$sum": 1},
|
|
}}})
|
|
cursor, err := store.runs.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return runSummaryAggregates{}, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
result := runSummaryAggregates{StatusCounts: emptyRunStatusCounts()}
|
|
for cursor.Next(ctx) {
|
|
var item struct {
|
|
Status domain.RunStatus `bson:"_id"`
|
|
Count int64 `bson:"count"`
|
|
}
|
|
if err := cursor.Decode(&item); err != nil {
|
|
return runSummaryAggregates{}, err
|
|
}
|
|
result.StatusCounts[item.Status] = item.Count
|
|
result.Total += item.Count
|
|
}
|
|
if err := cursor.Err(); err != nil {
|
|
return runSummaryAggregates{}, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (store *MongoStore) findRunSummaryPageItems(ctx context.Context, filter bson.M, limit int) ([]domain.RunSummary, error) {
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetProjection(runSummaryPageProjection()).
|
|
SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}).
|
|
SetLimit(int64(limit+1)),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.RunSummary](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) listRunSummariesPage(ctx context.Context, organizationID string, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) {
|
|
baseFilter := runSummaryBaseFilter(query)
|
|
if organizationID != "" {
|
|
baseFilter = organizationScopedFilter(organizationID, baseFilter)
|
|
}
|
|
|
|
itemFilter := bson.M{}
|
|
if query.Status != "" {
|
|
itemFilter["status"] = query.Status
|
|
}
|
|
limit := normalizedRunSummaryLimit(query.Limit)
|
|
itemFilter = combineRunSummaryFilters(baseFilter, itemFilter, runSummaryCursorFilter(query.Cursor))
|
|
|
|
queryCtx, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
var waitGroup sync.WaitGroup
|
|
var items []domain.RunSummary
|
|
var aggregates runSummaryAggregates
|
|
queryErrors := make(chan error, 2)
|
|
waitGroup.Add(1)
|
|
go func() {
|
|
defer waitGroup.Done()
|
|
foundItems, err := store.findRunSummaryPageItems(queryCtx, itemFilter, limit)
|
|
if err != nil {
|
|
queryErrors <- err
|
|
cancel()
|
|
return
|
|
}
|
|
items = foundItems
|
|
}()
|
|
includeAggregates := query.IncludeAggregates || !query.AggregatesExplicit
|
|
if includeAggregates {
|
|
waitGroup.Add(1)
|
|
go func() {
|
|
defer waitGroup.Done()
|
|
key := newRunSummaryAggregateCacheKey(organizationID, query)
|
|
foundAggregates, err := store.runSummaryAggregates.getOrLoad(queryCtx, key, func(loadCtx context.Context) (runSummaryAggregates, error) {
|
|
return store.loadRunSummaryAggregates(loadCtx, baseFilter)
|
|
})
|
|
if err != nil {
|
|
queryErrors <- err
|
|
cancel()
|
|
return
|
|
}
|
|
aggregates = foundAggregates
|
|
}()
|
|
}
|
|
waitGroup.Wait()
|
|
select {
|
|
case err := <-queryErrors:
|
|
return domain.RunSummaryPage{}, err
|
|
default:
|
|
}
|
|
|
|
hasMore := len(items) > limit
|
|
if hasMore {
|
|
items = items[:limit]
|
|
}
|
|
for index := range items {
|
|
items[index] = compactRunSummary(items[index])
|
|
}
|
|
var nextCursor *domain.RunSummaryCursor
|
|
if hasMore && len(items) > 0 {
|
|
last := items[len(items)-1]
|
|
nextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
|
}
|
|
return domain.RunSummaryPage{
|
|
Items: items,
|
|
Total: aggregates.Total,
|
|
StatusCounts: aggregates.StatusCounts,
|
|
NextCursor: nextCursor,
|
|
HasMore: hasMore,
|
|
}, nil
|
|
}
|
|
|
|
func (store *MongoStore) ListRunSummariesPage(ctx context.Context, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) {
|
|
return store.listRunSummariesPage(ctx, "", query)
|
|
}
|
|
|
|
func (store *MongoStore) ListRunSummariesPageByOrganization(ctx context.Context, organizationID string, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) {
|
|
return store.listRunSummariesPage(ctx, organizationID, query)
|
|
}
|
|
|
|
func (store *MongoStore) ListReconcileRunSummaries(ctx context.Context, limit int) ([]domain.RunSummary, error) {
|
|
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
|
|
filter := bson.M{"$or": bson.A{
|
|
bson.M{"status": bson.M{"$in": bson.A{domain.RunStatusRunning, domain.RunStatusStopping}}},
|
|
bson.M{
|
|
"status": domain.RunStatusQueued,
|
|
"currentNodeId": bson.M{"$exists": true, "$nin": bson.A{"", nil}},
|
|
},
|
|
}}
|
|
cursor, err := store.runs.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetProjection(runSummaryPageProjection()).
|
|
SetSort(bson.D{{Key: "updatedAt", Value: 1}, {Key: "id", Value: 1}}).
|
|
SetLimit(int64(limit)),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := decodeMany[domain.RunSummary](ctx, cursor)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range items {
|
|
items[index] = compactRunSummary(items[index])
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (store *MongoStore) CountWorkerRuns(ctx context.Context) (WorkerRunCounts, error) {
|
|
statuses := bson.A{
|
|
domain.RunStatusQueued,
|
|
domain.RunStatusRunning,
|
|
domain.RunStatusStopping,
|
|
domain.RunStatusWaitingForAudit,
|
|
domain.RunStatusWaitingForHuman,
|
|
}
|
|
pipeline := mongo.Pipeline{
|
|
bson.D{{Key: "$match", Value: bson.M{"status": bson.M{"$in": statuses}}}},
|
|
bson.D{{Key: "$group", Value: bson.M{
|
|
"_id": bson.M{
|
|
"workflowId": "$workflowId",
|
|
"status": "$status",
|
|
},
|
|
"count": bson.M{"$sum": 1},
|
|
}}},
|
|
}
|
|
read := options.Aggregate().SetHint(workerRunCountIndex)
|
|
cursor, err := store.runs.Aggregate(ctx, pipeline, read)
|
|
if poolSearchHintMissing(err) && ctx.Err() == nil {
|
|
cursor, err = store.runs.Aggregate(ctx, pipeline, read.SetHint(nil))
|
|
}
|
|
if err != nil {
|
|
return WorkerRunCounts{}, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
counts := newWorkerRunCounts()
|
|
for cursor.Next(ctx) {
|
|
var item struct {
|
|
Key struct {
|
|
WorkflowID string `bson:"workflowId"`
|
|
Status domain.RunStatus `bson:"status"`
|
|
} `bson:"_id"`
|
|
Count int `bson:"count"`
|
|
}
|
|
if err := cursor.Decode(&item); err != nil {
|
|
return WorkerRunCounts{}, err
|
|
}
|
|
addWorkerRunCount(&counts, item.Key.WorkflowID, item.Key.Status, item.Count)
|
|
}
|
|
if err := cursor.Err(); err != nil {
|
|
return WorkerRunCounts{}, err
|
|
}
|
|
return counts, nil
|
|
}
|
|
|
|
func (store *MongoStore) HasActiveRun(ctx context.Context, workflowID string) (bool, error) {
|
|
count, err := store.runs.CountDocuments(ctx, bson.M{
|
|
"workflowId": workflowID,
|
|
"status": bson.M{"$in": activeRunStatuses()},
|
|
}, options.Count().SetLimit(1))
|
|
return count > 0, err
|
|
}
|
|
|
|
func (store *MongoStore) UpdateRunProgress(ctx context.Context, runID string, progress *domain.RunProgress) error {
|
|
updatedAt := nowISO()
|
|
update := bson.M{"$set": bson.M{"updatedAt": updatedAt}}
|
|
if progress == nil {
|
|
update["$unset"] = bson.M{"progress": ""}
|
|
} else {
|
|
value := *progress
|
|
if strings.TrimSpace(value.UpdatedAt) == "" {
|
|
value.UpdatedAt = updatedAt
|
|
}
|
|
updatedAt = value.UpdatedAt
|
|
update["$set"] = bson.M{"progress": value, "updatedAt": updatedAt}
|
|
}
|
|
result, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, update)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if result.MatchedCount == 0 {
|
|
return fmt.Errorf("run %s not found", runID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (store *MongoStore) UpdateRunFailure(ctx context.Context, runID string, failure *domain.RunFailure) error {
|
|
update := bson.M{"$set": bson.M{"updatedAt": nowISO()}}
|
|
if failure == nil {
|
|
update["$unset"] = bson.M{"failure": ""}
|
|
} else {
|
|
update["$set"] = bson.M{"failure": *failure, "updatedAt": nowISO()}
|
|
}
|
|
result, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, update)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if result.MatchedCount == 0 {
|
|
return fmt.Errorf("run %s not found", runID)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunRecord, error) {
|
|
var item domain.RunRecord
|
|
err := store.runs.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hydrateRunDerivedFields(&item)
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
|
|
return store.CreateRunWithInputs(ctx, workflowID, trigger, nil, taskIDs...)
|
|
}
|
|
|
|
func (store *MongoStore) CreateRunWithInputs(ctx context.Context, workflowID string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) {
|
|
w, e := store.GetWorkflow(ctx, workflowID)
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if w == nil {
|
|
return nil, fmt.Errorf("workflow not found")
|
|
}
|
|
var result *domain.RunRecord
|
|
e = supplierGuardedMongoWrite(ctx, store.runs, "new-run", bson.M{"projectId": w.ProjectID, "organizationId": w.OrganizationID, "workflowId": workflowID, "inputs": inputs}, func(ctx context.Context, _ bson.M) error {
|
|
var e error
|
|
result, e = store.createRunSupplierGuarded(ctx, workflowID, trigger, inputs, taskIDs...)
|
|
return e
|
|
})
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if result == nil {
|
|
return nil, ErrSupplierDeleting
|
|
}
|
|
return result, nil
|
|
}
|
|
func (store *MongoStore) createRunSupplierGuarded(ctx context.Context, workflowID string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) {
|
|
workflow, err := store.GetWorkflow(ctx, workflowID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if workflow == nil {
|
|
return nil, fmt.Errorf("workflow %s not found", workflowID)
|
|
}
|
|
projectName := ""
|
|
if workflow.ProjectID != "" {
|
|
project, err := store.GetProject(ctx, workflow.ProjectID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if project == nil {
|
|
return nil, fmt.Errorf("project %s not found", workflow.ProjectID)
|
|
}
|
|
projectName = project.Name
|
|
if project.DeletionTaskID != "" {
|
|
return nil, fmt.Errorf("project deletion in progress")
|
|
}
|
|
}
|
|
now := nowISO()
|
|
availableAt := now
|
|
if trigger == domain.RunTriggerRerunSubtree || trigger == domain.RunTriggerHumanRelease {
|
|
// See MemoryStore.CreateRun: rerun-subtree creation is finalized by the
|
|
// API in a second write after reusable upstream executions are attached.
|
|
availableAt = time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339Nano)
|
|
}
|
|
previousRunAt := workflow.LastRunAt
|
|
resolvedInputs, resolvedBindings, err := domain.ResolveWorkflowRunInputs(*workflow, inputs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
run := domain.RunRecord{
|
|
ProcessingProfile: domain.WorkflowProcessingProfile(workflow.Nodes),
|
|
ID: domain.NewID("run"),
|
|
OrganizationID: workflow.OrganizationID,
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: workflow.ProjectID,
|
|
ProjectName: projectName,
|
|
Trigger: trigger,
|
|
Mode: domain.RunModeProduction,
|
|
Attempt: 1,
|
|
DefinitionVersion: workflow.DefinitionVersion,
|
|
WorkflowVersion: workflow.DefinitionVersion,
|
|
Inputs: resolvedInputs,
|
|
ResolvedInputBindings: resolvedBindings,
|
|
AvailableAt: availableAt,
|
|
Status: domain.RunStatusQueued,
|
|
PreviousRunAt: previousRunAt,
|
|
CurrentRunAt: now,
|
|
NodeExecutions: []domain.NodeExecutionRecord{},
|
|
Logs: []domain.LogEntryRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
K8sRefs: []domain.K8sRef{},
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
if len(taskIDs) > 0 {
|
|
run.TaskID = strings.TrimSpace(taskIDs[0])
|
|
}
|
|
run.CaptureDirectories = domain.CaptureDirectoriesFromRun(run)
|
|
if err := store.UpsertRun(ctx, run); err != nil {
|
|
return nil, err
|
|
}
|
|
workflow.LastRunID = run.ID
|
|
workflow.LastRunAt = now
|
|
workflow.Status = domain.WorkflowStatusRunning
|
|
workflow.UpdatedAt = now
|
|
if err := store.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
return &run, nil
|
|
}
|
|
|
|
func mongoRunLeaseAvailableFilter(now string) []bson.M {
|
|
return []bson.M{
|
|
{"executionLeaseOwner": bson.M{"$exists": false}},
|
|
{"executionLeaseOwner": ""},
|
|
{"executionLeaseExpiresAt": bson.M{"$exists": false}},
|
|
{"executionLeaseExpiresAt": ""},
|
|
{"executionLeaseExpiresAt": bson.M{"$lte": now}},
|
|
}
|
|
}
|
|
|
|
func mongoRunLeaseSet(owner string, ttl time.Duration, now time.Time) bson.M {
|
|
if ttl <= 0 {
|
|
ttl = 5 * time.Minute
|
|
}
|
|
return bson.M{
|
|
"executionLeaseOwner": strings.TrimSpace(owner),
|
|
"executionLeaseExpiresAt": now.Add(ttl).UTC().Format(time.RFC3339Nano),
|
|
}
|
|
}
|
|
|
|
func (store *MongoStore) ClaimNextQueuedRun(ctx context.Context, leaseOwner string, leaseTTL time.Duration) (*domain.RunRecord, error) {
|
|
nowTime := time.Now().UTC()
|
|
now := nowTime.Format(time.RFC3339Nano)
|
|
set := bson.M{
|
|
"status": string(domain.RunStatusRunning),
|
|
"startedAt": now,
|
|
"updatedAt": now,
|
|
}
|
|
for key, value := range mongoRunLeaseSet(leaseOwner, leaseTTL, nowTime) {
|
|
set[key] = value
|
|
}
|
|
result := store.runs.FindOneAndUpdate(
|
|
ctx,
|
|
bson.M{
|
|
"status": string(domain.RunStatusQueued),
|
|
"$or": []bson.M{
|
|
{"availableAt": bson.M{"$exists": false}},
|
|
{"availableAt": ""},
|
|
{"availableAt": bson.M{"$lte": now}},
|
|
},
|
|
"$and": []bson.M{
|
|
{"$or": mongoRunLeaseAvailableFilter(now)},
|
|
},
|
|
},
|
|
bson.M{"$set": set},
|
|
options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After),
|
|
)
|
|
var item domain.RunRecord
|
|
err := result.Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hydrateRunDerivedFields(&item)
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) ClaimQueuedRun(ctx context.Context, runID string, leaseOwner string, leaseTTL time.Duration) (*domain.RunRecord, error) {
|
|
nowTime := time.Now().UTC()
|
|
now := nowTime.Format(time.RFC3339Nano)
|
|
set := bson.M{
|
|
"status": string(domain.RunStatusRunning),
|
|
"startedAt": now,
|
|
"updatedAt": now,
|
|
}
|
|
for key, value := range mongoRunLeaseSet(leaseOwner, leaseTTL, nowTime) {
|
|
set[key] = value
|
|
}
|
|
result := store.runs.FindOneAndUpdate(
|
|
ctx,
|
|
bson.M{
|
|
"id": runID,
|
|
"status": string(domain.RunStatusQueued),
|
|
"$or": []bson.M{
|
|
{"availableAt": bson.M{"$exists": false}},
|
|
{"availableAt": ""},
|
|
{"availableAt": bson.M{"$lte": now}},
|
|
},
|
|
"$and": []bson.M{
|
|
{"$or": mongoRunLeaseAvailableFilter(now)},
|
|
},
|
|
},
|
|
bson.M{"$set": set},
|
|
options.FindOneAndUpdate().SetReturnDocument(options.After),
|
|
)
|
|
var item domain.RunRecord
|
|
err := result.Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
hydrateRunDerivedFields(&item)
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) AcquireRunLease(ctx context.Context, runID string, owner string, ttl time.Duration) (*domain.RunRecord, bool, error) {
|
|
nowTime := time.Now().UTC()
|
|
now := nowTime.Format(time.RFC3339Nano)
|
|
set := bson.M{"updatedAt": now}
|
|
for key, value := range mongoRunLeaseSet(owner, ttl, nowTime) {
|
|
set[key] = value
|
|
}
|
|
result := store.runs.FindOneAndUpdate(
|
|
ctx,
|
|
bson.M{
|
|
"id": runID,
|
|
"$or": append(
|
|
mongoRunLeaseAvailableFilter(now),
|
|
bson.M{"executionLeaseOwner": strings.TrimSpace(owner)},
|
|
),
|
|
},
|
|
bson.M{"$set": set},
|
|
options.FindOneAndUpdate().SetReturnDocument(options.After),
|
|
)
|
|
var item domain.RunRecord
|
|
err := result.Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
current, getErr := store.GetRun(ctx, runID)
|
|
return current, false, getErr
|
|
}
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
hydrateRunDerivedFields(&item)
|
|
return &item, true, nil
|
|
}
|
|
|
|
func (store *MongoStore) RenewRunLease(ctx context.Context, runID string, owner string, ttl time.Duration) (bool, error) {
|
|
nowTime := time.Now().UTC()
|
|
now := nowTime.Format(time.RFC3339Nano)
|
|
set := bson.M{"updatedAt": now}
|
|
for key, value := range mongoRunLeaseSet(owner, ttl, nowTime) {
|
|
set[key] = value
|
|
}
|
|
result, err := store.runs.UpdateOne(
|
|
ctx,
|
|
bson.M{"id": runID, "executionLeaseOwner": strings.TrimSpace(owner)},
|
|
bson.M{"$set": set},
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.MatchedCount > 0, nil
|
|
}
|
|
|
|
func (store *MongoStore) ReleaseRunLease(ctx context.Context, runID string, owner string) error {
|
|
_, err := store.runs.UpdateOne(
|
|
ctx,
|
|
bson.M{"id": runID, "executionLeaseOwner": strings.TrimSpace(owner)},
|
|
bson.M{
|
|
"$set": bson.M{"updatedAt": nowISO()},
|
|
"$unset": bson.M{
|
|
"executionLeaseOwner": "",
|
|
"executionLeaseExpiresAt": "",
|
|
},
|
|
},
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) RequestRunStop(ctx context.Context, runID string, mode domain.RunStopMode) (*domain.RunRecord, error) {
|
|
run, err := store.GetRun(ctx, runID)
|
|
if err != nil || run == nil {
|
|
return run, err
|
|
}
|
|
if run.Status == domain.RunStatusCompleted || run.Status == domain.RunStatusFailed || run.Status == domain.RunStatusCancelled {
|
|
return run, nil
|
|
}
|
|
|
|
now := nowISO()
|
|
set := bson.M{
|
|
"stopMode": string(mode),
|
|
"stopRequestedAt": now,
|
|
"updatedAt": now,
|
|
}
|
|
switch mode {
|
|
case domain.RunStopModeGraceful:
|
|
if run.Status == domain.RunStatusQueued || run.Status == domain.RunStatusWaitingForAudit || run.Status == domain.RunStatusWaitingForHuman {
|
|
set["status"] = string(domain.RunStatusCancelled)
|
|
set["endedAt"] = now
|
|
run.EndedAt = now
|
|
finalizeNodeExecutionsForStop(run, mode, now)
|
|
set["nodeExecutions"] = run.NodeExecutions
|
|
if strings.TrimSpace(run.Error) == "" {
|
|
set["error"] = "用户已停止工作流"
|
|
}
|
|
} else {
|
|
set["status"] = string(domain.RunStatusStopping)
|
|
}
|
|
case domain.RunStopModeForce:
|
|
set["status"] = string(domain.RunStatusCancelled)
|
|
set["endedAt"] = now
|
|
run.EndedAt = now
|
|
finalizeNodeExecutionsForStop(run, mode, now)
|
|
set["nodeExecutions"] = run.NodeExecutions
|
|
if strings.TrimSpace(run.Error) == "" {
|
|
set["error"] = "用户已强制停止工作流"
|
|
}
|
|
default:
|
|
return run, nil
|
|
}
|
|
computeRunDuration(run)
|
|
if run.DurationMS > 0 {
|
|
set["durationMs"] = run.DurationMS
|
|
}
|
|
|
|
if _, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, bson.M{"$set": set}); err != nil {
|
|
return nil, err
|
|
}
|
|
return store.GetRun(ctx, runID)
|
|
}
|
|
|
|
func (store *MongoStore) UpsertRun(ctx context.Context, item domain.RunRecord) error {
|
|
return supplierGuardedMongoWrite(ctx, store.runs, item.ID, item, func(ctx context.Context, d bson.M) error {
|
|
raw, e := bson.Marshal(d)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var next domain.RunRecord
|
|
if e = bson.Unmarshal(raw, &next); e != nil {
|
|
return e
|
|
}
|
|
return store.upsertRunSupplierGuarded(ctx, next)
|
|
})
|
|
}
|
|
func (store *MongoStore) upsertRunSupplierGuarded(ctx context.Context, item domain.RunRecord) error {
|
|
allow, err := projectAllowsUpsert(ctx, store.db, item.ProjectID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
computeRunDuration(&item)
|
|
stored := compactRunForStorage(item)
|
|
unsetFields := runStorageFieldsToUnset(item)
|
|
update, err := annotationSafeRunUpdate(stored, unsetFields)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = store.runs.UpdateOne(ctx, bson.M{"id": item.ID}, update, options.Update().SetUpsert(allow))
|
|
return err
|
|
}
|
|
|
|
// AppendNodeExecutionResourceSample pushes a single resource sample onto a running
|
|
// node's resourceSeries without rewriting the whole run document. The launcher
|
|
// flushes a node's full series at completion; this lets the worker stream live
|
|
// samples DURING execution so the resource monitor reflects custom-container GPU
|
|
// usage before the node finishes. The positional $ targets the nodeExecutions
|
|
// element matching nodeID; UpdateOne is a no-op if no such element exists yet.
|
|
func (store *MongoStore) AppendNodeExecutionResourceSample(ctx context.Context, runID string, nodeID string, sample domain.ResourceSample) error {
|
|
if strings.TrimSpace(runID) == "" || strings.TrimSpace(nodeID) == "" {
|
|
return fmt.Errorf("runID and nodeID are required")
|
|
}
|
|
_, err := store.runs.UpdateOne(ctx,
|
|
bson.M{"id": runID, "nodeExecutions.nodeId": nodeID},
|
|
bson.M{"$push": bson.M{"nodeExecutions.$.resourceSeries": sample}},
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) GetSourceState(ctx context.Context, workflowID string, nodeID string, sourceFingerprint string) (*domain.SourceStateRecord, error) {
|
|
var item domain.SourceStateRecord
|
|
err := store.sourceStates.FindOne(ctx, bson.M{
|
|
"workflowId": workflowID,
|
|
"nodeId": nodeID,
|
|
"sourceFingerprint": sourceFingerprint,
|
|
}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) ListSourceStates(ctx context.Context, workflowID string, nodeID string) ([]domain.SourceStateRecord, error) {
|
|
filter := bson.M{"workflowId": workflowID}
|
|
if nodeID != "" {
|
|
filter["nodeId"] = nodeID
|
|
}
|
|
cursor, err := store.sourceStates.Find(ctx, filter)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.SourceStateRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) UpsertSourceState(ctx context.Context, item domain.SourceStateRecord) error {
|
|
return supplierGuardedMongoWrite(ctx, store.sourceStates, item.ID, item, func(ctx context.Context, d bson.M) error {
|
|
raw, e := bson.Marshal(d)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
var next domain.SourceStateRecord
|
|
if e = bson.Unmarshal(raw, &next); e != nil {
|
|
return e
|
|
}
|
|
return store.upsertSourceStateSupplierGuarded(ctx, next)
|
|
})
|
|
}
|
|
func (store *MongoStore) upsertSourceStateSupplierGuarded(ctx context.Context, item domain.SourceStateRecord) error {
|
|
if item.ID == "" {
|
|
item.ID = item.WorkflowID + ":" + item.NodeID + ":" + item.SourceFingerprint
|
|
}
|
|
_, err := store.sourceStates.UpdateOne(
|
|
ctx,
|
|
bson.M{"id": item.ID},
|
|
bson.M{"$set": item},
|
|
options.Update().SetUpsert(true),
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) DeleteSourceState(ctx context.Context, workflowID string, nodeID string, sourceFingerprint string) error {
|
|
_, err := store.sourceStates.DeleteOne(ctx, bson.M{
|
|
"workflowId": workflowID,
|
|
"nodeId": nodeID,
|
|
"sourceFingerprint": sourceFingerprint,
|
|
})
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) DeleteSourceStatesByWorkflow(ctx context.Context, workflowID string) error {
|
|
_, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": workflowID})
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListAuditTasks(ctx context.Context, status string) ([]domain.AuditTaskRecord, error) {
|
|
filter := bson.M{}
|
|
if status != "" && status != "all" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.auditTasks.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.AuditTaskRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListAuditTasksByOrganization(ctx context.Context, organizationID string, status string) ([]domain.AuditTaskRecord, error) {
|
|
filter := bson.M{}
|
|
if status != "" && status != "all" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.auditTasks.Find(ctx, organizationScopedFilter(organizationID, filter), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.AuditTaskRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetAuditTask(ctx context.Context, id string) (*domain.AuditTaskRecord, error) {
|
|
var item domain.AuditTaskRecord
|
|
err := store.auditTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateAuditTask(ctx context.Context, item domain.AuditTaskRecord) (*domain.AuditTaskRecord, error) {
|
|
if err := store.UpsertAuditTask(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertAuditTask(ctx context.Context, item domain.AuditTaskRecord) error {
|
|
_, err := store.auditTasks.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListSamplingTasks(ctx context.Context, status string) ([]domain.SamplingTaskRecord, error) {
|
|
filter := bson.M{}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.samplingTasks.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.SamplingTaskRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListSamplingTasksByOrganization(ctx context.Context, organizationID string, status string) ([]domain.SamplingTaskRecord, error) {
|
|
filter := bson.M{}
|
|
if status != "" {
|
|
filter["status"] = status
|
|
}
|
|
cursor, err := store.samplingTasks.Find(ctx, organizationScopedFilter(organizationID, filter), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.SamplingTaskRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetSamplingTask(ctx context.Context, id string) (*domain.SamplingTaskRecord, error) {
|
|
var item domain.SamplingTaskRecord
|
|
err := store.samplingTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateSamplingTask(ctx context.Context, item domain.SamplingTaskRecord) (*domain.SamplingTaskRecord, error) {
|
|
if strings.TrimSpace(item.ID) == "" {
|
|
item.ID = domain.NewID("sampling")
|
|
}
|
|
if err := store.UpsertSamplingTask(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertSamplingTask(ctx context.Context, item domain.SamplingTaskRecord) error {
|
|
_, err := store.samplingTasks.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListUsers(ctx context.Context) ([]domain.UserRecord, error) {
|
|
cursor, err := store.users.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.UserRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListUsersByOrganization(ctx context.Context, organizationID string) ([]domain.UserRecord, error) {
|
|
cursor, err := store.users.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.UserRecord](ctx, cursor)
|
|
}
|
|
|
|
// ListProjectMemberUsers uses one scoped query and never loads credential fields.
|
|
func (store *MongoStore) ListProjectMemberUsers(ctx context.Context, organizationID string, ids []string) ([]domain.UserRecord, error) {
|
|
if len(ids) == 0 {
|
|
return []domain.UserRecord{}, nil
|
|
}
|
|
filter := bson.M{"$and": bson.A{orgCompatibilityFilter(organizationID), bson.M{"id": bson.M{"$in": ids}}}}
|
|
cursor, err := store.users.Find(ctx, filter, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "name": 1, "phone": 1, "status": 1, "role": 1, "isSuperAdmin": 1, "supplierId": 1, "organizationId": 1}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.UserRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetUser(ctx context.Context, id string) (*domain.UserRecord, error) {
|
|
var item domain.UserRecord
|
|
err := store.users.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
// FindUserByPhone looks up an undeleted login identity by phone number. Phone numbers are not
|
|
// case-folded (unlike the removed email field).
|
|
func (store *MongoStore) FindUserByPhone(ctx context.Context, phone string) (*domain.UserRecord, error) {
|
|
var item domain.UserRecord
|
|
err := store.users.FindOne(ctx, bson.M{"phone": strings.TrimSpace(phone), "deletedAt": bson.M{"$in": bson.A{"", nil}}}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
// FindUserByLegacyEmail keeps accounts created before the phone-only identity
|
|
// migration login-capable. New account management never writes this field and
|
|
// callers must not expose it as a current account identifier.
|
|
func (store *MongoStore) FindUserByLegacyEmail(ctx context.Context, email string) (*domain.UserRecord, error) {
|
|
var item domain.UserRecord
|
|
err := store.users.FindOne(ctx, bson.M{"email": strings.ToLower(strings.TrimSpace(email)), "deletedAt": bson.M{"$in": bson.A{"", nil}}}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateUser(ctx context.Context, input domain.CreateUserInput) (*domain.UserRecord, error) {
|
|
if input.Role == domain.UserRoleAcceptanceReviewer && strings.TrimSpace(input.SupplierID) == "" {
|
|
return nil, ErrAcceptanceAuthorizationRequired
|
|
}
|
|
if err := store.SnapshotLegacyAcceptanceAccounts(ctx, input.OrganizationID); err != nil {
|
|
return nil, err
|
|
}
|
|
now := nowISO()
|
|
item := domain.UserRecord{
|
|
AcceptanceAccessMode: "managed",
|
|
DefaultProjectRoles: input.DefaultProjectRoles,
|
|
ID: domain.NewID("user"),
|
|
OrganizationID: input.OrganizationID,
|
|
Name: input.Name,
|
|
Phone: strings.TrimSpace(input.Phone),
|
|
Role: input.Role,
|
|
SupplierID: strings.TrimSpace(input.SupplierID),
|
|
IsSuperAdmin: input.IsSuperAdmin,
|
|
PasswordHash: input.PasswordHash,
|
|
Status: input.Status,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
if item.Status == "" {
|
|
item.Status = domain.UserStatusActive
|
|
}
|
|
if item.SupplierID != "" {
|
|
_, err := store.runAccountTransaction(ctx, "suppliers:"+normalizedOrganizationID(item.OrganizationID), func(sc context.Context) (any, error) {
|
|
if item.Role == domain.UserRoleAcceptanceReviewer {
|
|
grant, err := store.GetSupplierAcceptanceAuthorization(sc, item.OrganizationID, item.SupplierID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !grant.Enabled {
|
|
return nil, ErrAcceptanceAuthorizationRequired
|
|
}
|
|
}
|
|
result, e := store.suppliers.UpdateOne(sc, bson.M{"id": item.SupplierID, "organizationId": item.OrganizationID, "status": "active"}, bson.M{"$inc": bson.M{"referenceVersion": 1}})
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
if result.MatchedCount != 1 {
|
|
return nil, ErrSupplierUnavailable
|
|
}
|
|
return nil, store.UpsertUser(sc, item)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
} else if err := store.UpsertUser(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteUser(ctx context.Context, id string) (bool, error) {
|
|
result, err := store.users.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount > 0, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertUser(ctx context.Context, item domain.UserRecord) error {
|
|
return store.upsertUser(ctx, item, nil)
|
|
}
|
|
func (store *MongoStore) UpdateUser(ctx context.Context, item, expected domain.UserRecord) error {
|
|
if item.Role != expected.Role {
|
|
_, err := store.withAccountMutationLock(ctx, "suppliers:"+normalizedOrganizationID(item.OrganizationID), func(ctx context.Context) (any, error) {
|
|
if item.Role == domain.UserRoleAcceptanceReviewer {
|
|
grant, err := store.GetSupplierAcceptanceAuthorization(ctx, item.OrganizationID, item.SupplierID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !grant.Enabled {
|
|
return nil, ErrAcceptanceAuthorizationRequired
|
|
}
|
|
}
|
|
item.AcceptanceAccessMode = "managed"
|
|
if removesActiveAdmin(expected, item) {
|
|
_, err := store.withAccountMutationLock(ctx, "user:"+item.ID, func(ctx context.Context) (any, error) { return nil, store.upsertUser(ctx, item, &expected) })
|
|
return nil, err
|
|
}
|
|
return nil, store.updateAccountRole(ctx, item, expected)
|
|
})
|
|
return err
|
|
}
|
|
_, err := store.withAccountMutationLock(ctx, "user:"+item.ID, func(ctx context.Context) (any, error) { return nil, store.upsertUser(ctx, item, &expected) })
|
|
return err
|
|
}
|
|
func (store *MongoStore) upsertUser(ctx context.Context, item domain.UserRecord, expected *domain.UserRecord) error {
|
|
if item.SupplierID != "" {
|
|
supplier, e := store.GetSupplier(ctx, item.SupplierID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if supplier != nil && (supplier.Status == "deleting" || supplier.Status == "deleted") {
|
|
return ErrSupplierDeleting
|
|
}
|
|
}
|
|
if item.Status == "" {
|
|
item.Status = domain.UserStatusActive
|
|
}
|
|
item.Phone = strings.TrimSpace(item.Phone)
|
|
item.SupplierID = strings.TrimSpace(item.SupplierID)
|
|
existing, err := store.GetUser(ctx, item.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if existing != nil && existing.DeletionRequestedAt != "" && (item.DeletionRequestedAt != existing.DeletionRequestedAt || item.Status != domain.UserStatusDisabled) {
|
|
return ErrAccountDeleted
|
|
}
|
|
if existing != nil && existing.DeletedAt != "" {
|
|
return ErrAccountDeleted
|
|
}
|
|
if expected != nil && (existing == nil || !sameUserMutationSnapshot(*existing, *expected)) {
|
|
return ErrUserMutationConflict
|
|
}
|
|
if existing != nil && removesActiveAdmin(*existing, item) {
|
|
return store.updateAdministrator(ctx, item, expected)
|
|
}
|
|
if existing == nil {
|
|
_, err = store.users.InsertOne(ctx, item)
|
|
} else {
|
|
if existing.SupplierID != item.SupplierID {
|
|
return ErrSupplierImmutable
|
|
}
|
|
fields, encodeErr := bsonDocumentWithoutSupplier(item)
|
|
if encodeErr != nil {
|
|
return encodeErr
|
|
}
|
|
if item.DeletedAt != "" {
|
|
fields["passwordHash"] = ""
|
|
}
|
|
var result *mongo.UpdateResult
|
|
result, err = store.users.UpdateOne(ctx, userMutationFilter(*existing), bson.M{"$set": fields})
|
|
if err == nil && result.MatchedCount == 0 {
|
|
return ErrUserMutationConflict
|
|
}
|
|
}
|
|
if isPhoneUniqueDuplicateKeyError(err) {
|
|
return ErrDuplicateUserPhone
|
|
}
|
|
return err
|
|
}
|
|
|
|
// isPhoneUniqueDuplicateKeyError reports whether err is a MongoDB duplicate-key
|
|
// error specifically from the users collection's phone_unique index, not just
|
|
// any unique-index conflict on that collection — a stale or unrelated index
|
|
// (e.g. a pre-migration email_unique index left over in an older database)
|
|
// would otherwise be misreported to the caller as "phone must be unique".
|
|
func isPhoneUniqueDuplicateKeyError(err error) bool {
|
|
return mongo.IsDuplicateKeyError(err) && (strings.Contains(err.Error(), "phone_unique") || strings.Contains(err.Error(), liveUserPhoneIndex))
|
|
}
|
|
|
|
func (store *MongoStore) CreateSession(ctx context.Context, item domain.SessionRecord) (*domain.SessionRecord, error) {
|
|
if strings.TrimSpace(item.ID) == "" {
|
|
item.ID = domain.NewID("session")
|
|
}
|
|
now := nowISO()
|
|
if item.CreatedAt == "" {
|
|
item.CreatedAt = now
|
|
}
|
|
if item.LastSeenAt == "" {
|
|
item.LastSeenAt = now
|
|
}
|
|
if item.ExpiresAtTime.IsZero() {
|
|
if parsed, err := time.Parse(time.RFC3339, item.ExpiresAt); err == nil {
|
|
item.ExpiresAtTime = parsed
|
|
}
|
|
}
|
|
if _, err := store.sessions.InsertOne(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) GetSessionByTokenHash(ctx context.Context, tokenHash string) (*domain.SessionRecord, error) {
|
|
var item domain.SessionRecord
|
|
err := store.sessions.FindOne(ctx, bson.M{"tokenHash": tokenHash}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteSession(ctx context.Context, id string) (bool, error) {
|
|
result, err := store.sessions.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount > 0, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteSessionsByUser(ctx context.Context, userID string) error {
|
|
_, err := store.sessions.DeleteMany(ctx, bson.M{"userId": userID})
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListStorageConnections(ctx context.Context) ([]domain.StorageConnectionRecord, error) {
|
|
cursor, err := store.storageConnections.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.StorageConnectionRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListStorageConnectionsByOrganization(ctx context.Context, organizationID string) ([]domain.StorageConnectionRecord, error) {
|
|
cursor, err := store.storageConnections.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.StorageConnectionRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) {
|
|
var item domain.StorageConnectionRecord
|
|
err := store.storageConnections.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateStorageConnection(ctx context.Context, input domain.CreateStorageConnectionInput) (*domain.StorageConnectionRecord, error) {
|
|
now := nowISO()
|
|
item := domain.StorageConnectionRecord{
|
|
ID: domain.NewID("storage"),
|
|
OrganizationID: input.OrganizationID,
|
|
Name: input.Name,
|
|
Description: input.Description,
|
|
Provider: input.Provider,
|
|
Endpoint: input.Endpoint,
|
|
Bucket: input.Bucket,
|
|
AccessKeyID: input.AccessKeyID,
|
|
SecretAccessKey: input.SecretAccessKey,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
if err := store.UpsertStorageConnection(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteStorageConnection(ctx context.Context, id string) (bool, error) {
|
|
result, err := store.storageConnections.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount > 0, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertStorageConnection(ctx context.Context, item domain.StorageConnectionRecord) error {
|
|
_, err := store.storageConnections.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) GetSettings(ctx context.Context) (*domain.SettingsRecord, error) {
|
|
var item domain.SettingsRecord
|
|
err := store.settings.FindOne(ctx, bson.M{"id": "system"}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
fallback := defaultSettings()
|
|
if err := store.UpsertSettings(ctx, fallback); err != nil {
|
|
return nil, err
|
|
}
|
|
return &fallback, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
normalized := ensureSettingsDefaults(item)
|
|
return &normalized, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertSettings(ctx context.Context, item domain.SettingsRecord) error {
|
|
item = ensureSettingsDefaults(item)
|
|
_, err := store.settings.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) GetArtifactStorageSettings(ctx context.Context, organizationID string) (domain.ArtifactStorageSettings, error) {
|
|
organizationID = normalizedSettingsOrganizationID(organizationID)
|
|
var item domain.SettingsRecord
|
|
err := store.settings.FindOne(ctx, bson.M{"id": settingsIDForOrganization(organizationID)}).Decode(&item)
|
|
if err != nil && err != mongo.ErrNoDocuments {
|
|
return domain.ArtifactStorageSettings{}, err
|
|
}
|
|
if err == nil && (strings.TrimSpace(item.ArtifactStorage.ConnectionID) != "" || strings.TrimSpace(item.ArtifactStorage.Bucket) != "") {
|
|
return item.ArtifactStorage, nil
|
|
}
|
|
if organizationID == domain.DefaultOrganizationID {
|
|
return domain.ArtifactStorageSettings{}, nil
|
|
}
|
|
global, err := store.GetSettings(ctx)
|
|
if err != nil {
|
|
return domain.ArtifactStorageSettings{}, err
|
|
}
|
|
return global.ArtifactStorage, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertArtifactStorageSettings(ctx context.Context, organizationID string, item domain.ArtifactStorageSettings) error {
|
|
organizationID = normalizedSettingsOrganizationID(organizationID)
|
|
id := settingsIDForOrganization(organizationID)
|
|
_, err := store.settings.UpdateOne(
|
|
ctx,
|
|
bson.M{"id": id},
|
|
bson.M{
|
|
"$set": bson.M{
|
|
"organizationId": organizationID,
|
|
"artifactStorage": item,
|
|
"updatedAt": nowISO(),
|
|
},
|
|
},
|
|
options.Update().SetUpsert(true),
|
|
)
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) ListMongoConnections(ctx context.Context) ([]domain.MongoConnectionRecord, error) {
|
|
cursor, err := store.mongoConnections.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.MongoConnectionRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) ListMongoConnectionsByOrganization(ctx context.Context, organizationID string) ([]domain.MongoConnectionRecord, error) {
|
|
cursor, err := store.mongoConnections.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return decodeMany[domain.MongoConnectionRecord](ctx, cursor)
|
|
}
|
|
|
|
func (store *MongoStore) GetMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) {
|
|
var item domain.MongoConnectionRecord
|
|
err := store.mongoConnections.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateMongoConnection(ctx context.Context, input domain.CreateMongoConnectionInput) (*domain.MongoConnectionRecord, error) {
|
|
now := nowISO()
|
|
item := domain.MongoConnectionRecord{
|
|
ID: domain.NewID("mongo"),
|
|
OrganizationID: input.OrganizationID,
|
|
Name: input.Name,
|
|
Description: input.Description,
|
|
URI: input.URI,
|
|
Database: input.Database,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
if _, err := store.mongoConnections.InsertOne(ctx, item); err != nil {
|
|
return nil, err
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteMongoConnection(ctx context.Context, id string) (bool, error) {
|
|
result, err := store.mongoConnections.DeleteOne(ctx, bson.M{"id": id})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return result.DeletedCount > 0, nil
|
|
}
|
|
|
|
func (store *MongoStore) UpsertMongoConnection(ctx context.Context, item domain.MongoConnectionRecord) error {
|
|
_, err := store.mongoConnections.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) GetDashboardSummary(ctx context.Context) (domain.DashboardSummary, error) {
|
|
projects, err := store.ListProjects(ctx)
|
|
if err != nil {
|
|
return domain.DashboardSummary{}, err
|
|
}
|
|
operators, err := store.ListOperators(ctx)
|
|
if err != nil {
|
|
return domain.DashboardSummary{}, err
|
|
}
|
|
runs, err := store.ListRunSummaries(ctx, "")
|
|
if err != nil {
|
|
return domain.DashboardSummary{}, err
|
|
}
|
|
auditTasks, err := store.ListAuditTasks(ctx, "all")
|
|
if err != nil {
|
|
return domain.DashboardSummary{}, err
|
|
}
|
|
settings, err := store.GetSettings(ctx)
|
|
if err != nil {
|
|
return domain.DashboardSummary{}, err
|
|
}
|
|
summary := BuildDashboardSummaryAt(projects, operators, runs, auditTasks, settings.Timezone, time.Now())
|
|
summary.Jobs.SuccessRate = float64(int(summary.Jobs.SuccessRate*10)) / 10
|
|
return summary, nil
|
|
}
|
|
|
|
func (store *MongoStore) ListStorageBuckets(ctx context.Context, sourcePath string) ([]string, error) {
|
|
return listBucketsFromSourcePath(sourcePath)
|
|
}
|
|
|
|
func (store *MongoStore) ListStorageObjects(ctx context.Context, sourcePath string, bucket string, prefix string) ([]string, error) {
|
|
items, err := listObjectsFromSourcePath(sourcePath, strings.TrimSpace(bucket), strings.TrimSpace(prefix))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Strings(items)
|
|
return items, nil
|
|
}
|
|
|
|
func mongoBrowserFilter(value any) any {
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
filter := bson.M{}
|
|
for key, item := range typed {
|
|
filter[key] = mongoBrowserFilter(item)
|
|
}
|
|
return filter
|
|
case []any:
|
|
items := make([]any, 0, len(typed))
|
|
for _, item := range typed {
|
|
items = append(items, mongoBrowserFilter(item))
|
|
}
|
|
return items
|
|
default:
|
|
return typed
|
|
}
|
|
}
|
|
|
|
func mongoBrowserDocument(document bson.M) map[string]any {
|
|
if len(document) == 0 {
|
|
return map[string]any{}
|
|
}
|
|
payload, err := bson.MarshalExtJSON(document, false, false)
|
|
if err != nil {
|
|
return sanitizeDatabaseBrowserDocument(map[string]any{})
|
|
}
|
|
result := map[string]any{}
|
|
if err := json.Unmarshal(payload, &result); err != nil {
|
|
return sanitizeDatabaseBrowserDocument(map[string]any{})
|
|
}
|
|
return sanitizeDatabaseBrowserDocument(result)
|
|
}
|
|
|
|
func mongoBrowserSampleDocument(ctx context.Context, collection mongoCollection) (map[string]any, error) {
|
|
var sample bson.M
|
|
err := collection.FindOne(ctx, bson.D{}, options.FindOne().SetSort(bson.D{{Key: "_id", Value: -1}})).Decode(&sample)
|
|
if err == mongo.ErrNoDocuments {
|
|
return map[string]any{}, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return mongoBrowserDocument(sample), nil
|
|
}
|
|
|
|
func mongoBrowserCollectionSummary(ctx context.Context, collection mongoCollection, name string) (domain.DatabaseCollectionSummary, error) {
|
|
count, err := collection.EstimatedDocumentCount(ctx)
|
|
if err != nil {
|
|
count, err = collection.CountDocuments(ctx, bson.D{})
|
|
if err != nil {
|
|
return domain.DatabaseCollectionSummary{}, err
|
|
}
|
|
}
|
|
sample, err := mongoBrowserSampleDocument(ctx, collection)
|
|
if err != nil {
|
|
return domain.DatabaseCollectionSummary{}, err
|
|
}
|
|
return domain.DatabaseCollectionSummary{
|
|
Name: name,
|
|
DocumentCount: count,
|
|
SampleFields: databaseBrowserSampleFields(sample),
|
|
}, nil
|
|
}
|
|
|
|
func mongoBrowserDocumentIDFilter(documentID string) (bson.M, error) {
|
|
trimmed := strings.TrimSpace(documentID)
|
|
if trimmed == "" {
|
|
return nil, fmt.Errorf("document id is required")
|
|
}
|
|
if oid, err := primitive.ObjectIDFromHex(trimmed); err == nil {
|
|
return bson.M{"_id": oid}, nil
|
|
}
|
|
return bson.M{"_id": trimmed}, nil
|
|
}
|
|
|
|
func mongoBrowserWritableDocument(document map[string]any) bson.M {
|
|
writable := clone(document)
|
|
delete(writable, "_id")
|
|
return mongoBrowserFilter(writable).(bson.M)
|
|
}
|
|
|
|
func mongoBrowserWritablePatch(document map[string]any) bson.M {
|
|
writable := clone(document)
|
|
delete(writable, "_id")
|
|
return mongoBrowserFilter(writable).(bson.M)
|
|
}
|
|
|
|
func mongoBrowserFindDocumentByFilter(ctx context.Context, collection mongoCollection, filter bson.M) (map[string]any, error) {
|
|
document := bson.M{}
|
|
if err := collection.FindOne(ctx, filter).Decode(&document); err != nil {
|
|
if err == mongo.ErrNoDocuments {
|
|
return nil, fmt.Errorf("document not found")
|
|
}
|
|
return nil, err
|
|
}
|
|
return mongoBrowserDocument(document), nil
|
|
}
|
|
|
|
func mongoBrowserIndexSummary(document bson.M) domain.DatabaseIndexSummary {
|
|
summary := domain.DatabaseIndexSummary{
|
|
Name: strings.TrimSpace(fmt.Sprintf("%v", document["name"])),
|
|
Keys: map[string]any{},
|
|
}
|
|
if key, ok := document["key"].(bson.M); ok {
|
|
summary.Keys = mongoBrowserDocument(key)
|
|
}
|
|
if unique, ok := document["unique"].(bool); ok {
|
|
summary.Unique = unique
|
|
}
|
|
return summary
|
|
}
|
|
|
|
func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.DatabaseOverview, error) {
|
|
names, err := store.db.ListCollectionNames(ctx, bson.D{})
|
|
if err != nil {
|
|
return domain.DatabaseOverview{}, err
|
|
}
|
|
sort.Strings(names)
|
|
|
|
overview := domain.DatabaseOverview{
|
|
Provider: "mongodb",
|
|
Database: store.db.Name(),
|
|
ReadOnly: true,
|
|
Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)),
|
|
}
|
|
for _, name := range names {
|
|
if IsPrivateDerivedCollection(name) {
|
|
continue
|
|
}
|
|
summary, err := mongoBrowserCollectionSummary(ctx, store.queryCollection(name), name)
|
|
if err != nil {
|
|
return domain.DatabaseOverview{}, err
|
|
}
|
|
overview.Collections = append(overview.Collections, summary)
|
|
}
|
|
return overview, nil
|
|
}
|
|
|
|
func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) {
|
|
collectionName := strings.TrimSpace(input.Collection)
|
|
if collectionName == "" {
|
|
return domain.DatabaseQueryResult{}, fmt.Errorf("collection is required")
|
|
}
|
|
if IsPrivateDerivedCollection(collectionName) {
|
|
return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", collectionName)
|
|
}
|
|
limit := input.Limit
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
skip := input.Skip
|
|
if skip < 0 {
|
|
skip = 0
|
|
}
|
|
|
|
collection := store.queryCollection(collectionName)
|
|
filter, _ := mongoBrowserFilter(input.Filter).(bson.M)
|
|
if filter == nil {
|
|
filter = bson.M{}
|
|
}
|
|
total, err := collection.CountDocuments(ctx, filter)
|
|
if err != nil {
|
|
return domain.DatabaseQueryResult{}, err
|
|
}
|
|
cursor, err := collection.Find(
|
|
ctx,
|
|
filter,
|
|
options.Find().
|
|
SetLimit(int64(limit)).
|
|
SetSkip(int64(skip)).
|
|
SetSort(bson.D{{Key: "_id", Value: -1}}),
|
|
)
|
|
if err != nil {
|
|
return domain.DatabaseQueryResult{}, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
|
|
documents := make([]map[string]any, 0)
|
|
for cursor.Next(ctx) {
|
|
document := bson.M{}
|
|
if err := cursor.Decode(&document); err != nil {
|
|
return domain.DatabaseQueryResult{}, err
|
|
}
|
|
documents = append(documents, mongoBrowserDocument(document))
|
|
}
|
|
if err := cursor.Err(); err != nil {
|
|
return domain.DatabaseQueryResult{}, err
|
|
}
|
|
|
|
result := domain.DatabaseQueryResult{
|
|
Provider: "mongodb",
|
|
Database: store.db.Name(),
|
|
Collection: collectionName,
|
|
Filter: clone(input.Filter),
|
|
Limit: limit,
|
|
Skip: skip,
|
|
Total: total,
|
|
Documents: documents,
|
|
}
|
|
if len(documents) > 0 {
|
|
result.SampleFields = databaseBrowserSampleFields(documents[0])
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateDatabaseCollection(ctx context.Context, name string) (domain.DatabaseCollectionSummary, error) {
|
|
collectionName := strings.TrimSpace(name)
|
|
if collectionName == "" {
|
|
return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required")
|
|
}
|
|
if err := store.db.CreateCollection(ctx, collectionName); err != nil {
|
|
return domain.DatabaseCollectionSummary{}, err
|
|
}
|
|
return mongoBrowserCollectionSummary(ctx, store.queryCollection(collectionName), collectionName)
|
|
}
|
|
|
|
func (store *MongoStore) ListDatabaseIndexes(ctx context.Context, collectionName string) ([]domain.DatabaseIndexSummary, error) {
|
|
name := strings.TrimSpace(collectionName)
|
|
if name == "" {
|
|
return nil, fmt.Errorf("collection is required")
|
|
}
|
|
cursor, err := store.queryCollection(name).Indexes().List(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cursor.Close(ctx)
|
|
items := make([]domain.DatabaseIndexSummary, 0)
|
|
for cursor.Next(ctx) {
|
|
document := bson.M{}
|
|
if err := cursor.Decode(&document); err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, mongoBrowserIndexSummary(document))
|
|
}
|
|
if err := cursor.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
sort.Slice(items, func(left int, right int) bool {
|
|
return items[left].Name < items[right].Name
|
|
})
|
|
return items, nil
|
|
}
|
|
|
|
func (store *MongoStore) CreateDatabaseIndex(ctx context.Context, collectionName string, input domain.DatabaseIndexCreateInput) (domain.DatabaseIndexSummary, error) {
|
|
name := strings.TrimSpace(collectionName)
|
|
if name == "" {
|
|
return domain.DatabaseIndexSummary{}, fmt.Errorf("collection is required")
|
|
}
|
|
if len(input.Keys) == 0 {
|
|
return domain.DatabaseIndexSummary{}, fmt.Errorf("index keys are required")
|
|
}
|
|
model := mongo.IndexModel{
|
|
Keys: mongoBrowserFilter(input.Keys),
|
|
}
|
|
indexName := strings.TrimSpace(input.Name)
|
|
if indexName != "" || input.Unique {
|
|
opts := options.Index()
|
|
if indexName != "" {
|
|
opts.SetName(indexName)
|
|
}
|
|
if input.Unique {
|
|
opts.SetUnique(true)
|
|
}
|
|
model.Options = opts
|
|
}
|
|
createdName, err := store.queryCollection(name).Indexes().CreateOne(ctx, model)
|
|
if err != nil {
|
|
return domain.DatabaseIndexSummary{}, err
|
|
}
|
|
indexes, err := store.ListDatabaseIndexes(ctx, name)
|
|
if err != nil {
|
|
return domain.DatabaseIndexSummary{}, err
|
|
}
|
|
for _, item := range indexes {
|
|
if item.Name == createdName {
|
|
return item, nil
|
|
}
|
|
}
|
|
return domain.DatabaseIndexSummary{Name: createdName, Keys: clone(input.Keys), Unique: input.Unique}, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteDatabaseIndex(ctx context.Context, collectionName string, indexName string) error {
|
|
name := strings.TrimSpace(collectionName)
|
|
target := strings.TrimSpace(indexName)
|
|
if name == "" {
|
|
return fmt.Errorf("collection is required")
|
|
}
|
|
if target == "" {
|
|
return fmt.Errorf("index name is required")
|
|
}
|
|
if target == "_id_" {
|
|
return fmt.Errorf("index %s cannot be deleted", target)
|
|
}
|
|
_, err := store.queryCollection(name).Indexes().DropOne(ctx, target)
|
|
return err
|
|
}
|
|
|
|
func (store *MongoStore) RenameDatabaseCollection(ctx context.Context, name string, newName string) (domain.DatabaseCollectionSummary, error) {
|
|
collectionName := strings.TrimSpace(name)
|
|
nextName := strings.TrimSpace(newName)
|
|
if store.querySyncEnabled.Load() && (queryReplicaFields[collectionName] != "" || queryReplicaFields[nextName] != "") {
|
|
return domain.DatabaseCollectionSummary{}, ErrQueryReplicaUnsupported
|
|
}
|
|
if collectionName == "" {
|
|
return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required")
|
|
}
|
|
if nextName == "" {
|
|
return domain.DatabaseCollectionSummary{}, fmt.Errorf("new collection name is required")
|
|
}
|
|
command := bson.D{
|
|
{Key: "renameCollection", Value: store.db.Name() + "." + collectionName},
|
|
{Key: "to", Value: store.db.Name() + "." + nextName},
|
|
}
|
|
if err := store.db.Client().Database("admin").RunCommand(ctx, command).Err(); err != nil {
|
|
return domain.DatabaseCollectionSummary{}, err
|
|
}
|
|
return mongoBrowserCollectionSummary(ctx, store.queryCollection(nextName), nextName)
|
|
}
|
|
|
|
func (store *MongoStore) DeleteDatabaseCollection(ctx context.Context, name string) error {
|
|
collectionName := strings.TrimSpace(name)
|
|
if collectionName == "" {
|
|
return fmt.Errorf("collection is required")
|
|
}
|
|
if err := store.queryCollection(collectionName).Drop(ctx); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (store *MongoStore) InsertDatabaseDocument(ctx context.Context, collectionName string, document map[string]any) (map[string]any, error) {
|
|
name := strings.TrimSpace(collectionName)
|
|
if name == "" {
|
|
return nil, fmt.Errorf("collection is required")
|
|
}
|
|
collection := store.queryCollection(name)
|
|
payload := mongoBrowserWritableDocument(document)
|
|
result, err := collection.InsertOne(ctx, payload)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return mongoBrowserFindDocumentByFilter(ctx, collection, bson.M{"_id": result.InsertedID})
|
|
}
|
|
|
|
func (store *MongoStore) ReplaceDatabaseDocument(ctx context.Context, collectionName string, documentID string, document map[string]any) (map[string]any, error) {
|
|
name := strings.TrimSpace(collectionName)
|
|
if name == "" {
|
|
return nil, fmt.Errorf("collection is required")
|
|
}
|
|
filter, err := mongoBrowserDocumentIDFilter(documentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
collection := store.queryCollection(name)
|
|
result, err := collection.ReplaceOne(ctx, filter, mongoBrowserWritableDocument(document))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if result.MatchedCount == 0 {
|
|
return nil, fmt.Errorf("document %s not found", documentID)
|
|
}
|
|
return mongoBrowserFindDocumentByFilter(ctx, collection, filter)
|
|
}
|
|
|
|
func (store *MongoStore) PatchDatabaseDocument(ctx context.Context, collectionName string, documentID string, patch map[string]any) (map[string]any, error) {
|
|
name := strings.TrimSpace(collectionName)
|
|
if name == "" {
|
|
return nil, fmt.Errorf("collection is required")
|
|
}
|
|
filter, err := mongoBrowserDocumentIDFilter(documentID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
collection := store.queryCollection(name)
|
|
result, err := collection.UpdateOne(ctx, filter, bson.M{"$set": mongoBrowserWritablePatch(patch)})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if result.MatchedCount == 0 {
|
|
return nil, fmt.Errorf("document %s not found", documentID)
|
|
}
|
|
return mongoBrowserFindDocumentByFilter(ctx, collection, filter)
|
|
}
|
|
|
|
func (store *MongoStore) UpsertDatabaseDocument(ctx context.Context, collectionName string, match map[string]any, patch map[string]any, setOnInsert map[string]any) (map[string]any, bool, error) {
|
|
name := strings.TrimSpace(collectionName)
|
|
if name == "" {
|
|
return nil, false, fmt.Errorf("collection is required")
|
|
}
|
|
filter, _ := mongoBrowserFilter(match).(bson.M)
|
|
if filter == nil {
|
|
filter = bson.M{}
|
|
}
|
|
update := bson.M{"$set": mongoBrowserWritablePatch(patch)}
|
|
if len(setOnInsert) > 0 {
|
|
update["$setOnInsert"] = mongoBrowserWritablePatch(setOnInsert)
|
|
}
|
|
collection := store.queryCollection(name)
|
|
result, err := collection.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true))
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
created := result.UpsertedCount > 0
|
|
if created && result.UpsertedID != nil {
|
|
document, err := mongoBrowserFindDocumentByFilter(ctx, collection, bson.M{"_id": result.UpsertedID})
|
|
return document, true, err
|
|
}
|
|
document, err := mongoBrowserFindDocumentByFilter(ctx, collection, filter)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
return document, false, nil
|
|
}
|
|
|
|
func (store *MongoStore) DeleteDatabaseDocument(ctx context.Context, collectionName string, documentID string) error {
|
|
name := strings.TrimSpace(collectionName)
|
|
if name == "" {
|
|
return fmt.Errorf("collection is required")
|
|
}
|
|
filter, err := mongoBrowserDocumentIDFilter(documentID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
result, err := store.queryCollection(name).DeleteOne(ctx, filter)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if result.DeletedCount == 0 {
|
|
return fmt.Errorf("document %s not found", documentID)
|
|
}
|
|
return nil
|
|
}
|