2026-11-13 22:31:44 -08:00

3127 lines
104 KiB
Go

package store
import (
"context"
"encoding/json"
"fmt"
"golang.org/x/sync/singleflight"
"regexp"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type MongoStore struct {
querySyncEnabled atomic.Bool
queryReplicaReads *queryReplicaReads
queryReplicaFactStore bool
queryPeriodic bool
queryPublishedResults bool
workforcePublicationTouches workforcePublicationTouches
workforcePublicationBuilds workforcePublicationBuildMemo
queryBuildWrites *queryBuildWriteGate
queryReplicaCounts queryCountCache
queryReplicaPageFlights singleflight.Group
queryReplicaBatchFacts queryBatchFacts
queryServing *queryServingOptions
productionReadFlights singleflight.Group
claimReadFlights singleflight.Group
poolSearchReadFlights singleflight.Group
claimReadOnce sync.Once
claimReadSlots chan struct{}
db *mongo.Database
organizations mongoCollection
projects mongoCollection
operators mongoCollection
builtinOperatorDeletions mongoCollection
workflows mongoCollection
workflowRuntimePolicies mongoCollection
templates mongoCollection
workflowReleases mongoCollection
projectWorkflowInstances mongoCollection
runs mongoCollection
sourceStates mongoCollection
captureRecords mongoCollection
collectionTasks mongoCollection
datasetVersions mongoCollection
dataMetricFacts mongoCollection
auditTasks mongoCollection
samplingTasks mongoCollection
annotationWorkgroups mongoCollection
annotationMemberships mongoCollection
suppliers mongoCollection
projectMembers mongoCollection
supplierProjectAuthorizations mongoCollection
supplierProjectAuthorizationInit mongoCollection
annotationLabels mongoCollection
projectAnnotationLabels mongoCollection
annotationTemplates mongoCollection
annotationTasks mongoCollection
annotationRevisions mongoCollection
sliceRevisions mongoCollection
annotationReviews mongoCollection
annotationWorklogEvents mongoCollection
organizationPlugins mongoCollection
humanBatches mongoCollection
humanItems mongoCollection
humanStageAttempts mongoCollection
humanEvents mongoCollection
humanRejects mongoCollection
releasePartitions mongoCollection
deliveryPackages mongoCollection
users mongoCollection
sessions mongoCollection
storageConnections mongoCollection
uploadSessions mongoCollection
mongoConnections mongoCollection
objectSyncSpaces mongoCollection
objectSyncDevices mongoCollection
objectSyncAuthorizations mongoCollection
objectSyncCredentials mongoCollection
objectSyncSessions mongoCollection
objectSyncItems mongoCollection
objectSyncLeases mongoCollection
objectSyncHeads mongoCollection
objectSyncVersions mongoCollection
objectSyncAuditEvents mongoCollection
objectSyncAuditSeals mongoCollection
objectSyncReleases mongoCollection
objectSyncClientReleases mongoCollection
objectSyncReconciles mongoCollection
objectSyncIdempotency mongoCollection
objectSyncRateLimits mongoCollection
settings mongoCollection
runSummaryAggregates *runSummaryAggregateCache
}
func NewMongoStore(db *mongo.Database) *MongoStore {
out := &MongoStore{db: db, runSummaryAggregates: newRunSummaryAggregateCache(runSummaryAggregateCacheTTL, runSummaryAggregateCacheMaxEntries)}
out.organizations = out.queryCollection("organizations")
out.projects = out.queryCollection("projects")
out.operators = out.queryCollection("operators")
out.builtinOperatorDeletions = out.queryCollection("operatorTombstones")
out.workflows = out.queryCollection("workflows")
out.workflowRuntimePolicies = out.queryCollection("workflowRuntimePolicies")
out.templates = out.queryCollection("templates")
out.workflowReleases = out.queryCollection("workflowReleases")
out.projectWorkflowInstances = out.queryCollection("projectWorkflowInstances")
out.runs = out.queryCollection("runs")
out.sourceStates = out.queryCollection("sourceStates")
out.captureRecords = out.queryCollection("captureRecords")
out.collectionTasks = out.queryCollection("collectionTasks")
out.datasetVersions = out.queryCollection("datasetVersions")
out.dataMetricFacts = out.queryCollection("dataMetricFacts")
out.auditTasks = out.queryCollection("auditTasks")
out.samplingTasks = out.queryCollection("samplingTasks")
out.annotationWorkgroups = out.queryCollection("annotationWorkgroups")
out.annotationMemberships = out.queryCollection("annotationMemberships")
out.suppliers = out.queryCollection("suppliers")
out.projectMembers = out.queryCollection("projectMembers")
out.supplierProjectAuthorizations = out.queryCollection("supplierProjectAuthorizations")
out.supplierProjectAuthorizationInit = out.queryCollection("supplierProjectAuthorizationInit")
out.annotationLabels = out.queryCollection("annotationLabels")
out.projectAnnotationLabels = out.queryCollection("projectAnnotationLabels")
out.annotationTemplates = out.queryCollection("annotationTemplates")
out.annotationTasks = out.queryCollection("annotationTasks")
out.annotationRevisions = out.queryCollection("annotationRevisions")
out.sliceRevisions = out.queryCollection("sliceRevisions")
out.annotationReviews = out.queryCollection("annotationReviews")
out.annotationWorklogEvents = out.queryCollection("annotationWorklogEvents")
out.organizationPlugins = out.queryCollection("organizationPlugins")
out.humanBatches = out.queryCollection("humanBatches")
out.humanItems = out.queryCollection("humanItems")
out.humanStageAttempts = out.queryCollection("humanStageAttempts")
out.humanEvents = out.queryCollection("humanEvents")
out.humanRejects = out.queryCollection("humanRejects")
out.releasePartitions = out.queryCollection("humanReleasePartitions")
out.deliveryPackages = out.queryCollection("deliveryPackages")
out.users = out.queryCollection("users")
out.sessions = out.queryCollection("sessions")
out.storageConnections = out.queryCollection("storageConnections")
out.uploadSessions = out.queryCollection("objectStorageUploadSessions")
out.mongoConnections = out.queryCollection("mongoConnections")
out.objectSyncSpaces = out.queryCollection("objectSyncSpaces")
out.objectSyncDevices = out.queryCollection("objectSyncDevices")
out.objectSyncAuthorizations = out.queryCollection("objectSyncAuthorizations")
out.objectSyncCredentials = out.queryCollection("objectSyncCredentials")
out.objectSyncSessions = out.queryCollection("objectSyncSessions")
out.objectSyncItems = out.queryCollection("objectSyncItems")
out.objectSyncLeases = out.queryCollection("objectSyncLeases")
out.objectSyncHeads = out.queryCollection("objectSyncHeads")
out.objectSyncVersions = out.queryCollection("objectSyncVersions")
out.objectSyncAuditEvents = out.queryCollection("objectSyncAuditEvents")
out.objectSyncAuditSeals = out.queryCollection("objectSyncAuditSeals")
out.objectSyncReleases = out.queryCollection("objectSyncReleases")
out.objectSyncClientReleases = out.queryCollection("objectSyncClientReleases")
out.objectSyncReconciles = out.queryCollection("objectSyncReconciles")
out.objectSyncIdempotency = out.queryCollection("objectSyncIdempotency")
out.objectSyncRateLimits = out.queryCollection("objectSyncRateLimits")
out.settings = out.queryCollection("settings")
return out
}
func (store *MongoStore) Ready(ctx context.Context) error {
return store.db.Client().Ping(ctx, nil)
}
func decodeMany[T any](ctx context.Context, cursor *mongo.Cursor) ([]T, error) {
defer cursor.Close(ctx)
items := make([]T, 0)
for cursor.Next(ctx) {
var item T
if err := cursor.Decode(&item); err != nil {
return nil, err
}
items = append(items, item)
}
return items, cursor.Err()
}
func orgCompatibilityFilter(organizationID string) bson.M {
normalized := normalizedOrganizationID(organizationID)
if normalized == domain.DefaultOrganizationID {
return bson.M{"$or": bson.A{
bson.M{"organizationId": normalized},
bson.M{"organizationId": bson.M{"$exists": false}},
bson.M{"organizationId": ""},
}}
}
return bson.M{"organizationId": normalized}
}
func operatorOrgCompatibilityFilter(organizationID string) bson.M {
return bson.M{"$or": bson.A{
orgCompatibilityFilter(organizationID),
bson.M{"type": "builtin"},
}}
}
func organizationScopedFilter(organizationID string, extra bson.M) bson.M {
orgFilter := orgCompatibilityFilter(organizationID)
if len(extra) == 0 {
return orgFilter
}
return bson.M{"$and": bson.A{orgFilter, extra}}
}
func (store *MongoStore) EnsureDefaultOrganization(ctx context.Context) (*domain.OrganizationRecord, error) {
existing, err := store.GetOrganization(ctx, domain.DefaultOrganizationID)
if err != nil {
return nil, err
}
if existing != nil {
return existing, nil
}
now := nowISO()
item := domain.OrganizationRecord{
ID: domain.DefaultOrganizationID,
Name: "Default Organization",
Slug: "default",
Status: domain.OrganizationStatusActive,
CreatedAt: now,
UpdatedAt: now,
}
if err := store.UpsertOrganization(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) ListOrganizations(ctx context.Context) ([]domain.OrganizationRecord, error) {
if _, err := store.EnsureDefaultOrganization(ctx); err != nil {
return nil, err
}
cursor, err := store.organizations.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.OrganizationRecord](ctx, cursor)
}
func (store *MongoStore) GetOrganization(ctx context.Context, id string) (*domain.OrganizationRecord, error) {
var item domain.OrganizationRecord
err := store.organizations.FindOne(ctx, bson.M{"id": normalizedOrganizationID(id)}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) UpsertOrganization(ctx context.Context, item domain.OrganizationRecord) error {
if strings.TrimSpace(item.ID) == "" {
item.ID = domain.NewID("org")
}
item.Slug = strings.TrimSpace(strings.ToLower(item.Slug))
now := nowISO()
if item.CreatedAt == "" {
item.CreatedAt = now
}
if item.Status == "" {
item.Status = domain.OrganizationStatusActive
}
item.UpdatedAt = now
_, err := store.organizations.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
if mongo.IsDuplicateKeyError(err) {
return ErrDuplicateOrganizationSlug
}
if err == nil {
_, err = store.EnsureDefaultSupplier(ctx, item.ID)
}
return err
}
func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id string) (bool, error) {
organizationID := normalizedOrganizationID(id)
if organizationID == domain.DefaultOrganizationID {
return false, fmt.Errorf("default organization cannot be deleted")
}
if existing, err := store.GetOrganization(ctx, organizationID); err != nil {
return false, err
} else if existing == nil {
return false, nil
}
filter := orgCompatibilityFilter(organizationID)
for _, collection := range []mongoCollection{
store.users,
store.sessions,
store.projects,
store.operators,
store.builtinOperatorDeletions,
store.workflows,
store.templates,
store.workflowReleases,
store.projectWorkflowInstances,
store.runs,
store.sourceStates,
store.captureRecords,
store.auditTasks,
store.samplingTasks,
store.annotationWorkgroups,
store.annotationMemberships,
store.suppliers,
store.projectMembers,
store.annotationTemplates,
store.annotationTasks,
store.annotationRevisions,
store.sliceRevisions,
store.annotationReviews,
store.annotationWorklogEvents,
store.organizationPlugins,
store.humanBatches,
store.humanItems,
store.humanStageAttempts,
store.humanEvents,
store.humanRejects,
store.releasePartitions,
store.deliveryPackages,
store.storageConnections,
store.uploadSessions,
store.mongoConnections,
store.objectSyncSpaces,
store.objectSyncDevices,
store.objectSyncAuthorizations,
store.objectSyncCredentials,
store.objectSyncSessions,
store.objectSyncItems,
store.objectSyncLeases,
store.objectSyncHeads,
store.objectSyncVersions,
store.objectSyncAuditEvents,
store.objectSyncReleases,
store.objectSyncClientReleases,
store.annotationLabels,
store.settings,
} {
if _, err := collection.DeleteMany(ctx, filter); err != nil {
return false, err
}
}
_, err := store.organizations.DeleteOne(ctx, bson.M{"id": organizationID})
return err == nil, err
}
func (store *MongoStore) ListProjects(ctx context.Context) ([]domain.ProjectRecord, error) {
cursor, err := store.projects.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.ProjectRecord](ctx, cursor)
}
func (store *MongoStore) ListProjectsByOrganization(ctx context.Context, organizationID string) ([]domain.ProjectRecord, error) {
cursor, err := store.projects.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.ProjectRecord](ctx, cursor)
}
func (store *MongoStore) GetProject(ctx context.Context, id string) (*domain.ProjectRecord, error) {
var item domain.ProjectRecord
err := store.projects.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateProject(ctx context.Context, input domain.CreateProjectInput) (*domain.ProjectRecord, error) {
now := nowISO()
item := domain.ProjectRecord{
ID: domain.NewID("project"),
OrganizationID: input.OrganizationID,
Name: input.Name,
Description: input.Description,
Scene: input.Scene,
OwnerUserID: input.OwnerUserID,
Customer: input.Customer,
DataBatch: input.DataBatch,
StorageConnectionID: input.StorageConnectionID,
StorageBucket: input.StorageBucket,
StoragePrefix: input.StoragePrefix,
AnnotationStandard: input.AnnotationStandard,
QualityPolicy: input.QualityPolicy,
DeliveryConfig: input.DeliveryConfig,
Status: domain.ProjectStatusActive,
CreatedAt: now,
UpdatedAt: now,
}
// The marker is created before any default-member compatibility sync. A
// crash may leave an orphan marker for a never-created ID, which is safe.
_, err := store.supplierProjectAuthorizationInit.UpdateOne(ctx, bson.M{"id": supplierAuthorizationInitID(item.OrganizationID, item.ID)}, bson.M{"$setOnInsert": bson.M{"id": supplierAuthorizationInitID(item.OrganizationID, item.ID), "organizationId": normalizedOrganizationID(item.OrganizationID), "projectId": item.ID, "initializedAt": now}}, options.Update().SetUpsert(true))
if err != nil {
return nil, err
}
if err := store.UpsertProject(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteProject(ctx context.Context, id string) (bool, error) {
id = strings.TrimSpace(id)
if id == "" {
return false, nil
}
workflowCursor, err := store.workflows.Find(ctx, bson.M{"projectId": id}, options.Find().SetProjection(bson.M{"id": 1}))
if err != nil {
return false, err
}
workflowDocs, err := decodeMany[struct {
ID string `bson:"id"`
}](ctx, workflowCursor)
if err != nil {
return false, err
}
for _, item := range workflowDocs {
if _, err := store.DeleteWorkflow(ctx, item.ID); err != nil {
return false, err
}
}
taskIDs, err := store.annotationTasks.Distinct(ctx, "id", bson.M{"projectId": id})
if err != nil {
return false, err
}
itemIDs, err := store.humanItems.Distinct(ctx, "id", bson.M{"projectId": id})
if err != nil {
return false, err
}
batchIDs, err := store.humanBatches.Distinct(ctx, "id", bson.M{"projectId": id})
if err != nil {
return false, err
}
if len(taskIDs) > 0 {
taskFilter := bson.M{"taskId": bson.M{"$in": taskIDs}}
for _, collection := range []mongoCollection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} {
if _, err := collection.DeleteMany(ctx, taskFilter); err != nil {
return false, err
}
}
}
if len(itemIDs) > 0 {
itemFilter := bson.M{"itemId": bson.M{"$in": itemIDs}}
for _, collection := range []mongoCollection{store.humanStageAttempts, store.humanEvents} {
if _, err := collection.DeleteMany(ctx, itemFilter); err != nil {
return false, err
}
}
}
if len(batchIDs) > 0 {
if _, err := store.releasePartitions.DeleteMany(ctx, bson.M{"batchId": bson.M{"$in": batchIDs}}); err != nil {
return false, err
}
}
for _, operation := range []struct {
collection mongoCollection
filter bson.M
}{
{store.annotationTasks, bson.M{"projectId": id}},
{store.queryCollection("annotationTimelineIndexes"), bson.M{"projectId": id}},
{store.queryCollection("annotationIndexJobs"), bson.M{"projectId": id}},
{store.humanRejects, bson.M{"projectId": id}},
{store.humanItems, bson.M{"projectId": id}},
{store.humanBatches, bson.M{"projectId": id}},
{store.deliveryPackages, bson.M{"projectId": id}},
{store.annotationMemberships, bson.M{"projectId": id}},
{store.annotationTemplates, bson.M{"projectId": id}},
{store.projectMembers, bson.M{"projectId": id}},
{store.workflows, bson.M{"projectId": id}},
{store.projectWorkflowInstances, bson.M{"projectId": id}},
{store.runs, bson.M{"projectId": id}},
{store.captureRecords, bson.M{"projectId": id}},
{store.auditTasks, bson.M{"projectId": id}},
{store.samplingTasks, bson.M{"projectId": id}},
} {
if _, err := operation.collection.DeleteMany(ctx, operation.filter); err != nil {
return false, err
}
}
result, err := store.projects.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount == 1, nil
}
func projectPersistenceDocument(item domain.ProjectRecord) bson.M {
// ProjectRecord has bson,omitempty fields for compact reads. Passing the
// struct directly to $set therefore skips empty strings and makes it
// impossible to clear optional project fields through the edit API. Build
// the persisted document explicitly so empty values replace stale values.
return bson.M{
"id": item.ID,
"organizationId": item.OrganizationID,
"name": item.Name,
"description": item.Description,
"scene": item.Scene,
"ownerUserId": item.OwnerUserID,
"customer": item.Customer,
"dataBatch": item.DataBatch,
"storageConnectionId": item.StorageConnectionID,
"storageBucket": item.StorageBucket,
"storagePrefix": item.StoragePrefix,
"workflowInstanceId": item.WorkflowInstanceID,
"annotationStandard": item.AnnotationStandard,
"qualityPolicy": item.QualityPolicy,
"deliveryConfig": item.DeliveryConfig,
"progressStatus": item.ProgressStatus,
"qualityStatus": item.QualityStatus,
"deliveryStatus": item.DeliveryStatus,
"status": item.Status,
"storagePathFavorites": item.StoragePathFavorites,
"createdAt": item.CreatedAt,
"updatedAt": item.UpdatedAt,
}
}
func (store *MongoStore) UpsertProject(ctx context.Context, item domain.ProjectRecord) error {
allow, err := projectAllowsUpsert(ctx, store.db, item.ID)
if err != nil {
return err
}
persisted := projectPersistenceDocument(item)
stampPoolSearchMetadata("projects", persisted)
_, err = store.projects.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": persisted}, options.Update().SetUpsert(allow))
return err
}
func (store *MongoStore) ListOperators(ctx context.Context) ([]domain.OperatorRecord, error) {
cursor, err := store.operators.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.OperatorRecord](ctx, cursor)
}
func (store *MongoStore) ListOperatorsByOrganization(ctx context.Context, organizationID string) ([]domain.OperatorRecord, error) {
cursor, err := store.operators.Find(ctx, operatorOrgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.OperatorRecord](ctx, cursor)
}
func (store *MongoStore) GetOperator(ctx context.Context, id string) (*domain.OperatorRecord, error) {
var item domain.OperatorRecord
err := store.operators.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateOperator(ctx context.Context, input domain.CreateOperatorInput) (*domain.OperatorRecord, error) {
now := nowISO()
item := domain.OperatorRecord{
ID: domain.NewID("operator"),
OrganizationID: input.OrganizationID,
Name: input.Name,
Type: input.Type,
Category: input.Category,
Description: input.Description,
Inputs: input.Inputs,
Outputs: input.Outputs,
ConfigSchema: input.ConfigSchema,
Version: input.Version,
Image: input.Image,
ImageID: input.ImageID,
ContainerCommand: input.ContainerCommand,
ContainerArgs: input.ContainerArgs,
RuntimeContract: input.RuntimeContract,
SkillDefinition: input.SkillDefinition,
Hyperparams: input.Hyperparams,
Resources: input.Resources,
BusinessStage: input.BusinessStage,
PublicationStatus: input.PublicationStatus,
GenerationSource: input.GenerationSource,
PublicationEvidence: input.PublicationEvidence,
CreatedAt: now,
UpdatedAt: now,
}
if err := store.UpsertOperator(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteOperator(ctx context.Context, id string) (bool, error) {
result, err := store.operators.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount == 1, nil
}
func (store *MongoStore) UpsertOperator(ctx context.Context, item domain.OperatorRecord) error {
_, err := store.operators.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) ListBuiltinOperatorDeletions(ctx context.Context) ([]domain.BuiltinOperatorDeletionRecord, error) {
cursor, err := store.builtinOperatorDeletions.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "deletedAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.BuiltinOperatorDeletionRecord](ctx, cursor)
}
func (store *MongoStore) UpsertBuiltinOperatorDeletion(ctx context.Context, item domain.BuiltinOperatorDeletionRecord) error {
_, err := store.builtinOperatorDeletions.UpdateOne(ctx, bson.M{"key": item.Key}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) ListWorkflowsByProject(ctx context.Context, projectID string) ([]domain.WorkflowRecord, error) {
cursor, err := store.workflows.Find(ctx, bson.M{"projectId": projectID}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.WorkflowRecord](ctx, cursor)
}
func (store *MongoStore) ListAllWorkflows(ctx context.Context) ([]domain.WorkflowRecord, error) {
cursor, err := store.workflows.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.WorkflowRecord](ctx, cursor)
}
func (store *MongoStore) ListAllWorkflowsByOrganization(ctx context.Context, organizationID string) ([]domain.WorkflowRecord, error) {
cursor, err := store.workflows.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.WorkflowRecord](ctx, cursor)
}
func (store *MongoStore) ListWorkerWorkflows(ctx context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) {
filters := bson.A{bson.M{
"type": domain.WorkflowTypeStreaming,
"schedule": bson.M{"$exists": true, "$ne": nil},
}, bson.M{"captureAutomation.enabled": true}}
if len(workflowIDs) > 0 {
filters = append(filters, bson.M{"id": bson.M{"$in": workflowIDs}})
}
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
cursor, err := store.workflows.Find(
ctx,
bson.M{"$and": bson.A{
bson.M{"$or": filters},
bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}},
}},
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
return decodeMany[domain.WorkflowRecord](ctx, cursor)
}
func (store *MongoStore) ListEnabledCaptureAutomations(ctx context.Context, organizationID string) ([]domain.WorkflowRecord, error) {
filter := organizationScopedFilter(organizationID, bson.M{
"captureAutomation.enabled": true,
"$or": bson.A{
bson.M{"archivedAt": bson.M{"$exists": false}},
bson.M{"archivedAt": ""},
},
})
cursor, err := store.workflows.Find(
ctx,
filter,
options.Find().SetProjection(bson.M{"id": 1, "name": 1, "organizationId": 1, "captureAutomation": 1}).SetSort(bson.D{{Key: "updatedAt", Value: -1}}),
)
if err != nil {
return nil, err
}
return decodeMany[domain.WorkflowRecord](ctx, cursor)
}
func (store *MongoStore) GetWorkflow(ctx context.Context, id string) (*domain.WorkflowRecord, error) {
var item domain.WorkflowRecord
err := store.workflows.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
domain.NormalizeWorkflowDefinitionMetadata(&item)
return &item, nil
}
func (store *MongoStore) CreateWorkflow(ctx context.Context, input domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) {
if input.ProjectID != "" {
p, err := store.GetProject(ctx, input.ProjectID)
if err != nil {
return nil, err
}
if p == nil || p.DeletionTaskID != "" {
return nil, fmt.Errorf("project missing or deletion in progress")
}
}
now := nowISO()
item := domain.WorkflowRecord{
ID: domain.NewID("workflow"),
OrganizationID: input.OrganizationID,
ProjectID: input.ProjectID,
Name: input.Name,
Description: input.Description,
Type: input.Type,
Kind: input.Kind,
Hidden: input.Hidden,
ExecutionScope: input.ExecutionScope,
ScanConfig: input.ScanConfig,
CaptureAutomation: input.CaptureAutomation,
Hyperparams: input.Hyperparams,
MaxConcurrentRuns: input.MaxConcurrentRuns,
NodeTimeoutSeconds: input.NodeTimeoutSeconds,
RunTimeoutSeconds: input.RunTimeoutSeconds,
MaxRetryAttempts: input.MaxRetryAttempts,
RetryBackoffSeconds: input.RetryBackoffSeconds,
Schedule: input.Schedule,
DefinitionState: domain.WorkflowDefinitionStateDraft,
DefinitionVersion: 1,
WorkflowInputs: input.WorkflowInputs,
ArtifactStorage: input.ArtifactStorage,
Nodes: input.Nodes,
Edges: input.Edges,
Status: domain.WorkflowStatusDraft,
CreatedAt: now,
UpdatedAt: now,
}
item.DefinitionFamilyID = item.ID
if err := store.UpsertWorkflow(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteWorkflow(ctx context.Context, id string) (bool, error) {
id = strings.TrimSpace(id)
if id == "" {
return false, nil
}
count, err := store.workflows.CountDocuments(ctx, bson.M{"id": id}, options.Count().SetLimit(1))
if err != nil || count == 0 {
return false, err
}
taskIDs, err := store.annotationTasks.Distinct(ctx, "id", bson.M{"workflowId": id})
if err != nil {
return false, err
}
itemIDs, err := store.humanItems.Distinct(ctx, "id", bson.M{"workflowId": id})
if err != nil {
return false, err
}
batchIDs, err := store.humanBatches.Distinct(ctx, "id", bson.M{"workflowId": id})
if err != nil {
return false, err
}
if len(taskIDs) > 0 {
taskFilter := bson.M{"taskId": bson.M{"$in": taskIDs}}
for _, collection := range []mongoCollection{store.annotationRevisions, store.sliceRevisions, store.annotationReviews, store.annotationWorklogEvents} {
if _, err := collection.DeleteMany(ctx, taskFilter); err != nil {
return false, err
}
}
}
if len(itemIDs) > 0 {
itemFilter := bson.M{"itemId": bson.M{"$in": itemIDs}}
for _, collection := range []mongoCollection{store.humanStageAttempts, store.humanEvents} {
if _, err := collection.DeleteMany(ctx, itemFilter); err != nil {
return false, err
}
}
}
if len(batchIDs) > 0 {
if _, err := store.releasePartitions.DeleteMany(ctx, bson.M{"batchId": bson.M{"$in": batchIDs}}); err != nil {
return false, err
}
}
for _, operation := range []struct {
collection mongoCollection
filter bson.M
}{
{store.projectWorkflowInstances, bson.M{"workflowId": id}},
{store.annotationTasks, bson.M{"workflowId": id}},
{store.queryCollection("annotationTimelineIndexes"), bson.M{"workflowId": id}},
{store.queryCollection("annotationIndexJobs"), bson.M{"workflowId": id}},
{store.humanRejects, bson.M{"workflowId": id}},
{store.humanItems, bson.M{"workflowId": id}},
{store.humanBatches, bson.M{"workflowId": id}},
{store.deliveryPackages, bson.M{"workflowId": id}},
} {
if _, err := operation.collection.DeleteMany(ctx, operation.filter); err != nil {
return false, err
}
}
if _, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
return false, err
}
if _, err := store.captureRecords.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
return false, err
}
if _, err := store.runs.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
return false, err
}
if _, err := store.auditTasks.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
return false, err
}
if _, err := store.samplingTasks.DeleteMany(ctx, bson.M{"workflowId": id}); err != nil {
return false, err
}
result, err := store.workflows.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount == 1, nil
}
func (store *MongoStore) UpsertWorkflow(ctx context.Context, item domain.WorkflowRecord) error {
return supplierGuardedMongoWrite(ctx, store.workflows, item.ID, item, func(ctx context.Context, d bson.M) error {
raw, e := bson.Marshal(d)
if e != nil {
return e
}
var next domain.WorkflowRecord
if e = bson.Unmarshal(raw, &next); e != nil {
return e
}
return store.upsertWorkflowSupplierGuarded(ctx, next)
})
}
func (store *MongoStore) upsertWorkflowSupplierGuarded(ctx context.Context, item domain.WorkflowRecord) error {
allow, err := projectAllowsUpsert(ctx, store.db, item.ProjectID)
if err != nil {
return err
}
if !allow && item.CaptureAutomation != nil {
item.CaptureAutomation.Enabled = false
}
_, err = store.workflows.UpdateOne(ctx, bson.M{"id": item.ID}, workflowUpsertUpdate(item), options.Update().SetUpsert(allow))
return err
}
func workflowUpsertUpdate(item domain.WorkflowRecord) bson.M {
update := bson.M{"$set": item}
// WorkflowInputs is bson:"omitempty", so a normal $set silently leaves the
// previous array behind when a workflow version intentionally removes all
// root inputs. Explicitly unset it to keep the stored definition identical
// to the validated API payload.
if len(item.WorkflowInputs) == 0 {
update["$unset"] = bson.M{"workflowInputs": ""}
}
return update
}
// UpdateWorkflowRunState does a field-level $set of only status/lastRunId/updatedAt
// so the worker can bump a workflow's run state without overwriting concurrent PUT
// edits to Nodes/Edges (lost-update fix; see store.Store interface comment).
func (store *MongoStore) UpdateWorkflowRunState(ctx context.Context, workflowID string, status domain.WorkflowStatus, lastRunID string, updatedAt string) error {
_, err := store.workflows.UpdateOne(ctx, bson.M{"id": workflowID}, bson.M{"$set": bson.M{
"status": status,
"lastRunId": lastRunID,
"updatedAt": updatedAt,
}})
return err
}
func (store *MongoStore) ListTemplates(ctx context.Context) ([]domain.TemplateRecord, error) {
cursor, err := store.templates.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.TemplateRecord](ctx, cursor)
}
func (store *MongoStore) ListTemplatesByOrganization(ctx context.Context, organizationID string) ([]domain.TemplateRecord, error) {
cursor, err := store.templates.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.TemplateRecord](ctx, cursor)
}
func (store *MongoStore) GetTemplate(ctx context.Context, id string) (*domain.TemplateRecord, error) {
var item domain.TemplateRecord
err := store.templates.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateTemplate(ctx context.Context, input domain.CreateTemplateInput) (*domain.TemplateRecord, error) {
now := nowISO()
templateType := input.Type
if templateType == "" {
templateType = domain.WorkflowTypeBatch
}
item := domain.TemplateRecord{
ID: domain.NewID("template"),
OrganizationID: input.OrganizationID,
Name: input.Name,
Description: input.Description,
Type: templateType,
Kind: input.Kind,
ExecutionScope: input.ExecutionScope,
ScanConfig: input.ScanConfig,
Hyperparams: input.Hyperparams,
MaxConcurrentRuns: input.MaxConcurrentRuns,
NodeTimeoutSeconds: input.NodeTimeoutSeconds,
RunTimeoutSeconds: input.RunTimeoutSeconds,
MaxRetryAttempts: input.MaxRetryAttempts,
RetryBackoffSeconds: input.RetryBackoffSeconds,
Schedule: input.Schedule,
DefinitionState: domain.WorkflowDefinitionStateDraft,
DefinitionVersion: 1,
WorkflowInputs: input.WorkflowInputs,
ArtifactStorage: input.ArtifactStorage,
Nodes: input.Nodes,
Edges: input.Edges,
CreatedAt: now,
UpdatedAt: now,
}
item.DefinitionFamilyID = item.ID
if err := store.UpsertTemplate(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteTemplate(ctx context.Context, id string) (bool, error) {
id = strings.TrimSpace(id)
if id == "" {
return false, nil
}
// Removing the template entry must not invalidate immutable snapshots
// still used by project instances or historical/active Runs.
referenced := []any{}
for _, collection := range []mongoCollection{store.projectWorkflowInstances, store.runs} {
ids, err := collection.Distinct(ctx, "releaseId", bson.M{"releaseId": bson.M{"$exists": true, "$ne": ""}})
if err != nil {
return false, err
}
referenced = append(referenced, ids...)
}
if _, err := store.workflowReleases.DeleteMany(ctx, bson.M{"templateId": id, "id": bson.M{"$nin": referenced}}); err != nil {
return false, err
}
result, err := store.templates.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount == 1, nil
}
func (store *MongoStore) UpsertTemplate(ctx context.Context, item domain.TemplateRecord) error {
_, err := store.templates.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) ListRuns(ctx context.Context, status string) ([]domain.RunRecord, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
func (store *MongoStore) ListRunsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(ctx, organizationScopedFilter(organizationID, filter), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
func runArtifactProjection() bson.M {
return bson.M{
"id": 1,
"organizationId": 1,
"workflowId": 1,
"workflowName": 1,
"projectId": 1,
"projectName": 1,
"status": 1,
"artifacts": 1,
"nodeExecutions.nodeId": 1,
"nodeExecutions.artifacts": 1,
"createdAt": 1,
"updatedAt": 1,
}
}
func (store *MongoStore) ListRunArtifacts(ctx context.Context, status string) ([]domain.RunRecord, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runArtifactProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
// ListRunArtifactsForProject scopes the query to runs whose projectId
// matches projectID directly, or whose workflowId is in workflowIDs, using
// the existing organizationId_projectId_createdAt_id and
// organizationId_workflowId_createdAt_id indexes instead of scanning every
// run in the organization.
func (store *MongoStore) ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error) {
or := bson.A{}
if strings.TrimSpace(projectID) != "" {
or = append(or, bson.M{"projectId": projectID})
}
if len(workflowIDs) > 0 {
or = append(or, bson.M{"workflowId": bson.M{"$in": workflowIDs}})
}
if len(or) == 0 {
return []domain.RunRecord{}, nil
}
cursor, err := store.runs.Find(
ctx,
organizationScopedFilter(organizationID, bson.M{"$or": or}),
options.Find().
SetProjection(runArtifactProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
// Excludes archived runs: this is the query behind validateDeletionReferences'
// org-wide safety scan, and an organization's run history is unbounded and
// never otherwise shrinks, so a run old enough to be archived is excluded
// here to keep that scan's cost from growing forever (see ArchiveTerminalRuns).
func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
filter := bson.M{"$or": bson.A{bson.M{"archivedAt": bson.M{"$exists": false}}, bson.M{"archivedAt": ""}}}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(
ctx,
organizationScopedFilter(organizationID, filter),
options.Find().
SetProjection(runArtifactProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
// runRecencySort orders candidates exactly like the in-memory selector's
// freshness key (dashboardLatestAcceptancePayload / latestPocGPUWorkflowRun:
// UpdatedAt when present, CreatedAt otherwise), so even runs that never got
// an UpdatedAt written sort the same on both sides of the store boundary.
func runRecencySort() bson.D {
return bson.D{
{Key: "updatedAt", Value: bson.M{"$ifNull": bson.A{"$updatedAt", "$createdAt"}}},
{Key: "createdAt", Value: -1},
}
}
// ListRunsAcceptanceCandidates implements Store.ListRunsAcceptanceCandidates.
// The $or clauses mirror dashboardAcceptancePayloadFromArtifact's four
// detection paths (artifact name + data/preview.value/value.acceptanceScope),
// so the latest acceptance run is always inside the bounded result set while
// the in-memory selector keeps its exact picking semantics.
func (store *MongoStore) ListRunsAcceptanceCandidates(ctx context.Context, organizationID string, limit int) ([]domain.RunRecord, error) {
filter := bson.M{"$or": bson.A{
bson.M{"artifacts.name": "poc-operator-full-acceptance-result.json"},
bson.M{"artifacts.data.acceptanceScope": bson.M{"$exists": true}},
bson.M{"artifacts.data.value.acceptanceScope": bson.M{"$exists": true}},
bson.M{"artifacts.data.preview.value.acceptanceScope": bson.M{"$exists": true}},
}}
if organizationID != "" {
filter = organizationScopedFilter(organizationID, filter)
}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runArtifactProjection()).
SetSort(runRecencySort()).
SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
// ListRunsWorkflowCandidates implements Store.ListRunsWorkflowCandidates.
// Candidates need full documents (logs + node executions feed the preflight
// gate's log scan), but the name filters keep the result set tiny.
func (store *MongoStore) ListRunsWorkflowCandidates(ctx context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error) {
or := bson.A{}
if len(workflowNames) > 0 {
or = append(or, bson.M{"workflowName": bson.M{"$in": workflowNames}})
}
if len(projectNames) > 0 {
or = append(or, bson.M{"projectName": bson.M{"$in": projectNames}})
}
if len(or) == 0 {
return []domain.RunRecord{}, nil
}
filter := bson.M{"$or": or}
if organizationID != "" {
filter = organizationScopedFilter(organizationID, filter)
}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetSort(runRecencySort()).
SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) {
filter := bson.M{
"status": bson.M{"$in": bson.A{string(domain.RunStatusCompleted), string(domain.RunStatusFailed), string(domain.RunStatusCancelled)}},
"endedAt": bson.M{"$ne": "", "$lt": olderThan.UTC().Format(time.RFC3339Nano)},
"archivedAt": bson.M{"$in": bson.A{nil, ""}},
}
result, err := store.runs.UpdateMany(ctx, filter, bson.M{"$set": bson.M{"archivedAt": nowISO()}})
if err != nil {
return 0, err
}
return int(result.ModifiedCount), nil
}
func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) {
cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{
{{Key: "$group", Value: bson.M{
"_id": "$status",
"count": bson.M{"$sum": 1},
}}},
})
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
counts := map[domain.RunStatus]int{}
for cursor.Next(ctx) {
var item struct {
Status domain.RunStatus `bson:"_id"`
Count int `bson:"count"`
}
if err := cursor.Decode(&item); err != nil {
return nil, err
}
counts[item.Status] = item.Count
}
return counts, cursor.Err()
}
func runSummaryProjection() bson.M {
return bson.M{
"id": 1,
"organizationId": 1,
"workflowId": 1,
"workflowName": 1,
"projectId": 1,
"projectName": 1,
"captureDirectories": 1,
"taskId": 1,
"trigger": 1,
"attempt": 1,
"mode": 1,
"releaseId": 1,
"workflowVersion": 1,
"gitCommitSha": 1,
"imageDigest": 1,
"parentRunId": 1,
"restartFromNodeId": 1,
"availableAt": 1,
"status": 1,
"stopMode": 1,
"stopRequestedAt": 1,
"startedAt": 1,
"endedAt": 1,
"durationMs": 1,
"currentNodeId": 1,
"failure": 1,
"progress": 1,
"error": 1,
"createdAt": 1,
"updatedAt": 1,
}
}
func runSummaryPageProjection() bson.M {
projection := runSummaryProjection()
projection["error"] = bson.M{"$substrCP": bson.A{
bson.M{"$ifNull": bson.A{"$error", ""}},
0,
runSummaryTextLimit,
}}
return projection
}
func (store *MongoStore) ListRunSummaries(ctx context.Context, status string) ([]domain.RunSummary, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runSummaryProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
return decodeMany[domain.RunSummary](ctx, cursor)
}
func (store *MongoStore) ListRunSummariesByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunSummary, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(
ctx,
organizationScopedFilter(organizationID, filter),
options.Find().
SetProjection(runSummaryProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
return decodeMany[domain.RunSummary](ctx, cursor)
}
func combineRunSummaryFilters(filters ...bson.M) bson.M {
items := make(bson.A, 0, len(filters))
for _, filter := range filters {
if len(filter) > 0 {
items = append(items, filter)
}
}
if len(items) == 0 {
return bson.M{}
}
if len(items) == 1 {
return items[0].(bson.M)
}
return bson.M{"$and": items}
}
func runSummaryBaseFilter(query domain.RunSummaryQuery) bson.M {
filters := make([]bson.M, 0, 4)
if query.ProjectID != "" {
filters = append(filters, bson.M{"projectId": query.ProjectID})
}
if query.WorkflowID != "" {
filters = append(filters, bson.M{"workflowId": query.WorkflowID})
}
if query.Mode == domain.RunModeProduction {
filters = append(filters, bson.M{"$or": bson.A{
bson.M{"mode": string(domain.RunModeProduction)},
bson.M{"mode": ""},
bson.M{"mode": bson.M{"$exists": false}},
bson.M{"mode": nil},
}})
} else if query.Mode != "" && query.Mode != domain.RunModeAll && query.Mode != "all" {
filters = append(filters, bson.M{"mode": string(query.Mode)})
}
if search := strings.TrimSpace(query.Search); search != "" {
pattern := primitive.Regex{Pattern: regexp.QuoteMeta(search), Options: "i"}
filters = append(filters, bson.M{"$or": bson.A{
bson.M{"id": pattern},
bson.M{"workflowId": pattern},
bson.M{"workflowName": pattern},
bson.M{"projectId": pattern},
bson.M{"projectName": pattern},
bson.M{"captureDirectories": pattern},
bson.M{"currentNodeId": pattern},
bson.M{"error": pattern},
}})
}
return combineRunSummaryFilters(filters...)
}
func runSummaryCursorFilter(cursor *domain.RunSummaryCursor) bson.M {
if cursor == nil {
return bson.M{}
}
return bson.M{"$or": bson.A{
bson.M{"createdAt": bson.M{"$lt": cursor.CreatedAt}},
bson.M{"createdAt": cursor.CreatedAt, "id": bson.M{"$lt": cursor.ID}},
}}
}
func (store *MongoStore) loadRunSummaryAggregates(ctx context.Context, filter bson.M) (runSummaryAggregates, error) {
pipeline := mongo.Pipeline{}
if len(filter) > 0 {
pipeline = append(pipeline, bson.D{{Key: "$match", Value: filter}})
}
pipeline = append(pipeline, bson.D{{Key: "$group", Value: bson.M{
"_id": "$status",
"count": bson.M{"$sum": 1},
}}})
cursor, err := store.runs.Aggregate(ctx, pipeline)
if err != nil {
return runSummaryAggregates{}, err
}
defer cursor.Close(ctx)
result := runSummaryAggregates{StatusCounts: emptyRunStatusCounts()}
for cursor.Next(ctx) {
var item struct {
Status domain.RunStatus `bson:"_id"`
Count int64 `bson:"count"`
}
if err := cursor.Decode(&item); err != nil {
return runSummaryAggregates{}, err
}
result.StatusCounts[item.Status] = item.Count
result.Total += item.Count
}
if err := cursor.Err(); err != nil {
return runSummaryAggregates{}, err
}
return result, nil
}
func (store *MongoStore) findRunSummaryPageItems(ctx context.Context, filter bson.M, limit int) ([]domain.RunSummary, error) {
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runSummaryPageProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}).
SetLimit(int64(limit+1)),
)
if err != nil {
return nil, err
}
return decodeMany[domain.RunSummary](ctx, cursor)
}
func (store *MongoStore) listRunSummariesPage(ctx context.Context, organizationID string, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) {
baseFilter := runSummaryBaseFilter(query)
if organizationID != "" {
baseFilter = organizationScopedFilter(organizationID, baseFilter)
}
itemFilter := bson.M{}
if query.Status != "" {
itemFilter["status"] = query.Status
}
limit := normalizedRunSummaryLimit(query.Limit)
itemFilter = combineRunSummaryFilters(baseFilter, itemFilter, runSummaryCursorFilter(query.Cursor))
queryCtx, cancel := context.WithCancel(ctx)
defer cancel()
var waitGroup sync.WaitGroup
var items []domain.RunSummary
var aggregates runSummaryAggregates
queryErrors := make(chan error, 2)
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
foundItems, err := store.findRunSummaryPageItems(queryCtx, itemFilter, limit)
if err != nil {
queryErrors <- err
cancel()
return
}
items = foundItems
}()
includeAggregates := query.IncludeAggregates || !query.AggregatesExplicit
if includeAggregates {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
key := newRunSummaryAggregateCacheKey(organizationID, query)
foundAggregates, err := store.runSummaryAggregates.getOrLoad(queryCtx, key, func(loadCtx context.Context) (runSummaryAggregates, error) {
return store.loadRunSummaryAggregates(loadCtx, baseFilter)
})
if err != nil {
queryErrors <- err
cancel()
return
}
aggregates = foundAggregates
}()
}
waitGroup.Wait()
select {
case err := <-queryErrors:
return domain.RunSummaryPage{}, err
default:
}
hasMore := len(items) > limit
if hasMore {
items = items[:limit]
}
for index := range items {
items[index] = compactRunSummary(items[index])
}
var nextCursor *domain.RunSummaryCursor
if hasMore && len(items) > 0 {
last := items[len(items)-1]
nextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
return domain.RunSummaryPage{
Items: items,
Total: aggregates.Total,
StatusCounts: aggregates.StatusCounts,
NextCursor: nextCursor,
HasMore: hasMore,
}, nil
}
func (store *MongoStore) ListRunSummariesPage(ctx context.Context, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) {
return store.listRunSummariesPage(ctx, "", query)
}
func (store *MongoStore) ListRunSummariesPageByOrganization(ctx context.Context, organizationID string, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) {
return store.listRunSummariesPage(ctx, organizationID, query)
}
func (store *MongoStore) ListReconcileRunSummaries(ctx context.Context, limit int) ([]domain.RunSummary, error) {
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
filter := bson.M{"$or": bson.A{
bson.M{"status": bson.M{"$in": bson.A{domain.RunStatusRunning, domain.RunStatusStopping}}},
bson.M{
"status": domain.RunStatusQueued,
"currentNodeId": bson.M{"$exists": true, "$nin": bson.A{"", nil}},
},
}}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runSummaryPageProjection()).
SetSort(bson.D{{Key: "updatedAt", Value: 1}, {Key: "id", Value: 1}}).
SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunSummary](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
items[index] = compactRunSummary(items[index])
}
return items, nil
}
func (store *MongoStore) CountWorkerRuns(ctx context.Context) (WorkerRunCounts, error) {
statuses := bson.A{
domain.RunStatusQueued,
domain.RunStatusRunning,
domain.RunStatusStopping,
domain.RunStatusWaitingForAudit,
domain.RunStatusWaitingForHuman,
}
pipeline := mongo.Pipeline{
bson.D{{Key: "$match", Value: bson.M{"status": bson.M{"$in": statuses}}}},
bson.D{{Key: "$group", Value: bson.M{
"_id": bson.M{
"workflowId": "$workflowId",
"status": "$status",
},
"count": bson.M{"$sum": 1},
}}},
}
read := options.Aggregate().SetHint(workerRunCountIndex)
cursor, err := store.runs.Aggregate(ctx, pipeline, read)
if poolSearchHintMissing(err) && ctx.Err() == nil {
cursor, err = store.runs.Aggregate(ctx, pipeline, read.SetHint(nil))
}
if err != nil {
return WorkerRunCounts{}, err
}
defer cursor.Close(ctx)
counts := newWorkerRunCounts()
for cursor.Next(ctx) {
var item struct {
Key struct {
WorkflowID string `bson:"workflowId"`
Status domain.RunStatus `bson:"status"`
} `bson:"_id"`
Count int `bson:"count"`
}
if err := cursor.Decode(&item); err != nil {
return WorkerRunCounts{}, err
}
addWorkerRunCount(&counts, item.Key.WorkflowID, item.Key.Status, item.Count)
}
if err := cursor.Err(); err != nil {
return WorkerRunCounts{}, err
}
return counts, nil
}
func (store *MongoStore) HasActiveRun(ctx context.Context, workflowID string) (bool, error) {
count, err := store.runs.CountDocuments(ctx, bson.M{
"workflowId": workflowID,
"status": bson.M{"$in": activeRunStatuses()},
}, options.Count().SetLimit(1))
return count > 0, err
}
func (store *MongoStore) UpdateRunProgress(ctx context.Context, runID string, progress *domain.RunProgress) error {
updatedAt := nowISO()
update := bson.M{"$set": bson.M{"updatedAt": updatedAt}}
if progress == nil {
update["$unset"] = bson.M{"progress": ""}
} else {
value := *progress
if strings.TrimSpace(value.UpdatedAt) == "" {
value.UpdatedAt = updatedAt
}
updatedAt = value.UpdatedAt
update["$set"] = bson.M{"progress": value, "updatedAt": updatedAt}
}
result, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, update)
if err != nil {
return err
}
if result.MatchedCount == 0 {
return fmt.Errorf("run %s not found", runID)
}
return nil
}
func (store *MongoStore) UpdateRunFailure(ctx context.Context, runID string, failure *domain.RunFailure) error {
update := bson.M{"$set": bson.M{"updatedAt": nowISO()}}
if failure == nil {
update["$unset"] = bson.M{"failure": ""}
} else {
update["$set"] = bson.M{"failure": *failure, "updatedAt": nowISO()}
}
result, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, update)
if err != nil {
return err
}
if result.MatchedCount == 0 {
return fmt.Errorf("run %s not found", runID)
}
return nil
}
func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunRecord, error) {
var item domain.RunRecord
err := store.runs.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
hydrateRunDerivedFields(&item)
return &item, nil
}
func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
return store.CreateRunWithInputs(ctx, workflowID, trigger, nil, taskIDs...)
}
func (store *MongoStore) CreateRunWithInputs(ctx context.Context, workflowID string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) {
w, e := store.GetWorkflow(ctx, workflowID)
if e != nil {
return nil, e
}
if w == nil {
return nil, fmt.Errorf("workflow not found")
}
var result *domain.RunRecord
e = supplierGuardedMongoWrite(ctx, store.runs, "new-run", bson.M{"projectId": w.ProjectID, "organizationId": w.OrganizationID, "workflowId": workflowID, "inputs": inputs}, func(ctx context.Context, _ bson.M) error {
var e error
result, e = store.createRunSupplierGuarded(ctx, workflowID, trigger, inputs, taskIDs...)
return e
})
if e != nil {
return nil, e
}
if result == nil {
return nil, ErrSupplierDeleting
}
return result, nil
}
func (store *MongoStore) createRunSupplierGuarded(ctx context.Context, workflowID string, trigger domain.RunTrigger, inputs map[string]domain.ArtifactPayload, taskIDs ...string) (*domain.RunRecord, error) {
workflow, err := store.GetWorkflow(ctx, workflowID)
if err != nil {
return nil, err
}
if workflow == nil {
return nil, fmt.Errorf("workflow %s not found", workflowID)
}
projectName := ""
if workflow.ProjectID != "" {
project, err := store.GetProject(ctx, workflow.ProjectID)
if err != nil {
return nil, err
}
if project == nil {
return nil, fmt.Errorf("project %s not found", workflow.ProjectID)
}
projectName = project.Name
if project.DeletionTaskID != "" {
return nil, fmt.Errorf("project deletion in progress")
}
}
now := nowISO()
availableAt := now
if trigger == domain.RunTriggerRerunSubtree || trigger == domain.RunTriggerHumanRelease {
// See MemoryStore.CreateRun: rerun-subtree creation is finalized by the
// API in a second write after reusable upstream executions are attached.
availableAt = time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339Nano)
}
previousRunAt := workflow.LastRunAt
resolvedInputs, resolvedBindings, err := domain.ResolveWorkflowRunInputs(*workflow, inputs)
if err != nil {
return nil, err
}
run := domain.RunRecord{
ProcessingProfile: domain.WorkflowProcessingProfile(workflow.Nodes),
ID: domain.NewID("run"),
OrganizationID: workflow.OrganizationID,
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: workflow.ProjectID,
ProjectName: projectName,
Trigger: trigger,
Mode: domain.RunModeProduction,
Attempt: 1,
DefinitionVersion: workflow.DefinitionVersion,
WorkflowVersion: workflow.DefinitionVersion,
Inputs: resolvedInputs,
ResolvedInputBindings: resolvedBindings,
AvailableAt: availableAt,
Status: domain.RunStatusQueued,
PreviousRunAt: previousRunAt,
CurrentRunAt: now,
NodeExecutions: []domain.NodeExecutionRecord{},
Logs: []domain.LogEntryRecord{},
ResourceSeries: []domain.ResourceSample{},
Artifacts: []domain.ExecutionArtifact{},
K8sRefs: []domain.K8sRef{},
CreatedAt: now,
UpdatedAt: now,
}
if len(taskIDs) > 0 {
run.TaskID = strings.TrimSpace(taskIDs[0])
}
run.CaptureDirectories = domain.CaptureDirectoriesFromRun(run)
if err := store.UpsertRun(ctx, run); err != nil {
return nil, err
}
workflow.LastRunID = run.ID
workflow.LastRunAt = now
workflow.Status = domain.WorkflowStatusRunning
workflow.UpdatedAt = now
if err := store.UpsertWorkflow(ctx, *workflow); err != nil {
return nil, err
}
return &run, nil
}
func mongoRunLeaseAvailableFilter(now string) []bson.M {
return []bson.M{
{"executionLeaseOwner": bson.M{"$exists": false}},
{"executionLeaseOwner": ""},
{"executionLeaseExpiresAt": bson.M{"$exists": false}},
{"executionLeaseExpiresAt": ""},
{"executionLeaseExpiresAt": bson.M{"$lte": now}},
}
}
func mongoRunLeaseSet(owner string, ttl time.Duration, now time.Time) bson.M {
if ttl <= 0 {
ttl = 5 * time.Minute
}
return bson.M{
"executionLeaseOwner": strings.TrimSpace(owner),
"executionLeaseExpiresAt": now.Add(ttl).UTC().Format(time.RFC3339Nano),
}
}
func (store *MongoStore) ClaimNextQueuedRun(ctx context.Context, leaseOwner string, leaseTTL time.Duration) (*domain.RunRecord, error) {
nowTime := time.Now().UTC()
now := nowTime.Format(time.RFC3339Nano)
set := bson.M{
"status": string(domain.RunStatusRunning),
"startedAt": now,
"updatedAt": now,
}
for key, value := range mongoRunLeaseSet(leaseOwner, leaseTTL, nowTime) {
set[key] = value
}
result := store.runs.FindOneAndUpdate(
ctx,
bson.M{
"status": string(domain.RunStatusQueued),
"$or": []bson.M{
{"availableAt": bson.M{"$exists": false}},
{"availableAt": ""},
{"availableAt": bson.M{"$lte": now}},
},
"$and": []bson.M{
{"$or": mongoRunLeaseAvailableFilter(now)},
},
},
bson.M{"$set": set},
options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After),
)
var item domain.RunRecord
err := result.Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
hydrateRunDerivedFields(&item)
return &item, nil
}
func (store *MongoStore) ClaimQueuedRun(ctx context.Context, runID string, leaseOwner string, leaseTTL time.Duration) (*domain.RunRecord, error) {
nowTime := time.Now().UTC()
now := nowTime.Format(time.RFC3339Nano)
set := bson.M{
"status": string(domain.RunStatusRunning),
"startedAt": now,
"updatedAt": now,
}
for key, value := range mongoRunLeaseSet(leaseOwner, leaseTTL, nowTime) {
set[key] = value
}
result := store.runs.FindOneAndUpdate(
ctx,
bson.M{
"id": runID,
"status": string(domain.RunStatusQueued),
"$or": []bson.M{
{"availableAt": bson.M{"$exists": false}},
{"availableAt": ""},
{"availableAt": bson.M{"$lte": now}},
},
"$and": []bson.M{
{"$or": mongoRunLeaseAvailableFilter(now)},
},
},
bson.M{"$set": set},
options.FindOneAndUpdate().SetReturnDocument(options.After),
)
var item domain.RunRecord
err := result.Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
hydrateRunDerivedFields(&item)
return &item, nil
}
func (store *MongoStore) AcquireRunLease(ctx context.Context, runID string, owner string, ttl time.Duration) (*domain.RunRecord, bool, error) {
nowTime := time.Now().UTC()
now := nowTime.Format(time.RFC3339Nano)
set := bson.M{"updatedAt": now}
for key, value := range mongoRunLeaseSet(owner, ttl, nowTime) {
set[key] = value
}
result := store.runs.FindOneAndUpdate(
ctx,
bson.M{
"id": runID,
"$or": append(
mongoRunLeaseAvailableFilter(now),
bson.M{"executionLeaseOwner": strings.TrimSpace(owner)},
),
},
bson.M{"$set": set},
options.FindOneAndUpdate().SetReturnDocument(options.After),
)
var item domain.RunRecord
err := result.Decode(&item)
if err == mongo.ErrNoDocuments {
current, getErr := store.GetRun(ctx, runID)
return current, false, getErr
}
if err != nil {
return nil, false, err
}
hydrateRunDerivedFields(&item)
return &item, true, nil
}
func (store *MongoStore) RenewRunLease(ctx context.Context, runID string, owner string, ttl time.Duration) (bool, error) {
nowTime := time.Now().UTC()
now := nowTime.Format(time.RFC3339Nano)
set := bson.M{"updatedAt": now}
for key, value := range mongoRunLeaseSet(owner, ttl, nowTime) {
set[key] = value
}
result, err := store.runs.UpdateOne(
ctx,
bson.M{"id": runID, "executionLeaseOwner": strings.TrimSpace(owner)},
bson.M{"$set": set},
)
if err != nil {
return false, err
}
return result.MatchedCount > 0, nil
}
func (store *MongoStore) ReleaseRunLease(ctx context.Context, runID string, owner string) error {
_, err := store.runs.UpdateOne(
ctx,
bson.M{"id": runID, "executionLeaseOwner": strings.TrimSpace(owner)},
bson.M{
"$set": bson.M{"updatedAt": nowISO()},
"$unset": bson.M{
"executionLeaseOwner": "",
"executionLeaseExpiresAt": "",
},
},
)
return err
}
func (store *MongoStore) RequestRunStop(ctx context.Context, runID string, mode domain.RunStopMode) (*domain.RunRecord, error) {
run, err := store.GetRun(ctx, runID)
if err != nil || run == nil {
return run, err
}
if run.Status == domain.RunStatusCompleted || run.Status == domain.RunStatusFailed || run.Status == domain.RunStatusCancelled {
return run, nil
}
now := nowISO()
set := bson.M{
"stopMode": string(mode),
"stopRequestedAt": now,
"updatedAt": now,
}
switch mode {
case domain.RunStopModeGraceful:
if run.Status == domain.RunStatusQueued || run.Status == domain.RunStatusWaitingForAudit || run.Status == domain.RunStatusWaitingForHuman {
set["status"] = string(domain.RunStatusCancelled)
set["endedAt"] = now
run.EndedAt = now
finalizeNodeExecutionsForStop(run, mode, now)
set["nodeExecutions"] = run.NodeExecutions
if strings.TrimSpace(run.Error) == "" {
set["error"] = "用户已停止工作流"
}
} else {
set["status"] = string(domain.RunStatusStopping)
}
case domain.RunStopModeForce:
set["status"] = string(domain.RunStatusCancelled)
set["endedAt"] = now
run.EndedAt = now
finalizeNodeExecutionsForStop(run, mode, now)
set["nodeExecutions"] = run.NodeExecutions
if strings.TrimSpace(run.Error) == "" {
set["error"] = "用户已强制停止工作流"
}
default:
return run, nil
}
computeRunDuration(run)
if run.DurationMS > 0 {
set["durationMs"] = run.DurationMS
}
if _, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, bson.M{"$set": set}); err != nil {
return nil, err
}
return store.GetRun(ctx, runID)
}
func (store *MongoStore) UpsertRun(ctx context.Context, item domain.RunRecord) error {
return supplierGuardedMongoWrite(ctx, store.runs, item.ID, item, func(ctx context.Context, d bson.M) error {
raw, e := bson.Marshal(d)
if e != nil {
return e
}
var next domain.RunRecord
if e = bson.Unmarshal(raw, &next); e != nil {
return e
}
return store.upsertRunSupplierGuarded(ctx, next)
})
}
func (store *MongoStore) upsertRunSupplierGuarded(ctx context.Context, item domain.RunRecord) error {
allow, err := projectAllowsUpsert(ctx, store.db, item.ProjectID)
if err != nil {
return err
}
computeRunDuration(&item)
stored := compactRunForStorage(item)
unsetFields := runStorageFieldsToUnset(item)
update, err := annotationSafeRunUpdate(stored, unsetFields)
if err != nil {
return err
}
_, err = store.runs.UpdateOne(ctx, bson.M{"id": item.ID}, update, options.Update().SetUpsert(allow))
return err
}
// AppendNodeExecutionResourceSample pushes a single resource sample onto a running
// node's resourceSeries without rewriting the whole run document. The launcher
// flushes a node's full series at completion; this lets the worker stream live
// samples DURING execution so the resource monitor reflects custom-container GPU
// usage before the node finishes. The positional $ targets the nodeExecutions
// element matching nodeID; UpdateOne is a no-op if no such element exists yet.
func (store *MongoStore) AppendNodeExecutionResourceSample(ctx context.Context, runID string, nodeID string, sample domain.ResourceSample) error {
if strings.TrimSpace(runID) == "" || strings.TrimSpace(nodeID) == "" {
return fmt.Errorf("runID and nodeID are required")
}
_, err := store.runs.UpdateOne(ctx,
bson.M{"id": runID, "nodeExecutions.nodeId": nodeID},
bson.M{"$push": bson.M{"nodeExecutions.$.resourceSeries": sample}},
)
return err
}
func (store *MongoStore) GetSourceState(ctx context.Context, workflowID string, nodeID string, sourceFingerprint string) (*domain.SourceStateRecord, error) {
var item domain.SourceStateRecord
err := store.sourceStates.FindOne(ctx, bson.M{
"workflowId": workflowID,
"nodeId": nodeID,
"sourceFingerprint": sourceFingerprint,
}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) ListSourceStates(ctx context.Context, workflowID string, nodeID string) ([]domain.SourceStateRecord, error) {
filter := bson.M{"workflowId": workflowID}
if nodeID != "" {
filter["nodeId"] = nodeID
}
cursor, err := store.sourceStates.Find(ctx, filter)
if err != nil {
return nil, err
}
return decodeMany[domain.SourceStateRecord](ctx, cursor)
}
func (store *MongoStore) UpsertSourceState(ctx context.Context, item domain.SourceStateRecord) error {
return supplierGuardedMongoWrite(ctx, store.sourceStates, item.ID, item, func(ctx context.Context, d bson.M) error {
raw, e := bson.Marshal(d)
if e != nil {
return e
}
var next domain.SourceStateRecord
if e = bson.Unmarshal(raw, &next); e != nil {
return e
}
return store.upsertSourceStateSupplierGuarded(ctx, next)
})
}
func (store *MongoStore) upsertSourceStateSupplierGuarded(ctx context.Context, item domain.SourceStateRecord) error {
if item.ID == "" {
item.ID = item.WorkflowID + ":" + item.NodeID + ":" + item.SourceFingerprint
}
_, err := store.sourceStates.UpdateOne(
ctx,
bson.M{"id": item.ID},
bson.M{"$set": item},
options.Update().SetUpsert(true),
)
return err
}
func (store *MongoStore) DeleteSourceState(ctx context.Context, workflowID string, nodeID string, sourceFingerprint string) error {
_, err := store.sourceStates.DeleteOne(ctx, bson.M{
"workflowId": workflowID,
"nodeId": nodeID,
"sourceFingerprint": sourceFingerprint,
})
return err
}
func (store *MongoStore) DeleteSourceStatesByWorkflow(ctx context.Context, workflowID string) error {
_, err := store.sourceStates.DeleteMany(ctx, bson.M{"workflowId": workflowID})
return err
}
func (store *MongoStore) ListAuditTasks(ctx context.Context, status string) ([]domain.AuditTaskRecord, error) {
filter := bson.M{}
if status != "" && status != "all" {
filter["status"] = status
}
cursor, err := store.auditTasks.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.AuditTaskRecord](ctx, cursor)
}
func (store *MongoStore) ListAuditTasksByOrganization(ctx context.Context, organizationID string, status string) ([]domain.AuditTaskRecord, error) {
filter := bson.M{}
if status != "" && status != "all" {
filter["status"] = status
}
cursor, err := store.auditTasks.Find(ctx, organizationScopedFilter(organizationID, filter), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.AuditTaskRecord](ctx, cursor)
}
func (store *MongoStore) GetAuditTask(ctx context.Context, id string) (*domain.AuditTaskRecord, error) {
var item domain.AuditTaskRecord
err := store.auditTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateAuditTask(ctx context.Context, item domain.AuditTaskRecord) (*domain.AuditTaskRecord, error) {
if err := store.UpsertAuditTask(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) UpsertAuditTask(ctx context.Context, item domain.AuditTaskRecord) error {
_, err := store.auditTasks.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) ListSamplingTasks(ctx context.Context, status string) ([]domain.SamplingTaskRecord, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.samplingTasks.Find(ctx, filter, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.SamplingTaskRecord](ctx, cursor)
}
func (store *MongoStore) ListSamplingTasksByOrganization(ctx context.Context, organizationID string, status string) ([]domain.SamplingTaskRecord, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.samplingTasks.Find(ctx, organizationScopedFilter(organizationID, filter), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.SamplingTaskRecord](ctx, cursor)
}
func (store *MongoStore) GetSamplingTask(ctx context.Context, id string) (*domain.SamplingTaskRecord, error) {
var item domain.SamplingTaskRecord
err := store.samplingTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateSamplingTask(ctx context.Context, item domain.SamplingTaskRecord) (*domain.SamplingTaskRecord, error) {
if strings.TrimSpace(item.ID) == "" {
item.ID = domain.NewID("sampling")
}
if err := store.UpsertSamplingTask(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) UpsertSamplingTask(ctx context.Context, item domain.SamplingTaskRecord) error {
_, err := store.samplingTasks.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) ListUsers(ctx context.Context) ([]domain.UserRecord, error) {
cursor, err := store.users.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.UserRecord](ctx, cursor)
}
func (store *MongoStore) ListUsersByOrganization(ctx context.Context, organizationID string) ([]domain.UserRecord, error) {
cursor, err := store.users.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: 1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.UserRecord](ctx, cursor)
}
// ListProjectMemberUsers uses one scoped query and never loads credential fields.
func (store *MongoStore) ListProjectMemberUsers(ctx context.Context, organizationID string, ids []string) ([]domain.UserRecord, error) {
if len(ids) == 0 {
return []domain.UserRecord{}, nil
}
filter := bson.M{"$and": bson.A{orgCompatibilityFilter(organizationID), bson.M{"id": bson.M{"$in": ids}}}}
cursor, err := store.users.Find(ctx, filter, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "name": 1, "phone": 1, "status": 1, "role": 1, "isSuperAdmin": 1, "supplierId": 1, "organizationId": 1}))
if err != nil {
return nil, err
}
return decodeMany[domain.UserRecord](ctx, cursor)
}
func (store *MongoStore) GetUser(ctx context.Context, id string) (*domain.UserRecord, error) {
var item domain.UserRecord
err := store.users.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
// FindUserByPhone looks up an undeleted login identity by phone number. Phone numbers are not
// case-folded (unlike the removed email field).
func (store *MongoStore) FindUserByPhone(ctx context.Context, phone string) (*domain.UserRecord, error) {
var item domain.UserRecord
err := store.users.FindOne(ctx, bson.M{"phone": strings.TrimSpace(phone), "deletedAt": bson.M{"$in": bson.A{"", nil}}}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
// FindUserByLegacyEmail keeps accounts created before the phone-only identity
// migration login-capable. New account management never writes this field and
// callers must not expose it as a current account identifier.
func (store *MongoStore) FindUserByLegacyEmail(ctx context.Context, email string) (*domain.UserRecord, error) {
var item domain.UserRecord
err := store.users.FindOne(ctx, bson.M{"email": strings.ToLower(strings.TrimSpace(email)), "deletedAt": bson.M{"$in": bson.A{"", nil}}}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateUser(ctx context.Context, input domain.CreateUserInput) (*domain.UserRecord, error) {
if input.Role == domain.UserRoleAcceptanceReviewer && strings.TrimSpace(input.SupplierID) == "" {
return nil, ErrAcceptanceAuthorizationRequired
}
if err := store.SnapshotLegacyAcceptanceAccounts(ctx, input.OrganizationID); err != nil {
return nil, err
}
now := nowISO()
item := domain.UserRecord{
AcceptanceAccessMode: "managed",
DefaultProjectRoles: input.DefaultProjectRoles,
ID: domain.NewID("user"),
OrganizationID: input.OrganizationID,
Name: input.Name,
Phone: strings.TrimSpace(input.Phone),
Role: input.Role,
SupplierID: strings.TrimSpace(input.SupplierID),
IsSuperAdmin: input.IsSuperAdmin,
PasswordHash: input.PasswordHash,
Status: input.Status,
CreatedAt: now,
UpdatedAt: now,
}
if item.Status == "" {
item.Status = domain.UserStatusActive
}
if item.SupplierID != "" {
_, err := store.runAccountTransaction(ctx, "suppliers:"+normalizedOrganizationID(item.OrganizationID), func(sc context.Context) (any, error) {
if item.Role == domain.UserRoleAcceptanceReviewer {
grant, err := store.GetSupplierAcceptanceAuthorization(sc, item.OrganizationID, item.SupplierID)
if err != nil {
return nil, err
}
if !grant.Enabled {
return nil, ErrAcceptanceAuthorizationRequired
}
}
result, e := store.suppliers.UpdateOne(sc, bson.M{"id": item.SupplierID, "organizationId": item.OrganizationID, "status": "active"}, bson.M{"$inc": bson.M{"referenceVersion": 1}})
if e != nil {
return nil, e
}
if result.MatchedCount != 1 {
return nil, ErrSupplierUnavailable
}
return nil, store.UpsertUser(sc, item)
})
if err != nil {
return nil, err
}
} else if err := store.UpsertUser(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteUser(ctx context.Context, id string) (bool, error) {
result, err := store.users.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount > 0, nil
}
func (store *MongoStore) UpsertUser(ctx context.Context, item domain.UserRecord) error {
return store.upsertUser(ctx, item, nil)
}
func (store *MongoStore) UpdateUser(ctx context.Context, item, expected domain.UserRecord) error {
if item.Role != expected.Role {
_, err := store.withAccountMutationLock(ctx, "suppliers:"+normalizedOrganizationID(item.OrganizationID), func(ctx context.Context) (any, error) {
if item.Role == domain.UserRoleAcceptanceReviewer {
grant, err := store.GetSupplierAcceptanceAuthorization(ctx, item.OrganizationID, item.SupplierID)
if err != nil {
return nil, err
}
if !grant.Enabled {
return nil, ErrAcceptanceAuthorizationRequired
}
}
item.AcceptanceAccessMode = "managed"
if removesActiveAdmin(expected, item) {
_, err := store.withAccountMutationLock(ctx, "user:"+item.ID, func(ctx context.Context) (any, error) { return nil, store.upsertUser(ctx, item, &expected) })
return nil, err
}
return nil, store.updateAccountRole(ctx, item, expected)
})
return err
}
_, err := store.withAccountMutationLock(ctx, "user:"+item.ID, func(ctx context.Context) (any, error) { return nil, store.upsertUser(ctx, item, &expected) })
return err
}
func (store *MongoStore) upsertUser(ctx context.Context, item domain.UserRecord, expected *domain.UserRecord) error {
if item.SupplierID != "" {
supplier, e := store.GetSupplier(ctx, item.SupplierID)
if e != nil {
return e
}
if supplier != nil && (supplier.Status == "deleting" || supplier.Status == "deleted") {
return ErrSupplierDeleting
}
}
if item.Status == "" {
item.Status = domain.UserStatusActive
}
item.Phone = strings.TrimSpace(item.Phone)
item.SupplierID = strings.TrimSpace(item.SupplierID)
existing, err := store.GetUser(ctx, item.ID)
if err != nil {
return err
}
if existing != nil && existing.DeletionRequestedAt != "" && (item.DeletionRequestedAt != existing.DeletionRequestedAt || item.Status != domain.UserStatusDisabled) {
return ErrAccountDeleted
}
if existing != nil && existing.DeletedAt != "" {
return ErrAccountDeleted
}
if expected != nil && (existing == nil || !sameUserMutationSnapshot(*existing, *expected)) {
return ErrUserMutationConflict
}
if existing != nil && removesActiveAdmin(*existing, item) {
return store.updateAdministrator(ctx, item, expected)
}
if existing == nil {
_, err = store.users.InsertOne(ctx, item)
} else {
if existing.SupplierID != item.SupplierID {
return ErrSupplierImmutable
}
fields, encodeErr := bsonDocumentWithoutSupplier(item)
if encodeErr != nil {
return encodeErr
}
if item.DeletedAt != "" {
fields["passwordHash"] = ""
}
var result *mongo.UpdateResult
result, err = store.users.UpdateOne(ctx, userMutationFilter(*existing), bson.M{"$set": fields})
if err == nil && result.MatchedCount == 0 {
return ErrUserMutationConflict
}
}
if isPhoneUniqueDuplicateKeyError(err) {
return ErrDuplicateUserPhone
}
return err
}
// isPhoneUniqueDuplicateKeyError reports whether err is a MongoDB duplicate-key
// error specifically from the users collection's phone_unique index, not just
// any unique-index conflict on that collection — a stale or unrelated index
// (e.g. a pre-migration email_unique index left over in an older database)
// would otherwise be misreported to the caller as "phone must be unique".
func isPhoneUniqueDuplicateKeyError(err error) bool {
return mongo.IsDuplicateKeyError(err) && (strings.Contains(err.Error(), "phone_unique") || strings.Contains(err.Error(), liveUserPhoneIndex))
}
func (store *MongoStore) CreateSession(ctx context.Context, item domain.SessionRecord) (*domain.SessionRecord, error) {
if strings.TrimSpace(item.ID) == "" {
item.ID = domain.NewID("session")
}
now := nowISO()
if item.CreatedAt == "" {
item.CreatedAt = now
}
if item.LastSeenAt == "" {
item.LastSeenAt = now
}
if item.ExpiresAtTime.IsZero() {
if parsed, err := time.Parse(time.RFC3339, item.ExpiresAt); err == nil {
item.ExpiresAtTime = parsed
}
}
if _, err := store.sessions.InsertOne(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) GetSessionByTokenHash(ctx context.Context, tokenHash string) (*domain.SessionRecord, error) {
var item domain.SessionRecord
err := store.sessions.FindOne(ctx, bson.M{"tokenHash": tokenHash}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteSession(ctx context.Context, id string) (bool, error) {
result, err := store.sessions.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount > 0, nil
}
func (store *MongoStore) DeleteSessionsByUser(ctx context.Context, userID string) error {
_, err := store.sessions.DeleteMany(ctx, bson.M{"userId": userID})
return err
}
func (store *MongoStore) ListStorageConnections(ctx context.Context) ([]domain.StorageConnectionRecord, error) {
cursor, err := store.storageConnections.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.StorageConnectionRecord](ctx, cursor)
}
func (store *MongoStore) ListStorageConnectionsByOrganization(ctx context.Context, organizationID string) ([]domain.StorageConnectionRecord, error) {
cursor, err := store.storageConnections.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.StorageConnectionRecord](ctx, cursor)
}
func (store *MongoStore) GetStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) {
var item domain.StorageConnectionRecord
err := store.storageConnections.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateStorageConnection(ctx context.Context, input domain.CreateStorageConnectionInput) (*domain.StorageConnectionRecord, error) {
now := nowISO()
item := domain.StorageConnectionRecord{
ID: domain.NewID("storage"),
OrganizationID: input.OrganizationID,
Name: input.Name,
Description: input.Description,
Provider: input.Provider,
Endpoint: input.Endpoint,
Bucket: input.Bucket,
AccessKeyID: input.AccessKeyID,
SecretAccessKey: input.SecretAccessKey,
CreatedAt: now,
UpdatedAt: now,
}
if err := store.UpsertStorageConnection(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteStorageConnection(ctx context.Context, id string) (bool, error) {
result, err := store.storageConnections.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount > 0, nil
}
func (store *MongoStore) UpsertStorageConnection(ctx context.Context, item domain.StorageConnectionRecord) error {
_, err := store.storageConnections.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) GetSettings(ctx context.Context) (*domain.SettingsRecord, error) {
var item domain.SettingsRecord
err := store.settings.FindOne(ctx, bson.M{"id": "system"}).Decode(&item)
if err == mongo.ErrNoDocuments {
fallback := defaultSettings()
if err := store.UpsertSettings(ctx, fallback); err != nil {
return nil, err
}
return &fallback, nil
}
if err != nil {
return nil, err
}
normalized := ensureSettingsDefaults(item)
return &normalized, nil
}
func (store *MongoStore) UpsertSettings(ctx context.Context, item domain.SettingsRecord) error {
item = ensureSettingsDefaults(item)
_, err := store.settings.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) GetArtifactStorageSettings(ctx context.Context, organizationID string) (domain.ArtifactStorageSettings, error) {
organizationID = normalizedSettingsOrganizationID(organizationID)
var item domain.SettingsRecord
err := store.settings.FindOne(ctx, bson.M{"id": settingsIDForOrganization(organizationID)}).Decode(&item)
if err != nil && err != mongo.ErrNoDocuments {
return domain.ArtifactStorageSettings{}, err
}
if err == nil && (strings.TrimSpace(item.ArtifactStorage.ConnectionID) != "" || strings.TrimSpace(item.ArtifactStorage.Bucket) != "") {
return item.ArtifactStorage, nil
}
if organizationID == domain.DefaultOrganizationID {
return domain.ArtifactStorageSettings{}, nil
}
global, err := store.GetSettings(ctx)
if err != nil {
return domain.ArtifactStorageSettings{}, err
}
return global.ArtifactStorage, nil
}
func (store *MongoStore) UpsertArtifactStorageSettings(ctx context.Context, organizationID string, item domain.ArtifactStorageSettings) error {
organizationID = normalizedSettingsOrganizationID(organizationID)
id := settingsIDForOrganization(organizationID)
_, err := store.settings.UpdateOne(
ctx,
bson.M{"id": id},
bson.M{
"$set": bson.M{
"organizationId": organizationID,
"artifactStorage": item,
"updatedAt": nowISO(),
},
},
options.Update().SetUpsert(true),
)
return err
}
func (store *MongoStore) ListMongoConnections(ctx context.Context) ([]domain.MongoConnectionRecord, error) {
cursor, err := store.mongoConnections.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.MongoConnectionRecord](ctx, cursor)
}
func (store *MongoStore) ListMongoConnectionsByOrganization(ctx context.Context, organizationID string) ([]domain.MongoConnectionRecord, error) {
cursor, err := store.mongoConnections.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.MongoConnectionRecord](ctx, cursor)
}
func (store *MongoStore) GetMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) {
var item domain.MongoConnectionRecord
err := store.mongoConnections.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) CreateMongoConnection(ctx context.Context, input domain.CreateMongoConnectionInput) (*domain.MongoConnectionRecord, error) {
now := nowISO()
item := domain.MongoConnectionRecord{
ID: domain.NewID("mongo"),
OrganizationID: input.OrganizationID,
Name: input.Name,
Description: input.Description,
URI: input.URI,
Database: input.Database,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := store.mongoConnections.InsertOne(ctx, item); err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) DeleteMongoConnection(ctx context.Context, id string) (bool, error) {
result, err := store.mongoConnections.DeleteOne(ctx, bson.M{"id": id})
if err != nil {
return false, err
}
return result.DeletedCount > 0, nil
}
func (store *MongoStore) UpsertMongoConnection(ctx context.Context, item domain.MongoConnectionRecord) error {
_, err := store.mongoConnections.UpdateOne(ctx, bson.M{"id": item.ID}, bson.M{"$set": item}, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) GetDashboardSummary(ctx context.Context) (domain.DashboardSummary, error) {
projects, err := store.ListProjects(ctx)
if err != nil {
return domain.DashboardSummary{}, err
}
operators, err := store.ListOperators(ctx)
if err != nil {
return domain.DashboardSummary{}, err
}
runs, err := store.ListRunSummaries(ctx, "")
if err != nil {
return domain.DashboardSummary{}, err
}
auditTasks, err := store.ListAuditTasks(ctx, "all")
if err != nil {
return domain.DashboardSummary{}, err
}
settings, err := store.GetSettings(ctx)
if err != nil {
return domain.DashboardSummary{}, err
}
summary := BuildDashboardSummaryAt(projects, operators, runs, auditTasks, settings.Timezone, time.Now())
summary.Jobs.SuccessRate = float64(int(summary.Jobs.SuccessRate*10)) / 10
return summary, nil
}
func (store *MongoStore) ListStorageBuckets(ctx context.Context, sourcePath string) ([]string, error) {
return listBucketsFromSourcePath(sourcePath)
}
func (store *MongoStore) ListStorageObjects(ctx context.Context, sourcePath string, bucket string, prefix string) ([]string, error) {
items, err := listObjectsFromSourcePath(sourcePath, strings.TrimSpace(bucket), strings.TrimSpace(prefix))
if err != nil {
return nil, err
}
sort.Strings(items)
return items, nil
}
func mongoBrowserFilter(value any) any {
switch typed := value.(type) {
case map[string]any:
filter := bson.M{}
for key, item := range typed {
filter[key] = mongoBrowserFilter(item)
}
return filter
case []any:
items := make([]any, 0, len(typed))
for _, item := range typed {
items = append(items, mongoBrowserFilter(item))
}
return items
default:
return typed
}
}
func mongoBrowserDocument(document bson.M) map[string]any {
if len(document) == 0 {
return map[string]any{}
}
payload, err := bson.MarshalExtJSON(document, false, false)
if err != nil {
return sanitizeDatabaseBrowserDocument(map[string]any{})
}
result := map[string]any{}
if err := json.Unmarshal(payload, &result); err != nil {
return sanitizeDatabaseBrowserDocument(map[string]any{})
}
return sanitizeDatabaseBrowserDocument(result)
}
func mongoBrowserSampleDocument(ctx context.Context, collection mongoCollection) (map[string]any, error) {
var sample bson.M
err := collection.FindOne(ctx, bson.D{}, options.FindOne().SetSort(bson.D{{Key: "_id", Value: -1}})).Decode(&sample)
if err == mongo.ErrNoDocuments {
return map[string]any{}, nil
}
if err != nil {
return nil, err
}
return mongoBrowserDocument(sample), nil
}
func mongoBrowserCollectionSummary(ctx context.Context, collection mongoCollection, name string) (domain.DatabaseCollectionSummary, error) {
count, err := collection.EstimatedDocumentCount(ctx)
if err != nil {
count, err = collection.CountDocuments(ctx, bson.D{})
if err != nil {
return domain.DatabaseCollectionSummary{}, err
}
}
sample, err := mongoBrowserSampleDocument(ctx, collection)
if err != nil {
return domain.DatabaseCollectionSummary{}, err
}
return domain.DatabaseCollectionSummary{
Name: name,
DocumentCount: count,
SampleFields: databaseBrowserSampleFields(sample),
}, nil
}
func mongoBrowserDocumentIDFilter(documentID string) (bson.M, error) {
trimmed := strings.TrimSpace(documentID)
if trimmed == "" {
return nil, fmt.Errorf("document id is required")
}
if oid, err := primitive.ObjectIDFromHex(trimmed); err == nil {
return bson.M{"_id": oid}, nil
}
return bson.M{"_id": trimmed}, nil
}
func mongoBrowserWritableDocument(document map[string]any) bson.M {
writable := clone(document)
delete(writable, "_id")
return mongoBrowserFilter(writable).(bson.M)
}
func mongoBrowserWritablePatch(document map[string]any) bson.M {
writable := clone(document)
delete(writable, "_id")
return mongoBrowserFilter(writable).(bson.M)
}
func mongoBrowserFindDocumentByFilter(ctx context.Context, collection mongoCollection, filter bson.M) (map[string]any, error) {
document := bson.M{}
if err := collection.FindOne(ctx, filter).Decode(&document); err != nil {
if err == mongo.ErrNoDocuments {
return nil, fmt.Errorf("document not found")
}
return nil, err
}
return mongoBrowserDocument(document), nil
}
func mongoBrowserIndexSummary(document bson.M) domain.DatabaseIndexSummary {
summary := domain.DatabaseIndexSummary{
Name: strings.TrimSpace(fmt.Sprintf("%v", document["name"])),
Keys: map[string]any{},
}
if key, ok := document["key"].(bson.M); ok {
summary.Keys = mongoBrowserDocument(key)
}
if unique, ok := document["unique"].(bool); ok {
summary.Unique = unique
}
return summary
}
func (store *MongoStore) GetDatabaseOverview(ctx context.Context) (domain.DatabaseOverview, error) {
names, err := store.db.ListCollectionNames(ctx, bson.D{})
if err != nil {
return domain.DatabaseOverview{}, err
}
sort.Strings(names)
overview := domain.DatabaseOverview{
Provider: "mongodb",
Database: store.db.Name(),
ReadOnly: true,
Collections: make([]domain.DatabaseCollectionSummary, 0, len(names)),
}
for _, name := range names {
if IsPrivateDerivedCollection(name) {
continue
}
summary, err := mongoBrowserCollectionSummary(ctx, store.queryCollection(name), name)
if err != nil {
return domain.DatabaseOverview{}, err
}
overview.Collections = append(overview.Collections, summary)
}
return overview, nil
}
func (store *MongoStore) QueryDatabaseCollection(ctx context.Context, input domain.DatabaseQueryInput) (domain.DatabaseQueryResult, error) {
collectionName := strings.TrimSpace(input.Collection)
if collectionName == "" {
return domain.DatabaseQueryResult{}, fmt.Errorf("collection is required")
}
if IsPrivateDerivedCollection(collectionName) {
return domain.DatabaseQueryResult{}, fmt.Errorf("collection %s not found", collectionName)
}
limit := input.Limit
if limit <= 0 {
limit = 50
}
skip := input.Skip
if skip < 0 {
skip = 0
}
collection := store.queryCollection(collectionName)
filter, _ := mongoBrowserFilter(input.Filter).(bson.M)
if filter == nil {
filter = bson.M{}
}
total, err := collection.CountDocuments(ctx, filter)
if err != nil {
return domain.DatabaseQueryResult{}, err
}
cursor, err := collection.Find(
ctx,
filter,
options.Find().
SetLimit(int64(limit)).
SetSkip(int64(skip)).
SetSort(bson.D{{Key: "_id", Value: -1}}),
)
if err != nil {
return domain.DatabaseQueryResult{}, err
}
defer cursor.Close(ctx)
documents := make([]map[string]any, 0)
for cursor.Next(ctx) {
document := bson.M{}
if err := cursor.Decode(&document); err != nil {
return domain.DatabaseQueryResult{}, err
}
documents = append(documents, mongoBrowserDocument(document))
}
if err := cursor.Err(); err != nil {
return domain.DatabaseQueryResult{}, err
}
result := domain.DatabaseQueryResult{
Provider: "mongodb",
Database: store.db.Name(),
Collection: collectionName,
Filter: clone(input.Filter),
Limit: limit,
Skip: skip,
Total: total,
Documents: documents,
}
if len(documents) > 0 {
result.SampleFields = databaseBrowserSampleFields(documents[0])
}
return result, nil
}
func (store *MongoStore) CreateDatabaseCollection(ctx context.Context, name string) (domain.DatabaseCollectionSummary, error) {
collectionName := strings.TrimSpace(name)
if collectionName == "" {
return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required")
}
if err := store.db.CreateCollection(ctx, collectionName); err != nil {
return domain.DatabaseCollectionSummary{}, err
}
return mongoBrowserCollectionSummary(ctx, store.queryCollection(collectionName), collectionName)
}
func (store *MongoStore) ListDatabaseIndexes(ctx context.Context, collectionName string) ([]domain.DatabaseIndexSummary, error) {
name := strings.TrimSpace(collectionName)
if name == "" {
return nil, fmt.Errorf("collection is required")
}
cursor, err := store.queryCollection(name).Indexes().List(ctx)
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
items := make([]domain.DatabaseIndexSummary, 0)
for cursor.Next(ctx) {
document := bson.M{}
if err := cursor.Decode(&document); err != nil {
return nil, err
}
items = append(items, mongoBrowserIndexSummary(document))
}
if err := cursor.Err(); err != nil {
return nil, err
}
sort.Slice(items, func(left int, right int) bool {
return items[left].Name < items[right].Name
})
return items, nil
}
func (store *MongoStore) CreateDatabaseIndex(ctx context.Context, collectionName string, input domain.DatabaseIndexCreateInput) (domain.DatabaseIndexSummary, error) {
name := strings.TrimSpace(collectionName)
if name == "" {
return domain.DatabaseIndexSummary{}, fmt.Errorf("collection is required")
}
if len(input.Keys) == 0 {
return domain.DatabaseIndexSummary{}, fmt.Errorf("index keys are required")
}
model := mongo.IndexModel{
Keys: mongoBrowserFilter(input.Keys),
}
indexName := strings.TrimSpace(input.Name)
if indexName != "" || input.Unique {
opts := options.Index()
if indexName != "" {
opts.SetName(indexName)
}
if input.Unique {
opts.SetUnique(true)
}
model.Options = opts
}
createdName, err := store.queryCollection(name).Indexes().CreateOne(ctx, model)
if err != nil {
return domain.DatabaseIndexSummary{}, err
}
indexes, err := store.ListDatabaseIndexes(ctx, name)
if err != nil {
return domain.DatabaseIndexSummary{}, err
}
for _, item := range indexes {
if item.Name == createdName {
return item, nil
}
}
return domain.DatabaseIndexSummary{Name: createdName, Keys: clone(input.Keys), Unique: input.Unique}, nil
}
func (store *MongoStore) DeleteDatabaseIndex(ctx context.Context, collectionName string, indexName string) error {
name := strings.TrimSpace(collectionName)
target := strings.TrimSpace(indexName)
if name == "" {
return fmt.Errorf("collection is required")
}
if target == "" {
return fmt.Errorf("index name is required")
}
if target == "_id_" {
return fmt.Errorf("index %s cannot be deleted", target)
}
_, err := store.queryCollection(name).Indexes().DropOne(ctx, target)
return err
}
func (store *MongoStore) RenameDatabaseCollection(ctx context.Context, name string, newName string) (domain.DatabaseCollectionSummary, error) {
collectionName := strings.TrimSpace(name)
nextName := strings.TrimSpace(newName)
if store.querySyncEnabled.Load() && (queryReplicaFields[collectionName] != "" || queryReplicaFields[nextName] != "") {
return domain.DatabaseCollectionSummary{}, ErrQueryReplicaUnsupported
}
if collectionName == "" {
return domain.DatabaseCollectionSummary{}, fmt.Errorf("collection is required")
}
if nextName == "" {
return domain.DatabaseCollectionSummary{}, fmt.Errorf("new collection name is required")
}
command := bson.D{
{Key: "renameCollection", Value: store.db.Name() + "." + collectionName},
{Key: "to", Value: store.db.Name() + "." + nextName},
}
if err := store.db.Client().Database("admin").RunCommand(ctx, command).Err(); err != nil {
return domain.DatabaseCollectionSummary{}, err
}
return mongoBrowserCollectionSummary(ctx, store.queryCollection(nextName), nextName)
}
func (store *MongoStore) DeleteDatabaseCollection(ctx context.Context, name string) error {
collectionName := strings.TrimSpace(name)
if collectionName == "" {
return fmt.Errorf("collection is required")
}
if err := store.queryCollection(collectionName).Drop(ctx); err != nil {
return err
}
return nil
}
func (store *MongoStore) InsertDatabaseDocument(ctx context.Context, collectionName string, document map[string]any) (map[string]any, error) {
name := strings.TrimSpace(collectionName)
if name == "" {
return nil, fmt.Errorf("collection is required")
}
collection := store.queryCollection(name)
payload := mongoBrowserWritableDocument(document)
result, err := collection.InsertOne(ctx, payload)
if err != nil {
return nil, err
}
return mongoBrowserFindDocumentByFilter(ctx, collection, bson.M{"_id": result.InsertedID})
}
func (store *MongoStore) ReplaceDatabaseDocument(ctx context.Context, collectionName string, documentID string, document map[string]any) (map[string]any, error) {
name := strings.TrimSpace(collectionName)
if name == "" {
return nil, fmt.Errorf("collection is required")
}
filter, err := mongoBrowserDocumentIDFilter(documentID)
if err != nil {
return nil, err
}
collection := store.queryCollection(name)
result, err := collection.ReplaceOne(ctx, filter, mongoBrowserWritableDocument(document))
if err != nil {
return nil, err
}
if result.MatchedCount == 0 {
return nil, fmt.Errorf("document %s not found", documentID)
}
return mongoBrowserFindDocumentByFilter(ctx, collection, filter)
}
func (store *MongoStore) PatchDatabaseDocument(ctx context.Context, collectionName string, documentID string, patch map[string]any) (map[string]any, error) {
name := strings.TrimSpace(collectionName)
if name == "" {
return nil, fmt.Errorf("collection is required")
}
filter, err := mongoBrowserDocumentIDFilter(documentID)
if err != nil {
return nil, err
}
collection := store.queryCollection(name)
result, err := collection.UpdateOne(ctx, filter, bson.M{"$set": mongoBrowserWritablePatch(patch)})
if err != nil {
return nil, err
}
if result.MatchedCount == 0 {
return nil, fmt.Errorf("document %s not found", documentID)
}
return mongoBrowserFindDocumentByFilter(ctx, collection, filter)
}
func (store *MongoStore) UpsertDatabaseDocument(ctx context.Context, collectionName string, match map[string]any, patch map[string]any, setOnInsert map[string]any) (map[string]any, bool, error) {
name := strings.TrimSpace(collectionName)
if name == "" {
return nil, false, fmt.Errorf("collection is required")
}
filter, _ := mongoBrowserFilter(match).(bson.M)
if filter == nil {
filter = bson.M{}
}
update := bson.M{"$set": mongoBrowserWritablePatch(patch)}
if len(setOnInsert) > 0 {
update["$setOnInsert"] = mongoBrowserWritablePatch(setOnInsert)
}
collection := store.queryCollection(name)
result, err := collection.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true))
if err != nil {
return nil, false, err
}
created := result.UpsertedCount > 0
if created && result.UpsertedID != nil {
document, err := mongoBrowserFindDocumentByFilter(ctx, collection, bson.M{"_id": result.UpsertedID})
return document, true, err
}
document, err := mongoBrowserFindDocumentByFilter(ctx, collection, filter)
if err != nil {
return nil, false, err
}
return document, false, nil
}
func (store *MongoStore) DeleteDatabaseDocument(ctx context.Context, collectionName string, documentID string) error {
name := strings.TrimSpace(collectionName)
if name == "" {
return fmt.Errorf("collection is required")
}
filter, err := mongoBrowserDocumentIDFilter(documentID)
if err != nil {
return err
}
result, err := store.queryCollection(name).DeleteOne(ctx, filter)
if err != nil {
return err
}
if result.DeletedCount == 0 {
return fmt.Errorf("document %s not found", documentID)
}
return nil
}