forked from longtao.wu/droboticflow
82 lines
6.0 KiB
Go
82 lines
6.0 KiB
Go
package store
|
|
|
|
import (
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
)
|
|
|
|
// Batch independent id joins before restoring individual rows. The original
|
|
// batch/stage and source-admission decisions still run on every candidate.
|
|
// Only routing/search metadata participates in blocking sorts, never drafts.
|
|
func adminPoolPagePipeline(org string, q AdminAnnotationPoolQuery) mongo.Pipeline {
|
|
conditions := bson.A{adminPoolScopeFilter(org, q)}
|
|
if q.Pool == "available" {
|
|
conditions = append(conditions, adminPoolAvailableFilter())
|
|
} else if q.Pool == "mine" {
|
|
conditions = append(conditions, adminPoolMineFilter(q.UserID))
|
|
}
|
|
if q.Cursor != nil {
|
|
conditions = append(conditions, bson.M{"$or": bson.A{bson.M{"createdAt": bson.M{"$lt": q.Cursor.CreatedAt}}, bson.M{"createdAt": q.Cursor.CreatedAt, "id": bson.M{"$lt": q.Cursor.ID}}}})
|
|
}
|
|
projection := annotationSourceAdmissionProjection()
|
|
for _, key := range []string{"createdAt", "updatedAt", "poolSearch", "subject.itemKey", "subject.artifactId", "subject.payload.sliceWorkflow.status", "subject.payload.sliceWorkflow.phase", "pendingCombinedSubmissionId", "assignedUserId"} {
|
|
projection[key] = 1
|
|
}
|
|
projection["_id"] = 0
|
|
p := mongo.Pipeline{{{Key: "$match", Value: bson.M{"$and": conditions}}}, {{Key: "$project", Value: projection}},
|
|
{{Key: "$setWindowFields", Value: bson.M{"sortBy": bson.D{{Key: "createdAt", Value: -1}}, "output": bson.M{"_row": bson.M{"$documentNumber": bson.M{}}}}}},
|
|
{{Key: "$group", Value: bson.M{"_id": bson.M{"$floor": bson.M{"$divide": bson.A{bson.M{"$subtract": bson.A{"$_row", 1}}, 64}}}, "poolItems": bson.M{"$push": "$$ROOT"}}}}}
|
|
// Task/source joins intentionally retain foreign scope fields: admission
|
|
// rejects mismatching jobs rather than treating them as absent.
|
|
lookup := func(from, local, as string, fields bson.M) bson.D {
|
|
return bson.D{{Key: "$lookup", Value: bson.M{"from": from, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$project": fields}}, "as": as}}}
|
|
}
|
|
p = append(p, adminPoolLookup("humanBatches", "poolItems.batchId", "poolBatches", org, bson.M{"_id": 0, "id": 1, "stages.id": 1}),
|
|
lookup("annotationTasks", "poolItems.legacyTaskId", "poolTasks", bson.M{"_id": 0, "id": 1, "organizationId": 1, "projectId": 1, "runId": 1, "status": 1, "nodeId": 1, "artifactId": 1, "rightArtifactId": 1, "sourceVideo": 1, "sourceIsFullVideo": 1, "templateVersion": 1}))
|
|
jobFields := bson.M{"_id": 0, "id": 1, "status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1}
|
|
p = append(p, lookup("annotationIndexJobs", "poolItems.legacyTaskId", "poolTimeline", jobFields),
|
|
bson.D{{Key: "$set", Value: bson.M{"_sourceIDs": bson.M{"$map": bson.M{"input": "$poolItems", "as": "item", "in": bson.M{"$concat": bson.A{"source-v1:", "$$item.organizationId", ":", bson.M{"$ifNull": bson.A{"$$item.processingJob.sourceFingerprint", ""}}}}}}}}},
|
|
lookup("annotationIndexJobs", "_sourceIDs", "poolPrepared", jobFields))
|
|
fields := []string{"poolBatches", "poolTasks", "poolTimeline", "poolPrepared"}
|
|
names := q.Search != "" || q.Capture != ""
|
|
if names {
|
|
p = append(p, humanPoolBatchLookup("runs", "poolItems.runId", "poolRuns", org), adminPoolLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org, bson.M{"_id": 0, "id": 1, "captureId": 1, "prefix": 1}), adminPoolLookup("projects", "poolItems.projectId", "poolProjects", org, bson.M{"_id": 0, "id": 1, "name": 1}))
|
|
fields = append(fields, "poolRuns", "poolCaptures", "poolProjects")
|
|
}
|
|
indexes := bson.M{}
|
|
for _, field := range fields {
|
|
indexes[field] = humanPoolJoinIndex(field)
|
|
}
|
|
p = append(p, bson.D{{Key: "$set", Value: indexes}}, bson.D{{Key: "$unwind", Value: "$poolItems"}})
|
|
sourceTask := humanPoolItemJoin("poolTasks", "$poolItems.legacyTaskId")
|
|
joined := bson.M{"poolBatches": humanPoolItemJoin("poolBatches", "$poolItems.batchId"), "_sourceTask": sourceTask, "_sourceTimeline": humanPoolItemJoin("poolTimeline", "$poolItems.legacyTaskId"),
|
|
"_preparedCandidate": bson.M{"$let": bson.M{"vars": bson.M{"sourceID": bson.M{"$concat": bson.A{"source-v1:", "$poolItems.organizationId", ":", bson.M{"$ifNull": bson.A{"$poolItems.processingJob.sourceFingerprint", ""}}}}}, "in": humanPoolItemJoin("poolPrepared", "$$sourceID")}},
|
|
"poolTasks": bson.M{"$filter": bson.M{"input": sourceTask, "as": "task", "cond": bson.M{"$eq": bson.A{"$$task.organizationId", normalizedOrganizationID(org)}}}}}
|
|
if names {
|
|
joined["poolRuns"] = humanPoolItemJoin("poolRuns", "$poolItems.runId")
|
|
joined["poolProjects"] = humanPoolItemJoin("poolProjects", "$poolItems.projectId")
|
|
joined["poolCaptures"] = bson.M{"$let": bson.M{"vars": bson.M{"run": humanPoolIndexedRecord("poolRuns", "$poolItems.runId")}, "in": humanPoolItemJoin("poolCaptures", "$$run.taskId")}}
|
|
}
|
|
p = append(p, bson.D{{Key: "$replaceWith", Value: bson.M{"$mergeObjects": bson.A{"$poolItems", joined}}}}, bson.D{{Key: "$unwind", Value: "$poolBatches"}}, bson.D{{Key: "$match", Value: bson.M{"$expr": bson.M{"$in": bson.A{"$currentStageId", bson.M{"$ifNull": bson.A{"$poolBatches.stages.id", bson.A{}}}}}}}})
|
|
// Reuse the existing source job selection and decision expressions. Replace
|
|
// only their lookups with this batch's indexed results.
|
|
for _, stage := range annotationSourceAdmissionStatusPipeline() {
|
|
if stage[0].Key == "$lookup" {
|
|
continue
|
|
}
|
|
p = append(p, stage)
|
|
if set, ok := stage[0].Value.(bson.M); ok {
|
|
if _, ok := set["_sourceJobID"]; ok {
|
|
p = append(p, bson.D{{Key: "$set", Value: bson.M{"_sourcePrepared": bson.M{"$cond": bson.A{bson.M{"$ne": bson.A{"$_sourceJobID", ""}}, "$_preparedCandidate", bson.A{}}}}}})
|
|
}
|
|
}
|
|
}
|
|
if q.Pool == "available" {
|
|
p = append(p, bson.D{{Key: "$match", Value: bson.M{"_sourceReady": true}}})
|
|
}
|
|
if q.Category == "source_failed" {
|
|
p = append(p, bson.D{{Key: "$match", Value: bson.M{"_sourceAdmissionStatus": "failed"}}})
|
|
}
|
|
return append(p, bson.D{{Key: "$unset", Value: bson.A{"_sourceReady", "_preparedCandidate", "poolBatches", "_row"}}}, bson.D{{Key: "$sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}})
|
|
}
|