droboticflow/internal/store/admin_annotation_pool.go

412 lines
14 KiB
Go

package store
import (
"context"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
"sort"
"strconv"
"strings"
"time"
)
// Server-authorized administrative read model. Search/category affect only the
// page; Capture is a scope and also affects the badge totals. No payload writes.
type AdminAnnotationPoolQuery struct {
ProjectID, WorkflowID, RunID, NodeID, ArtifactID string
UserID, Pool, Search, Capture, Category string
Limit int
Cursor *domain.HumanItemCursor
}
type AdminAnnotationPoolCounts struct {
Available int `json:"available"`
Mine int `json:"mine"`
All int `json:"all"`
}
func adminPoolScope(i domain.HumanItemRecord) bool {
return i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined" || i.CurrentStageID == "quality_review" || (i.CurrentStageID == "split_annotation" && i.Status == domain.HumanItemRework)
}
func adminPoolAvailable(i domain.HumanItemRecord) bool {
return i.SourceValidationStatus == "" && i.AssignedUserID == "" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemRework || i.Status == domain.HumanItemSubmitted) && (i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined")
}
func adminPoolMine(i domain.HumanItemRecord, user string) bool {
return i.AssignedUserID == user || (i.LatestAuthorUserID == user && i.CurrentStageID == "quality_review" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemSubmitted))
}
func adminPoolSelected(i domain.HumanItemRecord, q AdminAnnotationPoolQuery) bool {
switch q.Pool {
case "available":
return adminPoolAvailable(i)
case "mine":
return adminPoolMine(i, q.UserID)
}
return true
}
func adminPoolBaseQuery(q AdminAnnotationPoolQuery) domain.HumanItemQuery {
return domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: q.ProjectID, WorkflowID: q.WorkflowID, RunID: q.RunID}
}
func adminPoolValidBatch(i domain.HumanItemRecord, b domain.HumanBatchRecord, org string) bool {
if b.ID == "" || !humanOrganizationMatches(b.OrganizationID, org) {
return false
}
for _, st := range b.Stages {
if st.ID == i.CurrentStageID {
return true
}
}
return false
}
func adminPoolScopeFilter(org string, q AdminAnnotationPoolQuery) bson.M {
f := humanItemMongoFilter(org, adminPoolBaseQuery(q))
f["batchId"] = bson.M{"$ne": ""}
f["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$batchId"}, "string"}}, bson.M{"$eq": bson.A{bson.M{"$type": "$currentStageId"}, "string"}}}}
f["$or"] = bson.A{bson.M{"currentStageId": bson.M{"$in": bson.A{"semantic_annotation", "annotation_combined", "quality_review"}}}, bson.M{"currentStageId": "split_annotation", "status": domain.HumanItemRework}}
return f
}
func adminPoolAvailableFilter() bson.M {
return bson.M{"currentStageId": bson.M{"$in": bson.A{"semantic_annotation", "annotation_combined"}}, "assignedUserId": bson.M{"$in": bson.A{"", nil}}, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}}}
}
func adminPoolMineFilter(user string) bson.M {
return bson.M{"$or": bson.A{bson.M{"assignedUserId": user}, bson.M{"latestAuthorUserId": user, "currentStageId": "quality_review", "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemSubmitted}}}}}
}
// The same business classification as the workspace, using persisted state.
func adminPoolCategory(i domain.HumanItemRecord, t domain.AnnotationTaskRecord) string {
if i.Status == domain.HumanItemAccepted || t.Status == "accepted" {
return "accepted"
}
if i.PendingCombinedSubmissionID != "" {
prefix := "fanyu-v5-combined-" + i.ID + "-v"
if strings.HasPrefix(i.PendingCombinedSubmissionID, prefix) {
suffix := strings.TrimPrefix(i.PendingCombinedSubmissionID, prefix)
n, err := strconv.ParseInt(suffix, 10, 64)
if err == nil && strings.IndexFunc(suffix, func(r rune) bool { return r < '0' || r > '9' }) < 0 && n >= 0 && n <= 9007199254740991 && int64(i.Version) > n+1 {
return "submission_conflicted"
}
}
return "submitting"
}
if i.ReviewRound > 0 && (i.CurrentStageID == "quality_review" || i.CurrentStageID == "prepackage_acceptance" || i.CurrentStageID == "acceptance") {
return "rework_review"
}
if t.Status == "pending_quality" || i.CurrentStageID == "quality_review" || i.Status == domain.HumanItemSubmitted {
return "pending_quality"
}
if t.Status == "pending_acceptance" || i.CurrentStageID == "acceptance" || i.CurrentStageID == "prepackage_acceptance" {
return "pending_acceptance"
}
if t.Status == "rework" || i.Status == domain.HumanItemRework || (i.ReviewRound > 0 && i.CurrentStageID == "annotation_combined") {
return "rework"
}
workflow := adminPoolMap(i.Subject.Payload["sliceWorkflow"])
if workflow["status"] == "processing" {
return "slicing"
}
if workflow["status"] == "completed" || workflow["phase"] == "annotate" {
return "pending_annotation"
}
if workflow["status"] == "editing" || workflow["phase"] == "slice" || i.AssignedUserID != "" {
return "pending_slice"
}
return "available"
}
type adminPoolRow struct {
domain.HumanItemRecord `bson:",inline"`
SourceAdmissionStatus string `bson:"_sourceAdmissionStatus"`
Tasks []domain.AnnotationTaskRecord `bson:"poolTasks"`
Runs []domain.RunRecord `bson:"poolRuns"`
Captures []domain.CaptureRecord `bson:"poolCaptures"`
Projects []domain.ProjectRecord `bson:"poolProjects"`
}
func adminPoolMap(v any) map[string]any {
switch x := v.(type) {
case map[string]any:
return x
case bson.M:
return map[string]any(x)
case bson.D:
out := map[string]any{}
for _, e := range x {
out[e.Key] = e.Value
}
return out
}
return nil
}
func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
var task domain.AnnotationTaskRecord
if len(row.Tasks) > 0 {
task = row.Tasks[0]
}
if q.NodeID != "" && task.NodeID != q.NodeID {
return false
}
if q.ArtifactID != "" && task.ArtifactID != q.ArtifactID && task.RightArtifactID != q.ArtifactID {
return false
}
if q.Category == "source_failed" {
if row.SourceValidationStatus != "failed" {
return false
}
} else if q.Category != "" && q.Category != "all" && adminPoolCategory(row.HumanItemRecord, task) != q.Category {
return false
}
if q.Search == "" && q.Capture == "" {
return true
}
values := adminPoolCaptureValues(row)
if q.Capture != "" {
found := false
for _, v := range values {
if strings.EqualFold(strings.TrimSpace(v), q.Capture) {
found = true
}
}
if !found {
return false
}
}
if q.Search != "" && !strings.Contains(adminPoolSearchText(row), strings.ToLower(q.Search)) {
return false
}
return true
}
func adminPoolCaptureValues(row adminPoolRow) []string {
var task domain.AnnotationTaskRecord
if len(row.Tasks) > 0 {
task = row.Tasks[0]
}
values := []string{}
if len(row.Captures) > 0 {
c := row.Captures[0]
if c.CaptureID != "" {
values = append(values, c.CaptureID)
} else {
values = append(values, domain.CaptureDirectoryFromDirectoryValue(c.Prefix))
}
}
if len(values) == 0 {
values = append(values, domain.CaptureDirectoryFromArtifactValue(row.Subject.ItemKey))
if row.PoolSearchCurrent() {
values = append(values, row.PoolSearch.PayloadDirectories...)
} else {
domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values)
}
}
for _, v := range []string{task.ArtifactID, task.SourceVideo} {
values = append(values, domain.CaptureDirectoryFromArtifactValue(v))
}
if len(row.Runs) > 0 {
values = append(values, domain.CaptureDirectoriesFromRun(row.Runs[0])...)
}
return values
}
func adminPoolSearchText(row adminPoolRow) string {
values := adminPoolCaptureValues(row)
if len(row.Projects) > 0 {
values = append(values, row.Projects[0].Name)
} else if len(row.Runs) > 0 {
values = append(values, row.Runs[0].ProjectName)
}
return strings.ToLower(strings.Join(values, " "))
}
func (s *MemoryStore) adminPoolRowsLocked(org string, q AdminAnnotationPoolQuery) []adminPoolRow {
rows := []adminPoolRow{}
for _, i := range s.humanItems {
if !humanOrganizationMatches(i.OrganizationID, org) || !humanItemMatches(i, adminPoolBaseQuery(q)) || !adminPoolScope(i) || !adminPoolValidBatch(i, s.humanBatches[i.BatchID], org) {
continue
}
row := adminPoolRow{HumanItemRecord: humanItemListProjectionClone(i, humanItemListPayloadKeysRemoved)}
if t, ok := s.annotationTasks[i.LegacyTaskID]; ok && humanOrganizationMatches(t.OrganizationID, org) {
row.Tasks = []domain.AnnotationTaskRecord{t}
var timeline, source *domain.AnnotationIndexJob
if job, ok := s.annotationIndexJobs[t.ID]; ok {
timeline = &job
}
if job, ok := s.annotationIndexJobs[sourceJobForItem(i)]; ok {
source = &job
}
row.SourceValidationStatus = AnnotationSourceAdmissionStatus(t, i, timeline, source)
}
if r, ok := s.runs[i.RunID]; ok && humanOrganizationMatches(r.OrganizationID, org) {
row.Runs = []domain.RunRecord{r}
if c, ok := s.captureRecords[r.TaskID]; ok && humanOrganizationMatches(c.OrganizationID, org) {
row.Captures = []domain.CaptureRecord{c}
}
}
if p, ok := s.projects[i.ProjectID]; ok && humanOrganizationMatches(p.OrganizationID, org) {
row.Projects = []domain.ProjectRecord{p}
}
rows = append(rows, row)
}
return rows
}
func (s *MemoryStore) CountAdminAnnotationPools(_ context.Context, org string, q AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) {
s.mu.Lock()
defer s.mu.Unlock()
c := AdminAnnotationPoolCounts{}
q.Search = ""
q.Category = ""
for _, r := range s.adminPoolRowsLocked(org, q) {
if !adminPoolRowMatches(r, q) {
continue
}
c.All++
if adminPoolAvailable(r.HumanItemRecord) {
c.Available++
}
if adminPoolMine(r.HumanItemRecord, q.UserID) {
c.Mine++
}
}
return c, nil
}
func (s *MemoryStore) ListAdminAnnotationPool(_ context.Context, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
s.mu.Lock()
defer s.mu.Unlock()
rows := s.adminPoolRowsLocked(org, q)
sort.Slice(rows, func(i, j int) bool {
if rows[i].CreatedAt != rows[j].CreatedAt {
return rows[i].CreatedAt > rows[j].CreatedAt
}
return rows[i].ID > rows[j].ID
})
out := []domain.HumanItemRecord{}
for _, r := range rows {
if adminPoolSelected(r.HumanItemRecord, q) && humanItemAfterCursor(r.HumanItemRecord, q.Cursor) && adminPoolRowMatches(r, q) {
out = append(out, humanItemListProjectionClone(r.HumanItemRecord, humanItemListPayloadKeysRemoved))
if len(out) >= q.Limit {
break
}
}
}
return out, nil
}
func adminPoolLookup(from, local, as, org string, projection bson.M) bson.D {
return bson.D{{Key: "$lookup", Value: bson.M{"from": from, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$match": bson.M{"organizationId": normalizedOrganizationID(org)}}, bson.M{"$project": projection}}, "as": as}}}
}
// Stream a narrow metadata cursor; do not materialize the backlog to find an old
// search hit. Unfiltered reads stop after pageSize+1. Context bounds rare scans.
func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q AdminAnnotationPoolQuery, visit func(domain.HumanItemRecord) bool) error {
if adminPoolMetadataOnly(q) {
return s.scanAdminPoolMetadata(ctx, org, q, false, visit)
}
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
defer cancel()
pipeline := adminPoolPagePipeline(org, q)
if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready {
if len(ids) == 0 {
return nil
}
if scoped, withinBudget := queryCandidateFilter(pipeline[0][0].Value, ids); withinBudget {
pipeline[0] = bson.D{{Key: "$match", Value: scoped}}
}
}
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200))
if err != nil {
return err
}
defer cur.Close(ctx)
for {
rows := []adminPoolRow{}
for len(rows) < 200 && cur.Next(ctx) {
var row adminPoolRow
if err := cur.Decode(&row); err != nil {
return err
}
rows = append(rows, row)
}
if err := cur.Err(); err != nil {
return err
}
if len(rows) == 0 {
break
}
if err := s.humanPoolSearchLegacyPayloads(ctx, org, domain.HumanItemQuery{Search: q.Search, Capture: q.Capture}, rows); err != nil {
return err
}
for _, row := range rows {
row.SourceValidationStatus = row.SourceAdmissionStatus
if adminPoolRowMatches(row, q) && !visit(row.HumanItemRecord) {
return nil
}
}
}
return cur.Err()
}
func (s *MongoStore) ListAdminAnnotationPool(ctx context.Context, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
if target, err := s.poolReplica(ctx); err != nil {
return nil, err
} else if target != nil {
return s.replicaAdminPool(ctx, target, org, q)
}
ids := []string{}
statuses := map[string]string{}
err := s.scanAdminAnnotationPool(ctx, org, q, func(i domain.HumanItemRecord) bool {
ids = append(ids, i.ID)
statuses[i.ID] = i.SourceValidationStatus
return q.Limit <= 0 || len(ids) < q.Limit
})
if err != nil {
return nil, err
}
items, err := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids)
for n := range items {
items[n].SourceValidationStatus = statuses[items[n].ID]
}
return items, err
}
func (s *MongoStore) CountAdminAnnotationPools(ctx context.Context, org string, q AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) {
if target, err := s.poolReplica(ctx); err != nil {
return AdminAnnotationPoolCounts{}, err
} else if target != nil {
return target.CountAdminAnnotationPools(ctx, org, q)
}
if q.Capture == "" && q.NodeID == "" && q.ArtifactID == "" {
counts := AdminAnnotationPoolCounts{}
err := s.scanAdminPoolMetadata(ctx, org, q, true, func(item domain.HumanItemRecord) bool {
counts.All++
if adminPoolAvailable(item) {
counts.Available++
}
if adminPoolMine(item, q.UserID) {
counts.Mine++
}
return true
})
if err != nil {
return AdminAnnotationPoolCounts{}, err
}
return counts, nil
}
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
defer cancel()
all := q
all.Pool = "all"
all.Search = ""
all.Category = ""
all.Cursor = nil
all.Limit = int(^uint(0) >> 1)
c := AdminAnnotationPoolCounts{}
err := s.scanAdminAnnotationPool(ctx, org, all, func(i domain.HumanItemRecord) bool {
c.All++
if adminPoolAvailable(i) {
c.Available++
}
if adminPoolMine(i, q.UserID) {
c.Mine++
}
return true
})
return c, err
}