forked from longtao.wu/droboticflow
412 lines
14 KiB
Go
412 lines
14 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Server-authorized administrative read model. Search/category affect only the
|
|
// page; Capture is a scope and also affects the badge totals. No payload writes.
|
|
type AdminAnnotationPoolQuery struct {
|
|
ProjectID, WorkflowID, RunID, NodeID, ArtifactID string
|
|
UserID, Pool, Search, Capture, Category string
|
|
Limit int
|
|
Cursor *domain.HumanItemCursor
|
|
}
|
|
type AdminAnnotationPoolCounts struct {
|
|
Available int `json:"available"`
|
|
Mine int `json:"mine"`
|
|
All int `json:"all"`
|
|
}
|
|
|
|
func adminPoolScope(i domain.HumanItemRecord) bool {
|
|
return i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined" || i.CurrentStageID == "quality_review" || (i.CurrentStageID == "split_annotation" && i.Status == domain.HumanItemRework)
|
|
}
|
|
func adminPoolAvailable(i domain.HumanItemRecord) bool {
|
|
return i.SourceValidationStatus == "" && i.AssignedUserID == "" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemRework || i.Status == domain.HumanItemSubmitted) && (i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined")
|
|
}
|
|
func adminPoolMine(i domain.HumanItemRecord, user string) bool {
|
|
return i.AssignedUserID == user || (i.LatestAuthorUserID == user && i.CurrentStageID == "quality_review" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemSubmitted))
|
|
}
|
|
func adminPoolSelected(i domain.HumanItemRecord, q AdminAnnotationPoolQuery) bool {
|
|
switch q.Pool {
|
|
case "available":
|
|
return adminPoolAvailable(i)
|
|
case "mine":
|
|
return adminPoolMine(i, q.UserID)
|
|
}
|
|
return true
|
|
}
|
|
func adminPoolBaseQuery(q AdminAnnotationPoolQuery) domain.HumanItemQuery {
|
|
return domain.HumanItemQuery{PluginID: "annotation-qa", ProjectID: q.ProjectID, WorkflowID: q.WorkflowID, RunID: q.RunID}
|
|
}
|
|
func adminPoolValidBatch(i domain.HumanItemRecord, b domain.HumanBatchRecord, org string) bool {
|
|
if b.ID == "" || !humanOrganizationMatches(b.OrganizationID, org) {
|
|
return false
|
|
}
|
|
for _, st := range b.Stages {
|
|
if st.ID == i.CurrentStageID {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func adminPoolScopeFilter(org string, q AdminAnnotationPoolQuery) bson.M {
|
|
f := humanItemMongoFilter(org, adminPoolBaseQuery(q))
|
|
f["batchId"] = bson.M{"$ne": ""}
|
|
f["$expr"] = bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$batchId"}, "string"}}, bson.M{"$eq": bson.A{bson.M{"$type": "$currentStageId"}, "string"}}}}
|
|
f["$or"] = bson.A{bson.M{"currentStageId": bson.M{"$in": bson.A{"semantic_annotation", "annotation_combined", "quality_review"}}}, bson.M{"currentStageId": "split_annotation", "status": domain.HumanItemRework}}
|
|
return f
|
|
}
|
|
func adminPoolAvailableFilter() bson.M {
|
|
return bson.M{"currentStageId": bson.M{"$in": bson.A{"semantic_annotation", "annotation_combined"}}, "assignedUserId": bson.M{"$in": bson.A{"", nil}}, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}}}
|
|
}
|
|
func adminPoolMineFilter(user string) bson.M {
|
|
return bson.M{"$or": bson.A{bson.M{"assignedUserId": user}, bson.M{"latestAuthorUserId": user, "currentStageId": "quality_review", "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemSubmitted}}}}}
|
|
}
|
|
|
|
// The same business classification as the workspace, using persisted state.
|
|
func adminPoolCategory(i domain.HumanItemRecord, t domain.AnnotationTaskRecord) string {
|
|
if i.Status == domain.HumanItemAccepted || t.Status == "accepted" {
|
|
return "accepted"
|
|
}
|
|
if i.PendingCombinedSubmissionID != "" {
|
|
prefix := "fanyu-v5-combined-" + i.ID + "-v"
|
|
if strings.HasPrefix(i.PendingCombinedSubmissionID, prefix) {
|
|
suffix := strings.TrimPrefix(i.PendingCombinedSubmissionID, prefix)
|
|
n, err := strconv.ParseInt(suffix, 10, 64)
|
|
if err == nil && strings.IndexFunc(suffix, func(r rune) bool { return r < '0' || r > '9' }) < 0 && n >= 0 && n <= 9007199254740991 && int64(i.Version) > n+1 {
|
|
return "submission_conflicted"
|
|
}
|
|
}
|
|
return "submitting"
|
|
}
|
|
if i.ReviewRound > 0 && (i.CurrentStageID == "quality_review" || i.CurrentStageID == "prepackage_acceptance" || i.CurrentStageID == "acceptance") {
|
|
return "rework_review"
|
|
}
|
|
if t.Status == "pending_quality" || i.CurrentStageID == "quality_review" || i.Status == domain.HumanItemSubmitted {
|
|
return "pending_quality"
|
|
}
|
|
if t.Status == "pending_acceptance" || i.CurrentStageID == "acceptance" || i.CurrentStageID == "prepackage_acceptance" {
|
|
return "pending_acceptance"
|
|
}
|
|
if t.Status == "rework" || i.Status == domain.HumanItemRework || (i.ReviewRound > 0 && i.CurrentStageID == "annotation_combined") {
|
|
return "rework"
|
|
}
|
|
workflow := adminPoolMap(i.Subject.Payload["sliceWorkflow"])
|
|
if workflow["status"] == "processing" {
|
|
return "slicing"
|
|
}
|
|
if workflow["status"] == "completed" || workflow["phase"] == "annotate" {
|
|
return "pending_annotation"
|
|
}
|
|
if workflow["status"] == "editing" || workflow["phase"] == "slice" || i.AssignedUserID != "" {
|
|
return "pending_slice"
|
|
}
|
|
return "available"
|
|
}
|
|
|
|
type adminPoolRow struct {
|
|
domain.HumanItemRecord `bson:",inline"`
|
|
SourceAdmissionStatus string `bson:"_sourceAdmissionStatus"`
|
|
Tasks []domain.AnnotationTaskRecord `bson:"poolTasks"`
|
|
Runs []domain.RunRecord `bson:"poolRuns"`
|
|
Captures []domain.CaptureRecord `bson:"poolCaptures"`
|
|
Projects []domain.ProjectRecord `bson:"poolProjects"`
|
|
}
|
|
|
|
func adminPoolMap(v any) map[string]any {
|
|
switch x := v.(type) {
|
|
case map[string]any:
|
|
return x
|
|
case bson.M:
|
|
return map[string]any(x)
|
|
case bson.D:
|
|
out := map[string]any{}
|
|
for _, e := range x {
|
|
out[e.Key] = e.Value
|
|
}
|
|
return out
|
|
}
|
|
return nil
|
|
}
|
|
func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
|
|
var task domain.AnnotationTaskRecord
|
|
if len(row.Tasks) > 0 {
|
|
task = row.Tasks[0]
|
|
}
|
|
if q.NodeID != "" && task.NodeID != q.NodeID {
|
|
return false
|
|
}
|
|
if q.ArtifactID != "" && task.ArtifactID != q.ArtifactID && task.RightArtifactID != q.ArtifactID {
|
|
return false
|
|
}
|
|
if q.Category == "source_failed" {
|
|
if row.SourceValidationStatus != "failed" {
|
|
return false
|
|
}
|
|
} else if q.Category != "" && q.Category != "all" && adminPoolCategory(row.HumanItemRecord, task) != q.Category {
|
|
return false
|
|
}
|
|
if q.Search == "" && q.Capture == "" {
|
|
return true
|
|
}
|
|
values := adminPoolCaptureValues(row)
|
|
if q.Capture != "" {
|
|
found := false
|
|
for _, v := range values {
|
|
if strings.EqualFold(strings.TrimSpace(v), q.Capture) {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
return false
|
|
}
|
|
}
|
|
if q.Search != "" && !strings.Contains(adminPoolSearchText(row), strings.ToLower(q.Search)) {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
func adminPoolCaptureValues(row adminPoolRow) []string {
|
|
var task domain.AnnotationTaskRecord
|
|
if len(row.Tasks) > 0 {
|
|
task = row.Tasks[0]
|
|
}
|
|
|
|
values := []string{}
|
|
if len(row.Captures) > 0 {
|
|
c := row.Captures[0]
|
|
if c.CaptureID != "" {
|
|
values = append(values, c.CaptureID)
|
|
} else {
|
|
values = append(values, domain.CaptureDirectoryFromDirectoryValue(c.Prefix))
|
|
}
|
|
}
|
|
if len(values) == 0 {
|
|
values = append(values, domain.CaptureDirectoryFromArtifactValue(row.Subject.ItemKey))
|
|
if row.PoolSearchCurrent() {
|
|
values = append(values, row.PoolSearch.PayloadDirectories...)
|
|
} else {
|
|
domain.AppendCaptureDirectoriesFromPayload(row.Subject.Payload, &values)
|
|
}
|
|
}
|
|
for _, v := range []string{task.ArtifactID, task.SourceVideo} {
|
|
values = append(values, domain.CaptureDirectoryFromArtifactValue(v))
|
|
}
|
|
if len(row.Runs) > 0 {
|
|
values = append(values, domain.CaptureDirectoriesFromRun(row.Runs[0])...)
|
|
}
|
|
return values
|
|
}
|
|
func adminPoolSearchText(row adminPoolRow) string {
|
|
values := adminPoolCaptureValues(row)
|
|
if len(row.Projects) > 0 {
|
|
values = append(values, row.Projects[0].Name)
|
|
} else if len(row.Runs) > 0 {
|
|
values = append(values, row.Runs[0].ProjectName)
|
|
}
|
|
return strings.ToLower(strings.Join(values, " "))
|
|
}
|
|
|
|
func (s *MemoryStore) adminPoolRowsLocked(org string, q AdminAnnotationPoolQuery) []adminPoolRow {
|
|
rows := []adminPoolRow{}
|
|
for _, i := range s.humanItems {
|
|
if !humanOrganizationMatches(i.OrganizationID, org) || !humanItemMatches(i, adminPoolBaseQuery(q)) || !adminPoolScope(i) || !adminPoolValidBatch(i, s.humanBatches[i.BatchID], org) {
|
|
continue
|
|
}
|
|
row := adminPoolRow{HumanItemRecord: humanItemListProjectionClone(i, humanItemListPayloadKeysRemoved)}
|
|
if t, ok := s.annotationTasks[i.LegacyTaskID]; ok && humanOrganizationMatches(t.OrganizationID, org) {
|
|
row.Tasks = []domain.AnnotationTaskRecord{t}
|
|
var timeline, source *domain.AnnotationIndexJob
|
|
if job, ok := s.annotationIndexJobs[t.ID]; ok {
|
|
timeline = &job
|
|
}
|
|
if job, ok := s.annotationIndexJobs[sourceJobForItem(i)]; ok {
|
|
source = &job
|
|
}
|
|
row.SourceValidationStatus = AnnotationSourceAdmissionStatus(t, i, timeline, source)
|
|
}
|
|
if r, ok := s.runs[i.RunID]; ok && humanOrganizationMatches(r.OrganizationID, org) {
|
|
row.Runs = []domain.RunRecord{r}
|
|
if c, ok := s.captureRecords[r.TaskID]; ok && humanOrganizationMatches(c.OrganizationID, org) {
|
|
row.Captures = []domain.CaptureRecord{c}
|
|
}
|
|
}
|
|
if p, ok := s.projects[i.ProjectID]; ok && humanOrganizationMatches(p.OrganizationID, org) {
|
|
row.Projects = []domain.ProjectRecord{p}
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
return rows
|
|
}
|
|
func (s *MemoryStore) CountAdminAnnotationPools(_ context.Context, org string, q AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
c := AdminAnnotationPoolCounts{}
|
|
q.Search = ""
|
|
q.Category = ""
|
|
for _, r := range s.adminPoolRowsLocked(org, q) {
|
|
if !adminPoolRowMatches(r, q) {
|
|
continue
|
|
}
|
|
c.All++
|
|
if adminPoolAvailable(r.HumanItemRecord) {
|
|
c.Available++
|
|
}
|
|
if adminPoolMine(r.HumanItemRecord, q.UserID) {
|
|
c.Mine++
|
|
}
|
|
}
|
|
return c, nil
|
|
}
|
|
func (s *MemoryStore) ListAdminAnnotationPool(_ context.Context, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
rows := s.adminPoolRowsLocked(org, q)
|
|
sort.Slice(rows, func(i, j int) bool {
|
|
if rows[i].CreatedAt != rows[j].CreatedAt {
|
|
return rows[i].CreatedAt > rows[j].CreatedAt
|
|
}
|
|
return rows[i].ID > rows[j].ID
|
|
})
|
|
out := []domain.HumanItemRecord{}
|
|
for _, r := range rows {
|
|
if adminPoolSelected(r.HumanItemRecord, q) && humanItemAfterCursor(r.HumanItemRecord, q.Cursor) && adminPoolRowMatches(r, q) {
|
|
out = append(out, humanItemListProjectionClone(r.HumanItemRecord, humanItemListPayloadKeysRemoved))
|
|
if len(out) >= q.Limit {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func adminPoolLookup(from, local, as, org string, projection bson.M) bson.D {
|
|
return bson.D{{Key: "$lookup", Value: bson.M{"from": from, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$match": bson.M{"organizationId": normalizedOrganizationID(org)}}, bson.M{"$project": projection}}, "as": as}}}
|
|
}
|
|
|
|
// Stream a narrow metadata cursor; do not materialize the backlog to find an old
|
|
// search hit. Unfiltered reads stop after pageSize+1. Context bounds rare scans.
|
|
func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q AdminAnnotationPoolQuery, visit func(domain.HumanItemRecord) bool) error {
|
|
if adminPoolMetadataOnly(q) {
|
|
return s.scanAdminPoolMetadata(ctx, org, q, false, visit)
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
|
defer cancel()
|
|
pipeline := adminPoolPagePipeline(org, q)
|
|
if ids, ready := s.querySearchCandidates(ctx, org, q.Search, q.Capture); ready {
|
|
if len(ids) == 0 {
|
|
return nil
|
|
}
|
|
if scoped, withinBudget := queryCandidateFilter(pipeline[0][0].Value, ids); withinBudget {
|
|
pipeline[0] = bson.D{{Key: "$match", Value: scoped}}
|
|
}
|
|
}
|
|
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cur.Close(ctx)
|
|
for {
|
|
rows := []adminPoolRow{}
|
|
for len(rows) < 200 && cur.Next(ctx) {
|
|
var row adminPoolRow
|
|
if err := cur.Decode(&row); err != nil {
|
|
return err
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
if err := cur.Err(); err != nil {
|
|
return err
|
|
}
|
|
if len(rows) == 0 {
|
|
break
|
|
}
|
|
if err := s.humanPoolSearchLegacyPayloads(ctx, org, domain.HumanItemQuery{Search: q.Search, Capture: q.Capture}, rows); err != nil {
|
|
return err
|
|
}
|
|
for _, row := range rows {
|
|
row.SourceValidationStatus = row.SourceAdmissionStatus
|
|
if adminPoolRowMatches(row, q) && !visit(row.HumanItemRecord) {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
return cur.Err()
|
|
}
|
|
func (s *MongoStore) ListAdminAnnotationPool(ctx context.Context, org string, q AdminAnnotationPoolQuery) ([]domain.HumanItemRecord, error) {
|
|
if target, err := s.poolReplica(ctx); err != nil {
|
|
return nil, err
|
|
} else if target != nil {
|
|
return s.replicaAdminPool(ctx, target, org, q)
|
|
}
|
|
ids := []string{}
|
|
statuses := map[string]string{}
|
|
err := s.scanAdminAnnotationPool(ctx, org, q, func(i domain.HumanItemRecord) bool {
|
|
ids = append(ids, i.ID)
|
|
statuses[i.ID] = i.SourceValidationStatus
|
|
return q.Limit <= 0 || len(ids) < q.Limit
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items, err := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{ListProjection: true}, ids)
|
|
for n := range items {
|
|
items[n].SourceValidationStatus = statuses[items[n].ID]
|
|
}
|
|
return items, err
|
|
}
|
|
|
|
func (s *MongoStore) CountAdminAnnotationPools(ctx context.Context, org string, q AdminAnnotationPoolQuery) (AdminAnnotationPoolCounts, error) {
|
|
if target, err := s.poolReplica(ctx); err != nil {
|
|
return AdminAnnotationPoolCounts{}, err
|
|
} else if target != nil {
|
|
return target.CountAdminAnnotationPools(ctx, org, q)
|
|
}
|
|
if q.Capture == "" && q.NodeID == "" && q.ArtifactID == "" {
|
|
counts := AdminAnnotationPoolCounts{}
|
|
err := s.scanAdminPoolMetadata(ctx, org, q, true, func(item domain.HumanItemRecord) bool {
|
|
counts.All++
|
|
if adminPoolAvailable(item) {
|
|
counts.Available++
|
|
}
|
|
if adminPoolMine(item, q.UserID) {
|
|
counts.Mine++
|
|
}
|
|
return true
|
|
})
|
|
if err != nil {
|
|
return AdminAnnotationPoolCounts{}, err
|
|
}
|
|
return counts, nil
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
|
defer cancel()
|
|
|
|
all := q
|
|
all.Pool = "all"
|
|
all.Search = ""
|
|
all.Category = ""
|
|
all.Cursor = nil
|
|
all.Limit = int(^uint(0) >> 1)
|
|
c := AdminAnnotationPoolCounts{}
|
|
err := s.scanAdminAnnotationPool(ctx, org, all, func(i domain.HumanItemRecord) bool {
|
|
c.All++
|
|
if adminPoolAvailable(i) {
|
|
c.Available++
|
|
}
|
|
if adminPoolMine(i, q.UserID) {
|
|
c.Mine++
|
|
}
|
|
return true
|
|
})
|
|
return c, err
|
|
}
|