forked from longtao.wu/droboticflow
652 lines
25 KiB
Go
652 lines
25 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/workforcestats"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
func workforceTaskIDs(d workforcestats.Snapshot) []string {
|
|
ids := map[string]bool{}
|
|
for _, e := range d.Events {
|
|
if e.TaskID != "" {
|
|
ids[e.TaskID] = true
|
|
}
|
|
}
|
|
for _, r := range d.Reviews {
|
|
if r.TaskID != "" {
|
|
ids[r.TaskID] = true
|
|
}
|
|
}
|
|
for _, o := range d.Operations {
|
|
if o.TaskID != "" {
|
|
ids[o.TaskID] = true
|
|
}
|
|
for _, e := range o.Effects {
|
|
if e.BeforeTask != nil {
|
|
ids[e.BeforeTask.ID] = true
|
|
}
|
|
if e.Item != nil && e.Item.LegacyTaskID != "" {
|
|
ids[e.Item.LegacyTaskID] = true
|
|
}
|
|
if e.BeforeItem != nil && e.BeforeItem.LegacyTaskID != "" {
|
|
ids[e.BeforeItem.LegacyTaskID] = true
|
|
}
|
|
}
|
|
}
|
|
for _, item := range d.SourceItems {
|
|
if item.LegacyTaskID != "" {
|
|
ids[item.LegacyTaskID] = true
|
|
}
|
|
}
|
|
out := []string{}
|
|
for id := range ids {
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// workforceAcceptanceSourceIDs lists the source items final-acceptance
|
|
// decisions were made on.
|
|
func workforceAcceptanceSourceIDs(ops []domain.ReviewOperation) []string {
|
|
ids := map[string]bool{}
|
|
for _, op := range ops {
|
|
for _, id := range op.SourceItemIDs {
|
|
if id != "" {
|
|
ids[id] = true
|
|
}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(ids))
|
|
for id := range ids {
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func workforceAcceptanceDecision(o domain.ReviewOperation) bool {
|
|
return o.Status == "completed" && (o.Decision == "approved" || o.Decision == "rejected") && (o.StageID == "acceptance" || o.StageID == "prepackage_acceptance")
|
|
}
|
|
|
|
func (s *MemoryStore) LoadWorkforceStats(_ context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
d := workforcestats.Snapshot{}
|
|
belongs := func(o, p string) bool { return organizationMatches(o, org) && (q.ProjectID == "" || p == q.ProjectID) }
|
|
for _, e := range s.annotationWorklogEvents {
|
|
if belongs(e.OrganizationID, e.ProjectID) && workforcestats.InRange(e.CreatedAt, q) && (q.SupplierID == "" || e.SupplierID == q.SupplierID) && (q.UserID == "" || e.UserID == q.UserID) {
|
|
d.Events = append(d.Events, clone(e))
|
|
}
|
|
}
|
|
for _, o := range s.reviewOperations {
|
|
if belongs(o.OrganizationID, o.ProjectID) && o.Decision == "rejected" && o.Status == "completed" && workforcestats.InRange(o.CompletedAt, q) {
|
|
d.Operations = append(d.Operations, cloneReviewOperation(o))
|
|
}
|
|
if belongs(o.OrganizationID, o.ProjectID) && workforceAcceptanceDecision(o) && workforcestats.InRange(o.CompletedAt, q) && (q.UserID == "" || o.UserID == q.UserID) {
|
|
d.AcceptanceDecisions = append(d.AcceptanceDecisions, cloneReviewOperation(o))
|
|
}
|
|
}
|
|
for _, id := range workforceAcceptanceSourceIDs(d.AcceptanceDecisions) {
|
|
if item, ok := s.humanItems[id]; ok && organizationMatches(item.OrganizationID, org) {
|
|
d.SourceItems = append(d.SourceItems, domain.HumanItemRecord{ID: item.ID, LegacyTaskID: item.LegacyTaskID})
|
|
}
|
|
}
|
|
for _, r := range s.annotationReviews {
|
|
if organizationMatches(r.OrganizationID, org) && workforcestats.InRange(r.CreatedAt, q) {
|
|
d.Reviews = append(d.Reviews, clone(r))
|
|
}
|
|
}
|
|
ids := map[string]bool{}
|
|
for _, id := range workforceTaskIDs(d) {
|
|
ids[id] = true
|
|
}
|
|
d.Reviews = nil
|
|
for _, e := range s.annotationWorklogEvents {
|
|
if organizationMatches(e.OrganizationID, org) && ids[e.TaskID] && (e.EventType == "combined_submitted" || e.EventType == "annotation_submitted" || e.EventType == "rework_submitted") {
|
|
d.SubmissionHistory = append(d.SubmissionHistory, clone(e))
|
|
}
|
|
}
|
|
for _, r := range s.annotationReviews {
|
|
if organizationMatches(r.OrganizationID, org) && ids[r.TaskID] {
|
|
d.Reviews = append(d.Reviews, clone(r))
|
|
}
|
|
}
|
|
for _, r := range s.annotationRevisions {
|
|
if organizationMatches(r.OrganizationID, org) && ids[r.TaskID] {
|
|
r.Segments = nil
|
|
r.ObjectHints = nil
|
|
d.Revisions = append(d.Revisions, r)
|
|
}
|
|
}
|
|
for _, r := range s.sliceRevisions {
|
|
if organizationMatches(r.OrganizationID, org) && ids[r.TaskID] {
|
|
r.Operations = nil
|
|
d.Slices = append(d.Slices, clone(r))
|
|
}
|
|
}
|
|
runs := map[string]bool{}
|
|
for _, t := range s.annotationTasks {
|
|
if organizationMatches(t.OrganizationID, org) && ids[t.ID] {
|
|
d.Tasks = append(d.Tasks, clone(t))
|
|
runs[t.RunID] = true
|
|
}
|
|
}
|
|
for _, r := range s.runs {
|
|
if organizationMatches(r.OrganizationID, org) && runs[r.ID] {
|
|
d.Runs = append(d.Runs, domain.RunRecord{ID: r.ID, Mode: r.Mode})
|
|
}
|
|
}
|
|
for _, u := range s.users {
|
|
if organizationMatches(u.OrganizationID, org) {
|
|
d.Users = append(d.Users, domain.UserRecord{ID: u.ID, Name: u.Name, SupplierID: u.SupplierID, Role: u.Role})
|
|
}
|
|
}
|
|
for _, v := range s.suppliers {
|
|
if organizationMatches(v.OrganizationID, org) {
|
|
d.Suppliers = append(d.Suppliers, clone(v))
|
|
}
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// DashboardHumanWorkStats carries only the counters the dashboard human-work
|
|
// enrichment needs. It lets the API avoid materializing every human item and
|
|
// delivery package just to count them (diagnosis report C4).
|
|
type DashboardHumanWorkStats struct {
|
|
PendingAnnotation int
|
|
PendingQuality int
|
|
InAnnotation int
|
|
PendingAcceptanceItems int
|
|
Accepted int
|
|
Rejected int
|
|
Escalated int
|
|
VideoCount int
|
|
CaptureDurationSeconds int64
|
|
PackageCount int
|
|
PackagesPendingAcceptance int
|
|
}
|
|
|
|
func classifyHumanItemForDashboard(status domain.HumanItemStatus, stageID string) string {
|
|
switch status {
|
|
case domain.HumanItemReady, domain.HumanItemRework:
|
|
if stageID == "quality_review" || stageID == domain.FanyuV5StageCutReview || stageID == domain.FanyuV5StageSemanticReview {
|
|
return "pendingQuality"
|
|
}
|
|
return "pendingAnnotation"
|
|
case domain.HumanItemClaimed, domain.HumanItemInProgress:
|
|
return "inAnnotation"
|
|
case domain.HumanItemSubmitted:
|
|
if stageID == "quality_review" || stageID == domain.FanyuV5StageCutReview || stageID == domain.FanyuV5StageSemanticReview {
|
|
return "pendingQuality"
|
|
}
|
|
if stageID == "acceptance" {
|
|
return "pendingAcceptance"
|
|
}
|
|
case domain.HumanItemAccepted:
|
|
return "accepted"
|
|
case domain.HumanItemRejected:
|
|
return "rejected"
|
|
case domain.HumanItemEscalated:
|
|
return "escalated"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// MemoryStore implementation mirrors the legacy enrich loop exactly so the
|
|
// in-memory replica stays a faithful parity oracle for the Mongo pipeline.
|
|
func (s *MemoryStore) DashboardHumanWorkStats(_ context.Context, organizationID string) (DashboardHumanWorkStats, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
stats := DashboardHumanWorkStats{}
|
|
seenVideos := map[string]struct{}{}
|
|
for _, item := range s.humanItems {
|
|
if !humanOrganizationMatches(item.OrganizationID, organizationID) {
|
|
continue
|
|
}
|
|
if key := strings.TrimSpace(item.Subject.ItemKey); key != "" {
|
|
seenVideos[key] = struct{}{}
|
|
}
|
|
if duration, ok := item.Subject.Payload["durationSeconds"].(float64); ok && duration > 0 {
|
|
stats.CaptureDurationSeconds += int64(duration)
|
|
}
|
|
switch classifyHumanItemForDashboard(item.Status, item.CurrentStageID) {
|
|
case "pendingQuality":
|
|
stats.PendingQuality++
|
|
case "pendingAnnotation":
|
|
stats.PendingAnnotation++
|
|
case "inAnnotation":
|
|
stats.InAnnotation++
|
|
case "pendingAcceptance":
|
|
stats.PendingAcceptanceItems++
|
|
case "accepted":
|
|
stats.Accepted++
|
|
case "rejected":
|
|
stats.Rejected++
|
|
case "escalated":
|
|
stats.Escalated++
|
|
}
|
|
}
|
|
stats.VideoCount = len(seenVideos)
|
|
packageQuery := domain.DeliveryPackageQuery{Mode: domain.RunModeProduction}
|
|
for _, item := range s.deliveryPackages {
|
|
if organizationMatches(item.OrganizationID, organizationID) && deliveryPackageMatches(item, packageQuery) {
|
|
stats.PackageCount++
|
|
if item.Status == domain.DeliveryPackagePendingAcceptance {
|
|
stats.PackagesPendingAcceptance++
|
|
}
|
|
}
|
|
}
|
|
return stats, nil
|
|
}
|
|
|
|
type humanWorkAggregateRow struct {
|
|
ID struct {
|
|
Status string `bson:"s"`
|
|
Stage string `bson:"c"`
|
|
} `bson:"_id"`
|
|
Count int32 `bson:"n"`
|
|
}
|
|
|
|
// MongoStore implementation pushes the counting into aggregation pipelines so
|
|
// the dashboard never materializes humanItems/deliveryPackages documents.
|
|
func (s *MongoStore) DashboardHumanWorkStats(ctx context.Context, organizationID string) (DashboardHumanWorkStats, error) {
|
|
if replica, err := s.reportReplica(ctx); err != nil {
|
|
return DashboardHumanWorkStats{}, err
|
|
} else if replica != nil {
|
|
return replica.DashboardHumanWorkStats(ctx, organizationID)
|
|
}
|
|
stats := DashboardHumanWorkStats{}
|
|
baseFilter := humanItemMongoFilter(organizationID, domain.HumanItemQuery{})
|
|
rows := []humanWorkAggregateRow{}
|
|
cursor, err := s.humanItems.Aggregate(ctx, bson.A{
|
|
bson.M{"$match": baseFilter},
|
|
bson.M{"$group": bson.M{"_id": bson.M{"s": "$status", "c": "$currentStageId"}, "n": bson.M{"$sum": 1}}},
|
|
})
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
if err := cursor.All(ctx, &rows); err != nil {
|
|
return stats, err
|
|
}
|
|
for _, row := range rows {
|
|
count := int(row.Count)
|
|
switch classifyHumanItemForDashboard(domain.HumanItemStatus(row.ID.Status), row.ID.Stage) {
|
|
case "pendingQuality":
|
|
stats.PendingQuality += count
|
|
case "pendingAnnotation":
|
|
stats.PendingAnnotation += count
|
|
case "inAnnotation":
|
|
stats.InAnnotation += count
|
|
case "pendingAcceptance":
|
|
stats.PendingAcceptanceItems += count
|
|
case "accepted":
|
|
stats.Accepted += count
|
|
case "rejected":
|
|
stats.Rejected += count
|
|
case "escalated":
|
|
stats.Escalated += count
|
|
}
|
|
}
|
|
videoRows := []struct {
|
|
Count int32 `bson:"n"`
|
|
}{}
|
|
cursor, err = s.humanItems.Aggregate(ctx, bson.A{
|
|
bson.M{"$match": baseFilter},
|
|
bson.M{"$group": bson.M{"_id": "$subject.itemKey"}},
|
|
bson.M{"$match": bson.M{"_id": bson.M{"$nin": bson.A{"", nil}}}},
|
|
bson.M{"$count": "n"},
|
|
})
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
if err := cursor.All(ctx, &videoRows); err != nil {
|
|
return stats, err
|
|
}
|
|
if len(videoRows) > 0 {
|
|
stats.VideoCount = int(videoRows[0].Count)
|
|
}
|
|
durationRows := []struct {
|
|
Total float64 `bson:"d"`
|
|
}{}
|
|
cursor, err = s.humanItems.Aggregate(ctx, bson.A{
|
|
bson.M{"$match": baseFilter},
|
|
// Legacy semantics: only positive numeric durationSeconds values count,
|
|
// each truncated toward zero exactly like the Go int64(duration) cast.
|
|
bson.M{"$group": bson.M{"_id": nil, "d": bson.M{"$sum": bson.M{"$trunc": bson.M{"$convert": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$gt": bson.A{"$subject.payload.durationSeconds", 0}}, "$subject.payload.durationSeconds", 0}}, "to": "double", "onError": 0, "onNull": 0}}}}}},
|
|
})
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
if err := cursor.All(ctx, &durationRows); err != nil {
|
|
return stats, err
|
|
}
|
|
if len(durationRows) > 0 {
|
|
stats.CaptureDurationSeconds = int64(durationRows[0].Total)
|
|
}
|
|
total, err := s.deliveryPackages.CountDocuments(ctx, deliveryPackageMongoFilter(organizationID, domain.DeliveryPackageQuery{Mode: domain.RunModeProduction}))
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
stats.PackageCount = int(total)
|
|
pending, err := s.deliveryPackages.CountDocuments(ctx, deliveryPackageMongoFilter(organizationID, domain.DeliveryPackageQuery{Mode: domain.RunModeProduction, Status: domain.DeliveryPackagePendingAcceptance}))
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
stats.PackagesPendingAcceptance = int(pending)
|
|
return stats, nil
|
|
}
|
|
|
|
// ttlResultCache is a tiny in-process TTL cache. Consistency policy (diagnosis
|
|
// report C4/S2): dashboard and production counters tolerate short staleness,
|
|
// so entries simply expire; every non-GET /api request additionally flushes the
|
|
// API-level response cache (see internal/api). Bounded to avoid growth.
|
|
type ttlResultCache[T any] struct {
|
|
mu sync.Mutex
|
|
entries map[string]ttlResultCacheEntry[T]
|
|
now func() time.Time
|
|
}
|
|
|
|
type ttlResultCacheEntry[T any] struct {
|
|
value T
|
|
expires time.Time
|
|
}
|
|
|
|
func (c *ttlResultCache[T]) get(key string, ttl time.Duration) (T, bool) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
var zero T
|
|
if c.entries == nil {
|
|
return zero, false
|
|
}
|
|
entry, ok := c.entries[key]
|
|
if !ok || c.now().After(entry.expires) {
|
|
return zero, false
|
|
}
|
|
return entry.value, true
|
|
}
|
|
|
|
func (c *ttlResultCache[T]) set(key string, value T, ttl time.Duration) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.entries == nil {
|
|
c.entries = map[string]ttlResultCacheEntry[T]{}
|
|
}
|
|
if len(c.entries) >= 128 {
|
|
c.entries = map[string]ttlResultCacheEntry[T]{}
|
|
}
|
|
c.entries[key] = ttlResultCacheEntry[T]{value: value, expires: c.now().Add(ttl)}
|
|
}
|
|
|
|
const dashboardStatsCacheTTL = 45 * time.Second
|
|
|
|
var workforceStatsCache = &ttlResultCache[workforcestats.Snapshot]{now: time.Now}
|
|
|
|
// LoadWorkforceStatsCached caches the 30-day workforce snapshot per (org, query)
|
|
// for 45s. The window key is quantized to one minute so the default rolling
|
|
// dashboard range still hits the cache; the cached snapshot may therefore lag
|
|
// boundary events by up to TTL plus one minute. Falls back to an exact query on
|
|
// miss. MemoryStore intentionally has no cached variant: in-process tests and
|
|
// the local replica expect read-after-write freshness.
|
|
func (s *MongoStore) LoadWorkforceStatsCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) {
|
|
if replica, err := s.reportReplica(ctx); err != nil {
|
|
return workforcestats.Snapshot{}, err
|
|
} else if replica != nil {
|
|
return s.loadReplicaWorkforce(ctx, replica, org, q, true)
|
|
}
|
|
allowed := make([]string, 0, len(q.AllowedProjects))
|
|
for id, on := range q.AllowedProjects {
|
|
if on {
|
|
allowed = append(allowed, id)
|
|
}
|
|
}
|
|
sort.Strings(allowed)
|
|
fromKey, toKey := int64(0), int64(0)
|
|
if !q.From.IsZero() {
|
|
fromKey = q.From.Unix() / 60
|
|
}
|
|
if !q.To.IsZero() {
|
|
toKey = q.To.Unix() / 60
|
|
}
|
|
epoch := int64(0)
|
|
if s.queryReplicaFactStore {
|
|
var err error
|
|
epoch, err = s.countFactEpoch(ctx)
|
|
if err != nil {
|
|
return workforcestats.Snapshot{}, err
|
|
}
|
|
}
|
|
// A committed fact change must invalidate the independent read snapshot,
|
|
// including when projector and API run in separate processes. Authority
|
|
// readers keep their existing TTL; this epoch is query-database metadata.
|
|
key := fmt.Sprintf("%p|%s|%d|%s|%d|%d|%s|%s|%s|%v|%v|%s", s.db.Client(), s.db.Name(), epoch, org, fromKey, toKey, q.ProjectID, q.SupplierID, q.UserID, q.MatchUser, q.Details, strings.Join(allowed, ","))
|
|
if snapshot, ok := workforceStatsCache.get(key, dashboardStatsCacheTTL); ok {
|
|
return snapshot, nil
|
|
}
|
|
snapshot, err := s.LoadWorkforceStats(ctx, org, q)
|
|
if err != nil {
|
|
return snapshot, err
|
|
}
|
|
workforceStatsCache.set(key, snapshot, dashboardStatsCacheTTL)
|
|
return snapshot, nil
|
|
}
|
|
|
|
// Read-only, time-bounded event queries and batched joins. No index migrations or
|
|
// customer data rewrites. History queries project out media and annotation text.
|
|
func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) {
|
|
if replica, err := s.reportReplica(ctx); err != nil {
|
|
return workforcestats.Snapshot{}, err
|
|
} else if replica != nil {
|
|
return s.loadReplicaWorkforce(ctx, replica, org, q, false)
|
|
}
|
|
d := workforcestats.Snapshot{}
|
|
fields := func(names ...string) bson.M {
|
|
m := bson.M{}
|
|
for _, n := range names {
|
|
m[n] = 1
|
|
}
|
|
return m
|
|
}
|
|
read := func(c mongoCollection, filter, projection bson.M, target any) error {
|
|
if hook, ok := ctx.Value(queryBuildReadKey{}).(queryBuildReadFunc); ok {
|
|
return hook(ctx, c, filter, projection, target)
|
|
}
|
|
cur, err := c.Find(ctx, filter, options.Find().SetProjection(projection).SetMaxTime(15*time.Second))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cur.Close(ctx)
|
|
if budget, ok := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget); ok {
|
|
return workforceBoundedRead(ctx, cur, target, budget)
|
|
}
|
|
return cur.All(ctx, target)
|
|
}
|
|
window := func(field string) bson.M {
|
|
f := bson.M{}
|
|
t := bson.M{}
|
|
// Stored RFC3339 values have mixed fractional precision. Broaden by one second;
|
|
// the shared calculator applies exact parsed UTC bounds after this DB filter.
|
|
if !q.From.IsZero() {
|
|
t["$gte"] = q.From.Add(-time.Second).Format(time.RFC3339)
|
|
}
|
|
if !q.To.IsZero() {
|
|
t["$lt"] = q.To.Add(time.Second).Format(time.RFC3339)
|
|
}
|
|
if len(t) > 0 {
|
|
f[field] = t
|
|
}
|
|
return f
|
|
}
|
|
projects := bson.A{}
|
|
for id, allowed := range q.AllowedProjects {
|
|
if allowed {
|
|
projects = append(projects, id)
|
|
}
|
|
}
|
|
ef := window("createdAt")
|
|
if q.AllowedProjects != nil {
|
|
ef["projectId"] = bson.M{"$in": projects}
|
|
}
|
|
if q.ProjectID != "" {
|
|
ef["projectId"] = q.ProjectID
|
|
}
|
|
if q.SupplierID != "" {
|
|
ef["supplierId"] = q.SupplierID
|
|
}
|
|
if q.MatchUser || q.UserID != "" {
|
|
ef["userId"] = q.UserID
|
|
}
|
|
if err := read(s.annotationWorklogEvents, organizationScopedFilter(org, ef), fields("submissionId", "submissionKind", "videoSeconds", "id", "organizationId", "projectId", "taskId", "userId", "supplierId", "eventType", "mode", "temporary", "revision", "templateVersion", "durationSeconds", "outputDelta", "createdAt"), &d.Events); err != nil {
|
|
return d, err
|
|
}
|
|
of := window("completedAt")
|
|
if q.AllowedProjects != nil {
|
|
of["projectId"] = bson.M{"$in": projects}
|
|
}
|
|
of["status"] = "completed"
|
|
of["decision"] = "rejected"
|
|
if q.ProjectID != "" {
|
|
of["projectId"] = q.ProjectID
|
|
}
|
|
if q.SupplierID != "" {
|
|
of["$or"] = bson.A{bson.M{"effects.beforeTask.supplierId": q.SupplierID}, bson.M{"effects.beforeItem.supplierId": q.SupplierID}, bson.M{"effects.worklog.supplierId": q.SupplierID}, bson.M{"effects.item.supplierId": q.SupplierID}}
|
|
}
|
|
projection := workforceOperationProjection(q.Details)
|
|
if err := read(s.queryCollection("reviewOperations"), organizationScopedFilter(org, of), projection, &d.Operations); err != nil {
|
|
return d, err
|
|
}
|
|
af := window("completedAt")
|
|
if q.AllowedProjects != nil {
|
|
af["projectId"] = bson.M{"$in": projects}
|
|
}
|
|
if q.ProjectID != "" {
|
|
af["projectId"] = q.ProjectID
|
|
}
|
|
if q.MatchUser || q.UserID != "" {
|
|
af["userId"] = q.UserID
|
|
}
|
|
af["status"] = "completed"
|
|
af["decision"] = bson.M{"$in": bson.A{"approved", "rejected"}}
|
|
af["stageId"] = bson.M{"$in": bson.A{"acceptance", "prepackage_acceptance"}}
|
|
if err := read(s.queryCollection("reviewOperations"), organizationScopedFilter(org, af), fields("_id", "organizationId", "projectId", "userId", "stageId", "status", "decision", "createdAt", "completedAt", "sourceItemIds", "reviewVideoSources"), &d.AcceptanceDecisions); err != nil {
|
|
return d, err
|
|
}
|
|
if sourceIDs := workforceAcceptanceSourceIDs(d.AcceptanceDecisions); len(sourceIDs) > 0 {
|
|
if err := read(s.humanItems, organizationScopedFilter(org, bson.M{"id": bson.M{"$in": sourceIDs}}), fields("id", "legacyTaskId"), &d.SourceItems); err != nil {
|
|
return d, err
|
|
}
|
|
}
|
|
// Legacy review records do not have projectId; their task association is checked
|
|
// before aggregation. First read only time-bounded identities, then batch join.
|
|
// Only legacy rejections contribute on their own (approvals are looked up by
|
|
// task for tasks already joined), so approved reviews must not widen the join
|
|
// to every task reviewed org-wide in the window.
|
|
reviewWindow := window("createdAt")
|
|
reviewWindow["decision"] = string(domain.AnnotationReviewRejected)
|
|
if err := read(s.annotationReviews, organizationScopedFilter(org, reviewWindow), fields("id", "taskId"), &d.Reviews); err != nil {
|
|
return d, err
|
|
}
|
|
ids := workforceTaskIDs(d)
|
|
d.Reviews = nil
|
|
if len(ids) > 0 {
|
|
related := organizationScopedFilter(org, bson.M{"taskId": bson.M{"$in": ids}})
|
|
historyFilter := organizationScopedFilter(org, bson.M{"taskId": bson.M{"$in": ids}, "eventType": bson.M{"$in": bson.A{"combined_submitted", "annotation_submitted", "rework_submitted"}}})
|
|
if workforceCanReuseCompleteHistory(ctx, q) {
|
|
d.SubmissionHistory = workforceCompleteHistory(d.Events)
|
|
} else if err := read(s.annotationWorklogEvents, historyFilter, fields("id", "taskId", "projectId", "userId", "supplierId", "eventType", "revision", "templateVersion", "mode", "outputDelta", "createdAt", "durationSeconds", "submissionId", "submissionKind", "videoSeconds"), &d.SubmissionHistory); err != nil {
|
|
return d, err
|
|
}
|
|
rp := fields("id", "taskId", "revision", "reviewRound", "stage", "decision", "reviewerUserId", "createdAt", "problemSliceIds")
|
|
if q.Details {
|
|
rp["comment"] = 1
|
|
}
|
|
if err := read(s.annotationReviews, related, rp, &d.Reviews); err != nil {
|
|
return d, err
|
|
}
|
|
if err := read(s.annotationRevisions, related, fields("id", "taskId", "revision", "authorUserId", "isRework", "countsAsOutput", "createdAt"), &d.Revisions); err != nil {
|
|
return d, err
|
|
}
|
|
// Slice ranges only price submissions and the scope's own review
|
|
// decisions, never every task joined for return attribution.
|
|
// Price only versions the shared calculator can actually consult.
|
|
// Reading every historical slice version of these tasks grows with
|
|
// autosaves rather than the report's output and can dominate the load.
|
|
versions := workforcestats.SliceVersions(d)
|
|
_, grouped := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget)
|
|
for _, filter := range workforceSliceReadFilters(versions, grouped) {
|
|
var rows []domain.SliceRevisionRecord
|
|
if err := read(s.sliceRevisions, organizationScopedFilter(org, filter), fields("taskId", "revision", "segments.start", "segments.end"), &rows); err != nil {
|
|
return d, err
|
|
}
|
|
d.Slices = append(d.Slices, rows...)
|
|
}
|
|
|
|
if err := read(s.annotationTasks, organizationScopedFilter(org, bson.M{"id": bson.M{"$in": ids}}), fields("id", "projectId", "runId", "supplierId", "templateVersion"), &d.Tasks); err != nil {
|
|
return d, err
|
|
}
|
|
runIDs := []string{}
|
|
for _, t := range d.Tasks {
|
|
if t.RunID != "" {
|
|
runIDs = append(runIDs, t.RunID)
|
|
}
|
|
}
|
|
if len(runIDs) > 0 {
|
|
if err := read(s.runs, organizationScopedFilter(org, bson.M{"id": bson.M{"$in": runIDs}}), fields("id", "mode"), &d.Runs); err != nil {
|
|
return d, err
|
|
}
|
|
}
|
|
}
|
|
if err := read(s.users, organizationScopedFilter(org, bson.M{}), fields("id", "name", "supplierId", "role"), &d.Users); err != nil {
|
|
return d, err
|
|
}
|
|
if err := read(s.suppliers, organizationScopedFilter(org, bson.M{}), fields("id", "name"), &d.Suppliers); err != nil {
|
|
return d, err
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
func workforceFields(names ...string) bson.M {
|
|
m := bson.M{}
|
|
for _, name := range names {
|
|
m[name] = 1
|
|
}
|
|
return m
|
|
}
|
|
|
|
func workforceOperationProjection(details bool) bson.M {
|
|
projection := workforceFields("_id", "operationId", "organizationId", "projectId", "itemId", "taskId", "stageId", "reviewRound", "decision", "status", "createdAt", "completedAt", "draft.modelVersion", "draft.problems.id", "draft.problems.scope", "draft.problems.sourceItemId", "draft.problems.sliceId", "draft.problems.actionId", "draft.problems.start", "draft.problems.end", "draft.problems.timeBasis", "effects.review.id", "effects.review.taskId", "effects.review.revision", "effects.worklog.supplierId", "effects.beforeTask.id", "effects.beforeTask.supplierId", "effects.beforeTask.projectId", "effects.beforeTask.templateVersion", "effects.beforeItem.id", "effects.beforeItem.legacyTaskId", "effects.beforeItem.latestAuthorUserId", "effects.beforeItem.acceptedByUserId", "effects.beforeItem.supplierId", "effects.beforeItem.projectId", "effects.beforeItem.currentRevision", "effects.beforeItem.subject.start", "effects.beforeItem.subject.end", "effects.item.id", "effects.item.legacyTaskId", "effects.item.latestAuthorUserId", "effects.item.supplierId", "effects.item.currentRevision", "effects.item.acceptedByUserId")
|
|
if details {
|
|
projection["draft.problems.text"] = 1
|
|
projection["effects.review.comment"] = 1
|
|
projection["effects.reject.comment"] = 1
|
|
}
|
|
return projection
|
|
}
|
|
|
|
// Only the private whole-organization materializer has already loaded every
|
|
// history event. Filtered/date/permission-scoped requests still need the join.
|
|
func workforceCanReuseCompleteHistory(ctx context.Context, q workforcestats.Query) bool {
|
|
budget, ok := ctx.Value(workforceBuildBudgetKey{}).(*workforceBuildBudget)
|
|
return ok && budget != nil && q.Mode == domain.RunModeAll && q.From.IsZero() && q.To.IsZero() && !q.RelativeFrom && !q.RelativeTo && q.ProjectID == "" && q.SupplierID == "" && q.UserID == "" && !q.MatchUser && q.AllowedProjects == nil && q.AllowedSuppliers == nil && q.TemplateVersion == nil && q.Role == "" && q.GroupBy == "" && !q.Details && !q.GroupTemplate
|
|
}
|
|
|
|
func workforceCompleteHistory(events []domain.AnnotationWorklogEventRecord) []domain.AnnotationWorklogEventRecord {
|
|
var history []domain.AnnotationWorklogEventRecord
|
|
for _, e := range events {
|
|
if e.TaskID == "" || (e.EventType != "combined_submitted" && e.EventType != "annotation_submitted" && e.EventType != "rework_submitted") {
|
|
continue
|
|
}
|
|
// Match the existing history projection exactly. In particular it does not
|
|
// include organizationId, temporary or private assignment fields.
|
|
history = append(history, domain.AnnotationWorklogEventRecord{ID: e.ID, TaskID: e.TaskID, ProjectID: e.ProjectID, UserID: e.UserID, SupplierID: e.SupplierID, EventType: e.EventType, Revision: e.Revision, TemplateVersion: e.TemplateVersion, Mode: e.Mode, OutputDelta: e.OutputDelta, CreatedAt: e.CreatedAt, DurationSeconds: e.DurationSeconds, SubmissionID: e.SubmissionID, SubmissionKind: e.SubmissionKind, VideoSeconds: e.VideoSeconds})
|
|
}
|
|
return history
|
|
}
|