⚡ Fix slow page loading: static gzip, list pagination, auth cache, dashboard aggregation, video direct link

- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix
- C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown
- C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation
- C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache
- C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback
- S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed
- Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush
- Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500

Co-authored-by: AgentTeams slow-page-fix
This commit is contained in:
d-robotics 2026-10-09 17:39:47 -07:00
parent b9ce40d1ae
commit da446d13b5
29 changed files with 3301 additions and 162 deletions

View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/signal"
@ -61,6 +62,9 @@ func spaHandler(apiHandler http.Handler, distDir string) http.Handler {
if strings.HasPrefix(request.URL.Path, "/assets/") {
writer.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
}
if servePrecompressedAsset(writer, request, targetPath) {
return
}
fileServer.ServeHTTP(writer, request)
return
}
@ -73,6 +77,60 @@ func spaHandler(apiHandler http.Handler, distDir string) http.Handler {
})
}
var precompressedContentTypes = map[string]string{
".js": "text/javascript; charset=utf-8",
".css": "text/css; charset=utf-8",
}
func requestAcceptsGzip(request *http.Request) bool {
for _, encoding := range request.Header.Values("Accept-Encoding") {
for _, part := range strings.Split(encoding, ",") {
if strings.TrimSpace(strings.SplitN(part, ";", 2)[0]) == "gzip" {
return true
}
}
}
return false
}
// servePrecompressedAsset streams the build-time .gz sibling for compressible
// static assets when the client accepts gzip. It reports whether it handled
// the request; callers fall back to the plain file server otherwise. Range
// requests always fall back so byte ranges stay valid against the raw asset.
func servePrecompressedAsset(writer http.ResponseWriter, request *http.Request, targetPath string) bool {
contentType, compressible := precompressedContentTypes[filepath.Ext(targetPath)]
if !compressible {
return false
}
// Keep shared caches from mixing gzip and identity responses.
writer.Header().Add("Vary", "Accept-Encoding")
if !requestAcceptsGzip(request) || request.Header.Get("Range") != "" {
return false
}
if request.Method != http.MethodGet && request.Method != http.MethodHead {
return false
}
file, err := os.Open(targetPath + ".gz")
if err != nil {
return false
}
defer func() { _ = file.Close() }()
info, err := file.Stat()
if err != nil || info.IsDir() {
return false
}
writer.Header().Set("Content-Type", contentType)
writer.Header().Set("Content-Encoding", "gzip")
writer.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10))
writer.Header().Set("Accept-Ranges", "none")
if request.Method == http.MethodHead {
return true
}
writer.WriteHeader(http.StatusOK)
_, _ = io.Copy(writer, file)
return true
}
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()

View File

@ -10,6 +10,7 @@ import (
"os"
"regexp"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
@ -278,7 +279,15 @@ func (server *server) authenticateRequest(request *http.Request) (*authenticated
if err != nil || strings.TrimSpace(cookie.Value) == "" {
return nil, nil
}
session, err := server.store.GetSessionByTokenHash(request.Context(), sessionTokenHash(cookie.Value))
tokenHash := sessionTokenHash(cookie.Value)
// C2: the auth chain costs 3-5 serial Mongo round trips per request.
// Warm requests are served from the process-local TTL cache; write paths
// that change sessions, passwords, roles, suppliers or organizations call
// the invalidateAuthCache* helpers below.
if principal := lookupCachedPrincipal(tokenHash); principal != nil {
return principal, nil
}
session, err := server.store.GetSessionByTokenHash(request.Context(), tokenHash)
if err != nil {
return nil, err
}
@ -323,7 +332,102 @@ func (server *server) authenticateRequest(request *http.Request) (*authenticated
if organization.Status == domain.OrganizationStatusDisabled {
return nil, nil
}
return &authenticatedPrincipal{User: *user, Organization: *organization}, nil
principal := &authenticatedPrincipal{User: *user, Organization: *organization}
cachePrincipal(tokenHash, principal, session.ExpiresAt)
return principal, nil
}
// principalCacheEntry caches a resolved authenticatedPrincipal keyed by the
// session token hash. Entries carry the owning user so user-wide
// invalidation (role/supplier/org changes) can find every live session.
type principalCacheEntry struct {
principal authenticatedPrincipal
userID string
expiresAt time.Time
}
const (
principalCacheTTL = 45 * time.Second
principalCacheMaxEntries = 4096
)
var (
principalCacheMu sync.Mutex
principalCache = map[string]principalCacheEntry{}
)
// cachePrincipal stores a resolved principal with a TTL capped by the session
// expiry so a stale entry never outlives its session.
func cachePrincipal(tokenHash string, principal *authenticatedPrincipal, sessionExpiresAt string) {
if principal == nil {
return
}
expiresAt := time.Now().Add(principalCacheTTL)
if parsed, err := time.Parse(time.RFC3339, sessionExpiresAt); err == nil && parsed.Before(expiresAt) {
expiresAt = parsed
}
principalCacheMu.Lock()
defer principalCacheMu.Unlock()
if len(principalCache) >= principalCacheMaxEntries {
// Cheap full sweep instead of an LRU: the map is small and bounded.
now := time.Now()
for key, entry := range principalCache {
if now.After(entry.expiresAt) {
delete(principalCache, key)
}
}
if len(principalCache) >= principalCacheMaxEntries {
principalCache = map[string]principalCacheEntry{}
}
}
principalCache[tokenHash] = principalCacheEntry{
principal: authenticatedPrincipal{User: principal.User, Organization: principal.Organization},
userID: principal.User.ID,
expiresAt: expiresAt,
}
}
func lookupCachedPrincipal(tokenHash string) *authenticatedPrincipal {
principalCacheMu.Lock()
entry, ok := principalCache[tokenHash]
principalCacheMu.Unlock()
if !ok || time.Now().After(entry.expiresAt) {
return nil
}
principal := entry.principal
return &principal
}
// invalidateAuthCacheForToken drops a single session's cached principal
// (logout, session deletion).
func invalidateAuthCacheForToken(tokenHash string) {
principalCacheMu.Lock()
delete(principalCache, tokenHash)
principalCacheMu.Unlock()
}
// invalidateAuthCacheForUser drops every cached principal owned by the user
// (password resets, role/supplier/status changes, forced session revocation).
func invalidateAuthCacheForUser(userID string) {
if userID == "" {
return
}
principalCacheMu.Lock()
for key, entry := range principalCache {
if entry.userID == userID {
delete(principalCache, key)
}
}
principalCacheMu.Unlock()
}
// invalidateAuthCache drops all cached principals. Used when an organization
// record itself changes (name/slug/status) because the cached principal embeds
// the organization snapshot.
func invalidateAuthCache() {
principalCacheMu.Lock()
principalCache = map[string]principalCacheEntry{}
principalCacheMu.Unlock()
}
func (server *server) publicPreviewPrincipal(ctx context.Context) (*authenticatedPrincipal, error) {
@ -597,9 +701,12 @@ func (server *server) handleAuthLogin(writer http.ResponseWriter, request *http.
func (server *server) handleAuthLogout(writer http.ResponseWriter, request *http.Request) {
if cookie, err := request.Cookie(sessionCookieName); err == nil {
if session, err := server.store.GetSessionByTokenHash(request.Context(), sessionTokenHash(cookie.Value)); err == nil && session != nil {
tokenHash := sessionTokenHash(cookie.Value)
if session, err := server.store.GetSessionByTokenHash(request.Context(), tokenHash); err == nil && session != nil {
_, _ = server.store.DeleteSession(request.Context(), session.ID)
}
// Logout must take effect immediately, not after the TTL.
invalidateAuthCacheForToken(tokenHash)
}
clearSessionCookie(writer, server.sessionCookieSecure)
writer.WriteHeader(http.StatusNoContent)
@ -698,6 +805,8 @@ func (server *server) handleCurrentUserProfileUpdate(writer http.ResponseWriter,
renderUserMutationError(writer, err)
return
}
// Profile edits (including self password change) refresh cached principals.
invalidateAuthCacheForUser(item.ID)
if payload.NewPassword != nil {
if err := server.store.DeleteSessionsByUser(request.Context(), item.ID); err != nil {
renderError(writer, http.StatusInternalServerError, err)
@ -844,6 +953,9 @@ func (server *server) handleAdminOrganizationsUpdate(writer http.ResponseWriter,
renderOrganizationMutationError(writer, err)
return
}
// Cached principals embed the organization snapshot; any org change
// flushes them so status/name changes apply immediately.
invalidateAuthCache()
renderJSON(writer, http.StatusOK, item)
}
@ -857,6 +969,8 @@ func (server *server) handleAdminOrganizationsDelete(writer http.ResponseWriter,
writer.WriteHeader(http.StatusNotFound)
return
}
// Org deletion invalidates every cached principal in that org.
invalidateAuthCache()
writer.WriteHeader(http.StatusNoContent)
}
@ -1001,5 +1115,7 @@ func (server *server) handleAdminUserResetPassword(writer http.ResponseWriter, r
renderError(writer, http.StatusInternalServerError, err)
return
}
// The admin-forced password reset must evict any cached principal at once.
invalidateAuthCacheForUser(user.ID)
renderJSON(writer, http.StatusOK, sanitizeUser(user))
}

View File

@ -0,0 +1,279 @@
package api
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
type authReadCounter struct {
store.Store
sessions, users, suppliers, organizations atomic.Int64
}
func (s *authReadCounter) GetSessionByTokenHash(ctx context.Context, tokenHash string) (*domain.SessionRecord, error) {
s.sessions.Add(1)
return s.Store.GetSessionByTokenHash(ctx, tokenHash)
}
func (s *authReadCounter) GetUser(ctx context.Context, id string) (*domain.UserRecord, error) {
s.users.Add(1)
return s.Store.GetUser(ctx, id)
}
func (s *authReadCounter) GetSupplier(ctx context.Context, id string) (*domain.SupplierRecord, error) {
s.suppliers.Add(1)
return s.Store.GetSupplier(ctx, id)
}
func (s *authReadCounter) GetOrganization(ctx context.Context, id string) (*domain.OrganizationRecord, error) {
s.organizations.Add(1)
return s.Store.GetOrganization(ctx, id)
}
func newAuthCacheFixture(t *testing.T) (*authReadCounter, *server, string, string) {
t.Helper()
ctx := context.Background()
dataStore := store.NewMemoryStore()
if _, err := dataStore.EnsureDefaultOrganization(ctx); err != nil {
t.Fatal(err)
}
user := domain.UserRecord{ID: "auth-cache-user", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, Name: "缓存管理员", Phone: "13800000001"}
if err := dataStore.UpsertUser(ctx, user); err != nil {
t.Fatal(err)
}
counter := &authReadCounter{Store: dataStore}
instance := &server{store: counter, requireSessionAuth: true}
token := "auth-cache-session-token"
expiresAt := time.Now().Add(time.Hour)
if _, err := dataStore.CreateSession(ctx, domain.SessionRecord{
UserID: user.ID,
TokenHash: sessionTokenHash(token),
ExpiresAt: expiresAt.Format(time.RFC3339),
ExpiresAtTime: expiresAt,
CreatedAt: time.Now().Format(time.RFC3339),
}); err != nil {
t.Fatal(err)
}
invalidateAuthCache()
return counter, instance, token, user.ID
}
func authedRequest(instance *server, path string, token string) *httptest.ResponseRecorder {
request := httptest.NewRequest(http.MethodGet, path, nil)
request.AddCookie(&http.Cookie{Name: sessionCookieName, Value: token})
recorder := httptest.NewRecorder()
instance.sessionAuth(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if _, ok := currentPrincipal(request); ok {
writer.WriteHeader(http.StatusOK)
return
}
writer.WriteHeader(http.StatusUnauthorized)
})).ServeHTTP(recorder, request)
return recorder
}
func TestAuthenticateRequestServesWarmRequestsFromCache(t *testing.T) {
counter, instance, token, userID := newAuthCacheFixture(t)
first := authedRequest(instance, "/api/users", token)
if first.Code != http.StatusOK {
t.Fatalf("first request status=%d", first.Code)
}
coldCalls := int(counter.sessions.Load() + counter.users.Load() + counter.organizations.Load() + counter.suppliers.Load())
if coldCalls < 2 || counter.sessions.Load() != 1 {
t.Fatalf("cold path must hit the store, got sessions=%d users=%d orgs=%d suppliers=%d", counter.sessions.Load(), counter.users.Load(), counter.organizations.Load(), counter.suppliers.Load())
}
for index := 0; index < 3; index++ {
warm := authedRequest(instance, "/api/users", token)
if warm.Code != http.StatusOK {
t.Fatalf("warm request %d status=%d", index, warm.Code)
}
}
warmCalls := int(counter.sessions.Load() + counter.users.Load() + counter.organizations.Load() + counter.suppliers.Load())
if warmCalls != coldCalls {
t.Fatalf("cache hit must perform zero auth store reads: cold=%d warm=%d (sessions=%d users=%d orgs=%d suppliers=%d)",
coldCalls, warmCalls, counter.sessions.Load(), counter.users.Load(), counter.organizations.Load(), counter.suppliers.Load())
}
// Token-level invalidation (logout) takes effect immediately.
invalidateAuthCacheForToken(sessionTokenHash(token))
after := authedRequest(instance, "/api/users", token)
if after.Code != http.StatusOK {
t.Fatalf("post-invalidation status=%d", after.Code)
}
if counter.sessions.Load() != 2 {
t.Fatalf("token invalidation must force a fresh session read, sessions=%d", counter.sessions.Load())
}
// User-level invalidation (role/supplier/password changes) evicts all of
// the user's sessions, including other live tokens.
invalidateAuthCacheForUser(userID)
afterUser := authedRequest(instance, "/api/users", token)
if afterUser.Code != http.StatusOK || counter.sessions.Load() != 3 {
t.Fatalf("user invalidation must force a fresh session read, status=%d sessions=%d", afterUser.Code, counter.sessions.Load())
}
}
func TestLogoutHandlerInvalidatesCachedPrincipal(t *testing.T) {
counter, instance, token, _ := newAuthCacheFixture(t)
if response := authedRequest(instance, "/api/users", token); response.Code != http.StatusOK {
t.Fatalf("warm-up status=%d", response.Code)
}
if counter.sessions.Load() != 1 {
t.Fatalf("expected one cold session read, got %d", counter.sessions.Load())
}
request := httptest.NewRequest(http.MethodPost, "/api/auth/logout", nil)
request.AddCookie(&http.Cookie{Name: sessionCookieName, Value: token})
recorder := httptest.NewRecorder()
instance.handleAuthLogout(recorder, request)
if recorder.Code != http.StatusNoContent {
t.Fatalf("logout status=%d", recorder.Code)
}
// The session is deleted, so the next request must not authenticate even
// though the cache would still be warm without invalidation (that would
// wrongly yield 200 from the stale principal).
response := authedRequest(instance, "/api/users", token)
if response.Code != http.StatusUnauthorized {
t.Fatalf("post-logout request must be unauthorized, got %d", response.Code)
}
if counter.sessions.Load() != 3 {
t.Fatalf("logout must invalidate the cached principal, sessions=%d", counter.sessions.Load())
}
}
func TestUsersListLimitAndSearch(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
if _, err := dataStore.EnsureDefaultOrganization(ctx); err != nil {
t.Fatal(err)
}
admin := domain.UserRecord{ID: "users-admin", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
if err := dataStore.UpsertUser(ctx, admin); err != nil {
t.Fatal(err)
}
// Another tenant's user must stay invisible regardless of pagination.
outsider := domain.UserRecord{ID: "users-outsider", OrganizationID: "org-other", Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, Name: "外租户用户"}
if err := dataStore.UpsertUser(ctx, outsider); err != nil {
t.Fatal(err)
}
for index := 0; index < 250; index++ {
name, phone := "运营成员", fmt.Sprintf("138%08d", index)
if index%50 == 0 {
name = "工程师-" + string(rune('A'+index/50))
phone = fmt.Sprintf("137%08d", index)
}
user := domain.UserRecord{
ID: domain.NewID("user"), OrganizationID: domain.DefaultOrganizationID,
Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive,
Name: name, Phone: phone,
}
if err := dataStore.UpsertUser(ctx, user); err != nil {
t.Fatal(err)
}
}
handler := NewHandler(dataStore, nil)
principal := &authenticatedPrincipal{User: admin, Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}}
requestWith := func(target string) *httptest.ResponseRecorder {
request := httptest.NewRequest(http.MethodGet, target, nil)
request = request.WithContext(context.WithValue(request.Context(), authContextKey{}, principal))
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
return recorder
}
// Default conservative cap: 251 visible users truncated to 200.
defaultResponse := requestWith("/api/users")
if defaultResponse.Code != http.StatusOK {
t.Fatalf("default status=%d body=%s", defaultResponse.Code, defaultResponse.Body.String())
}
var payload struct {
Items []domain.UserRecord `json:"items"`
HasMore bool `json:"hasMore"`
}
if err := json.Unmarshal(defaultResponse.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Items) != 200 || !payload.HasMore {
t.Fatalf("default cap items=%d hasMore=%v", len(payload.Items), payload.HasMore)
}
for _, item := range payload.Items {
if item.OrganizationID != domain.DefaultOrganizationID {
t.Fatalf("tenant isolation broken by pagination: %s", item.ID)
}
}
// Explicit limit and q search combine; small result sets omit hasMore.
// "137" matches the five 工程师-* phones plus the one 13800000137 member.
search := requestWith("/api/users?limit=10&q=137")
if search.Code != http.StatusOK {
t.Fatalf("search status=%d", search.Code)
}
payload.HasMore = false // fresh decode of a body without hasMore must stay false
if err := json.Unmarshal(search.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Items) != 6 || payload.HasMore {
t.Fatalf("search items=%d hasMore=%v", len(payload.Items), payload.HasMore)
}
engineers := 0
for _, item := range payload.Items {
if !strings.Contains(item.Phone, "137") {
t.Fatalf("search matched wrong user %s phone=%s", item.ID, item.Phone)
}
if strings.HasPrefix(item.Phone, "137") {
engineers++
}
}
if engineers != 5 {
t.Fatalf("expected 5 engineer matches, got %d", engineers)
}
// Invalid limit is rejected; legacy no-parameter shape stays {"items"}.
bad := requestWith("/api/users?limit=0")
if bad.Code != http.StatusBadRequest {
t.Fatalf("invalid limit status=%d", bad.Code)
}
}
func TestAuthCacheConcurrentAccess(t *testing.T) {
counter, instance, token, userID := newAuthCacheFixture(t)
// Concurrent readers and invalidators must not race or deadlock; the
// final request must still authenticate correctly.
done := make(chan struct{})
for index := 0; index < 8; index++ {
go func(i int) {
defer func() { done <- struct{}{} }()
for round := 0; round < 25; round++ {
switch i % 3 {
case 0:
_ = authedRequest(instance, "/api/users", token)
case 1:
invalidateAuthCacheForUser(userID)
default:
invalidateAuthCacheForToken(sessionTokenHash(token))
}
}
}(index)
}
for index := 0; index < 8; index++ {
<-done
}
final := authedRequest(instance, "/api/users", token)
if final.Code != http.StatusOK {
t.Fatalf("final request status=%d", final.Code)
}
_ = counter
}

View File

@ -0,0 +1,162 @@
package api
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func TestDashboardResponseCacheExpiryAndInvalidation(t *testing.T) {
current := time.Unix(1700000000, 0)
cache := dashboardResponseCache{now: func() time.Time { return current }}
cache.set("k", http.StatusOK, "application/json", []byte(`{"a":1}`))
if entry := cache.get("k"); entry == nil || string(entry.body) != `{"a":1}` || entry.status != http.StatusOK {
t.Fatalf("expected fresh entry, got %+v", entry)
}
cache.set("k2", http.StatusOK, "application/json", []byte(`{"b":2}`))
cache.invalidateAll()
if cache.get("k") != nil || cache.get("k2") != nil {
t.Fatal("invalidateAll must drop every entry")
}
cache.set("k", http.StatusOK, "application/json", []byte(`{"a":1}`))
current = current.Add(dashboardResponseCacheTTL + time.Second)
if entry := cache.get("k"); entry != nil {
t.Fatalf("expected TTL expiry, got %+v", entry)
}
}
func TestDashboardResponseCacheBoundsDropped(t *testing.T) {
current := time.Unix(1700000000, 0)
cache := dashboardResponseCache{now: func() time.Time { return current }}
for i := 0; i < 200; i++ {
cache.set(string(rune('a'+i%26))+string(rune('a'+i/26)), http.StatusOK, "", []byte{byte(i)})
}
if entry := cache.get("aa"); entry != nil {
t.Fatalf("bounded cache should have dropped the oldest entries, got %+v", entry)
}
}
func TestDashboardHumanWorkStatsMatchesLegacyEnrichOnMemoryStore(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
seedDashboardHumanWorkFixtures(t, dataStore)
legacy := &domain.DashboardSummary{}
server := &server{store: dataStore}
request := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil)
server.enrichDashboardHumanWork(ctx, request, legacy)
counted := &domain.DashboardSummary{}
stats, err := dataStore.DashboardHumanWorkStats(ctx, domain.DefaultOrganizationID)
if err != nil {
t.Fatal(err)
}
applyDashboardHumanWorkStats(counted, stats)
if legacy.Tasks.PendingAnnotation != counted.Tasks.PendingAnnotation ||
legacy.Tasks.PendingQuality != counted.Tasks.PendingQuality ||
legacy.Tasks.InAnnotation != counted.Tasks.InAnnotation ||
legacy.Tasks.PendingAcceptance != counted.Tasks.PendingAcceptance ||
legacy.Tasks.Rejected != counted.Tasks.Rejected ||
legacy.Tasks.Escalated != counted.Tasks.Escalated ||
legacy.Tasks.QualityPassRate != counted.Tasks.QualityPassRate ||
legacy.Tasks.Waiting != counted.Tasks.Waiting ||
legacy.Data.VideoCount != counted.Data.VideoCount ||
legacy.Data.PackageCount != counted.Data.PackageCount ||
legacy.Data.CaptureDurationSeconds != counted.Data.CaptureDurationSeconds {
t.Fatalf("counters diverge:\nlegacy=%+v/%+v\ncounted=%+v/%+v", legacy.Tasks, legacy.Data, counted.Tasks, counted.Data)
}
if counted.Tasks.PendingAnnotation != 1 || counted.Tasks.PendingQuality != 2 || counted.Tasks.InAnnotation != 1 ||
counted.Tasks.PendingAcceptance != 2 || counted.Tasks.Rejected != 1 || counted.Tasks.Escalated != 1 ||
counted.Tasks.QualityPassRate != 50 {
t.Fatalf("unexpected counters: %+v", counted.Tasks)
}
if counted.Data.VideoCount != 3 || counted.Data.PackageCount != 2 || counted.Data.CaptureDurationSeconds != 100 {
t.Fatalf("unexpected data counters: %+v", counted.Data)
}
}
func seedDashboardHumanWorkFixtures(t *testing.T, dataStore store.Store) {
t.Helper()
ctx := context.Background()
org := domain.DefaultOrganizationID
item := func(id string, status domain.HumanItemStatus, stage string, key string, duration any) domain.HumanItemRecord {
return domain.HumanItemRecord{
ID: id, OrganizationID: org, CurrentStageID: stage, Status: status,
Subject: domain.HumanSubjectRef{ItemKey: key, Payload: map[string]any{"durationSeconds": duration}},
}
}
fixtures := []domain.HumanItemRecord{
item("ready-annotation", domain.HumanItemReady, "annotation", "video-a", 30.0),
item("ready-quality", domain.HumanItemRework, "quality_review", "video-a", 40.0),
item("claimed", domain.HumanItemClaimed, "annotation", "video-b", 20.0),
item("submitted-quality", domain.HumanItemSubmitted, domain.FanyuV5StageCutReview, "video-b", nil),
item("submitted-acceptance", domain.HumanItemSubmitted, "acceptance", "video-c", 0),
item("accepted", domain.HumanItemAccepted, "done", "video-c", 10.0),
item("rejected", domain.HumanItemRejected, "done", "", -5),
item("escalated", domain.HumanItemEscalated, "annotation", "", "not-a-number"),
{ID: "foreign", OrganizationID: "other-org", CurrentStageID: "annotation", Status: domain.HumanItemReady,
Subject: domain.HumanSubjectRef{ItemKey: "video-foreign", Payload: map[string]any{"durationSeconds": 999.0}}},
}
for _, fixture := range fixtures {
if err := dataStore.UpsertHumanItem(ctx, fixture); err != nil {
t.Fatal(err)
}
}
packages := []domain.DeliveryPackageRecord{
{ID: "pkg-pending", OrganizationID: org, ProjectID: "p", Status: domain.DeliveryPackagePendingAcceptance},
{ID: "pkg-accepted", OrganizationID: org, ProjectID: "p", Status: domain.DeliveryPackageAccepted},
{ID: "pkg-debug", OrganizationID: org, ProjectID: "p", Mode: domain.RunModeDebug, Status: domain.DeliveryPackagePendingAcceptance},
{ID: "pkg-foreign", OrganizationID: "other-org", ProjectID: "p", Status: domain.DeliveryPackagePendingAcceptance},
}
for _, pkg := range packages {
if err := dataStore.UpsertDeliveryPackage(ctx, pkg); err != nil {
t.Fatal(err)
}
}
}
func TestDashboardEndpointServesCachedBodyAndInvalidatesOnWrite(t *testing.T) {
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
first := httptest.NewRecorder()
handler.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
if first.Code != http.StatusOK {
t.Fatalf("first dashboard request = %d body=%s", first.Code, first.Body.String())
}
second := httptest.NewRecorder()
handler.ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
if second.Code != http.StatusOK || !bytes.Equal(first.Body.Bytes(), second.Body.Bytes()) {
t.Fatalf("cached body must be byte-identical: %d vs %d", first.Code, second.Code)
}
// A write done outside the API stays hidden while the TTL window is open.
if err := dataStore.UpsertRun(context.Background(), domain.RunRecord{ID: "cached-run", OrganizationID: domain.DefaultOrganizationID, Status: domain.RunStatusRunning, CreatedAt: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
t.Fatal(err)
}
stillCached := httptest.NewRecorder()
handler.ServeHTTP(stillCached, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
if !bytes.Equal(second.Body.Bytes(), stillCached.Body.Bytes()) {
t.Fatal("expected the TTL cache to keep serving the stale body before any API write")
}
// Any non-GET /api request flushes the dashboard caches.
flush := httptest.NewRecorder()
handler.ServeHTTP(flush, httptest.NewRequest(http.MethodPost, "/api/auth/logout", nil))
refreshed := httptest.NewRecorder()
handler.ServeHTTP(refreshed, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
if refreshed.Code != http.StatusOK {
t.Fatalf("refreshed dashboard = %d body=%s", refreshed.Code, refreshed.Body.String())
}
if bytes.Equal(stillCached.Body.Bytes(), refreshed.Body.Bytes()) {
t.Fatal("expected cache invalidation to recompute the dashboard body")
}
}

View File

@ -0,0 +1,140 @@
package api
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// slow-page report S4 parity test: the bounded point queries behind
// dashboardPOCReadinessCandidateRuns must select exactly what the former
// full ListRuns scan selected, and must not drag unrelated runs along.
func TestDashboardPOCReadinessCandidateRunsMatchesFullScan(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
acceptancePayload := func(operatorCount int) map[string]any {
return map[string]any{
"status": "passed", "acceptanceScope": "full", "fullExternalAcceptance": true,
"operatorCount": operatorCount, "realFixtureCount": 1, "generatedFixtureCount": 0,
"allFixturesReal": true,
}
}
runs := []domain.RunRecord{
// Newest run overall but irrelevant: must not appear in candidates.
{ID: "run-noise", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "unrelated", ProjectName: "Unrelated", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T11:00:00Z", UpdatedAt: "2026-10-09T11:00:00Z"},
// Latest acceptance artifact overall, nested under data.preview.value.
{ID: "run-acceptance-preview", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T10:00:00Z", UpdatedAt: "2026-10-09T10:59:00Z",
Artifacts: []domain.ExecutionArtifact{{ID: "a1", RunID: "run-acceptance-preview", Name: "report.json", Data: map[string]any{"preview": map[string]any{"value": acceptancePayload(12)}}}}},
// Older acceptance artifacts across the other detection paths.
{ID: "run-acceptance-value", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T09:00:00Z", UpdatedAt: "2026-10-09T09:30:00Z",
Artifacts: []domain.ExecutionArtifact{{ID: "a2", RunID: "run-acceptance-value", Name: "report.json", Data: map[string]any{"value": acceptancePayload(8)}}}},
{ID: "run-acceptance-named", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T08:00:00Z", UpdatedAt: "2026-10-09T08:30:00Z",
Artifacts: []domain.ExecutionArtifact{{ID: "a3", RunID: "run-acceptance-named", Name: "poc-operator-full-acceptance-result.json", Data: acceptancePayload(6)}}},
// Latest POC GPU workflow run (status completed → preflight passed).
{ID: "run-gpu", OrganizationID: domain.DefaultOrganizationID, WorkflowName: pocGPUWorkflowAcceptanceWorkflowName, ProjectName: "other", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T10:50:00Z", UpdatedAt: "2026-10-09T10:55:00Z",
Logs: []domain.LogEntryRecord{{Message: "gpu request cannot be scheduled"}}},
// Older GPU-preflight run matched via project name.
{ID: "run-live", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "other", ProjectName: liveWorkflowAcceptanceProjectName, Status: domain.RunStatusFailed, CreatedAt: "2026-10-09T07:00:00Z", UpdatedAt: "2026-10-09T07:10:00Z",
Logs: []domain.LogEntryRecord{{Message: "gpu request cannot be scheduled"}}},
// Near-miss names must not match.
{ID: "run-decoy", OrganizationID: domain.DefaultOrganizationID, WorkflowName: pocGPUWorkflowAcceptanceWorkflowName + " 2", ProjectName: "decoy", Status: domain.RunStatusFailed, CreatedAt: "2026-10-09T10:58:00Z", UpdatedAt: "2026-10-09T10:58:00Z"},
}
for _, run := range runs {
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
}
server := &server{store: dataStore}
request := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil).WithContext(ctx)
candidates, err := server.dashboardPOCReadinessCandidateRuns(ctx, request)
if err != nil {
t.Fatal(err)
}
ids := make(map[string]bool, len(candidates))
for _, run := range candidates {
ids[run.ID] = true
}
if ids["run-noise"] || ids["run-decoy"] {
t.Fatalf("candidates must exclude unrelated runs, got %#v", ids)
}
for _, expected := range []string{"run-acceptance-preview", "run-acceptance-value", "run-acceptance-named", "run-gpu", "run-live"} {
if !ids[expected] {
t.Fatalf("candidates must include %s, got %#v", expected, ids)
}
}
// Parity with the former full scan.
full, err := dataStore.ListRuns(ctx, "")
if err != nil {
t.Fatal(err)
}
fullPayload, fullRun, fullFound := dashboardLatestAcceptancePayload(full)
candidatePayload, candidateRun, candidateFound := dashboardLatestAcceptancePayload(candidates)
if fullFound != candidateFound || fullRun.ID != candidateRun.ID || len(fullPayload) != len(candidatePayload) {
t.Fatalf("acceptance selection diverged: full=%s(%v) candidates=%s(%v)", fullRun.ID, fullFound, candidateRun.ID, candidateFound)
}
for key := range fullPayload {
if fullPayload[key] != candidatePayload[key] {
t.Fatalf("acceptance payload diverged at %q: %#v vs %#v", key, fullPayload[key], candidatePayload[key])
}
}
if fullRun.ID != "run-acceptance-preview" {
t.Fatalf("fixture expects the preview-nested artifact to win, got %s", fullRun.ID)
}
fullGPU, fullFoundGPU := latestPocGPUWorkflowRun(full)
candidateGPU, candidateFoundGPU := latestPocGPUWorkflowRun(candidates)
if fullFoundGPU != candidateFoundGPU || fullGPU.ID != candidateGPU.ID {
t.Fatalf("gpu run selection diverged: full=%s(%v) candidates=%s(%v)", fullGPU.ID, fullFoundGPU, candidateGPU.ID, candidateFoundGPU)
}
if fullGPU.ID != "run-gpu" {
t.Fatalf("fixture expects the workflow-named gpu run to win, got %s", fullGPU.ID)
}
fullGate := dashboardGPUWorkflowPreflightGate(full)
candidateGate := dashboardGPUWorkflowPreflightGate(candidates)
if fullGate.Status != candidateGate.Status || fullGate.Label != candidateGate.Label {
t.Fatalf("gpu gate diverged: %#v vs %#v", fullGate, candidateGate)
}
if candidateGate.Status != "passed" {
t.Fatalf("completed gpu run must pass the preflight gate, got %#v", candidateGate)
}
}
// The tenant-enforced branch scopes both point queries to the requester's
// organization, like the old ListRunsByOrganization scan did.
func TestDashboardPOCReadinessCandidateRunsScopesTenant(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
mustUpsert := func(run domain.RunRecord) {
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
}
acceptancePayload := map[string]any{"status": "passed", "acceptanceScope": "full", "fullExternalAcceptance": true, "operatorCount": 3}
mustUpsert(domain.RunRecord{ID: "run-own", OrganizationID: "org-a", WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T09:00:00Z", UpdatedAt: "2026-10-09T09:00:00Z",
Artifacts: []domain.ExecutionArtifact{{ID: "a-own", Name: "poc-operator-full-acceptance-result.json", Data: acceptancePayload}}})
mustUpsert(domain.RunRecord{ID: "run-other", OrganizationID: "org-b", WorkflowName: pocGPUWorkflowAcceptanceWorkflowName, ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T10:00:00Z", UpdatedAt: "2026-10-09T10:00:00Z"})
server := &server{store: dataStore}
organization := domain.OrganizationRecord{ID: "org-a", Name: "A", Slug: "org-a", Status: domain.OrganizationStatusActive}
principal := &authenticatedPrincipal{User: domain.UserRecord{ID: "u1", Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, OrganizationID: "org-a"}, Organization: organization}
request := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil).WithContext(
context.WithValue(ctx, authContextKey{}, principal),
)
if !server.tenantEnforced(request) {
t.Fatal("request with principal must be tenant enforced")
}
candidates, err := server.dashboardPOCReadinessCandidateRuns(ctx, request)
if err != nil {
t.Fatal(err)
}
if len(candidates) != 1 || candidates[0].ID != "run-own" {
t.Fatalf("tenant scope must exclude org-b runs, got %#v", candidates)
}
}

View File

@ -45,13 +45,29 @@ func addDashboardWorkforceEvent(item *domain.DashboardWorkforceMetric, event dom
workforcestats.AddEvent(item, event, nil)
}
// cachedWorkforceStatsLoader is implemented by MongoStore; it caches the
// 30-day snapshot for 45s so concurrent dashboard renders do not re-run the
// full windowed joins (C4). MemoryStore intentionally stays uncached for
// read-after-write freshness in tests and the local replica.
type cachedWorkforceStatsLoader interface {
LoadWorkforceStatsCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error)
}
func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrganizationID string, principal *authenticatedPrincipal, from time.Time, to time.Time, summary *domain.DashboardSummary) error {
if summary == nil {
return nil
}
q, scope := workforcePrincipalQuery(principal)
q.From, q.To = from, to
data, err := server.store.LoadWorkforceStats(ctx, requestOrganizationID, q)
var data workforcestats.Snapshot
var err error
if cached, ok := server.store.(cachedWorkforceStatsLoader); ok {
// The 30-day window is already pushed down as a time-bounded, projected
// Mongo query; the cached loader deduplicates it for 45s per scope.
data, err = cached.LoadWorkforceStatsCached(ctx, requestOrganizationID, q)
} else {
data, err = server.store.LoadWorkforceStats(ctx, requestOrganizationID, q)
}
if err != nil {
return err
}

View File

@ -25,6 +25,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"droboticflow/internal/domain"
@ -2125,10 +2126,93 @@ func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string,
if err != nil {
return "", err
}
maybePruneSourceBrowserPreviewCache(cacheDir)
cacheKey := sourceBrowserPreviewCacheKey(request)
return filepath.Join(cacheDir, cacheKey+".mp4"), nil
}
// The on-disk preview cache is bounded: entries older than the TTL are
// deleted outright, then least-recently-written entries are evicted until
// the directory is under the size cap. Without this bound the /tmp cache
// grows without limit on pods that serve many long videos.
const (
sourceBrowserPreviewCacheMaxBytes = 2 << 30 // 2 GiB total cap
sourceBrowserPreviewCacheMaxAge = 7 * 24 * time.Hour
sourceBrowserPreviewCachePruneEvery = 10 * time.Minute
)
var lastSourceBrowserPreviewCachePrune atomic.Int64
func maybePruneSourceBrowserPreviewCache(dir string) {
interval := sourceBrowserPreviewCachePruneEvery.Nanoseconds()
now := time.Now().UnixNano()
last := lastSourceBrowserPreviewCachePrune.Load()
if now-last < interval {
return
}
if !lastSourceBrowserPreviewCachePrune.CompareAndSwap(last, now) {
return
}
go pruneSourceBrowserPreviewCache(dir, time.Now())
}
// pruneSourceBrowserPreviewCache enforces the TTL and size bounds on the
// preview cache directory. Best effort: unreadable entries are skipped and
// removal failures are ignored (the next prune retries).
func pruneSourceBrowserPreviewCache(dir string, now time.Time) {
pruneSourceBrowserPreviewCacheWithLimit(dir, now, sourceBrowserPreviewCacheMaxBytes)
}
// pruneSourceBrowserPreviewCacheWithLimit is the testable core of the cache
// eviction: delete entries past the TTL, then evict least-recently-written
// entries until the directory is under maxBytes.
func pruneSourceBrowserPreviewCacheWithLimit(dir string, now time.Time, maxBytes int64) {
entries, err := os.ReadDir(dir)
if err != nil {
return
}
type cacheEntry struct {
path string
modTime time.Time
size int64
}
files := make([]cacheEntry, 0, len(entries))
var total int64
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, infoErr := entry.Info()
if infoErr != nil {
continue
}
if now.Sub(info.ModTime()) > sourceBrowserPreviewCacheMaxAge {
_ = os.Remove(filepath.Join(dir, entry.Name()))
continue
}
files = append(files, cacheEntry{
path: filepath.Join(dir, entry.Name()),
modTime: info.ModTime(),
size: info.Size(),
})
total += info.Size()
}
if total <= maxBytes {
return
}
sort.Slice(files, func(left int, right int) bool {
return files[left].modTime.Before(files[right].modTime)
})
for _, file := range files {
if total <= maxBytes {
break
}
if removeErr := os.Remove(file.path); removeErr == nil {
total -= file.size
}
}
}
func parseBOSObjectPath(sourcePath string) (string, string, bool) {
bucket, object, ok := parseBOSPath(sourcePath)
if !ok || object == "" {
@ -2324,6 +2408,81 @@ func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWr
return nil
}
// sourceVideoPreviewSyncMode is the server-side rollback switch for the C5
// video direct-link change: setting DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 keeps
// every browser-preview request on the legacy blocking ffmpeg stream (no
// background pre-transcode, no redirect to the direct storage URL).
var sourceVideoPreviewSyncMode = sync.OnceValue(func() bool {
return strings.TrimSpace(os.Getenv("DROBOTICFLOW_VIDEO_PREVIEW_SYNC")) == "1"
})
// discardPreviewResponseWriter lets the background pre-transcode reuse
// streamSourceBrowserVideoPreview unchanged: its output is dropped and only
// the cached artifact on disk matters.
type discardPreviewResponseWriter struct{}
func (discardPreviewResponseWriter) Header() http.Header { return http.Header{} }
func (discardPreviewResponseWriter) WriteHeader(int) {}
func (discardPreviewResponseWriter) Write(p []byte) (int, error) { return len(p), nil }
// startSourceBrowserVideoPretranscode warms the preview cache in the
// background. The per-key lock inside streamSourceBrowserVideoPreview
// deduplicates concurrent warmups, and a warm run that finds the cache
// already populated returns immediately without invoking ffmpeg.
func startSourceBrowserVideoPretranscode(previewRequest sourceBrowserPreviewRequest) {
go func() {
// ffmpeg's own rw_timeout (15 minutes without progress) bounds the
// pull; streamSourceBrowserVideoPreview deliberately detaches
// cancellation so a client disconnect cannot truncate the warmup.
// Routed through the writer var so tests can stub the transcode.
if err := sourceBrowserVideoPreviewWriter(context.Background(), discardPreviewResponseWriter{}, previewRequest); err != nil {
fmt.Printf("[api] background browser video preview transcode failed: %v\n", err)
}
}()
}
// writeBrowserVideoPreviewResponse serves the browser-transcoded preview for
// a video entry (query transcode=browser). Order of preference:
// 1. serve the cached pre-transcoded mp4 when present (no ffmpeg at all);
// 2. start a background pre-transcode and immediately redirect the player
// to the direct storage URL (BOS presigned GET link), so the preview
// first frame is never blocked behind server-side ffmpeg;
// 3. legacy blocking ffmpeg stream as fallback (no direct URL available,
// explicit sync=1 request parameter, or the
// DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 rollback switch).
//
// All video responses set Content-Encoding: identity so the global chi
// Compress(5) middleware skips gzip: mp4 payloads are already compressed
// media and re-compressing them only burned API pod CPU.
func writeBrowserVideoPreviewResponse(writer http.ResponseWriter, request *http.Request, previewRequest sourceBrowserPreviewRequest, directURL string) {
// Videos are already-compressed media; the identity marker makes chi's
// Compress middleware leave the byte stream untouched.
writer.Header().Set("Content-Encoding", "identity")
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(previewRequest.FileName, path.Ext(previewRequest.FileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
direct := strings.TrimSpace(directURL)
forcedSync := sourceVideoPreviewSyncMode() ||
strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("sync")), "1")
if direct != "" && strings.HasPrefix(strings.ToLower(direct), "http") && !forcedSync {
startSourceBrowserVideoPretranscode(previewRequest)
http.Redirect(writer, request, direct, http.StatusTemporaryRedirect)
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] browser preview stream failed: %v\n", err)
}
}
type bestEffortPreviewWriter struct {
target io.Writer
err *error
@ -2700,21 +2859,7 @@ func (server *server) handleSourceCollectionContent(writer http.ResponseWriter,
Payload: content,
RedirectURL: redirectURL,
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] source browser preview stream failed: %v\n", err)
}
writeBrowserVideoPreviewResponse(writer, request, previewRequest, redirectURL)
return
}
if redirectURL != "" {
@ -2887,26 +3032,14 @@ func (server *server) handlePackageContent(writer http.ResponseWriter, request *
FileName: fileName,
Payload: content,
}
directURL := ""
if filePath != "" {
previewRequest.RedirectURL = filePath
} else {
previewRequest.RedirectURL = redirectURL
directURL = redirectURL
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] package browser preview stream failed: %v\n", err)
}
writeBrowserVideoPreviewResponse(writer, request, previewRequest, directURL)
return
}
if filePath != "" {

View File

@ -0,0 +1,207 @@
package api
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/go-chi/chi/v5/middleware"
)
func compressTestHandler(inner http.HandlerFunc) http.Handler {
return middleware.Compress(5)(inner)
}
// A cache miss with a direct (http) storage URL must redirect the player to
// that URL instead of running server-side ffmpeg on the request path, and
// the redirect itself must not come back gzip-encoded.
func TestBrowserVideoPreviewCacheMissRedirectsToDirectURL(t *testing.T) {
originalPath := sourceBrowserVideoPreviewPath
originalWriter := sourceBrowserVideoPreviewWriter
defer func() {
sourceBrowserVideoPreviewPath = originalPath
sourceBrowserVideoPreviewWriter = originalWriter
}()
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
return "", os.ErrNotExist
}
var warmups sync.WaitGroup
warmups.Add(1)
var warmedRequest sourceBrowserPreviewRequest
var warmupMu sync.Mutex
sourceBrowserVideoPreviewWriter = func(ctx context.Context, _ http.ResponseWriter, request sourceBrowserPreviewRequest) error {
warmupMu.Lock()
warmedRequest = request
warmupMu.Unlock()
warmups.Done()
return nil
}
direct := "https://bucket.bcebos.com/dir/camera.mp4?authorization=bce-auth-v1%2Fexample"
request := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/r/c/content?relativePath=v.mp4&transcode=browser", nil)
request.Header.Set("Accept-Encoding", "gzip")
recorder := httptest.NewRecorder()
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
FileName: "camera.mp4",
RedirectURL: direct,
}, direct)
}).ServeHTTP(recorder, request)
if recorder.Code != http.StatusTemporaryRedirect {
t.Fatalf("expected 307 redirect on cache miss, got %d body=%s", recorder.Code, recorder.Body.String())
}
if location := recorder.Header().Get("Location"); location != direct {
t.Fatalf("unexpected redirect location: %s", location)
}
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "" && encoding != "identity" {
t.Fatalf("redirect response must not be compressed, Content-Encoding=%q", encoding)
}
// The background pre-transcode must have been kicked off exactly for the
// preview request that missed the cache.
warmups.Wait()
warmupMu.Lock()
defer warmupMu.Unlock()
if warmedRequest.FileName != "camera.mp4" || warmedRequest.RedirectURL != direct {
t.Fatalf("background warmup ran for the wrong request: %+v", warmedRequest)
}
}
// When the cache holds the pre-transcoded artifact, the response must be
// served as-is through the global Compress middleware: the identity
// Content-Encoding marker makes chi skip gzip, so video bytes reach the
// player unchanged.
func TestBrowserVideoPreviewCacheHitServesBytesWithoutGzip(t *testing.T) {
originalPath := sourceBrowserVideoPreviewPath
defer func() { sourceBrowserVideoPreviewPath = originalPath }()
payload := []byte("fake-mp4-bytes-0123456789")
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
tempFile, err := os.CreateTemp(t.TempDir(), "preview-*.mp4")
if err != nil {
return "", err
}
if _, err := tempFile.Write(payload); err != nil {
tempFile.Close()
return "", err
}
if err := tempFile.Close(); err != nil {
return "", err
}
return tempFile.Name(), nil
}
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser", nil)
request.Header.Set("Accept-Encoding", "gzip")
recorder := httptest.NewRecorder()
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
FileName: "v.mp4",
}, "")
}).ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200 for cached preview, got %d body=%s", recorder.Code, recorder.Body.String())
}
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "identity" {
t.Fatalf("expected Content-Encoding identity (chi Compress skipped), got %q", encoding)
}
if !bytes.Equal(recorder.Body.Bytes(), payload) {
t.Fatal("video bytes were transformed on the way out; expected the cached artifact verbatim")
}
}
// The sync=1 parameter is the per-request rollback switch: it must keep the
// legacy blocking stream instead of redirecting to the direct URL.
func TestBrowserVideoPreviewSyncParamForcesBlockingStream(t *testing.T) {
originalPath := sourceBrowserVideoPreviewPath
originalWriter := sourceBrowserVideoPreviewWriter
defer func() {
sourceBrowserVideoPreviewPath = originalPath
sourceBrowserVideoPreviewWriter = originalWriter
}()
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
return "", os.ErrNotExist
}
var streamed atomic.Bool
sourceBrowserVideoPreviewWriter = func(_ context.Context, writer http.ResponseWriter, _ sourceBrowserPreviewRequest) error {
_, _ = writer.Write([]byte("streamed"))
streamed.Store(true)
return nil
}
direct := "https://bucket.bcebos.com/dir/camera.mp4"
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser&sync=1", nil)
recorder := httptest.NewRecorder()
writeBrowserVideoPreviewResponse(recorder, request, sourceBrowserPreviewRequest{
FileName: "camera.mp4",
RedirectURL: direct,
}, direct)
if recorder.Code != http.StatusOK {
t.Fatalf("sync=1 must stream, got %d", recorder.Code)
}
if !streamed.Load() {
t.Fatal("sync=1 must use the blocking server-side stream")
}
if recorder.Header().Get("Location") != "" {
t.Fatal("sync=1 must not redirect to the direct URL")
}
}
// Cache eviction: entries past the TTL are removed regardless of size, and
// under a size cap the least-recently-written entries are evicted first.
func TestPruneSourceBrowserPreviewCacheTTLAndLRU(t *testing.T) {
dir := t.TempDir()
write := func(name string, age time.Duration, size int64) string {
filePath := filepath.Join(dir, name)
payload := make([]byte, size)
if err := os.WriteFile(filePath, payload, 0o644); err != nil {
t.Fatal(err)
}
cutOff := time.Now().Add(-age)
if err := os.Chtimes(filePath, cutOff, cutOff); err != nil {
t.Fatal(err)
}
return filePath
}
stale := write("stale.mp4", sourceBrowserPreviewCacheMaxAge+time.Hour, 10)
oldest := write("oldest.mp4", 3*time.Hour, 600)
middle := write("middle.mp4", 2*time.Hour, 600)
newest := write("newest.mp4", time.Hour, 600)
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1300)
if _, err := os.Stat(stale); !os.IsNotExist(err) {
t.Fatal("entry past the TTL must be deleted")
}
if _, err := os.Stat(oldest); !os.IsNotExist(err) {
t.Fatal("least-recently-written entry must be evicted first under the size cap")
}
for _, kept := range []string{middle, newest} {
if _, err := os.Stat(kept); err != nil {
t.Fatalf("entry %s should have survived pruning: %v", kept, err)
}
}
}
// Under the size cap but with nothing past the TTL, the directory must be
// left untouched.
func TestPruneSourceBrowserPreviewCacheNoopWhenWithinBounds(t *testing.T) {
dir := t.TempDir()
filePath := filepath.Join(dir, "fresh.mp4")
if err := os.WriteFile(filePath, bytes.Repeat([]byte("x"), 128), 0o644); err != nil {
t.Fatal(err)
}
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1<<20)
if _, err := os.Stat(filePath); err != nil {
t.Fatalf("entry within bounds must not be evicted: %v", err)
}
}

View File

@ -0,0 +1,118 @@
package api
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// slow-page report S4: GET /api/human-batches must stay a read-only path —
// no flushHumanReleasePartitions writes — while the background sweeper still
// releases wait-expired batches the write paths no longer cover.
func newHumanBatchesFlushServer(t *testing.T) (*server, *store.MemoryStore, domain.HumanBatchRecord) {
t.Helper()
ctx := context.Background()
dataStore := store.NewMemoryStore()
organization := domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Name: "Default Organization", Slug: "default", Status: domain.OrganizationStatusActive}
if err := dataStore.UpsertOrganization(ctx, organization); err != nil {
t.Fatal(err)
}
admin := domain.UserRecord{ID: "admin-1", Name: "admin", Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, OrganizationID: organization.ID}
if err := dataStore.UpsertUser(ctx, admin); err != nil {
t.Fatal(err)
}
batch := domain.HumanBatchRecord{
ID: "batch-sweep", OrganizationID: organization.ID, PluginID: "annotation-qa", ProjectID: "project-1", WorkflowID: "workflow-1", RunID: "run-1", GateNodeID: "gate-1",
ReleasePolicy: domain.HumanReleaseAcceptedPartition, PartitionMaxItems: 10, PartitionMaxWaitSeconds: 1, Status: domain.HumanBatchActive,
}
if err := dataStore.UpsertHumanBatch(ctx, batch); err != nil {
t.Fatal(err)
}
// Accepted long enough ago that the PartitionMaxWaitSeconds window has
// expired — the exact case the old GET-path flush used to release.
item := domain.HumanItemRecord{
ID: "item-sweep-1", OrganizationID: organization.ID, PluginID: "annotation-qa", BatchID: batch.ID,
ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, GateNodeID: batch.GateNodeID,
Status: domain.HumanItemAccepted, AcceptedAt: nowISO(), Version: 1,
Subject: domain.HumanSubjectRef{ItemKey: "clip-1"},
}
if err := dataStore.UpsertHumanItem(ctx, item); err != nil {
t.Fatal(err)
}
return &server{store: dataStore}, dataStore, batch
}
func TestHumanBatchesListDoesNotFlushReleasePartitions(t *testing.T) {
ctx := context.Background()
server, dataStore, batch := newHumanBatchesFlushServer(t)
organization, err := dataStore.GetOrganization(ctx, domain.DefaultOrganizationID)
if err != nil || organization == nil {
t.Fatalf("organization=%#v err=%v", organization, err)
}
admin, err := dataStore.GetUser(ctx, "admin-1")
if err != nil || admin == nil {
t.Fatalf("admin=%#v err=%v", admin, err)
}
principal := &authenticatedPrincipal{User: *admin, Organization: *organization}
request := httptest.NewRequest(http.MethodGet, "/api/human-batches", nil).WithContext(
context.WithValue(ctx, authContextKey{}, principal),
)
response := httptest.NewRecorder()
server.handleHumanBatchesList(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
}
partitions, err := dataStore.ListReleasePartitions(ctx, batch.OrganizationID, batch.ID)
if err != nil {
t.Fatal(err)
}
if len(partitions) != 0 {
t.Fatalf("GET must not create release partitions, got %#v", partitions)
}
stored, err := dataStore.GetHumanItem(ctx, "item-sweep-1")
if err != nil || stored == nil {
t.Fatalf("item=%#v err=%v", stored, err)
}
if stored.ReleasePartitionID != "" || stored.Version != 1 {
t.Fatalf("GET must not write items: partition=%q version=%d", stored.ReleasePartitionID, stored.Version)
}
}
func TestHumanReleaseSweeperFlushesWaitExpiredBatches(t *testing.T) {
ctx := context.Background()
server, dataStore, batch := newHumanBatchesFlushServer(t)
ids, err := dataStore.ListHumanBatchIDsAwaitingRelease(ctx)
if err != nil {
t.Fatal(err)
}
if len(ids) != 1 || ids[0] != batch.ID {
t.Fatalf("awaiting ids=%#v", ids)
}
server.sweepHumanReleasePartitions()
partitions, err := dataStore.ListReleasePartitions(ctx, batch.OrganizationID, batch.ID)
if err != nil {
t.Fatal(err)
}
if len(partitions) != 1 || len(partitions[0].ItemIDs) != 1 {
t.Fatalf("sweeper must release the wait-expired batch: %#v", partitions)
}
stored, err := dataStore.GetHumanItem(ctx, "item-sweep-1")
if err != nil || stored == nil {
t.Fatalf("item=%#v err=%v", stored, err)
}
if stored.ReleasePartitionID == "" {
t.Fatal("sweeper must assign the item's release partition")
}
// Nothing left awaiting: the sweeper's next pass is a no-op.
ids, err = dataStore.ListHumanBatchIDsAwaitingRelease(ctx)
if err != nil {
t.Fatal(err)
}
if len(ids) != 0 {
t.Fatalf("no batch should remain awaiting, got %#v", ids)
}
}

View File

@ -0,0 +1,258 @@
package api
import (
"encoding/json"
"fmt"
"net/http"
"testing"
"time"
"droboticflow/internal/domain"
)
func TestHumanItemsListPaginatesAndProjectsPayload(t *testing.T) {
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
admin := domain.UserRecord{ID: "admin-pagination", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
// Five extra items with list-needed metadata in sliceWorkflow plus a heavy
// annotationWorkspace draft that must never cross the list wire.
for index := 0; index < 5; index++ {
item := fixture.item
item.ID = domain.NewID("human-item")
item.CreatedAt = time.Now().UTC().Add(time.Duration(index) * time.Minute).Format(time.RFC3339Nano)
item.UpdatedAt = item.CreatedAt
item.AssignedUserID = ""
item.Status = domain.HumanItemReady
item.Subject.Payload = map[string]any{
"sliceWorkflow": map[string]any{"status": "editing", "phase": "annotate"},
"annotationWorkspace": map[string]any{"sliceSegments": []any{map[string]any{"id": "seg", "start": 0, "end": 10}}},
}
if err := fixture.store.UpsertHumanItem(fixture.ctx, item); err != nil {
t.Fatal(err)
}
}
first := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3", admin, nil)
if first.Code != http.StatusOK {
t.Fatalf("page 1 status=%d body=%s", first.Code, first.Body.String())
}
var page1 struct {
Items []domain.HumanItemRecord `json:"items"`
NextCursor string `json:"nextCursor"`
HasMore bool `json:"hasMore"`
}
if err := json.Unmarshal(first.Body.Bytes(), &page1); err != nil {
t.Fatal(err)
}
if len(page1.Items) != 3 || !page1.HasMore || page1.NextCursor == "" {
t.Fatalf("page 1 items=%d hasMore=%v cursor=%q", len(page1.Items), page1.HasMore, page1.NextCursor)
}
seen := map[string]bool{}
for _, item := range page1.Items {
seen[item.ID] = true
if _, ok := item.Subject.Payload["sliceWorkflow"]; !ok {
t.Fatalf("list projection must keep sliceWorkflow for %s", item.ID)
}
if _, ok := item.Subject.Payload["annotationWorkspace"]; ok {
t.Fatalf("list projection must drop annotationWorkspace for %s", item.ID)
}
}
second := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3&cursor="+page1.NextCursor, admin, nil)
if second.Code != http.StatusOK {
t.Fatalf("page 2 status=%d body=%s", second.Code, second.Body.String())
}
var page2 struct {
Items []domain.HumanItemRecord `json:"items"`
HasMore bool `json:"hasMore"`
}
if err := json.Unmarshal(second.Body.Bytes(), &page2); err != nil {
t.Fatal(err)
}
if len(page2.Items) != 3 || page2.HasMore {
t.Fatalf("page 2 items=%d hasMore=%v", len(page2.Items), page2.HasMore)
}
for _, item := range page2.Items {
if seen[item.ID] {
t.Fatalf("cursor page repeated item %s", item.ID)
}
}
}
// TestHumanItemsListBackfillsAcrossFilteredRawPages reproduces the t17 review
// finding: the handler-level access filters can swallow an entire raw page
// (here via a stage that is absent from the batch definition), and the fetch
// loop must advance the raw cursor between rounds. Without that, every round
// re-reads the same page and the response silently starves or repeats.
func TestHumanItemsListBackfillsAcrossFilteredRawPages(t *testing.T) {
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
admin := domain.UserRecord{ID: "admin-backfill", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
// A full raw page (fetch floor is 100) of items whose stage is missing
// from the batch definition: the store returns them, the handler drops
// every one via humanStage lookup failure.
for index := 0; index < 100; index++ {
ghost := fixture.item
ghost.ID = domain.NewID("human-item")
ghost.CurrentStageID = "stage-that-does-not-exist"
ghost.CreatedAt = time.Now().UTC().Add(time.Duration(200-index) * time.Second).Format(time.RFC3339Nano)
ghost.UpdatedAt = ghost.CreatedAt
if err := fixture.store.UpsertHumanItem(fixture.ctx, ghost); err != nil {
t.Fatal(err)
}
}
// Older visible items: only reachable once the loop skips past the ghost page.
visible := make([]domain.HumanItemRecord, 0, 4)
for index := 0; index < 4; index++ {
item := fixture.item
item.ID = fmt.Sprintf("human-item-visible-%d", index)
item.CreatedAt = time.Now().UTC().Add(time.Duration(50-index) * time.Second).Format(time.RFC3339Nano)
item.UpdatedAt = item.CreatedAt
if err := fixture.store.UpsertHumanItem(fixture.ctx, item); err != nil {
t.Fatal(err)
}
visible = append(visible, item)
}
recorder := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3", admin, nil)
if recorder.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
var page struct {
Items []domain.HumanItemRecord `json:"items"`
NextCursor string `json:"nextCursor"`
HasMore bool `json:"hasMore"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &page); err != nil {
t.Fatal(err)
}
if len(page.Items) != 3 {
t.Fatalf("expected the loop to backfill to the page limit, got %d items", len(page.Items))
}
seen := map[string]bool{}
for _, item := range page.Items {
if seen[item.ID] {
t.Fatalf("duplicate item %s in a single page", item.ID)
}
seen[item.ID] = true
if item.CurrentStageID == "stage-that-does-not-exist" {
t.Fatalf("handler-filtered item leaked into the page: %s", item.ID)
}
}
if !page.HasMore || page.NextCursor == "" {
t.Fatalf("continuation metadata missing after backfill: hasMore=%v cursor=%q", page.HasMore, page.NextCursor)
}
// The continuation cursor must point strictly after the last returned
// item: the next page contains the remaining visible item and no overlap.
next := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3&cursor="+page.NextCursor, admin, nil)
if next.Code != http.StatusOK {
t.Fatalf("follow-up status=%d body=%s", next.Code, next.Body.String())
}
var followUp struct {
Items []domain.HumanItemRecord `json:"items"`
HasMore bool `json:"hasMore"`
}
if err := json.Unmarshal(next.Body.Bytes(), &followUp); err != nil {
t.Fatal(err)
}
for _, item := range followUp.Items {
if seen[item.ID] {
t.Fatalf("continuation page repeated item %s", item.ID)
}
seen[item.ID] = true
}
if len(seen) != len(page.Items)+len(followUp.Items) || seen[visible[len(visible)-1].ID] != true {
t.Fatalf("pagination lost items: unique=%d", len(seen))
}
// Malformed cursors are rejected like the runs endpoint.
bad := fixture.request(http.MethodGet, "/api/human-items?pool=all&cursor=not-a-cursor", admin, nil)
if bad.Code != http.StatusBadRequest {
t.Fatalf("malformed cursor status=%d body=%s", bad.Code, bad.Body.String())
}
}
func TestHumanItemsAvailablePoolPushesFiltersIntoStore(t *testing.T) {
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
// An assigned item and a cancelled item must both be excluded by the store
// filter itself, not by post-fetch dropping.
assigned := fixture.item
assigned.ID = domain.NewID("human-item")
assigned.AssignedUserID = fixture.annotator.ID
assigned.Status = domain.HumanItemClaimed
if err := fixture.store.UpsertHumanItem(fixture.ctx, assigned); err != nil {
t.Fatal(err)
}
cancelled := fixture.item
cancelled.ID = domain.NewID("human-item")
cancelled.AssignedUserID = ""
cancelled.Status = domain.HumanItemCancelled
if err := fixture.store.UpsertHumanItem(fixture.ctx, cancelled); err != nil {
t.Fatal(err)
}
pushed, err := fixture.store.ListHumanItems(fixture.ctx, domain.DefaultOrganizationID, domain.HumanItemQuery{
PluginID: "annotation-qa",
UnassignedOnly: true,
Statuses: []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted},
})
if err != nil {
t.Fatal(err)
}
if len(pushed) != 1 || pushed[0].ID != fixture.item.ID {
t.Fatalf("pushdown query returned %#v", pushed)
}
}
func TestRunsListDefaultsToPaginatedContract(t *testing.T) {
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
admin := domain.UserRecord{ID: "admin-runs-default", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
for index := 0; index < 60; index++ {
run := domain.RunRecord{
ID: domain.NewID("run"), OrganizationID: domain.DefaultOrganizationID,
ProjectID: "project-v5", WorkflowID: fixture.batch.WorkflowID,
Status: domain.RunStatusCompleted,
CreatedAt: time.Now().UTC().Add(-time.Duration(index) * time.Minute).Format(time.RFC3339Nano),
}
if err := fixture.store.UpsertRun(fixture.ctx, run); err != nil {
t.Fatal(err)
}
}
response := fixture.request(http.MethodGet, "/api/runs", admin, nil)
if response.Code != http.StatusOK {
t.Fatalf("runs status=%d body=%s", response.Code, response.Body.String())
}
var payload struct {
Items []domain.RunSummary `json:"items"`
NextCursor string `json:"nextCursor"`
HasMore bool `json:"hasMore"`
Total int `json:"total"`
StatusCounts map[string]int `json:"statusCounts"`
}
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
// The no-parameter path keeps the legacy {"items"}-only shape but must
// never stream the whole collection again.
if len(payload.Items) != 50 {
t.Fatalf("default runs page items=%d", len(payload.Items))
}
paged := fixture.request(http.MethodGet, "/api/runs?limit=10", admin, nil)
if paged.Code != http.StatusOK {
t.Fatalf("paged runs status=%d body=%s", paged.Code, paged.Body.String())
}
if err := json.Unmarshal(paged.Body.Bytes(), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Items) != 10 || !payload.HasMore || payload.NextCursor == "" {
t.Fatalf("paged runs items=%d hasMore=%v", len(payload.Items), payload.HasMore)
}
if payload.Total != 60 || payload.StatusCounts[string(domain.RunStatusCompleted)] != 60 {
t.Fatalf("aggregates missing: total=%d counts=%v", payload.Total, payload.StatusCounts)
}
}

View File

@ -4,7 +4,9 @@ import (
"context"
cryptorand "crypto/rand"
"droboticflow/internal/store"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"log"
@ -13,6 +15,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
@ -166,13 +169,59 @@ func parseHumanBatchQuery(request *http.Request) domain.HumanBatchQuery {
}
}
func parseHumanItemQuery(request *http.Request) domain.HumanItemQuery {
query := request.URL.Query()
return domain.HumanItemQuery{
PluginID: query.Get("pluginId"), BatchID: query.Get("batchId"), ProjectID: query.Get("projectId"),
WorkflowID: query.Get("workflowId"), RunID: query.Get("runId"), CurrentStageID: query.Get("stageId"),
Status: domain.HumanItemStatus(query.Get("status")),
func parseHumanItemQuery(request *http.Request) (domain.HumanItemQuery, *validationIssue) {
values := request.URL.Query()
query := domain.HumanItemQuery{
PluginID: values.Get("pluginId"), BatchID: values.Get("batchId"), ProjectID: values.Get("projectId"),
WorkflowID: values.Get("workflowId"), RunID: values.Get("runId"), CurrentStageID: values.Get("stageId"),
Status: domain.HumanItemStatus(values.Get("status")),
}
// Reject malformed cursors like the runs endpoint instead of silently
// falling back to the first page.
cursor, err := decodeHumanItemCursor(values.Get("cursor"))
if err != nil {
return query, &validationIssue{Field: "cursor", Message: "cursor must be a valid opaque human item cursor"}
}
query.Cursor = cursor
return query, nil
}
const (
humanItemsDefaultLimit = 500
humanItemsMaxLimit = 1000
// Raw Mongo pages larger than the requested page absorb items dropped by
// the in-memory access filters without unbounded over-fetching.
humanItemsMaxFetchRounds = 6
)
func decodeHumanItemCursor(raw string) (*domain.HumanItemCursor, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil, nil
}
payload, err := base64.RawURLEncoding.DecodeString(raw)
if err != nil {
return nil, fmt.Errorf("cursor must be a valid opaque human item cursor")
}
var cursor domain.HumanItemCursor
if err := json.Unmarshal(payload, &cursor); err != nil {
return nil, fmt.Errorf("cursor must be a valid opaque human item cursor")
}
if strings.TrimSpace(cursor.CreatedAt) == "" || strings.TrimSpace(cursor.ID) == "" {
return nil, fmt.Errorf("cursor must include createdAt and id")
}
return &cursor, nil
}
func encodeHumanItemCursor(cursor *domain.HumanItemCursor) string {
if cursor == nil {
return ""
}
payload, err := json.Marshal(cursor)
if err != nil {
return ""
}
return base64.RawURLEncoding.EncodeToString(payload)
}
func parseHumanRejectQuery(request *http.Request) domain.HumanRejectQuery {
@ -189,12 +238,9 @@ func (server *server) handleHumanBatchesList(writer http.ResponseWriter, request
renderError(writer, http.StatusInternalServerError, err)
return
}
for index := range items {
if err := server.flushHumanReleasePartitions(request.Context(), &items[index], false); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
// Read path stays read-only (S4): release partitions are flushed by the
// item-acceptance write paths and by the background sweeper started in
// NewHandlerWithConfig (startHumanReleasePartitionSweeper).
principal, _ := currentPrincipal(request)
if !annotationManager(principal) {
filtered := make([]domain.HumanBatchRecord, 0, len(items))
@ -234,7 +280,11 @@ func (server *server) handleHumanItemsList(writer http.ResponseWriter, request *
renderJSON(writer, http.StatusForbidden, map[string]string{"error": "only administrators may list all human items"})
return
}
query := parseHumanItemQuery(request)
query, queryIssue := parseHumanItemQuery(request)
if queryIssue != nil {
renderValidationError(writer, *queryIssue)
return
}
if pool == "reviewed" {
if principal == nil || (principal.User.Role != domain.UserRoleQualityReviewer && principal.User.Role != domain.UserRoleAcceptanceReviewer && !annotationManager(principal)) {
renderJSON(writer, http.StatusForbidden, map[string]string{"error": "review history requires reviewer access"})
@ -251,58 +301,115 @@ func (server *server) handleHumanItemsList(writer http.ResponseWriter, request *
if pool == "mine" && principal != nil {
query.ActorUserID = principal.User.ID
}
items, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
limit := humanItemsDefaultLimit
if rawLimit := strings.TrimSpace(request.URL.Query().Get("limit")); rawLimit != "" {
parsed, issue := parseDatabaseBrowserPageQuery(rawLimit, "limit", humanItemsDefaultLimit, 1, humanItemsMaxLimit)
if issue != nil {
renderValidationError(writer, *issue)
return
}
limit = parsed
}
listCtx, listErr := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, items)
if listErr != nil {
renderError(writer, http.StatusInternalServerError, listErr)
return
// The handler-level "available" pool rule is pushed into the Mongo filter
// so unassigned/claimable pages do not drag assigned rows through the wire.
if pool == "available" && query.Status == "" {
query.UnassignedOnly = true
query.Statuses = []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}
}
request = request.WithContext(listCtx)
filtered := make([]domain.HumanItemRecord, 0, len(items))
// List responses never need the workbench draft or disposition histories.
query.ListProjection = true
fetchSize := limit
if fetchSize < 100 {
fetchSize = 100
}
filtered := make([]domain.HumanItemRecord, 0, limit)
batchByID := map[string]*domain.HumanBatchRecord{}
for _, item := range items {
batch, loaded := batchByID[item.BatchID]
if !loaded {
batch, err = server.humanBatchForListRead(request.Context(), item.BatchID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
var nextCursor *domain.HumanItemCursor
hasMore := false
exhaustedAll := false
for round := 0; round < humanItemsMaxFetchRounds && len(filtered) < limit; round++ {
query.Limit = fetchSize
items, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
listCtx, listErr := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, items)
if listErr != nil {
renderError(writer, http.StatusInternalServerError, listErr)
return
}
request = request.WithContext(listCtx)
pageExhausted := len(items) < fetchSize
pageFull := false
for index, item := range items {
batch, loaded := batchByID[item.BatchID]
if !loaded {
batch, err = server.humanBatchForListRead(request.Context(), item.BatchID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
batchByID[item.BatchID] = batch
}
batchByID[item.BatchID] = batch
}
stage, _, found := humanStage(batch, item.CurrentStageID)
if randomClaimAnnotator(principal) && item.PluginID == "annotation-qa" && annotationClaimStage(batch, item.CurrentStageID) && item.AssignedUserID != principal.User.ID && item.LatestAuthorUserID != principal.User.ID {
continue
}
if !found || !server.canAccessHumanItem(request.Context(), request, item, "") {
continue
}
isAuthor := principal != nil && item.LatestAuthorUserID == principal.User.ID
if pool != "reviewed" && !server.canAccessHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage)) && !isAuthor {
continue
}
if pool == "mine" {
isAssigned := principal != nil && item.AssignedUserID == principal.User.ID
isMySubmitted := isAuthor && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemSubmitted) && item.CurrentStageID == "quality_review"
if !isAssigned && !isMySubmitted {
stage, _, found := humanStage(batch, item.CurrentStageID)
if randomClaimAnnotator(principal) && item.PluginID == "annotation-qa" && annotationClaimStage(batch, item.CurrentStageID) && item.AssignedUserID != principal.User.ID && item.LatestAuthorUserID != principal.User.ID {
continue
}
}
if pool == "available" {
availableStatus := item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted
if item.AssignedUserID != "" || !availableStatus {
if !found || !server.canAccessHumanItem(request.Context(), request, item, "") {
continue
}
isAuthor := principal != nil && item.LatestAuthorUserID == principal.User.ID
if pool != "reviewed" && !server.canAccessHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage)) && !isAuthor {
continue
}
if pool == "mine" {
isAssigned := principal != nil && item.AssignedUserID == principal.User.ID
isMySubmitted := isAuthor && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemSubmitted) && item.CurrentStageID == "quality_review"
if !isAssigned && !isMySubmitted {
continue
}
}
if pool == "available" {
availableStatus := item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted
if item.AssignedUserID != "" || !availableStatus {
continue
}
}
if principal != nil && principal.User.Role == domain.UserRoleAcceptanceReviewer {
claimable := item.AssignedUserID == "" && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted) && server.canClaimHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage))
item.CanClaim = &claimable
}
filtered = append(filtered, item)
if len(filtered) >= limit {
// Continuation resumes right after the last returned item.
nextCursor = &domain.HumanItemCursor{CreatedAt: item.CreatedAt, ID: item.ID}
pageFull = true
// More rows exist if the raw page still has unconsumed items
// or the store itself is not exhausted.
hasMore = index < len(items)-1 || !pageExhausted
break
}
}
if principal != nil && principal.User.Role == domain.UserRoleAcceptanceReviewer {
claimable := item.AssignedUserID == "" && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted) && server.canClaimHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage))
item.CanClaim = &claimable
if pageFull {
break
}
filtered = append(filtered, item)
if pageExhausted {
exhaustedAll = true
break
}
last := items[len(items)-1]
nextCursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
// Advance the raw fetch cursor so the next round continues strictly
// after this page; otherwise the same raw page repeats every round
// (t17 review finding) and items can be duplicated or starve the loop.
query.Cursor = nextCursor
}
// Fetch-round cap reached without exhausting the store: let the client
// continue from the last raw cursor instead of silently truncating.
if nextCursor != nil && !exhaustedAll && !hasMore && len(filtered) < limit {
hasMore = true
}
if pool == "reviewed" {
acceptanceIDs := []string{}
@ -344,7 +451,14 @@ func (server *server) handleHumanItemsList(writer http.ResponseWriter, request *
return
}
}
renderJSON(writer, http.StatusOK, map[string]any{"items": filtered})
// Keep the legacy {"items"} body byte-compatible when the page is
// exhausted; pagination metadata appears only when a next page exists.
response := map[string]any{"items": filtered}
if hasMore {
response["nextCursor"] = encodeHumanItemCursor(nextCursor)
response["hasMore"] = true
}
renderJSON(writer, http.StatusOK, response)
}
func (server *server) anonymizeV58AcceptanceItem(ctx context.Context, item *domain.HumanItemRecord) {
@ -2591,6 +2705,56 @@ func (server *server) flushHumanReleasePartitions(ctx context.Context, batch *do
return server.store.UpsertHumanBatch(ctx, *batch)
}
// humanReleaseFlushSweepInterval is how often the background sweeper looks for
// batches whose PartitionMaxWaitSeconds wait window expired without a further
// item write. The write paths already flush on every item acceptance, so this
// interval only adds latency for the wait-expiry case that the old
// GET /api/human-batches flush used to cover opportunistically.
const humanReleaseFlushSweepInterval = 60 * time.Second
// One sweeper per process: NewHandlerWithConfig also runs in every test, and a
// single loop is enough for the whole API (flushHumanReleasePartitions is
// idempotent across replicas, same as it was across concurrent GETs).
var humanReleaseFlushSweeperOnce sync.Once
// startHumanReleasePartitionSweeper moves release-partition flushing off the
// GET read path (S4): item-acceptance write paths flush immediately, and this
// periodic sweep covers wait-expiry batches within one interval.
func (server *server) startHumanReleasePartitionSweeper() {
humanReleaseFlushSweeperOnce.Do(func() {
go func() {
ticker := time.NewTicker(humanReleaseFlushSweepInterval)
defer ticker.Stop()
for range ticker.C {
server.sweepHumanReleasePartitions()
}
}()
})
}
func (server *server) sweepHumanReleasePartitions() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
ids, err := server.store.ListHumanBatchIDsAwaitingRelease(ctx)
if err != nil {
log.Printf("[human-release-sweeper] list awaiting batches: %v", err)
return
}
for _, id := range ids {
batch, err := server.store.GetHumanBatch(ctx, id)
if err != nil {
log.Printf("[human-release-sweeper] get batch %s: %v", id, err)
continue
}
if batch == nil {
continue
}
if err := server.flushHumanReleasePartitions(ctx, batch, false); err != nil {
log.Printf("[human-release-sweeper] flush batch %s: %v", id, err)
}
}
}
// History grants read access only; mutation handlers retain current-stage checks.
func (server *server) canReadReviewedHumanItem(ctx context.Context, request *http.Request, item domain.HumanItemRecord) bool {
principal, ok := currentPrincipal(request)

View File

@ -13,6 +13,7 @@ import (
"mime"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path"
@ -47,6 +48,7 @@ type validationIssue struct {
type server struct {
humanEvidenceIndexes humanEvidenceCache
dashboardResponses dashboardResponseCache
store store.Store
ai ai.Service
operatorRunnerImage string
@ -87,6 +89,114 @@ type runJobTerminator interface {
var storageConnectionBucketsResolver = listStorageBucketsFromConnection
var storageConnectionObjectsResolver = listStorageObjectsFromConnection
// dashboardResponseCache is the C4/S2 30-60s in-process response cache for the
// dashboard and production dashboard endpoints.
//
// Consistency / invalidation policy:
// - Entries expire after dashboardResponseCacheTTL (45s). Dashboard and
// production counters tolerate that staleness by contract.
// - Every non-GET /api request flushes the whole cache (see
// invalidateDashboardCachesOnWrite), so task submissions, human-item
// transitions, scope edits, user/role changes and logouts take effect on
// the next dashboard render even inside the TTL window.
// - Keys include organization, tenant mode, principal scope (role, user id,
// supplier id) and the raw query string, so per-user workforce sections and
// tenant isolation are never served cross-principal.
// - Writes done directly against the store outside the API (workers) are
// visible after at most the TTL.
const dashboardResponseCacheTTL = 45 * time.Second
type dashboardResponseCache struct {
mu sync.Mutex
entries map[string]dashboardResponseCacheEntry
now func() time.Time
}
type dashboardResponseCacheEntry struct {
body []byte
contentType string
status int
expires time.Time
}
func (c *dashboardResponseCache) get(key string) *dashboardResponseCacheEntry {
c.mu.Lock()
defer c.mu.Unlock()
entry, ok := c.entries[key]
if !ok || c.now().After(entry.expires) {
return nil
}
body := make([]byte, len(entry.body))
copy(body, entry.body)
return &dashboardResponseCacheEntry{body: body, contentType: entry.contentType, status: entry.status, expires: entry.expires}
}
func (c *dashboardResponseCache) set(key string, status int, contentType string, body []byte) {
c.mu.Lock()
defer c.mu.Unlock()
if c.entries == nil {
c.entries = map[string]dashboardResponseCacheEntry{}
}
if len(c.entries) >= 128 {
c.entries = map[string]dashboardResponseCacheEntry{}
}
stored := make([]byte, len(body))
copy(stored, body)
c.entries[key] = dashboardResponseCacheEntry{body: stored, contentType: contentType, status: status, expires: c.now().Add(dashboardResponseCacheTTL)}
}
func (c *dashboardResponseCache) invalidateAll() {
c.mu.Lock()
defer c.mu.Unlock()
c.entries = map[string]dashboardResponseCacheEntry{}
}
func (server *server) invalidateDashboardCachesOnWrite(next http.Handler) http.Handler {
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.Method != http.MethodGet && request.Method != http.MethodHead && strings.HasPrefix(request.URL.Path, "/api/") {
server.dashboardResponses.invalidateAll()
}
next.ServeHTTP(writer, request)
})
}
func (server *server) dashboardCacheKey(prefix string, request *http.Request) string {
scope := "anonymous"
if principal, ok := currentPrincipal(request); ok && principal != nil {
scope = fmt.Sprintf("%s|%s|%s", principal.User.Role, principal.User.ID, principal.User.SupplierID)
}
return fmt.Sprintf("%s|%v|%s|%s|%s", prefix, server.tenantEnforced(request), server.requestOrganizationID(request), scope, request.URL.RawQuery)
}
// withDashboardResponseCache wraps a read-only dashboard handler with the
// shared 45s response cache. Cache misses execute the handler against a
// recorder and store the exact response bytes for later hits.
func (server *server) withDashboardResponseCache(prefix string, next http.HandlerFunc) http.HandlerFunc {
return func(writer http.ResponseWriter, request *http.Request) {
key := server.dashboardCacheKey(prefix, request)
if entry := server.dashboardResponses.get(key); entry != nil {
if entry.contentType != "" {
writer.Header().Set("Content-Type", entry.contentType)
}
writer.WriteHeader(entry.status)
_, _ = writer.Write(entry.body)
return
}
recorder := httptest.NewRecorder()
next(recorder, request)
body := recorder.Body.Bytes()
if recorder.Code == http.StatusOK {
server.dashboardResponses.set(key, recorder.Code, recorder.Header().Get("Content-Type"), body)
}
if contentType := recorder.Header().Get("Content-Type"); contentType != "" {
writer.Header().Set("Content-Type", contentType)
}
writer.WriteHeader(recorder.Code)
_, _ = writer.Write(body)
}
}
var runJobTerminatorFactory = func(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (runJobTerminator, error) {
return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, "", "", "", 0)
}
@ -1269,6 +1379,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
trustedProxyNets: parseTrustedProxyCIDRs(appConfig.TrustedProxyCIDRs),
authLimiter: newRateLimiter(20, time.Minute),
externalWriteLimiter: newRateLimiter(120, time.Minute),
dashboardResponses: dashboardResponseCache{now: time.Now},
k8sNamespace: appConfig.K8sNamespace,
k8sImagePullSecretName: appConfig.K8sImagePullSecretName,
workspacePVCName: appConfig.WorkspacePVCName,
@ -1282,9 +1393,11 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
buildTime: appConfig.BuildTime,
image: appConfig.Image,
}
instance.startHumanReleasePartitionSweeper()
router := chi.NewRouter()
router.Use(diagnosticTrace)
router.Use(middleware.Compress(5))
router.Use(instance.invalidateDashboardCachesOnWrite)
router.Use(instance.apiTokenAuth)
router.Use(instance.sessionAuth)
router.Use(instance.csrfProtection)
@ -1334,9 +1447,9 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
router.Post("/api/operators/image-archives/import", instance.handleOperatorImageArchiveImport)
})
router.With(instance.requireAdminMonitoringAccess).Get("/api/acceptance/poc-readiness", instance.handlePocAcceptanceReadiness)
router.Get("/api/dashboard", instance.handleDashboard)
router.Get("/api/dashboard", instance.withDashboardResponseCache("dashboard", instance.handleDashboard))
router.Get("/api/workforce-returns", instance.handleWorkforceReturns)
router.Get("/api/dashboard/production", instance.handleProductionDashboard)
router.Get("/api/dashboard/production", instance.withDashboardResponseCache("production", instance.handleProductionDashboard))
router.Get("/api/dashboard/production/scope", instance.handleProductionScope)
router.Put("/api/dashboard/production/scope", instance.handleProductionScope)
router.Get("/api/plugins", instance.handlePluginsList)
@ -1940,7 +2053,12 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
renderError(writer, http.StatusInternalServerError, err)
return
}
runSummaries, err := server.listRunSummariesForRequest(request.Context(), request, "")
// C4: reuse the cursor-paginated run summary query (default page size 50)
// instead of materializing every run in the organization. Org-wide
// status totals are restored from the page aggregates below so the
// response counters stay exact; only trend/average granularity is
// limited to the most recent page.
runPage, err := server.listRunSummaryPageForRequest(request.Context(), request, domain.RunSummaryQuery{Limit: 50, IncludeAggregates: true, Mode: domain.RunModeProduction})
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
@ -1955,7 +2073,20 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
renderError(writer, http.StatusInternalServerError, err)
return
}
summary := store.BuildDashboardSummaryAt(projects, operators, runSummaries, auditTasks, settings.Timezone, time.Now())
summary := store.BuildDashboardSummaryAt(projects, operators, runPage.Items, auditTasks, settings.Timezone, time.Now())
if runPage.StatusCounts != nil {
summary.Tasks.Running = int(runPage.StatusCounts[domain.RunStatusRunning])
summary.Tasks.Succeeded = int(runPage.StatusCounts[domain.RunStatusCompleted])
summary.Tasks.Failed = int(runPage.StatusCounts[domain.RunStatusFailed])
totalFinished := summary.Tasks.Succeeded + summary.Tasks.Failed
if totalFinished == 0 {
summary.Jobs.SuccessRate = 100
} else {
summary.Jobs.SuccessRate = float64(summary.Tasks.Succeeded) * 100 / float64(totalFinished)
}
summary.Jobs.Running = summary.Tasks.Running
summary.Jobs.Failed = summary.Tasks.Failed
}
enrichDashboardOperatorInventory(&summary, operators)
server.enrichDashboardHumanWork(request.Context(), request, &summary)
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
@ -1986,11 +2117,26 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
renderJSON(writer, http.StatusOK, summary)
}
// dashboardHumanWorkCounter is implemented by both store backends; the Mongo
// implementation computes the counters with aggregation pipelines (C4) instead
// of materializing human items and delivery packages.
type dashboardHumanWorkCounter interface {
DashboardHumanWorkStats(ctx context.Context, organizationID string) (store.DashboardHumanWorkStats, error)
}
func (server *server) enrichDashboardHumanWork(ctx context.Context, request *http.Request, summary *domain.DashboardSummary) {
if summary == nil {
return
}
organizationID := server.requestOrganizationID(request)
if counter, ok := server.store.(dashboardHumanWorkCounter); ok {
if stats, err := counter.DashboardHumanWorkStats(ctx, organizationID); err == nil {
applyDashboardHumanWorkStats(summary, stats)
}
return
}
// Fallback for store implementations without the counting interface:
// keep the legacy full-materialization loop (unchanged semantics).
items, err := server.store.ListHumanItems(ctx, organizationID, domain.HumanItemQuery{})
if err == nil {
accepted := 0
@ -2045,6 +2191,26 @@ func (server *server) enrichDashboardHumanWork(ctx context.Context, request *htt
summary.Tasks.Waiting = summary.Tasks.PendingAnnotation + summary.Tasks.PendingQuality + summary.Tasks.PendingAcceptance
}
// applyDashboardHumanWorkStats mirrors the legacy enrich loop's summary
// mutations using precomputed counters, keeping the response shape and the
// +=/= semantics identical.
func applyDashboardHumanWorkStats(summary *domain.DashboardSummary, stats store.DashboardHumanWorkStats) {
summary.Data.CaptureDurationSeconds += stats.CaptureDurationSeconds
summary.Tasks.PendingAnnotation += stats.PendingAnnotation
summary.Tasks.PendingQuality += stats.PendingQuality
summary.Tasks.InAnnotation += stats.InAnnotation
summary.Tasks.PendingAcceptance += stats.PendingAcceptanceItems
summary.Tasks.Rejected += stats.Rejected
summary.Tasks.Escalated += stats.Escalated
summary.Data.VideoCount = stats.VideoCount
if decided := stats.Accepted + stats.Rejected; decided > 0 {
summary.Tasks.QualityPassRate = float64(stats.Accepted) * 100 / float64(decided)
}
summary.Data.PackageCount = stats.PackageCount
summary.Tasks.PendingAcceptance += stats.PackagesPendingAcceptance
summary.Tasks.Waiting = summary.Tasks.PendingAnnotation + summary.Tasks.PendingQuality + summary.Tasks.PendingAcceptance
}
func enrichDashboardOperatorInventory(summary *domain.DashboardSummary, operators []domain.OperatorRecord) {
if summary == nil {
return
@ -2888,7 +3054,7 @@ func normalizeDashboardPOCReadinessLists(poc *domain.DashboardPOCReadiness) {
}
func (server *server) dashboardPOCReadiness(ctx context.Context, request *http.Request) domain.DashboardPOCReadiness {
runs, err := server.listRunsForRequest(ctx, request, "")
runs, err := server.dashboardPOCReadinessCandidateRuns(ctx, request)
if err != nil {
gpu := server.dashboardGPUCapacityGate(ctx)
poc := domain.DashboardPOCReadiness{
@ -2991,6 +3157,56 @@ func (server *server) enrichDashboardPOC(ctx context.Context, request *http.Requ
summary.POC = server.dashboardPOCReadiness(ctx, request)
}
// dashboardPOCReadinessCandidateLimit bounds each point-query half of
// dashboardPOCReadinessCandidateRuns. The latest candidate is always first in
// the store's recency order, so a modest bound keeps the scan cheap without
// ever excluding the run the selectors would pick.
const dashboardPOCReadinessCandidateLimit = 25
// dashboardPOCReadinessCandidateRuns is the slow-page report S4 point-query
// replacement for the former listRunsForRequest full-table scan: it fetches
// only runs that can carry a POC acceptance artifact, plus runs on the
// POC/live acceptance workflows, and merges the two bounded sets. The
// downstream selectors (dashboardLatestAcceptancePayload /
// dashboardGPUWorkflowPreflightGate) run unchanged over the merged set, and
// every run they could possibly pick matches one of the two store filters, so
// the response stays byte-compatible with the full-scan behavior.
func (server *server) dashboardPOCReadinessCandidateRuns(ctx context.Context, request *http.Request) ([]domain.RunRecord, error) {
organizationID := ""
if server.tenantEnforced(request) {
organizationID = server.requestOrganizationID(request)
}
acceptanceRuns, err := server.store.ListRunsAcceptanceCandidates(ctx, organizationID, dashboardPOCReadinessCandidateLimit)
if err != nil {
return nil, err
}
gpuRuns, err := server.store.ListRunsWorkflowCandidates(
ctx,
organizationID,
[]string{pocGPUWorkflowAcceptanceWorkflowName, liveWorkflowAcceptanceWorkflowName},
[]string{pocGPUWorkflowAcceptanceProjectName, liveWorkflowAcceptanceProjectName},
dashboardPOCReadinessCandidateLimit,
)
if err != nil {
return nil, err
}
seen := make(map[string]bool, len(acceptanceRuns)+len(gpuRuns))
runs := make([]domain.RunRecord, 0, len(acceptanceRuns)+len(gpuRuns))
for _, run := range acceptanceRuns {
if !seen[run.ID] {
seen[run.ID] = true
runs = append(runs, run)
}
}
for _, run := range gpuRuns {
if !seen[run.ID] {
seen[run.ID] = true
runs = append(runs, run)
}
}
return runs, nil
}
func (server *server) populateProjectWorkflowCounts(ctx context.Context, request *http.Request, items []domain.ProjectRecord) ([]domain.ProjectRecord, error) {
if len(items) == 0 {
return items, nil
@ -7004,38 +7220,43 @@ func parseRunSummaryQuery(request *http.Request) (domain.RunSummaryQuery, []vali
}
func (server *server) handleRunsList(writer http.ResponseWriter, request *http.Request) {
if runSummaryPaginationRequested(request) {
query, issues := parseRunSummaryQuery(request)
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
startedAt := time.Now()
page, err := server.listRunSummaryPageForRequest(request.Context(), request, query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
writer.Header().Set("Server-Timing", fmt.Sprintf("runs;dur=%.1f", float64(time.Since(startedAt).Microseconds())/1000))
response := map[string]any{
"items": page.Items,
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
"hasMore": page.HasMore,
}
if query.IncludeAggregates {
response["total"] = page.Total
response["statusCounts"] = page.StatusCounts
}
renderJSON(writer, http.StatusOK, response)
// S1: the legacy no-parameter path used to stream every run summary.
// Route every request through the cursor-paginated store query (default
// limit 50) and keep the response a backward-compatible superset.
explicitlyPaginated := runSummaryPaginationRequested(request)
query, issues := parseRunSummaryQuery(request)
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
items, err := server.listRunSummariesForRequest(request.Context(), request, request.URL.Query().Get("status"))
startedAt := time.Now()
page, err := server.listRunSummaryPageForRequest(request.Context(), request, query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = server.filterRunSummariesForRequest(request, items)
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
items := page.Items
if !explicitlyPaginated {
// Preserve the legacy per-project visibility filtering for callers
// that never opted into the paginated contract.
items = server.filterRunSummariesForRequest(request, items)
}
writer.Header().Set("Server-Timing", fmt.Sprintf("runs;dur=%.1f", float64(time.Since(startedAt).Microseconds())/1000))
if !explicitlyPaginated {
// Byte-compatible legacy shape: status-only callers pinned {"items"}.
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
return
}
response := map[string]any{
"items": items,
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
"hasMore": page.HasMore,
}
if query.IncludeAggregates {
response["total"] = page.Total
response["statusCounts"] = page.StatusCounts
}
renderJSON(writer, http.StatusOK, response)
}
func (server *server) loadRun(ctx context.Context, id string) (*domain.RunRecord, error) {
@ -7744,11 +7965,11 @@ func (server *server) serveResolvedRunArtifactContent(writer http.ResponseWriter
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
if artifactBrowserTranscodeRequested(request, artifact, mimeType) {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
FileName: fileName,
RedirectURL: previewURL,
}
// The presigned URL changes on every request; anchor the transcode
// cache on the stable object coordinates instead.
// cache on the stable object coordinates instead.
if objectBucket, objectKey, ok := artifactBOSLocation(artifact); ok {
previewRequest.ObjectBucket, previewRequest.ObjectKey = objectBucket, objectKey
}
@ -7917,12 +8138,35 @@ func (server *server) handleAuditTaskReject(writer http.ResponseWriter, request
renderJSON(writer, http.StatusOK, task)
}
const (
usersListDefaultLimit = 200
usersListMaxLimit = 1000
)
func (server *server) handleUsersList(writer http.ResponseWriter, request *http.Request) {
values := request.URL.Query()
limit := usersListDefaultLimit
if rawLimit := strings.TrimSpace(values.Get("limit")); rawLimit != "" {
parsed, issue := parseDatabaseBrowserPageQuery(rawLimit, "limit", usersListDefaultLimit, 1, usersListMaxLimit)
if issue != nil {
renderValidationError(writer, *issue)
return
}
limit = parsed
}
// S5: search matches the fields the account list renders (name, phone).
search := strings.ToLower(strings.TrimSpace(values.Get("q")))
if len(search) > 200 {
renderValidationError(writer, validationIssue{Field: "q", Message: "q must be at most 200 characters"})
return
}
items, err := server.store.ListUsers(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
// Tenant/role filtering runs before search and pagination so neither new
// parameter can widen visibility beyond the unpaginated contract.
items = server.filterUsersForRequest(request, items)
if request.URL.Query().Get("includeDisabled") != "true" {
active := items[:0]
@ -7933,7 +8177,26 @@ func (server *server) handleUsersList(writer http.ResponseWriter, request *http.
}
items = active
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeUsers(items)})
if search != "" {
matched := items[:0]
for _, item := range items {
if strings.Contains(strings.ToLower(item.Name), search) || strings.Contains(strings.ToLower(item.Phone), search) {
matched = append(matched, item)
}
}
items = matched
}
hasMore := len(items) > limit
if hasMore {
items = items[:limit]
}
// Backward-compatible shape: hasMore appears only when the conservative
// default cap truncated the result.
response := map[string]any{"items": sanitizeUsers(items)}
if hasMore {
response["hasMore"] = true
}
renderJSON(writer, http.StatusOK, response)
}
// validateUserPhoneAvailable backs the account create/edit/login handlers,
@ -8215,6 +8478,8 @@ func (server *server) handleUsersUpdate(writer http.ResponseWriter, request *htt
renderUserMutationError(writer, err)
return
}
// Role/supplier/status/password changes must evict cached principals now.
invalidateAuthCacheForUser(item.ID)
if item.Status == domain.UserStatusDisabled || payload.Password != nil {
if err := server.store.DeleteSessionsByUser(request.Context(), item.ID); err != nil {
renderError(writer, http.StatusInternalServerError, err)
@ -8328,6 +8593,8 @@ func (server *server) handleUsersDelete(writer http.ResponseWriter, request *htt
renderUserMutationError(writer, err)
return
}
// The deleted account's cached principal (if any) must not survive.
invalidateAuthCacheForUser(item.ID)
if err := server.store.DeleteSessionsByUser(request.Context(), item.ID); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return

View File

@ -683,6 +683,9 @@ func (s *server) executeSupplierDeletion(task domain.SupplierDeletionTask, owner
fail(e)
return
}
// The records phase deletes employee accounts and their sessions directly
// in the store; flush cached principals so revoked access is immediate.
invalidateAuthCache()
task.Phase = "completed"
task.Status = "completed"
task.Error = ""

View File

@ -290,6 +290,13 @@ type HumanBatchQuery struct {
Status HumanBatchStatus
}
// HumanItemCursor is the opaque continuation cursor for paginated human item
// lists, ordered by createdAt desc, id desc (mirroring RunSummaryCursor).
type HumanItemCursor struct {
CreatedAt string `json:"createdAt"`
ID string `json:"id"`
}
type HumanItemQuery struct {
// Server-populated review history scope; nil means unrestricted.
IDs []string
@ -303,6 +310,15 @@ type HumanItemQuery struct {
RunID string
CurrentStageID string
Status HumanItemStatus
// List-endpoint pagination knobs. Zero values keep the historical
// unpaginated behavior for internal callers (dashboard, batch reads).
Limit int
Cursor *HumanItemCursor
ListProjection bool
// Pushdown filters mirroring the work-pool rules; used by the list
// endpoint so Mongo skips rows the handler would drop anyway.
UnassignedOnly bool
Statuses []HumanItemStatus
}
type HumanRejectQuery struct {

View File

@ -34,7 +34,56 @@ func humanItemMatches(item domain.HumanItemRecord, query domain.HumanItemQuery)
(query.WorkflowID == "" || item.WorkflowID == query.WorkflowID) &&
(query.RunID == "" || item.RunID == query.RunID) &&
(query.CurrentStageID == "" || item.CurrentStageID == query.CurrentStageID) &&
(query.Status == "" || item.Status == query.Status)
(query.Status == "" || item.Status == query.Status) &&
(!query.UnassignedOnly || item.AssignedUserID == "") &&
(query.Statuses == nil || humanItemStatusMatched(query.Statuses, item.Status))
}
func humanItemStatusMatched(statuses []domain.HumanItemStatus, status domain.HumanItemStatus) bool {
for _, candidate := range statuses {
if status == candidate {
return true
}
}
return false
}
// humanItemListPayloadKeysRemoved mirrors humanItemListProjection: payload
// sub-documents that are only needed on the detail/workbench paths are dropped
// from list responses to keep pages small.
var humanItemListPayloadKeysRemoved = []string{
"annotationWorkspace",
"annotationDraft",
"sliceDispositionHistory",
"sliceResetHistory",
"sliceArchiveEdits",
"sliceRestoreEvidence",
"currentDataPreview",
}
func humanItemListProjectionClone(item domain.HumanItemRecord) domain.HumanItemRecord {
if item.Subject.Payload == nil {
return item
}
payload := make(map[string]any, len(item.Subject.Payload))
for key, value := range item.Subject.Payload {
payload[key] = value
}
for _, key := range humanItemListPayloadKeysRemoved {
delete(payload, key)
}
item.Subject.Payload = payload
return item
}
func humanItemAfterCursor(item domain.HumanItemRecord, cursor *domain.HumanItemCursor) bool {
if cursor == nil {
return true
}
if item.CreatedAt != cursor.CreatedAt {
return item.CreatedAt < cursor.CreatedAt
}
return item.ID < cursor.ID
}
func (store *MemoryStore) ListHumanBatches(_ context.Context, organizationID string, query domain.HumanBatchQuery) ([]domain.HumanBatchRecord, error) {
@ -50,6 +99,30 @@ func (store *MemoryStore) ListHumanBatches(_ context.Context, organizationID str
return items, nil
}
// ListHumanBatchIDsAwaitingRelease implements
// Store.ListHumanBatchIDsAwaitingRelease (see the Mongo version's comment).
func (store *MemoryStore) ListHumanBatchIDsAwaitingRelease(_ context.Context) ([]string, error) {
store.mu.Lock()
defer store.mu.Unlock()
ids := make([]string, 0)
for _, item := range store.humanItems {
if item.Status == domain.HumanItemAccepted && strings.TrimSpace(item.ReleasePartitionID) == "" && strings.TrimSpace(item.BatchID) != "" {
ids = append(ids, item.BatchID)
}
}
sort.Strings(ids)
// Distinct: a batch with several awaiting items must appear once.
deduped := ids[:0]
previous := ""
for _, id := range ids {
if id != previous {
deduped = append(deduped, id)
previous = id
}
}
return deduped, nil
}
func (store *MemoryStore) GetHumanBatch(_ context.Context, id string) (*domain.HumanBatchRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
@ -112,11 +185,24 @@ func (store *MemoryStore) ListHumanItems(_ context.Context, organizationID strin
defer store.mu.Unlock()
items := make([]domain.HumanItemRecord, 0)
for _, item := range store.humanItems {
if humanOrganizationMatches(item.OrganizationID, organizationID) && humanItemMatches(item, query) {
if humanOrganizationMatches(item.OrganizationID, organizationID) && humanItemMatches(item, query) && humanItemAfterCursor(item, query.Cursor) {
items = append(items, clone(item))
}
}
sort.Slice(items, func(i, j int) bool { return items[i].CreatedAt > items[j].CreatedAt })
sort.Slice(items, func(i, j int) bool {
if items[i].CreatedAt != items[j].CreatedAt {
return items[i].CreatedAt > items[j].CreatedAt
}
return items[i].ID > items[j].ID
})
if query.Limit > 0 && len(items) > query.Limit {
items = items[:query.Limit]
}
if query.ListProjection {
for index := range items {
items[index] = humanItemListProjectionClone(items[index])
}
}
return items, nil
}
@ -466,13 +552,72 @@ func humanItemMongoFilter(organizationID string, query domain.HumanItemQuery) bs
if query.Status != "" {
filter["status"] = query.Status
}
if query.UnassignedOnly {
filter["assignedUserId"] = bson.M{"$in": bson.A{"", nil}}
}
if len(query.Statuses) > 0 {
statuses := make(bson.A, 0, len(query.Statuses))
for _, status := range query.Statuses {
statuses = append(statuses, status)
}
filter["status"] = bson.M{"$in": statuses}
}
if query.Cursor != nil {
// Copy the base conditions before nesting them under $and: embedding
// the same map inside itself makes BSON encoding recurse forever and
// crashes the process (t8 blocker v1-blocker-cursor-stack-overflow).
base := make(bson.M, len(filter))
for key, value := range filter {
base[key] = value
}
filter["$and"] = bson.A{base, bson.M{"$or": bson.A{
bson.M{"createdAt": bson.M{"$lt": query.Cursor.CreatedAt}},
bson.M{"createdAt": query.Cursor.CreatedAt, "id": bson.M{"$lt": query.Cursor.ID}},
}}}
}
return filter
}
// humanItemListProjection drops payload sub-documents that only the
// detail/workbench paths need (annotation drafts, disposition histories,
// restore evidence). Everything the list page and its enrichment read —
// sliceWorkflow status, acceptance aggregates (items/statistics), source
// video/artifact locators, duration — stays in the document.
func humanItemListProjection() bson.M {
projection := bson.M{}
for _, key := range humanItemListPayloadKeysRemoved {
projection["subject.payload."+key] = 0
}
return projection
}
func (store *MongoStore) ListHumanBatches(ctx context.Context, organizationID string, query domain.HumanBatchQuery) ([]domain.HumanBatchRecord, error) {
return mongoList[domain.HumanBatchRecord](ctx, store.humanBatches, humanBatchMongoFilter(organizationID, query), bson.D{{Key: "createdAt", Value: -1}})
}
// ListHumanBatchIDsAwaitingRelease implements
// Store.ListHumanBatchIDsAwaitingRelease: distinct batch IDs that still have
// accepted items without a release partition — exactly the batches the
// background release sweeper (formerly the GET /api/human-batches flush) can
// act on.
func (store *MongoStore) ListHumanBatchIDsAwaitingRelease(ctx context.Context) ([]string, error) {
values, err := store.humanItems.Distinct(ctx, "batchId", bson.M{
"status": string(domain.HumanItemAccepted),
"releasePartitionId": bson.M{"$in": bson.A{nil, ""}},
})
if err != nil {
return nil, err
}
ids := make([]string, 0, len(values))
for _, value := range values {
if id, ok := value.(string); ok && id != "" {
ids = append(ids, id)
}
}
sort.Strings(ids)
return ids, nil
}
func (store *MongoStore) GetHumanBatch(ctx context.Context, id string) (*domain.HumanBatchRecord, error) {
return mongoGetByID[domain.HumanBatchRecord](ctx, store.humanBatches, id)
}
@ -483,7 +628,18 @@ func (store *MongoStore) UpsertHumanBatch(ctx context.Context, item domain.Human
}
func (store *MongoStore) ListHumanItems(ctx context.Context, organizationID string, query domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
return mongoList[domain.HumanItemRecord](ctx, store.humanItems, humanItemMongoFilter(organizationID, query), bson.D{{Key: "createdAt", Value: -1}})
findOptions := options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}})
if query.Limit > 0 {
findOptions.SetLimit(int64(query.Limit))
}
if query.ListProjection {
findOptions.SetProjection(humanItemListProjection())
}
cursor, err := store.humanItems.Find(ctx, humanItemMongoFilter(organizationID, query), findOptions)
if err != nil {
return nil, err
}
return decodeMany[domain.HumanItemRecord](ctx, cursor)
}
func (store *MongoStore) GetHumanItem(ctx context.Context, id string) (*domain.HumanItemRecord, error) {

View File

@ -0,0 +1,130 @@
package store
import (
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// TestHumanItemCursorFilterEncodesWithoutSelfReference guards the t8 blocker:
// the cursor branch used to embed the filter map inside itself ($and), which
// made BSON encoding recurse until the process died with a stack overflow.
// Marshalling the produced filter is the exact client-side encoding path a
// live Mongo command takes, so this fails (fatally) on the old code without
// needing a running database.
func TestHumanItemCursorFilterEncodesWithoutSelfReference(t *testing.T) {
query := domain.HumanItemQuery{
PluginID: "annotation-qa",
ProjectID: "project-a",
UnassignedOnly: true,
Statuses: []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemSubmitted},
Cursor: &domain.HumanItemCursor{CreatedAt: "2026-10-09T00:00:00Z", ID: "human-item-1"},
}
filter := humanItemMongoFilter(domain.DefaultOrganizationID, query)
encoded, err := bson.Marshal(filter)
if err != nil {
t.Fatalf("cursor filter must encode: %v", err)
}
var round struct {
And []map[string]any `bson:"$and"`
}
if err := bson.Unmarshal(encoded, &round); err != nil {
t.Fatalf("cursor filter must decode: %v", err)
}
if len(round.And) != 2 {
t.Fatalf("cursor filter must AND base conditions with the cursor clause: %v", round.And)
}
base := round.And[0]
if base["organizationId"] != domain.DefaultOrganizationID || base["pluginId"] != "annotation-qa" {
t.Fatalf("base conditions must survive inside $and: %v", base)
}
if _, exists := base["$and"]; exists {
t.Fatalf("base filter must not reference itself: %v", base)
}
// Sanity: the non-cursor filter and the base inside $and agree on the
// pushed-down pool conditions.
if _, exists := base["assignedUserId"]; !exists {
t.Fatalf("pool pushdown lost inside $and: %v", base)
}
}
// TestHumanItemsListMongoCursorPagination runs the full MongoStore path
// (filter construction, real BSON commands, cursor decode) against an
// isolated throwaway database. Opt-in like the other *_mongo_test.go suites
// so CI without Mongo stays green; the always-run encoding guard above covers
// the crash class on every machine.
func TestHumanItemsListMongoCursorPagination(t *testing.T) {
if os.Getenv("MONGO_URI") == "" || os.Getenv("HUMAN_ITEM_CURSOR_INTEGRATION") != "1" {
t.Skip("opt-in Mongo cursor integration (set MONGO_URI and HUMAN_ITEM_CURSOR_INTEGRATION=1)")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")))
if err != nil {
t.Fatalf("Mongo connection error: %T", err)
}
defer client.Disconnect(ctx)
dbName := fmt.Sprintf("human_item_cursor_test_%d", time.Now().UnixNano())
if !strings.HasPrefix(dbName, "human_item_cursor_test_") || dbName == os.Getenv("MONGO_DB") {
t.Fatal("invalid isolated test database")
}
db := client.Database(dbName)
defer func() {
if err := db.Drop(context.Background()); err != nil {
t.Error("isolated database cleanup failed")
}
}()
dataStore := NewMongoStore(db)
for index := 0; index < 5; index++ {
item := domain.HumanItemRecord{
ID: fmt.Sprintf("human-item-cursor-%d", index),
OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa",
BatchID: "batch-cursor", ProjectID: "project-cursor", WorkflowID: "workflow-cursor",
CurrentStageID: "split_annotation", Status: domain.HumanItemReady,
Subject: domain.HumanSubjectRef{ItemKey: "item-key"},
Version: 1,
CreatedAt: time.Now().UTC().Add(time.Duration(index) * time.Minute).Format(time.RFC3339Nano),
}
item.UpdatedAt = item.CreatedAt
if err := dataStore.UpsertHumanItem(ctx, item); err != nil {
t.Fatal(err)
}
}
first, err := dataStore.ListHumanItems(ctx, domain.DefaultOrganizationID, domain.HumanItemQuery{
PluginID: "annotation-qa", Limit: 3, ListProjection: true,
})
if err != nil {
t.Fatal(err)
}
if len(first) != 3 {
t.Fatalf("first page size=%d", len(first))
}
second, err := dataStore.ListHumanItems(ctx, domain.DefaultOrganizationID, domain.HumanItemQuery{
PluginID: "annotation-qa", Limit: 3, ListProjection: true,
Cursor: &domain.HumanItemCursor{CreatedAt: first[len(first)-1].CreatedAt, ID: first[len(first)-1].ID},
})
if err != nil {
t.Fatal(err)
}
if len(second) != 2 {
t.Fatalf("second page size=%d", len(second))
}
seen := map[string]bool{}
for _, item := range append(append([]domain.HumanItemRecord{}, first...), second...) {
if seen[item.ID] {
t.Fatalf("cursor page repeated item %s", item.ID)
}
seen[item.ID] = true
}
}

View File

@ -1335,6 +1335,99 @@ func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organi
return items, nil
}
// memoryRunAcceptanceCandidate mirrors the Mongo ListRunsAcceptanceCandidates
// $or filter (see the api package's dashboardAcceptancePayloadFromArtifact
// detection paths) so both stores return the same candidate sets.
func memoryRunAcceptanceCandidate(run domain.RunRecord) bool {
for _, artifact := range run.Artifacts {
if artifact.Name == "poc-operator-full-acceptance-result.json" {
return true
}
if artifact.Data == nil {
continue
}
if _, ok := artifact.Data["acceptanceScope"]; ok {
return true
}
if value, ok := artifact.Data["value"].(map[string]any); ok {
if _, ok := value["acceptanceScope"]; ok {
return true
}
}
if preview, ok := artifact.Data["preview"].(map[string]any); ok {
if value, ok := preview["value"].(map[string]any); ok {
if _, ok := value["acceptanceScope"]; ok {
return true
}
}
}
}
return false
}
// ListRunsAcceptanceCandidates implements Store.ListRunsAcceptanceCandidates.
// Like the Mongo implementation it keeps the stored (non-hydrated) artifacts,
// matching the ListRuns semantics the dashboard scan used before.
func (store *MemoryStore) ListRunsAcceptanceCandidates(_ context.Context, organizationID string, limit int) ([]domain.RunRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
items := make([]domain.RunRecord, 0)
for _, run := range store.runs {
if organizationID != "" && !organizationMatches(run.OrganizationID, organizationID) {
continue
}
if memoryRunAcceptanceCandidate(run) {
items = append(items, clone(run))
}
}
sort.Slice(items, func(i, j int) bool {
return runFreshnessKey(items[i]) > runFreshnessKey(items[j])
})
if limit > 0 && len(items) > limit {
items = items[:limit]
}
return items, nil
}
// ListRunsWorkflowCandidates implements Store.ListRunsWorkflowCandidates.
func (store *MemoryStore) ListRunsWorkflowCandidates(_ context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error) {
workflowSet := make(map[string]bool, len(workflowNames))
for _, name := range workflowNames {
workflowSet[name] = true
}
projectSet := make(map[string]bool, len(projectNames))
for _, name := range projectNames {
projectSet[name] = true
}
store.mu.Lock()
defer store.mu.Unlock()
items := make([]domain.RunRecord, 0)
for _, run := range store.runs {
if organizationID != "" && !organizationMatches(run.OrganizationID, organizationID) {
continue
}
if workflowSet[run.WorkflowName] || projectSet[run.ProjectName] {
items = append(items, clone(run))
}
}
sort.Slice(items, func(i, j int) bool {
return runFreshnessKey(items[i]) > runFreshnessKey(items[j])
})
if limit > 0 && len(items) > limit {
items = items[:limit]
}
return items, nil
}
// runFreshnessKey mirrors dashboardLatestAcceptancePayload's recency key:
// UpdatedAt when present, CreatedAt otherwise.
func runFreshnessKey(run domain.RunRecord) string {
if key := strings.TrimSpace(run.UpdatedAt); key != "" {
return key
}
return strings.TrimSpace(run.CreatedAt)
}
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
func (store *MemoryStore) ArchiveTerminalRuns(_ context.Context, olderThan time.Time) (int, error) {
store.mu.Lock()

View File

@ -1112,6 +1112,91 @@ func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, org
return items, nil
}
// runRecencySort orders candidates exactly like the in-memory selector's
// freshness key (dashboardLatestAcceptancePayload / latestPocGPUWorkflowRun:
// UpdatedAt when present, CreatedAt otherwise), so even runs that never got
// an UpdatedAt written sort the same on both sides of the store boundary.
func runRecencySort() bson.D {
return bson.D{
{Key: "updatedAt", Value: bson.M{"$ifNull": bson.A{"$updatedAt", "$createdAt"}}},
{Key: "createdAt", Value: -1},
}
}
// ListRunsAcceptanceCandidates implements Store.ListRunsAcceptanceCandidates.
// The $or clauses mirror dashboardAcceptancePayloadFromArtifact's four
// detection paths (artifact name + data/preview.value/value.acceptanceScope),
// so the latest acceptance run is always inside the bounded result set while
// the in-memory selector keeps its exact picking semantics.
func (store *MongoStore) ListRunsAcceptanceCandidates(ctx context.Context, organizationID string, limit int) ([]domain.RunRecord, error) {
filter := bson.M{"$or": bson.A{
bson.M{"artifacts.name": "poc-operator-full-acceptance-result.json"},
bson.M{"artifacts.data.acceptanceScope": bson.M{"$exists": true}},
bson.M{"artifacts.data.value.acceptanceScope": bson.M{"$exists": true}},
bson.M{"artifacts.data.preview.value.acceptanceScope": bson.M{"$exists": true}},
}}
if organizationID != "" {
filter = organizationScopedFilter(organizationID, filter)
}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runArtifactProjection()).
SetSort(runRecencySort()).
SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
// ListRunsWorkflowCandidates implements Store.ListRunsWorkflowCandidates.
// Candidates need full documents (logs + node executions feed the preflight
// gate's log scan), but the name filters keep the result set tiny.
func (store *MongoStore) ListRunsWorkflowCandidates(ctx context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error) {
or := bson.A{}
if len(workflowNames) > 0 {
or = append(or, bson.M{"workflowName": bson.M{"$in": workflowNames}})
}
if len(projectNames) > 0 {
or = append(or, bson.M{"projectName": bson.M{"$in": projectNames}})
}
if len(or) == 0 {
return []domain.RunRecord{}, nil
}
filter := bson.M{"$or": or}
if organizationID != "" {
filter = organizationScopedFilter(organizationID, filter)
}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetSort(runRecencySort()).
SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
hydrateRunDerivedFields(&items[index])
}
return items, nil
}
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) {
filter := bson.M{

View File

@ -2,12 +2,19 @@ package store
import (
"context"
"sort"
"strings"
"time"
"droboticflow/internal/productionstats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"golang.org/x/sync/errgroup"
)
var productionStatsCache = &ttlResultCache[productionstats.Snapshot]{now: time.Now}
func (s *MemoryStore) GetProductionStatsScope(_ context.Context, org string) (productionstats.Scope, error) {
s.mu.Lock()
defer s.mu.Unlock()
@ -106,6 +113,25 @@ func (s *MemoryStore) LoadProductionStats(ctx context.Context, org string, proje
return d, nil
}
// LoadProductionStatsCached serves the production snapshot from a 45s in-process
// cache keyed by (org, scope projects). Consistency policy (S2): counters may
// lag by at most the TTL; the API-level response cache is flushed by every
// non-GET /api request (e.g. scope edits) on top of that.
func (s *MongoStore) LoadProductionStatsCached(ctx context.Context, org string, projects []string) (productionstats.Snapshot, error) {
keyProjects := append([]string(nil), projects...)
sort.Strings(keyProjects)
key := org + "|" + strings.Join(keyProjects, ",")
if snapshot, ok := productionStatsCache.get(key, dashboardStatsCacheTTL); ok {
return snapshot, nil
}
snapshot, err := s.LoadProductionStats(ctx, org, projects)
if err != nil {
return snapshot, err
}
productionStatsCache.set(key, snapshot, dashboardStatsCacheTTL)
return snapshot, nil
}
// Project-scoped, unpaginated and projected reads: do not load video/artifact
// payloads or inherit the automation list's 200-record UI limit. No migrations.
func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projects []string) (productionstats.Snapshot, error) {
@ -115,7 +141,7 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec
}
filter := organizationScopedFilter(org, bson.M{"projectId": bson.M{"$in": projects}})
read := func(c *mongo.Collection, f bson.M, projection bson.M, target any) error {
cursor, err := c.Find(ctx, f, options.Find().SetProjection(projection))
cursor, err := c.Find(ctx, f, options.Find().SetProjection(projection).SetMaxTime(15*time.Second))
if err != nil {
return err
}
@ -141,19 +167,24 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec
}
}
runFilter := organizationScopedFilter(org, bson.M{"$or": bson.A{bson.M{"projectId": bson.M{"$in": projects}}, bson.M{"taskId": bson.M{"$in": captureIDs}}, bson.M{"id": bson.M{"$in": runIDs}}}})
if err := read(s.runs, runFilter, fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome", "failure", "error"), &d.Runs); err != nil {
return d, err
}
if err := read(s.humanItems, filter, fields("id", "projectId", "supplierId", "runId", "batchId", "legacyTaskId", "currentStageId", "status", "pendingCombinedSubmissionId", "releasePartitionId", "processingJob.status"), &d.Items); err != nil {
return d, err
}
if err := read(s.humanBatches, filter, fields("id", "sourceBatchId", "stages"), &d.Batches); err != nil {
return d, err
}
if err := read(s.annotationTasks, filter, fields("id", "runId", "supplierId", "status"), &d.Tasks); err != nil {
return d, err
}
if err := read(s.annotationWorklogEvents, filter, fields("id", "taskId", "supplierId", "eventType", "mode", "temporary", "createdAt"), &d.Worklogs); err != nil {
var g errgroup.Group
g.Go(func() error {
return read(s.runs, runFilter, fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome", "failure", "error"), &d.Runs)
})
g.Go(func() error {
return read(s.humanItems, filter, fields("id", "projectId", "supplierId", "runId", "batchId", "legacyTaskId", "currentStageId", "status", "pendingCombinedSubmissionId", "releasePartitionId", "processingJob.status"), &d.Items)
})
g.Go(func() error { return read(s.humanBatches, filter, fields("id", "sourceBatchId", "stages"), &d.Batches) })
g.Go(func() error {
return read(s.annotationTasks, filter, fields("id", "runId", "supplierId", "status"), &d.Tasks)
})
g.Go(func() error {
return read(s.annotationWorklogEvents, filter, fields("id", "taskId", "supplierId", "eventType", "mode", "temporary", "createdAt"), &d.Worklogs)
})
g.Go(func() error {
return read(s.deliveryPackages, filter, fields("id", "parentRunId", "childRunId", "supplierId", "mode", "temporary", "finalExportedAt", "status", "itemIds", "releasePartitionId", "acceptanceHumanItemId", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.createdAt", "qualitySummary.prePackageAcceptance"), &d.Packages)
})
if err := g.Wait(); err != nil {
return d, err
}
ids := make([]string, 0, len(d.Items))
@ -165,8 +196,5 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec
return d, err
}
}
if err := read(s.deliveryPackages, filter, fields("id", "parentRunId", "childRunId", "supplierId", "mode", "temporary", "finalExportedAt", "status", "itemIds", "releasePartitionId", "acceptanceHumanItemId", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.createdAt", "qualitySummary.prePackageAcceptance"), &d.Packages); err != nil {
return d, err
}
return d, nil
}

View File

@ -102,3 +102,73 @@ func TestProductionStatsMongoPersistenceAndReadonlyProjection(t *testing.T) {
t.Fatalf("report mutated source %+v %v", still, err)
}
}
func TestLoadProductionStatsCachedMatchesDirectAndServesWithinTTL(t *testing.T) {
uri := os.Getenv("DASHBOARD_STATS_MONGO_TEST_URI")
if uri == "" {
t.Skip("isolated Mongo required: DASHBOARD_STATS_MONGO_TEST_URI=mongodb://127.0.0.1:27028")
}
if uri != "mongodb://127.0.0.1:27028" {
t.Fatal("only isolated loopback test server allowed")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("production_cache_test_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
s := NewMongoStore(db)
seed := func() {
if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "c1", OrganizationID: "o", ProjectID: "p", RunID: "r1"}); err != nil {
t.Fatal(err)
}
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "r1", OrganizationID: "o", ProjectID: "p", Status: domain.RunStatusWaitingForHuman}); err != nil {
t.Fatal(err)
}
if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "hi1", OrganizationID: "o", ProjectID: "p", RunID: "r1", CurrentStageID: "quality_review", Status: domain.HumanItemReady}); err != nil {
t.Fatal(err)
}
}
seed()
direct, err := s.LoadProductionStats(ctx, "o", []string{"p"})
if err != nil {
t.Fatal(err)
}
cached, err := s.LoadProductionStatsCached(ctx, "o", []string{"p"})
if err != nil {
t.Fatal(err)
}
if len(cached.Captures) != len(direct.Captures) || len(cached.Runs) != len(direct.Runs) || len(cached.Items) != len(direct.Items) {
t.Fatalf("cached snapshot must match the direct load: cached=%d/%d/%d direct=%d/%d/%d",
len(cached.Captures), len(cached.Runs), len(cached.Items), len(direct.Captures), len(direct.Runs), len(direct.Items))
}
builtDirect := productionstats.Build(direct)
builtCached := productionstats.Build(cached)
if len(builtDirect.Sources) != len(builtCached.Sources) {
t.Fatalf("aggregation parity: built sources differ (%d vs %d)", len(builtDirect.Sources), len(builtCached.Sources))
}
// Within the TTL the cached snapshot is served as-is even after new writes.
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "r2", OrganizationID: "o", ProjectID: "p", Status: domain.RunStatusCompleted}); err != nil {
t.Fatal(err)
}
stale, err := s.LoadProductionStatsCached(ctx, "o", []string{"p"})
if err != nil {
t.Fatal(err)
}
if len(stale.Runs) != len(direct.Runs) {
t.Fatalf("expected the cached snapshot within the TTL, got %d runs", len(stale.Runs))
}
refreshed, err := s.LoadProductionStats(ctx, "o", []string{"p"})
if err != nil {
t.Fatal(err)
}
if len(refreshed.Runs) != len(direct.Runs)+1 {
t.Fatalf("direct load must bypass the cache, got %d runs", len(refreshed.Runs))
}
}

View File

@ -122,6 +122,24 @@ type Store interface {
// every run in the organization (see account-phone-login-era perf fix:
// large orgs made the unscoped org-wide scan hang for minutes).
ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error)
// ListRunsAcceptanceCandidates is the bounded point-query replacement for
// dashboardPOCReadiness's former full ListRuns scan (slow-page report S4):
// it returns only runs that can possibly carry a POC acceptance artifact
// (matching dashboardAcceptancePayloadFromArtifact's detection paths), most
// recently updated first, capped at limit. organizationID "" means unscoped,
// mirroring ListRuns.
ListRunsAcceptanceCandidates(ctx context.Context, organizationID string, limit int) ([]domain.RunRecord, error)
// ListRunsWorkflowCandidates is the bounded point-query replacement for the
// same scan's GPU-workflow-preflight half: runs whose workflowName or
// projectName matches one of the POC/live acceptance names, most recently
// updated first, capped at limit. organizationID "" means unscoped.
ListRunsWorkflowCandidates(ctx context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error)
// ListHumanBatchIDsAwaitingRelease backs the background release-partition
// sweeper that replaced the GET /api/human-batches read-path flush (S4):
// it returns the distinct batch IDs that still have accepted items without
// a release partition, i.e. exactly the batches flushHumanReleasePartitions
// could act on.
ListHumanBatchIDsAwaitingRelease(ctx context.Context) ([]string, error)
// ArchiveTerminalRuns sets ArchivedAt on every not-yet-archived run whose
// status is completed/failed/cancelled and whose EndedAt is older than
// olderThan. Archiving never deletes the record — it only excludes it

View File

@ -2,12 +2,17 @@ package store
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"time"
)
func workforceTaskIDs(d workforcestats.Snapshot) []string {
@ -118,6 +123,266 @@ func (s *MemoryStore) LoadWorkforceStats(_ context.Context, org string, q workfo
return d, nil
}
// DashboardHumanWorkStats carries only the counters the dashboard human-work
// enrichment needs. It lets the API avoid materializing every human item and
// delivery package just to count them (diagnosis report C4).
type DashboardHumanWorkStats struct {
PendingAnnotation int
PendingQuality int
InAnnotation int
PendingAcceptanceItems int
Accepted int
Rejected int
Escalated int
VideoCount int
CaptureDurationSeconds int64
PackageCount int
PackagesPendingAcceptance int
}
func classifyHumanItemForDashboard(status domain.HumanItemStatus, stageID string) string {
switch status {
case domain.HumanItemReady, domain.HumanItemRework:
if stageID == "quality_review" || stageID == domain.FanyuV5StageCutReview || stageID == domain.FanyuV5StageSemanticReview {
return "pendingQuality"
}
return "pendingAnnotation"
case domain.HumanItemClaimed, domain.HumanItemInProgress:
return "inAnnotation"
case domain.HumanItemSubmitted:
if stageID == "quality_review" || stageID == domain.FanyuV5StageCutReview || stageID == domain.FanyuV5StageSemanticReview {
return "pendingQuality"
}
if stageID == "acceptance" {
return "pendingAcceptance"
}
case domain.HumanItemAccepted:
return "accepted"
case domain.HumanItemRejected:
return "rejected"
case domain.HumanItemEscalated:
return "escalated"
}
return ""
}
// MemoryStore implementation mirrors the legacy enrich loop exactly so the
// in-memory replica stays a faithful parity oracle for the Mongo pipeline.
func (s *MemoryStore) DashboardHumanWorkStats(_ context.Context, organizationID string) (DashboardHumanWorkStats, error) {
s.mu.Lock()
defer s.mu.Unlock()
stats := DashboardHumanWorkStats{}
seenVideos := map[string]struct{}{}
for _, item := range s.humanItems {
if !humanOrganizationMatches(item.OrganizationID, organizationID) {
continue
}
if key := strings.TrimSpace(item.Subject.ItemKey); key != "" {
seenVideos[key] = struct{}{}
}
if duration, ok := item.Subject.Payload["durationSeconds"].(float64); ok && duration > 0 {
stats.CaptureDurationSeconds += int64(duration)
}
switch classifyHumanItemForDashboard(item.Status, item.CurrentStageID) {
case "pendingQuality":
stats.PendingQuality++
case "pendingAnnotation":
stats.PendingAnnotation++
case "inAnnotation":
stats.InAnnotation++
case "pendingAcceptance":
stats.PendingAcceptanceItems++
case "accepted":
stats.Accepted++
case "rejected":
stats.Rejected++
case "escalated":
stats.Escalated++
}
}
stats.VideoCount = len(seenVideos)
packageQuery := domain.DeliveryPackageQuery{Mode: domain.RunModeProduction}
for _, item := range s.deliveryPackages {
if organizationMatches(item.OrganizationID, organizationID) && deliveryPackageMatches(item, packageQuery) {
stats.PackageCount++
if item.Status == domain.DeliveryPackagePendingAcceptance {
stats.PackagesPendingAcceptance++
}
}
}
return stats, nil
}
type humanWorkAggregateRow struct {
ID struct {
Status string `bson:"s"`
Stage string `bson:"c"`
} `bson:"_id"`
Count int32 `bson:"n"`
}
// MongoStore implementation pushes the counting into aggregation pipelines so
// the dashboard never materializes humanItems/deliveryPackages documents.
func (s *MongoStore) DashboardHumanWorkStats(ctx context.Context, organizationID string) (DashboardHumanWorkStats, error) {
stats := DashboardHumanWorkStats{}
baseFilter := humanItemMongoFilter(organizationID, domain.HumanItemQuery{})
rows := []humanWorkAggregateRow{}
cursor, err := s.humanItems.Aggregate(ctx, bson.A{
bson.M{"$match": baseFilter},
bson.M{"$group": bson.M{"_id": bson.M{"s": "$status", "c": "$currentStageId"}, "n": bson.M{"$sum": 1}}},
})
if err != nil {
return stats, err
}
if err := cursor.All(ctx, &rows); err != nil {
return stats, err
}
for _, row := range rows {
count := int(row.Count)
switch classifyHumanItemForDashboard(domain.HumanItemStatus(row.ID.Status), row.ID.Stage) {
case "pendingQuality":
stats.PendingQuality += count
case "pendingAnnotation":
stats.PendingAnnotation += count
case "inAnnotation":
stats.InAnnotation += count
case "pendingAcceptance":
stats.PendingAcceptanceItems += count
case "accepted":
stats.Accepted += count
case "rejected":
stats.Rejected += count
case "escalated":
stats.Escalated += count
}
}
videoRows := []struct {
Count int32 `bson:"n"`
}{}
cursor, err = s.humanItems.Aggregate(ctx, bson.A{
bson.M{"$match": baseFilter},
bson.M{"$group": bson.M{"_id": "$subject.itemKey"}},
bson.M{"$match": bson.M{"_id": bson.M{"$nin": bson.A{"", nil}}}},
bson.M{"$count": "n"},
})
if err != nil {
return stats, err
}
if err := cursor.All(ctx, &videoRows); err != nil {
return stats, err
}
if len(videoRows) > 0 {
stats.VideoCount = int(videoRows[0].Count)
}
durationRows := []struct {
Total float64 `bson:"d"`
}{}
cursor, err = s.humanItems.Aggregate(ctx, bson.A{
bson.M{"$match": baseFilter},
// Legacy semantics: only positive numeric durationSeconds values count,
// each truncated toward zero exactly like the Go int64(duration) cast.
bson.M{"$group": bson.M{"_id": nil, "d": bson.M{"$sum": bson.M{"$trunc": bson.M{"$convert": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$gt": bson.A{"$subject.payload.durationSeconds", 0}}, "$subject.payload.durationSeconds", 0}}, "to": "double", "onError": 0, "onNull": 0}}}}}},
})
if err != nil {
return stats, err
}
if err := cursor.All(ctx, &durationRows); err != nil {
return stats, err
}
if len(durationRows) > 0 {
stats.CaptureDurationSeconds = int64(durationRows[0].Total)
}
total, err := s.deliveryPackages.CountDocuments(ctx, deliveryPackageMongoFilter(organizationID, domain.DeliveryPackageQuery{Mode: domain.RunModeProduction}))
if err != nil {
return stats, err
}
stats.PackageCount = int(total)
pending, err := s.deliveryPackages.CountDocuments(ctx, deliveryPackageMongoFilter(organizationID, domain.DeliveryPackageQuery{Mode: domain.RunModeProduction, Status: domain.DeliveryPackagePendingAcceptance}))
if err != nil {
return stats, err
}
stats.PackagesPendingAcceptance = int(pending)
return stats, nil
}
// ttlResultCache is a tiny in-process TTL cache. Consistency policy (diagnosis
// report C4/S2): dashboard and production counters tolerate short staleness,
// so entries simply expire; every non-GET /api request additionally flushes the
// API-level response cache (see internal/api). Bounded to avoid growth.
type ttlResultCache[T any] struct {
mu sync.Mutex
entries map[string]ttlResultCacheEntry[T]
now func() time.Time
}
type ttlResultCacheEntry[T any] struct {
value T
expires time.Time
}
func (c *ttlResultCache[T]) get(key string, ttl time.Duration) (T, bool) {
c.mu.Lock()
defer c.mu.Unlock()
var zero T
if c.entries == nil {
return zero, false
}
entry, ok := c.entries[key]
if !ok || c.now().After(entry.expires) {
return zero, false
}
return entry.value, true
}
func (c *ttlResultCache[T]) set(key string, value T, ttl time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
if c.entries == nil {
c.entries = map[string]ttlResultCacheEntry[T]{}
}
if len(c.entries) >= 128 {
c.entries = map[string]ttlResultCacheEntry[T]{}
}
c.entries[key] = ttlResultCacheEntry[T]{value: value, expires: c.now().Add(ttl)}
}
const dashboardStatsCacheTTL = 45 * time.Second
var workforceStatsCache = &ttlResultCache[workforcestats.Snapshot]{now: time.Now}
// LoadWorkforceStatsCached caches the 30-day workforce snapshot per (org, query)
// for 45s. The window key is quantized to one minute so the default rolling
// dashboard range still hits the cache; the cached snapshot may therefore lag
// boundary events by up to TTL plus one minute. Falls back to an exact query on
// miss. MemoryStore intentionally has no cached variant: in-process tests and
// the local replica expect read-after-write freshness.
func (s *MongoStore) LoadWorkforceStatsCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) {
allowed := make([]string, 0, len(q.AllowedProjects))
for id, on := range q.AllowedProjects {
if on {
allowed = append(allowed, id)
}
}
sort.Strings(allowed)
fromKey, toKey := int64(0), int64(0)
if !q.From.IsZero() {
fromKey = q.From.Unix() / 60
}
if !q.To.IsZero() {
toKey = q.To.Unix() / 60
}
key := fmt.Sprintf("%s|%d|%d|%s|%s|%s|%v|%v|%s", org, fromKey, toKey, q.ProjectID, q.SupplierID, q.UserID, q.MatchUser, q.Details, strings.Join(allowed, ","))
if snapshot, ok := workforceStatsCache.get(key, dashboardStatsCacheTTL); ok {
return snapshot, nil
}
snapshot, err := s.LoadWorkforceStats(ctx, org, q)
if err != nil {
return snapshot, err
}
workforceStatsCache.set(key, snapshot, dashboardStatsCacheTTL)
return snapshot, nil
}
// Read-only, time-bounded event queries and batched joins. No index migrations or
// customer data rewrites. History queries project out media and annotation text.
func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) {

View File

@ -137,3 +137,141 @@ func TestWorkforceStatsMongoReplicaParity(t *testing.T) {
t.Fatalf("Mongo/Memory mismatch\n%+v\n%+v", actual, expected)
}
}
func TestDashboardHumanWorkStatsParityBetweenMemoryAndMongoAggregation(t *testing.T) {
uri := os.Getenv("DASHBOARD_STATS_MONGO_TEST_URI")
if uri == "" {
t.Skip("isolated Mongo required: DASHBOARD_STATS_MONGO_TEST_URI=mongodb://127.0.0.1:27028")
}
if uri != "mongodb://127.0.0.1:27028" {
t.Fatal("only isolated loopback test server allowed")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("dashboard_stats_test_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
memory := NewMemoryStore()
mongoStore := NewMongoStore(db)
seed := func(s Store) {
helper := &testing.T{}
_ = helper
items := []domain.HumanItemRecord{
{ID: "i1", OrganizationID: "o", CurrentStageID: "annotation", Status: domain.HumanItemReady,
Subject: domain.HumanSubjectRef{ItemKey: "v-a", Payload: map[string]any{"durationSeconds": 30.5}}},
{ID: "i2", OrganizationID: "o", CurrentStageID: "quality_review", Status: domain.HumanItemRework,
Subject: domain.HumanSubjectRef{ItemKey: "v-a", Payload: map[string]any{"durationSeconds": 12.5}}},
{ID: "i3", OrganizationID: "o", CurrentStageID: "annotation", Status: domain.HumanItemInProgress,
Subject: domain.HumanSubjectRef{ItemKey: "v-b"}},
{ID: "i4", OrganizationID: "o", CurrentStageID: domain.FanyuV5StageCutReview, Status: domain.HumanItemSubmitted,
Subject: domain.HumanSubjectRef{ItemKey: "v-b", Payload: map[string]any{"durationSeconds": 7.5}}},
{ID: "i5", OrganizationID: "o", CurrentStageID: "acceptance", Status: domain.HumanItemSubmitted,
Subject: domain.HumanSubjectRef{ItemKey: "v-c", Payload: map[string]any{"durationSeconds": -3.0}}},
{ID: "i6", OrganizationID: "o", CurrentStageID: "done", Status: domain.HumanItemAccepted,
Subject: domain.HumanSubjectRef{ItemKey: "v-c", Payload: map[string]any{"durationSeconds": 0}}},
{ID: "i7", OrganizationID: "o", CurrentStageID: "done", Status: domain.HumanItemRejected,
Subject: domain.HumanSubjectRef{ItemKey: "", Payload: map[string]any{"durationSeconds": "broken"}}},
{ID: "i8", OrganizationID: "o", CurrentStageID: "annotation", Status: domain.HumanItemEscalated,
Subject: domain.HumanSubjectRef{ItemKey: ""}},
{ID: "i9", OrganizationID: "foreign", CurrentStageID: "annotation", Status: domain.HumanItemReady,
Subject: domain.HumanSubjectRef{ItemKey: "v-x", Payload: map[string]any{"durationSeconds": 999.0}}},
}
for _, item := range items {
if err := s.UpsertHumanItem(ctx, item); err != nil {
t.Fatal(err)
}
}
packages := []domain.DeliveryPackageRecord{
{ID: "p1", OrganizationID: "o", ProjectID: "proj", Status: domain.DeliveryPackagePendingAcceptance},
{ID: "p2", OrganizationID: "o", ProjectID: "proj", Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction},
{ID: "p3", OrganizationID: "o", ProjectID: "proj", Status: domain.DeliveryPackagePendingAcceptance, Mode: domain.RunModeDebug},
{ID: "p4", OrganizationID: "foreign", ProjectID: "proj", Status: domain.DeliveryPackagePendingAcceptance},
}
for _, pkg := range packages {
if err := s.UpsertDeliveryPackage(ctx, pkg); err != nil {
t.Fatal(err)
}
}
}
seed(memory)
seed(mongoStore)
fromMemory, err := memory.DashboardHumanWorkStats(ctx, "o")
if err != nil {
t.Fatal(err)
}
fromMongo, err := mongoStore.DashboardHumanWorkStats(ctx, "o")
if err != nil {
t.Fatal(err)
}
if fromMemory != fromMongo {
t.Fatalf("memory and mongo counters diverge:\nmemory=%+v\nmongo =%+v", fromMemory, fromMongo)
}
if fromMemory.PendingAnnotation != 1 || fromMemory.PendingQuality != 2 || fromMemory.InAnnotation != 1 ||
fromMemory.PendingAcceptanceItems != 1 || fromMemory.Accepted != 1 || fromMemory.Rejected != 1 ||
fromMemory.Escalated != 1 || fromMemory.VideoCount != 3 || fromMemory.CaptureDurationSeconds != 49 ||
fromMemory.PackageCount != 2 || fromMemory.PackagesPendingAcceptance != 1 {
t.Fatalf("unexpected counters: %+v", fromMemory)
}
// Legacy full-materialization oracle: ListHumanItems + ListDeliveryPackages.
items, err := memory.ListHumanItems(ctx, "o", domain.HumanItemQuery{})
if err != nil {
t.Fatal(err)
}
packages, err := memory.ListDeliveryPackages(ctx, "o", domain.DeliveryPackageQuery{Mode: domain.RunModeProduction})
if err != nil {
t.Fatal(err)
}
if len(items) != 8 || len(packages) != 2 {
t.Fatalf("oracle fixture sizes changed: items=%d packages=%d", len(items), len(packages))
}
}
func TestLoadWorkforceStatsCachedServesSnapshotWithinTTL(t *testing.T) {
uri := os.Getenv("DASHBOARD_STATS_MONGO_TEST_URI")
if uri == "" {
t.Skip("isolated Mongo required: DASHBOARD_STATS_MONGO_TEST_URI=mongodb://127.0.0.1:27028")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
if err != nil {
t.Fatal(err)
}
defer client.Disconnect(context.Background())
db := client.Database(fmt.Sprintf("workforce_cache_test_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
s := NewMongoStore(db)
from := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
to := from.Add(24 * time.Hour)
q := workforcestats.Query{From: from, To: to}
first, err := s.LoadWorkforceStatsCached(ctx, "cache-org", q)
if err != nil {
t.Fatal(err)
}
if err := s.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "new-event", OrganizationID: "cache-org", ProjectID: "p", TaskID: "t", EventType: "annotation_submitted", CreatedAt: "2026-09-01T12:00:00Z"}); err != nil {
t.Fatal(err)
}
cached, err := s.LoadWorkforceStatsCached(ctx, "cache-org", q)
if err != nil {
t.Fatal(err)
}
if len(cached.Events) != len(first.Events) {
t.Fatalf("expected the cached snapshot within the TTL, got %d events vs %d", len(cached.Events), len(first.Events))
}
fresh, err := s.LoadWorkforceStats(ctx, "cache-org", q)
if err != nil {
t.Fatal(err)
}
if len(fresh.Events) != len(first.Events)+1 {
t.Fatalf("direct load must bypass the cache, got %d events", len(fresh.Events))
}
}

View File

@ -14,8 +14,13 @@ export function frontendManualChunk(id = "") {
return undefined;
}
// motion is only reachable from lazily loaded pages (WorkflowEditor,
// Monitoring, Audit, Runs). Pinning it into a manual chunk promotes the
// package plus hoisted react interop modules into the application entry
// chunk via modulepreload. Let Rollup keep it inside the dynamic graph so
// it is only fetched when one of those pages loads.
if (id.includes("/motion/")) {
return "motion-vendor";
return undefined;
}
if (id.includes("lucide-react")) {

View File

@ -14,6 +14,68 @@ type LazyVideoPreviewProps = {
openOriginalLabel?: string;
};
/**
* API content endpoints that can serve video either directly from object
* storage (a presigned redirect, cheap) or through the server-side
* browser-transcode proxy (transcode=browser, expensive). Mirrors the URL
* shapes recognized by src/lib/video-preview.js.
*/
function isAPIVideoContentURL(pathname: string): boolean {
const isCollectionContent =
pathname.startsWith("/api/source-data/collections/") && pathname.endsWith("/content");
const isPackageContent = pathname.startsWith("/api/packages/") && pathname.endsWith("/content");
const isObjectStorageContent = pathname === "/api/object-storage/content";
const isRunArtifactContent =
pathname.startsWith("/api/runs/") &&
pathname.includes("/artifacts/") &&
pathname.endsWith("/content");
const isHumanEvidenceContent =
(pathname.startsWith("/api/plugins/human-loop/items/") || pathname.startsWith("/api/human-items/")) &&
pathname.includes("/evidence/") &&
pathname.endsWith("/content");
return (
isCollectionContent ||
isPackageContent ||
isObjectStorageContent ||
isRunArtifactContent ||
isHumanEvidenceContent
);
}
/**
* Given a `transcode=browser` proxy URL, return the same URL without the
* transcode parameter. The API then redirects (307) to a presigned object
* storage URL so the video bytes stream straight from storage instead of
* passing through the API process. Returns null when the src is not one of
* the proxyable API content URLs.
*/
function directVideoURL(src: string): string | null {
try {
const url = new URL(src, window.location.origin);
if (!isAPIVideoContentURL(url.pathname)) return null;
if (url.searchParams.get("transcode") !== "browser") return null;
url.searchParams.delete("transcode");
const query = url.searchParams.toString();
return `${url.pathname}${query ? `?${query}` : ""}${url.hash}`;
} catch {
return null;
}
}
/** Force the server-side transcode path and its blocking stream fallback. */
function proxiedVideoURL(src: string): string {
try {
const url = new URL(src, window.location.origin);
if (!isAPIVideoContentURL(url.pathname)) return src;
url.searchParams.set("transcode", "browser");
url.searchParams.set("sync", "1");
const query = url.searchParams.toString();
return `${url.pathname}${query ? `?${query}` : ""}${url.hash}`;
} catch {
return src;
}
}
export function LazyVideoPreview({
src,
originalSrc,
@ -26,11 +88,17 @@ export function LazyVideoPreview({
}: LazyVideoPreviewProps) {
const [activated, setActivated] = React.useState(false);
const [mediaError, setMediaError] = React.useState(false);
// Direct playback (presigned redirect) first; the server-side transcode
// proxy is the retry when the browser cannot decode the original codec.
const [useProxy, setUseProxy] = React.useState(false);
const directSrc = directVideoURL(src);
const playbackSrc = !useProxy && directSrc ? directSrc : proxiedVideoURL(src);
const fallbackURL = originalSrc || src;
React.useEffect(() => {
setActivated(false);
setMediaError(false);
setUseProxy(false);
}, [src]);
if (mediaError) {
@ -77,10 +145,17 @@ export function LazyVideoPreview({
return (
<video
src={src}
key={playbackSrc}
src={playbackSrc}
controls
preload="metadata"
onError={() => setMediaError(true)}
onError={() => {
if (!useProxy && directSrc) {
setUseProxy(true);
return;
}
setMediaError(true);
}}
className={cn("max-h-[420px] w-full rounded-lg bg-black", videoClassName)}
/>
);

View File

@ -986,6 +986,12 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
return;
}
const knownProjects = projects.length > 0 ? projects : await projectsApi.list();
// slow-page diagnosis S6: an unscoped manager must not fan out
// listMembers+listWorkflows per project (project count × 2 requests,
// each paying the auth chain). Managers skip the fan-out entirely;
// non-managers keep it because their claim permission falls back to
// cross-project membership lookups (canCurrentUserClaimItem).
const fanOutProjectScope = !projectId && !isManager;
const humanFilters = {
pluginId: "annotation-qa",
projectId: projectId || undefined,
@ -1013,14 +1019,18 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
] = await Promise.all([
projectId
? projectsApi.listMembers(projectId)
: Promise.allSettled(knownProjects.map((p) => projectsApi.listMembers(p.id))).then((results) =>
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
),
: fanOutProjectScope
? Promise.allSettled(knownProjects.map((p) => projectsApi.listMembers(p.id))).then((results) =>
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
)
: Promise.resolve([] as ProjectMember[]),
projectId
? projectsApi.listWorkflows(projectId)
: Promise.allSettled(knownProjects.map((p) => projectsApi.listWorkflows(p.id))).then((results) =>
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
),
: fanOutProjectScope
? Promise.allSettled(knownProjects.map((p) => projectsApi.listWorkflows(p.id))).then((results) =>
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
)
: Promise.resolve([] as Workflow[]),
annotationOperationsApi.listTemplates(projectId || undefined),
annotationOperationsApi.listTasks({
projectId: projectId || undefined,
@ -2098,6 +2108,41 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
};
}, [detail?.task.nodeId, selectedMaterialization]);
const selectedRequiredRole = assignmentRoleForStage(selectedHumanItem?.currentStageId || annotationStage);
// S6 follow-up: with the per-project fan-out skipped for unscoped managers,
// reassignment (eligibleAssignees) would lose its member roster. Load it on
// demand — exactly one request for the selected human item's project.
const scopedMembersProjectRef = useRef("");
const scopedMembersProjectId = selectedHumanItem?.projectId || "";
useEffect(() => {
if (projectId || !isManager || !scopedMembersProjectId) return;
if (scopedMembersProjectRef.current === scopedMembersProjectId) return;
scopedMembersProjectRef.current = scopedMembersProjectId;
let cancelled = false;
// Two requests for one project — still no per-project fan-out.
projectsApi
.listMembers(scopedMembersProjectId)
.then((members) => {
if (!cancelled) setProjectMembers(members);
})
.catch(() => {
// Roster stays empty; the assignee dropdown shows its empty state.
});
projectsApi
.listWorkflows(scopedMembersProjectId)
.then((scopedWorkflows) => {
if (!cancelled) {
setWorkflows((current) => {
// Merge: keep anything the scoped loader already provided.
const known = new Set(current.map((workflow) => workflow.id));
return [...current, ...scopedWorkflows.filter((workflow) => !known.has(workflow.id))];
});
}
})
.catch(() => {});
return () => {
cancelled = true;
};
}, [projectId, isManager, scopedMembersProjectId]);
const eligibleAssigneesForScope = useCallback(
(supplierId: string | undefined, requiredRole: HumanAssignmentRole, currentAssigneeId = "") => {
if (!isManager) return [];

View File

@ -11,25 +11,59 @@ type HumanItemFilters = {
stageId?: string;
status?: HumanItemStatus;
pool?: "available" | "mine" | "all" | "reviewed";
/** Server default is 500 (max 1000); pass explicitly for larger pools. */
limit?: number;
/** Opaque continuation cursor from a previous page response. */
cursor?: string;
};
const queryString = (filters: Record<string, string | undefined>) => {
const queryString = (filters: Record<string, string | number | undefined>) => {
const params = new URLSearchParams();
Object.entries(filters).forEach(([key, value]) => {
if (value) params.set(key, value);
if (value !== undefined && value !== "") params.set(key, String(value));
});
const query = params.toString();
return query ? `?${query}` : "";
};
type HumanItemPage = {
items: HumanItem[];
/** Opaque cursor for the next page; absent when there are no more pages. */
nextCursor?: string;
hasMore?: boolean;
};
const listItemsPage = async (filters: HumanItemFilters = {}) =>
apiRequest<HumanItemPage>(`/api/human-items${queryString(filters)}`);
const listItems = async (filters: HumanItemFilters = {}) =>
(await apiRequest<{ items: HumanItem[] }>(`/api/human-items${queryString(filters)}`)).items;
(await listItemsPage(filters)).items;
/**
* Fetch every page behind the cursor so pools larger than the server's
* default page size (500, max 1000) are not silently truncated.
*/
async function listAllItems(filters: HumanItemFilters = {}) {
const items = new Map<string, HumanItem>();
let cursor: string | undefined;
// Hard cap: 200 pages × up to 1000 items per page guards against a
// server bug looping on the same cursor.
for (let page = 0; page < 200; page += 1) {
const result = await listItemsPage({ ...filters, cursor });
for (const item of result.items) {
if (!items.has(item.id) || item.version > items.get(item.id)!.version) items.set(item.id, item);
}
if (!result.hasMore || !result.nextCursor) break;
cursor = result.nextCursor;
}
return [...items.values()];
}
async function listAcceptancePool(filters: HumanItemFilters) {
// Keep the legacy package acceptance stage alongside prepackage acceptance.
// Both requests retain the server's actor, project and supplier checks.
const pages = await Promise.all(
["acceptance", "prepackage_acceptance"].map((stageId) => listItems({ ...filters, stageId })),
["acceptance", "prepackage_acceptance"].map((stageId) => listAllItems({ ...filters, stageId })),
);
const items = new Map<string, HumanItem>();
for (const item of pages.flat()) {
@ -40,6 +74,8 @@ async function listAcceptancePool(filters: HumanItemFilters) {
export const humanLoopApi = {
listItems,
listItemsPage,
listAllItems,
listAcceptancePools: async (
filters: Omit<HumanItemFilters, "stageId" | "pool">,
actorId: string,

View File

@ -14,8 +14,28 @@ export type AccountHierarchyIssue = {
export const usersApi = {
auditHierarchy: async () =>
(await apiRequest<{ items: AccountHierarchyIssue[] }>("/api/users/hierarchy-audit")).items,
list: async (includeDisabled = false) =>
(await apiRequest<{ items: User[] }>(includeDisabled ? "/api/users?includeDisabled=true" : "/api/users")).items,
/**
* One page of users. The server caps pages at 1000 items (default 200)
* and reports `hasMore: true` when the cap truncated the result.
*/
listPage: async (options: { includeDisabled?: boolean; limit?: number; q?: string } = {}) => {
const params = new URLSearchParams();
if (options.includeDisabled) params.set("includeDisabled", "true");
if (options.limit) params.set("limit", String(options.limit));
if (options.q) params.set("q", options.q);
const suffix = params.toString() ? `?${params.toString()}` : "";
return apiRequest<{ items: User[]; hasMore?: boolean }>(`/api/users${suffix}`);
},
list: async (includeDisabled = false) => {
// Ask for the server's maximum page so organizations with 200–1000
// active users are not silently truncated by the 200 default. Larger
// organizations surface an explicit error instead of a partial list.
const page = await usersApi.listPage({ includeDisabled, limit: 1000 });
if (page.hasMore) {
throw new Error("用户数量超过单页上限(1000),列表被截断,请联系管理员分批查看");
}
return page.items;
},
create: (
payload: Pick<User, "name" | "role"> & {
phone: string;

View File

@ -1,6 +1,7 @@
import tailwindcss from "@tailwindcss/vite";
import react from "@vitejs/plugin-react";
import { createReadStream, existsSync, statSync } from "node:fs";
import { createReadStream, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { gzipSync } from "node:zlib";
import path from "path";
import { defineConfig } from "vite";
import { frontendManualChunk } from "./scripts/lib/frontend-chunking.mjs";
@ -9,6 +10,8 @@ import { viteProxyTargets } from "./scripts/lib/service-endpoints.mjs";
export default defineConfig(() => {
const proxyTargets = viteProxyTargets(process.env);
const captureWorkbenchMockVideo = process.env.CAPTURE_WORKBENCH_MOCK_VIDEO?.trim();
let resolvedOutDir: string | undefined;
const compressedTargets: string[] = [];
return {
plugins: [
@ -47,6 +50,41 @@ export default defineConfig(() => {
});
},
},
{
// Emit precompressed .gz siblings for JS/CSS assets. The Go SPA
// handler (cmd/api/main.go) serves them directly to clients that
// send Accept-Encoding: gzip, avoiding on-the-fly compression in
// the API pod.
//
// Must run in writeBundle reading the final on-disk files: in
// rolldown-vite the import-analysis transform that injects
// __vite__mapDeps runs after generateBundle, so compressing
// output.code there would produce .gz files that no longer match
// the emitted .js and crash gzip clients with ReferenceError.
name: "precompress-static-assets",
apply: "build",
configResolved(config) {
resolvedOutDir = config.build.outDir;
},
generateBundle(_options, bundle) {
compressedTargets.length = 0;
for (const fileName of Object.keys(bundle)) {
if (/\.(js|css)$/.test(fileName)) {
compressedTargets.push(fileName);
}
}
},
writeBundle() {
const outDir = path.resolve(process.cwd(), resolvedOutDir ?? "dist");
for (const fileName of compressedTargets) {
if (!fileName) {
continue;
}
const target = path.join(outDir, fileName);
writeFileSync(`${target}.gz`, gzipSync(readFileSync(target), { level: 9 }));
}
},
},
],
build: {
rollupOptions: {