forked from longtao.wu/droboticflow
⚡ Fix slow page loading: static gzip, list pagination, auth cache, dashboard aggregation, video direct link
- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix - C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown - C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation - C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache - C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback - S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed - Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush - Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500 Co-authored-by: AgentTeams slow-page-fix
This commit is contained in:
parent
b9ce40d1ae
commit
da446d13b5
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
@ -61,6 +62,9 @@ func spaHandler(apiHandler http.Handler, distDir string) http.Handler {
|
||||
if strings.HasPrefix(request.URL.Path, "/assets/") {
|
||||
writer.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
|
||||
}
|
||||
if servePrecompressedAsset(writer, request, targetPath) {
|
||||
return
|
||||
}
|
||||
fileServer.ServeHTTP(writer, request)
|
||||
return
|
||||
}
|
||||
@ -73,6 +77,60 @@ func spaHandler(apiHandler http.Handler, distDir string) http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
var precompressedContentTypes = map[string]string{
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".css": "text/css; charset=utf-8",
|
||||
}
|
||||
|
||||
func requestAcceptsGzip(request *http.Request) bool {
|
||||
for _, encoding := range request.Header.Values("Accept-Encoding") {
|
||||
for _, part := range strings.Split(encoding, ",") {
|
||||
if strings.TrimSpace(strings.SplitN(part, ";", 2)[0]) == "gzip" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// servePrecompressedAsset streams the build-time .gz sibling for compressible
|
||||
// static assets when the client accepts gzip. It reports whether it handled
|
||||
// the request; callers fall back to the plain file server otherwise. Range
|
||||
// requests always fall back so byte ranges stay valid against the raw asset.
|
||||
func servePrecompressedAsset(writer http.ResponseWriter, request *http.Request, targetPath string) bool {
|
||||
contentType, compressible := precompressedContentTypes[filepath.Ext(targetPath)]
|
||||
if !compressible {
|
||||
return false
|
||||
}
|
||||
// Keep shared caches from mixing gzip and identity responses.
|
||||
writer.Header().Add("Vary", "Accept-Encoding")
|
||||
if !requestAcceptsGzip(request) || request.Header.Get("Range") != "" {
|
||||
return false
|
||||
}
|
||||
if request.Method != http.MethodGet && request.Method != http.MethodHead {
|
||||
return false
|
||||
}
|
||||
file, err := os.Open(targetPath + ".gz")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
info, err := file.Stat()
|
||||
if err != nil || info.IsDir() {
|
||||
return false
|
||||
}
|
||||
writer.Header().Set("Content-Type", contentType)
|
||||
writer.Header().Set("Content-Encoding", "gzip")
|
||||
writer.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10))
|
||||
writer.Header().Set("Accept-Ranges", "none")
|
||||
if request.Method == http.MethodHead {
|
||||
return true
|
||||
}
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
_, _ = io.Copy(writer, file)
|
||||
return true
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -278,7 +279,15 @@ func (server *server) authenticateRequest(request *http.Request) (*authenticated
|
||||
if err != nil || strings.TrimSpace(cookie.Value) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
session, err := server.store.GetSessionByTokenHash(request.Context(), sessionTokenHash(cookie.Value))
|
||||
tokenHash := sessionTokenHash(cookie.Value)
|
||||
// C2: the auth chain costs 3-5 serial Mongo round trips per request.
|
||||
// Warm requests are served from the process-local TTL cache; write paths
|
||||
// that change sessions, passwords, roles, suppliers or organizations call
|
||||
// the invalidateAuthCache* helpers below.
|
||||
if principal := lookupCachedPrincipal(tokenHash); principal != nil {
|
||||
return principal, nil
|
||||
}
|
||||
session, err := server.store.GetSessionByTokenHash(request.Context(), tokenHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -323,7 +332,102 @@ func (server *server) authenticateRequest(request *http.Request) (*authenticated
|
||||
if organization.Status == domain.OrganizationStatusDisabled {
|
||||
return nil, nil
|
||||
}
|
||||
return &authenticatedPrincipal{User: *user, Organization: *organization}, nil
|
||||
principal := &authenticatedPrincipal{User: *user, Organization: *organization}
|
||||
cachePrincipal(tokenHash, principal, session.ExpiresAt)
|
||||
return principal, nil
|
||||
}
|
||||
|
||||
// principalCacheEntry caches a resolved authenticatedPrincipal keyed by the
|
||||
// session token hash. Entries carry the owning user so user-wide
|
||||
// invalidation (role/supplier/org changes) can find every live session.
|
||||
type principalCacheEntry struct {
|
||||
principal authenticatedPrincipal
|
||||
userID string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
principalCacheTTL = 45 * time.Second
|
||||
principalCacheMaxEntries = 4096
|
||||
)
|
||||
|
||||
var (
|
||||
principalCacheMu sync.Mutex
|
||||
principalCache = map[string]principalCacheEntry{}
|
||||
)
|
||||
|
||||
// cachePrincipal stores a resolved principal with a TTL capped by the session
|
||||
// expiry so a stale entry never outlives its session.
|
||||
func cachePrincipal(tokenHash string, principal *authenticatedPrincipal, sessionExpiresAt string) {
|
||||
if principal == nil {
|
||||
return
|
||||
}
|
||||
expiresAt := time.Now().Add(principalCacheTTL)
|
||||
if parsed, err := time.Parse(time.RFC3339, sessionExpiresAt); err == nil && parsed.Before(expiresAt) {
|
||||
expiresAt = parsed
|
||||
}
|
||||
principalCacheMu.Lock()
|
||||
defer principalCacheMu.Unlock()
|
||||
if len(principalCache) >= principalCacheMaxEntries {
|
||||
// Cheap full sweep instead of an LRU: the map is small and bounded.
|
||||
now := time.Now()
|
||||
for key, entry := range principalCache {
|
||||
if now.After(entry.expiresAt) {
|
||||
delete(principalCache, key)
|
||||
}
|
||||
}
|
||||
if len(principalCache) >= principalCacheMaxEntries {
|
||||
principalCache = map[string]principalCacheEntry{}
|
||||
}
|
||||
}
|
||||
principalCache[tokenHash] = principalCacheEntry{
|
||||
principal: authenticatedPrincipal{User: principal.User, Organization: principal.Organization},
|
||||
userID: principal.User.ID,
|
||||
expiresAt: expiresAt,
|
||||
}
|
||||
}
|
||||
|
||||
func lookupCachedPrincipal(tokenHash string) *authenticatedPrincipal {
|
||||
principalCacheMu.Lock()
|
||||
entry, ok := principalCache[tokenHash]
|
||||
principalCacheMu.Unlock()
|
||||
if !ok || time.Now().After(entry.expiresAt) {
|
||||
return nil
|
||||
}
|
||||
principal := entry.principal
|
||||
return &principal
|
||||
}
|
||||
|
||||
// invalidateAuthCacheForToken drops a single session's cached principal
|
||||
// (logout, session deletion).
|
||||
func invalidateAuthCacheForToken(tokenHash string) {
|
||||
principalCacheMu.Lock()
|
||||
delete(principalCache, tokenHash)
|
||||
principalCacheMu.Unlock()
|
||||
}
|
||||
|
||||
// invalidateAuthCacheForUser drops every cached principal owned by the user
|
||||
// (password resets, role/supplier/status changes, forced session revocation).
|
||||
func invalidateAuthCacheForUser(userID string) {
|
||||
if userID == "" {
|
||||
return
|
||||
}
|
||||
principalCacheMu.Lock()
|
||||
for key, entry := range principalCache {
|
||||
if entry.userID == userID {
|
||||
delete(principalCache, key)
|
||||
}
|
||||
}
|
||||
principalCacheMu.Unlock()
|
||||
}
|
||||
|
||||
// invalidateAuthCache drops all cached principals. Used when an organization
|
||||
// record itself changes (name/slug/status) because the cached principal embeds
|
||||
// the organization snapshot.
|
||||
func invalidateAuthCache() {
|
||||
principalCacheMu.Lock()
|
||||
principalCache = map[string]principalCacheEntry{}
|
||||
principalCacheMu.Unlock()
|
||||
}
|
||||
|
||||
func (server *server) publicPreviewPrincipal(ctx context.Context) (*authenticatedPrincipal, error) {
|
||||
@ -597,9 +701,12 @@ func (server *server) handleAuthLogin(writer http.ResponseWriter, request *http.
|
||||
|
||||
func (server *server) handleAuthLogout(writer http.ResponseWriter, request *http.Request) {
|
||||
if cookie, err := request.Cookie(sessionCookieName); err == nil {
|
||||
if session, err := server.store.GetSessionByTokenHash(request.Context(), sessionTokenHash(cookie.Value)); err == nil && session != nil {
|
||||
tokenHash := sessionTokenHash(cookie.Value)
|
||||
if session, err := server.store.GetSessionByTokenHash(request.Context(), tokenHash); err == nil && session != nil {
|
||||
_, _ = server.store.DeleteSession(request.Context(), session.ID)
|
||||
}
|
||||
// Logout must take effect immediately, not after the TTL.
|
||||
invalidateAuthCacheForToken(tokenHash)
|
||||
}
|
||||
clearSessionCookie(writer, server.sessionCookieSecure)
|
||||
writer.WriteHeader(http.StatusNoContent)
|
||||
@ -698,6 +805,8 @@ func (server *server) handleCurrentUserProfileUpdate(writer http.ResponseWriter,
|
||||
renderUserMutationError(writer, err)
|
||||
return
|
||||
}
|
||||
// Profile edits (including self password change) refresh cached principals.
|
||||
invalidateAuthCacheForUser(item.ID)
|
||||
if payload.NewPassword != nil {
|
||||
if err := server.store.DeleteSessionsByUser(request.Context(), item.ID); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
@ -844,6 +953,9 @@ func (server *server) handleAdminOrganizationsUpdate(writer http.ResponseWriter,
|
||||
renderOrganizationMutationError(writer, err)
|
||||
return
|
||||
}
|
||||
// Cached principals embed the organization snapshot; any org change
|
||||
// flushes them so status/name changes apply immediately.
|
||||
invalidateAuthCache()
|
||||
renderJSON(writer, http.StatusOK, item)
|
||||
}
|
||||
|
||||
@ -857,6 +969,8 @@ func (server *server) handleAdminOrganizationsDelete(writer http.ResponseWriter,
|
||||
writer.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
// Org deletion invalidates every cached principal in that org.
|
||||
invalidateAuthCache()
|
||||
writer.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@ -1001,5 +1115,7 @@ func (server *server) handleAdminUserResetPassword(writer http.ResponseWriter, r
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
// The admin-forced password reset must evict any cached principal at once.
|
||||
invalidateAuthCacheForUser(user.ID)
|
||||
renderJSON(writer, http.StatusOK, sanitizeUser(user))
|
||||
}
|
||||
|
||||
279
internal/api/auth_cache_test.go
Normal file
279
internal/api/auth_cache_test.go
Normal file
@ -0,0 +1,279 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
type authReadCounter struct {
|
||||
store.Store
|
||||
sessions, users, suppliers, organizations atomic.Int64
|
||||
}
|
||||
|
||||
func (s *authReadCounter) GetSessionByTokenHash(ctx context.Context, tokenHash string) (*domain.SessionRecord, error) {
|
||||
s.sessions.Add(1)
|
||||
return s.Store.GetSessionByTokenHash(ctx, tokenHash)
|
||||
}
|
||||
|
||||
func (s *authReadCounter) GetUser(ctx context.Context, id string) (*domain.UserRecord, error) {
|
||||
s.users.Add(1)
|
||||
return s.Store.GetUser(ctx, id)
|
||||
}
|
||||
|
||||
func (s *authReadCounter) GetSupplier(ctx context.Context, id string) (*domain.SupplierRecord, error) {
|
||||
s.suppliers.Add(1)
|
||||
return s.Store.GetSupplier(ctx, id)
|
||||
}
|
||||
|
||||
func (s *authReadCounter) GetOrganization(ctx context.Context, id string) (*domain.OrganizationRecord, error) {
|
||||
s.organizations.Add(1)
|
||||
return s.Store.GetOrganization(ctx, id)
|
||||
}
|
||||
|
||||
func newAuthCacheFixture(t *testing.T) (*authReadCounter, *server, string, string) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
if _, err := dataStore.EnsureDefaultOrganization(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
user := domain.UserRecord{ID: "auth-cache-user", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, Name: "缓存管理员", Phone: "13800000001"}
|
||||
if err := dataStore.UpsertUser(ctx, user); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counter := &authReadCounter{Store: dataStore}
|
||||
instance := &server{store: counter, requireSessionAuth: true}
|
||||
token := "auth-cache-session-token"
|
||||
expiresAt := time.Now().Add(time.Hour)
|
||||
if _, err := dataStore.CreateSession(ctx, domain.SessionRecord{
|
||||
UserID: user.ID,
|
||||
TokenHash: sessionTokenHash(token),
|
||||
ExpiresAt: expiresAt.Format(time.RFC3339),
|
||||
ExpiresAtTime: expiresAt,
|
||||
CreatedAt: time.Now().Format(time.RFC3339),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
invalidateAuthCache()
|
||||
return counter, instance, token, user.ID
|
||||
}
|
||||
|
||||
func authedRequest(instance *server, path string, token string) *httptest.ResponseRecorder {
|
||||
request := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
request.AddCookie(&http.Cookie{Name: sessionCookieName, Value: token})
|
||||
recorder := httptest.NewRecorder()
|
||||
instance.sessionAuth(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if _, ok := currentPrincipal(request); ok {
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
writer.WriteHeader(http.StatusUnauthorized)
|
||||
})).ServeHTTP(recorder, request)
|
||||
return recorder
|
||||
}
|
||||
|
||||
func TestAuthenticateRequestServesWarmRequestsFromCache(t *testing.T) {
|
||||
counter, instance, token, userID := newAuthCacheFixture(t)
|
||||
|
||||
first := authedRequest(instance, "/api/users", token)
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("first request status=%d", first.Code)
|
||||
}
|
||||
coldCalls := int(counter.sessions.Load() + counter.users.Load() + counter.organizations.Load() + counter.suppliers.Load())
|
||||
if coldCalls < 2 || counter.sessions.Load() != 1 {
|
||||
t.Fatalf("cold path must hit the store, got sessions=%d users=%d orgs=%d suppliers=%d", counter.sessions.Load(), counter.users.Load(), counter.organizations.Load(), counter.suppliers.Load())
|
||||
}
|
||||
|
||||
for index := 0; index < 3; index++ {
|
||||
warm := authedRequest(instance, "/api/users", token)
|
||||
if warm.Code != http.StatusOK {
|
||||
t.Fatalf("warm request %d status=%d", index, warm.Code)
|
||||
}
|
||||
}
|
||||
warmCalls := int(counter.sessions.Load() + counter.users.Load() + counter.organizations.Load() + counter.suppliers.Load())
|
||||
if warmCalls != coldCalls {
|
||||
t.Fatalf("cache hit must perform zero auth store reads: cold=%d warm=%d (sessions=%d users=%d orgs=%d suppliers=%d)",
|
||||
coldCalls, warmCalls, counter.sessions.Load(), counter.users.Load(), counter.organizations.Load(), counter.suppliers.Load())
|
||||
}
|
||||
|
||||
// Token-level invalidation (logout) takes effect immediately.
|
||||
invalidateAuthCacheForToken(sessionTokenHash(token))
|
||||
after := authedRequest(instance, "/api/users", token)
|
||||
if after.Code != http.StatusOK {
|
||||
t.Fatalf("post-invalidation status=%d", after.Code)
|
||||
}
|
||||
if counter.sessions.Load() != 2 {
|
||||
t.Fatalf("token invalidation must force a fresh session read, sessions=%d", counter.sessions.Load())
|
||||
}
|
||||
|
||||
// User-level invalidation (role/supplier/password changes) evicts all of
|
||||
// the user's sessions, including other live tokens.
|
||||
invalidateAuthCacheForUser(userID)
|
||||
afterUser := authedRequest(instance, "/api/users", token)
|
||||
if afterUser.Code != http.StatusOK || counter.sessions.Load() != 3 {
|
||||
t.Fatalf("user invalidation must force a fresh session read, status=%d sessions=%d", afterUser.Code, counter.sessions.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogoutHandlerInvalidatesCachedPrincipal(t *testing.T) {
|
||||
counter, instance, token, _ := newAuthCacheFixture(t)
|
||||
if response := authedRequest(instance, "/api/users", token); response.Code != http.StatusOK {
|
||||
t.Fatalf("warm-up status=%d", response.Code)
|
||||
}
|
||||
if counter.sessions.Load() != 1 {
|
||||
t.Fatalf("expected one cold session read, got %d", counter.sessions.Load())
|
||||
}
|
||||
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/auth/logout", nil)
|
||||
request.AddCookie(&http.Cookie{Name: sessionCookieName, Value: token})
|
||||
recorder := httptest.NewRecorder()
|
||||
instance.handleAuthLogout(recorder, request)
|
||||
if recorder.Code != http.StatusNoContent {
|
||||
t.Fatalf("logout status=%d", recorder.Code)
|
||||
}
|
||||
|
||||
// The session is deleted, so the next request must not authenticate even
|
||||
// though the cache would still be warm without invalidation (that would
|
||||
// wrongly yield 200 from the stale principal).
|
||||
response := authedRequest(instance, "/api/users", token)
|
||||
if response.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("post-logout request must be unauthorized, got %d", response.Code)
|
||||
}
|
||||
if counter.sessions.Load() != 3 {
|
||||
t.Fatalf("logout must invalidate the cached principal, sessions=%d", counter.sessions.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsersListLimitAndSearch(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
if _, err := dataStore.EnsureDefaultOrganization(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := domain.UserRecord{ID: "users-admin", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
|
||||
if err := dataStore.UpsertUser(ctx, admin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Another tenant's user must stay invisible regardless of pagination.
|
||||
outsider := domain.UserRecord{ID: "users-outsider", OrganizationID: "org-other", Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, Name: "外租户用户"}
|
||||
if err := dataStore.UpsertUser(ctx, outsider); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for index := 0; index < 250; index++ {
|
||||
name, phone := "运营成员", fmt.Sprintf("138%08d", index)
|
||||
if index%50 == 0 {
|
||||
name = "工程师-" + string(rune('A'+index/50))
|
||||
phone = fmt.Sprintf("137%08d", index)
|
||||
}
|
||||
user := domain.UserRecord{
|
||||
ID: domain.NewID("user"), OrganizationID: domain.DefaultOrganizationID,
|
||||
Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive,
|
||||
Name: name, Phone: phone,
|
||||
}
|
||||
if err := dataStore.UpsertUser(ctx, user); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
handler := NewHandler(dataStore, nil)
|
||||
principal := &authenticatedPrincipal{User: admin, Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}}
|
||||
requestWith := func(target string) *httptest.ResponseRecorder {
|
||||
request := httptest.NewRequest(http.MethodGet, target, nil)
|
||||
request = request.WithContext(context.WithValue(request.Context(), authContextKey{}, principal))
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, request)
|
||||
return recorder
|
||||
}
|
||||
|
||||
// Default conservative cap: 251 visible users truncated to 200.
|
||||
defaultResponse := requestWith("/api/users")
|
||||
if defaultResponse.Code != http.StatusOK {
|
||||
t.Fatalf("default status=%d body=%s", defaultResponse.Code, defaultResponse.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
Items []domain.UserRecord `json:"items"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
if err := json.Unmarshal(defaultResponse.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Items) != 200 || !payload.HasMore {
|
||||
t.Fatalf("default cap items=%d hasMore=%v", len(payload.Items), payload.HasMore)
|
||||
}
|
||||
for _, item := range payload.Items {
|
||||
if item.OrganizationID != domain.DefaultOrganizationID {
|
||||
t.Fatalf("tenant isolation broken by pagination: %s", item.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Explicit limit and q search combine; small result sets omit hasMore.
|
||||
// "137" matches the five 工程师-* phones plus the one 13800000137 member.
|
||||
search := requestWith("/api/users?limit=10&q=137")
|
||||
if search.Code != http.StatusOK {
|
||||
t.Fatalf("search status=%d", search.Code)
|
||||
}
|
||||
payload.HasMore = false // fresh decode of a body without hasMore must stay false
|
||||
if err := json.Unmarshal(search.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Items) != 6 || payload.HasMore {
|
||||
t.Fatalf("search items=%d hasMore=%v", len(payload.Items), payload.HasMore)
|
||||
}
|
||||
engineers := 0
|
||||
for _, item := range payload.Items {
|
||||
if !strings.Contains(item.Phone, "137") {
|
||||
t.Fatalf("search matched wrong user %s phone=%s", item.ID, item.Phone)
|
||||
}
|
||||
if strings.HasPrefix(item.Phone, "137") {
|
||||
engineers++
|
||||
}
|
||||
}
|
||||
if engineers != 5 {
|
||||
t.Fatalf("expected 5 engineer matches, got %d", engineers)
|
||||
}
|
||||
|
||||
// Invalid limit is rejected; legacy no-parameter shape stays {"items"}.
|
||||
bad := requestWith("/api/users?limit=0")
|
||||
if bad.Code != http.StatusBadRequest {
|
||||
t.Fatalf("invalid limit status=%d", bad.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthCacheConcurrentAccess(t *testing.T) {
|
||||
counter, instance, token, userID := newAuthCacheFixture(t)
|
||||
// Concurrent readers and invalidators must not race or deadlock; the
|
||||
// final request must still authenticate correctly.
|
||||
done := make(chan struct{})
|
||||
for index := 0; index < 8; index++ {
|
||||
go func(i int) {
|
||||
defer func() { done <- struct{}{} }()
|
||||
for round := 0; round < 25; round++ {
|
||||
switch i % 3 {
|
||||
case 0:
|
||||
_ = authedRequest(instance, "/api/users", token)
|
||||
case 1:
|
||||
invalidateAuthCacheForUser(userID)
|
||||
default:
|
||||
invalidateAuthCacheForToken(sessionTokenHash(token))
|
||||
}
|
||||
}
|
||||
}(index)
|
||||
}
|
||||
for index := 0; index < 8; index++ {
|
||||
<-done
|
||||
}
|
||||
final := authedRequest(instance, "/api/users", token)
|
||||
if final.Code != http.StatusOK {
|
||||
t.Fatalf("final request status=%d", final.Code)
|
||||
}
|
||||
_ = counter
|
||||
}
|
||||
162
internal/api/dashboard_cache_test.go
Normal file
162
internal/api/dashboard_cache_test.go
Normal file
@ -0,0 +1,162 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func TestDashboardResponseCacheExpiryAndInvalidation(t *testing.T) {
|
||||
current := time.Unix(1700000000, 0)
|
||||
cache := dashboardResponseCache{now: func() time.Time { return current }}
|
||||
cache.set("k", http.StatusOK, "application/json", []byte(`{"a":1}`))
|
||||
if entry := cache.get("k"); entry == nil || string(entry.body) != `{"a":1}` || entry.status != http.StatusOK {
|
||||
t.Fatalf("expected fresh entry, got %+v", entry)
|
||||
}
|
||||
cache.set("k2", http.StatusOK, "application/json", []byte(`{"b":2}`))
|
||||
cache.invalidateAll()
|
||||
if cache.get("k") != nil || cache.get("k2") != nil {
|
||||
t.Fatal("invalidateAll must drop every entry")
|
||||
}
|
||||
cache.set("k", http.StatusOK, "application/json", []byte(`{"a":1}`))
|
||||
current = current.Add(dashboardResponseCacheTTL + time.Second)
|
||||
if entry := cache.get("k"); entry != nil {
|
||||
t.Fatalf("expected TTL expiry, got %+v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardResponseCacheBoundsDropped(t *testing.T) {
|
||||
current := time.Unix(1700000000, 0)
|
||||
cache := dashboardResponseCache{now: func() time.Time { return current }}
|
||||
for i := 0; i < 200; i++ {
|
||||
cache.set(string(rune('a'+i%26))+string(rune('a'+i/26)), http.StatusOK, "", []byte{byte(i)})
|
||||
}
|
||||
if entry := cache.get("aa"); entry != nil {
|
||||
t.Fatalf("bounded cache should have dropped the oldest entries, got %+v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardHumanWorkStatsMatchesLegacyEnrichOnMemoryStore(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
seedDashboardHumanWorkFixtures(t, dataStore)
|
||||
|
||||
legacy := &domain.DashboardSummary{}
|
||||
server := &server{store: dataStore}
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil)
|
||||
server.enrichDashboardHumanWork(ctx, request, legacy)
|
||||
|
||||
counted := &domain.DashboardSummary{}
|
||||
stats, err := dataStore.DashboardHumanWorkStats(ctx, domain.DefaultOrganizationID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyDashboardHumanWorkStats(counted, stats)
|
||||
|
||||
if legacy.Tasks.PendingAnnotation != counted.Tasks.PendingAnnotation ||
|
||||
legacy.Tasks.PendingQuality != counted.Tasks.PendingQuality ||
|
||||
legacy.Tasks.InAnnotation != counted.Tasks.InAnnotation ||
|
||||
legacy.Tasks.PendingAcceptance != counted.Tasks.PendingAcceptance ||
|
||||
legacy.Tasks.Rejected != counted.Tasks.Rejected ||
|
||||
legacy.Tasks.Escalated != counted.Tasks.Escalated ||
|
||||
legacy.Tasks.QualityPassRate != counted.Tasks.QualityPassRate ||
|
||||
legacy.Tasks.Waiting != counted.Tasks.Waiting ||
|
||||
legacy.Data.VideoCount != counted.Data.VideoCount ||
|
||||
legacy.Data.PackageCount != counted.Data.PackageCount ||
|
||||
legacy.Data.CaptureDurationSeconds != counted.Data.CaptureDurationSeconds {
|
||||
t.Fatalf("counters diverge:\nlegacy=%+v/%+v\ncounted=%+v/%+v", legacy.Tasks, legacy.Data, counted.Tasks, counted.Data)
|
||||
}
|
||||
if counted.Tasks.PendingAnnotation != 1 || counted.Tasks.PendingQuality != 2 || counted.Tasks.InAnnotation != 1 ||
|
||||
counted.Tasks.PendingAcceptance != 2 || counted.Tasks.Rejected != 1 || counted.Tasks.Escalated != 1 ||
|
||||
counted.Tasks.QualityPassRate != 50 {
|
||||
t.Fatalf("unexpected counters: %+v", counted.Tasks)
|
||||
}
|
||||
if counted.Data.VideoCount != 3 || counted.Data.PackageCount != 2 || counted.Data.CaptureDurationSeconds != 100 {
|
||||
t.Fatalf("unexpected data counters: %+v", counted.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func seedDashboardHumanWorkFixtures(t *testing.T, dataStore store.Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
org := domain.DefaultOrganizationID
|
||||
item := func(id string, status domain.HumanItemStatus, stage string, key string, duration any) domain.HumanItemRecord {
|
||||
return domain.HumanItemRecord{
|
||||
ID: id, OrganizationID: org, CurrentStageID: stage, Status: status,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: key, Payload: map[string]any{"durationSeconds": duration}},
|
||||
}
|
||||
}
|
||||
fixtures := []domain.HumanItemRecord{
|
||||
item("ready-annotation", domain.HumanItemReady, "annotation", "video-a", 30.0),
|
||||
item("ready-quality", domain.HumanItemRework, "quality_review", "video-a", 40.0),
|
||||
item("claimed", domain.HumanItemClaimed, "annotation", "video-b", 20.0),
|
||||
item("submitted-quality", domain.HumanItemSubmitted, domain.FanyuV5StageCutReview, "video-b", nil),
|
||||
item("submitted-acceptance", domain.HumanItemSubmitted, "acceptance", "video-c", 0),
|
||||
item("accepted", domain.HumanItemAccepted, "done", "video-c", 10.0),
|
||||
item("rejected", domain.HumanItemRejected, "done", "", -5),
|
||||
item("escalated", domain.HumanItemEscalated, "annotation", "", "not-a-number"),
|
||||
{ID: "foreign", OrganizationID: "other-org", CurrentStageID: "annotation", Status: domain.HumanItemReady,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "video-foreign", Payload: map[string]any{"durationSeconds": 999.0}}},
|
||||
}
|
||||
for _, fixture := range fixtures {
|
||||
if err := dataStore.UpsertHumanItem(ctx, fixture); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
packages := []domain.DeliveryPackageRecord{
|
||||
{ID: "pkg-pending", OrganizationID: org, ProjectID: "p", Status: domain.DeliveryPackagePendingAcceptance},
|
||||
{ID: "pkg-accepted", OrganizationID: org, ProjectID: "p", Status: domain.DeliveryPackageAccepted},
|
||||
{ID: "pkg-debug", OrganizationID: org, ProjectID: "p", Mode: domain.RunModeDebug, Status: domain.DeliveryPackagePendingAcceptance},
|
||||
{ID: "pkg-foreign", OrganizationID: "other-org", ProjectID: "p", Status: domain.DeliveryPackagePendingAcceptance},
|
||||
}
|
||||
for _, pkg := range packages {
|
||||
if err := dataStore.UpsertDeliveryPackage(ctx, pkg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardEndpointServesCachedBodyAndInvalidatesOnWrite(t *testing.T) {
|
||||
dataStore := store.NewMemoryStore()
|
||||
handler := NewHandler(dataStore, nil)
|
||||
|
||||
first := httptest.NewRecorder()
|
||||
handler.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("first dashboard request = %d body=%s", first.Code, first.Body.String())
|
||||
}
|
||||
|
||||
second := httptest.NewRecorder()
|
||||
handler.ServeHTTP(second, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
|
||||
if second.Code != http.StatusOK || !bytes.Equal(first.Body.Bytes(), second.Body.Bytes()) {
|
||||
t.Fatalf("cached body must be byte-identical: %d vs %d", first.Code, second.Code)
|
||||
}
|
||||
|
||||
// A write done outside the API stays hidden while the TTL window is open.
|
||||
if err := dataStore.UpsertRun(context.Background(), domain.RunRecord{ID: "cached-run", OrganizationID: domain.DefaultOrganizationID, Status: domain.RunStatusRunning, CreatedAt: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stillCached := httptest.NewRecorder()
|
||||
handler.ServeHTTP(stillCached, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
|
||||
if !bytes.Equal(second.Body.Bytes(), stillCached.Body.Bytes()) {
|
||||
t.Fatal("expected the TTL cache to keep serving the stale body before any API write")
|
||||
}
|
||||
|
||||
// Any non-GET /api request flushes the dashboard caches.
|
||||
flush := httptest.NewRecorder()
|
||||
handler.ServeHTTP(flush, httptest.NewRequest(http.MethodPost, "/api/auth/logout", nil))
|
||||
refreshed := httptest.NewRecorder()
|
||||
handler.ServeHTTP(refreshed, httptest.NewRequest(http.MethodGet, "/api/dashboard", nil))
|
||||
if refreshed.Code != http.StatusOK {
|
||||
t.Fatalf("refreshed dashboard = %d body=%s", refreshed.Code, refreshed.Body.String())
|
||||
}
|
||||
if bytes.Equal(stillCached.Body.Bytes(), refreshed.Body.Bytes()) {
|
||||
t.Fatal("expected cache invalidation to recompute the dashboard body")
|
||||
}
|
||||
}
|
||||
140
internal/api/dashboard_poc_readiness_test.go
Normal file
140
internal/api/dashboard_poc_readiness_test.go
Normal file
@ -0,0 +1,140 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
// slow-page report S4 parity test: the bounded point queries behind
|
||||
// dashboardPOCReadinessCandidateRuns must select exactly what the former
|
||||
// full ListRuns scan selected, and must not drag unrelated runs along.
|
||||
func TestDashboardPOCReadinessCandidateRunsMatchesFullScan(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
|
||||
acceptancePayload := func(operatorCount int) map[string]any {
|
||||
return map[string]any{
|
||||
"status": "passed", "acceptanceScope": "full", "fullExternalAcceptance": true,
|
||||
"operatorCount": operatorCount, "realFixtureCount": 1, "generatedFixtureCount": 0,
|
||||
"allFixturesReal": true,
|
||||
}
|
||||
}
|
||||
runs := []domain.RunRecord{
|
||||
// Newest run overall but irrelevant: must not appear in candidates.
|
||||
{ID: "run-noise", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "unrelated", ProjectName: "Unrelated", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T11:00:00Z", UpdatedAt: "2026-10-09T11:00:00Z"},
|
||||
// Latest acceptance artifact overall, nested under data.preview.value.
|
||||
{ID: "run-acceptance-preview", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T10:00:00Z", UpdatedAt: "2026-10-09T10:59:00Z",
|
||||
Artifacts: []domain.ExecutionArtifact{{ID: "a1", RunID: "run-acceptance-preview", Name: "report.json", Data: map[string]any{"preview": map[string]any{"value": acceptancePayload(12)}}}}},
|
||||
// Older acceptance artifacts across the other detection paths.
|
||||
{ID: "run-acceptance-value", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T09:00:00Z", UpdatedAt: "2026-10-09T09:30:00Z",
|
||||
Artifacts: []domain.ExecutionArtifact{{ID: "a2", RunID: "run-acceptance-value", Name: "report.json", Data: map[string]any{"value": acceptancePayload(8)}}}},
|
||||
{ID: "run-acceptance-named", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T08:00:00Z", UpdatedAt: "2026-10-09T08:30:00Z",
|
||||
Artifacts: []domain.ExecutionArtifact{{ID: "a3", RunID: "run-acceptance-named", Name: "poc-operator-full-acceptance-result.json", Data: acceptancePayload(6)}}},
|
||||
// Latest POC GPU workflow run (status completed → preflight passed).
|
||||
{ID: "run-gpu", OrganizationID: domain.DefaultOrganizationID, WorkflowName: pocGPUWorkflowAcceptanceWorkflowName, ProjectName: "other", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T10:50:00Z", UpdatedAt: "2026-10-09T10:55:00Z",
|
||||
Logs: []domain.LogEntryRecord{{Message: "gpu request cannot be scheduled"}}},
|
||||
// Older GPU-preflight run matched via project name.
|
||||
{ID: "run-live", OrganizationID: domain.DefaultOrganizationID, WorkflowName: "other", ProjectName: liveWorkflowAcceptanceProjectName, Status: domain.RunStatusFailed, CreatedAt: "2026-10-09T07:00:00Z", UpdatedAt: "2026-10-09T07:10:00Z",
|
||||
Logs: []domain.LogEntryRecord{{Message: "gpu request cannot be scheduled"}}},
|
||||
// Near-miss names must not match.
|
||||
{ID: "run-decoy", OrganizationID: domain.DefaultOrganizationID, WorkflowName: pocGPUWorkflowAcceptanceWorkflowName + " 2", ProjectName: "decoy", Status: domain.RunStatusFailed, CreatedAt: "2026-10-09T10:58:00Z", UpdatedAt: "2026-10-09T10:58:00Z"},
|
||||
}
|
||||
for _, run := range runs {
|
||||
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
server := &server{store: dataStore}
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil).WithContext(ctx)
|
||||
|
||||
candidates, err := server.dashboardPOCReadinessCandidateRuns(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := make(map[string]bool, len(candidates))
|
||||
for _, run := range candidates {
|
||||
ids[run.ID] = true
|
||||
}
|
||||
if ids["run-noise"] || ids["run-decoy"] {
|
||||
t.Fatalf("candidates must exclude unrelated runs, got %#v", ids)
|
||||
}
|
||||
for _, expected := range []string{"run-acceptance-preview", "run-acceptance-value", "run-acceptance-named", "run-gpu", "run-live"} {
|
||||
if !ids[expected] {
|
||||
t.Fatalf("candidates must include %s, got %#v", expected, ids)
|
||||
}
|
||||
}
|
||||
|
||||
// Parity with the former full scan.
|
||||
full, err := dataStore.ListRuns(ctx, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fullPayload, fullRun, fullFound := dashboardLatestAcceptancePayload(full)
|
||||
candidatePayload, candidateRun, candidateFound := dashboardLatestAcceptancePayload(candidates)
|
||||
if fullFound != candidateFound || fullRun.ID != candidateRun.ID || len(fullPayload) != len(candidatePayload) {
|
||||
t.Fatalf("acceptance selection diverged: full=%s(%v) candidates=%s(%v)", fullRun.ID, fullFound, candidateRun.ID, candidateFound)
|
||||
}
|
||||
for key := range fullPayload {
|
||||
if fullPayload[key] != candidatePayload[key] {
|
||||
t.Fatalf("acceptance payload diverged at %q: %#v vs %#v", key, fullPayload[key], candidatePayload[key])
|
||||
}
|
||||
}
|
||||
if fullRun.ID != "run-acceptance-preview" {
|
||||
t.Fatalf("fixture expects the preview-nested artifact to win, got %s", fullRun.ID)
|
||||
}
|
||||
|
||||
fullGPU, fullFoundGPU := latestPocGPUWorkflowRun(full)
|
||||
candidateGPU, candidateFoundGPU := latestPocGPUWorkflowRun(candidates)
|
||||
if fullFoundGPU != candidateFoundGPU || fullGPU.ID != candidateGPU.ID {
|
||||
t.Fatalf("gpu run selection diverged: full=%s(%v) candidates=%s(%v)", fullGPU.ID, fullFoundGPU, candidateGPU.ID, candidateFoundGPU)
|
||||
}
|
||||
if fullGPU.ID != "run-gpu" {
|
||||
t.Fatalf("fixture expects the workflow-named gpu run to win, got %s", fullGPU.ID)
|
||||
}
|
||||
fullGate := dashboardGPUWorkflowPreflightGate(full)
|
||||
candidateGate := dashboardGPUWorkflowPreflightGate(candidates)
|
||||
if fullGate.Status != candidateGate.Status || fullGate.Label != candidateGate.Label {
|
||||
t.Fatalf("gpu gate diverged: %#v vs %#v", fullGate, candidateGate)
|
||||
}
|
||||
if candidateGate.Status != "passed" {
|
||||
t.Fatalf("completed gpu run must pass the preflight gate, got %#v", candidateGate)
|
||||
}
|
||||
}
|
||||
|
||||
// The tenant-enforced branch scopes both point queries to the requester's
|
||||
// organization, like the old ListRunsByOrganization scan did.
|
||||
func TestDashboardPOCReadinessCandidateRunsScopesTenant(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
mustUpsert := func(run domain.RunRecord) {
|
||||
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
acceptancePayload := map[string]any{"status": "passed", "acceptanceScope": "full", "fullExternalAcceptance": true, "operatorCount": 3}
|
||||
mustUpsert(domain.RunRecord{ID: "run-own", OrganizationID: "org-a", WorkflowName: "w", ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T09:00:00Z", UpdatedAt: "2026-10-09T09:00:00Z",
|
||||
Artifacts: []domain.ExecutionArtifact{{ID: "a-own", Name: "poc-operator-full-acceptance-result.json", Data: acceptancePayload}}})
|
||||
mustUpsert(domain.RunRecord{ID: "run-other", OrganizationID: "org-b", WorkflowName: pocGPUWorkflowAcceptanceWorkflowName, ProjectName: "p", Status: domain.RunStatusCompleted, CreatedAt: "2026-10-09T10:00:00Z", UpdatedAt: "2026-10-09T10:00:00Z"})
|
||||
|
||||
server := &server{store: dataStore}
|
||||
organization := domain.OrganizationRecord{ID: "org-a", Name: "A", Slug: "org-a", Status: domain.OrganizationStatusActive}
|
||||
principal := &authenticatedPrincipal{User: domain.UserRecord{ID: "u1", Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, OrganizationID: "org-a"}, Organization: organization}
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil).WithContext(
|
||||
context.WithValue(ctx, authContextKey{}, principal),
|
||||
)
|
||||
if !server.tenantEnforced(request) {
|
||||
t.Fatal("request with principal must be tenant enforced")
|
||||
}
|
||||
candidates, err := server.dashboardPOCReadinessCandidateRuns(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(candidates) != 1 || candidates[0].ID != "run-own" {
|
||||
t.Fatalf("tenant scope must exclude org-b runs, got %#v", candidates)
|
||||
}
|
||||
}
|
||||
@ -45,13 +45,29 @@ func addDashboardWorkforceEvent(item *domain.DashboardWorkforceMetric, event dom
|
||||
workforcestats.AddEvent(item, event, nil)
|
||||
}
|
||||
|
||||
// cachedWorkforceStatsLoader is implemented by MongoStore; it caches the
|
||||
// 30-day snapshot for 45s so concurrent dashboard renders do not re-run the
|
||||
// full windowed joins (C4). MemoryStore intentionally stays uncached for
|
||||
// read-after-write freshness in tests and the local replica.
|
||||
type cachedWorkforceStatsLoader interface {
|
||||
LoadWorkforceStatsCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error)
|
||||
}
|
||||
|
||||
func (server *server) populateDashboardWorkforce(ctx context.Context, requestOrganizationID string, principal *authenticatedPrincipal, from time.Time, to time.Time, summary *domain.DashboardSummary) error {
|
||||
if summary == nil {
|
||||
return nil
|
||||
}
|
||||
q, scope := workforcePrincipalQuery(principal)
|
||||
q.From, q.To = from, to
|
||||
data, err := server.store.LoadWorkforceStats(ctx, requestOrganizationID, q)
|
||||
var data workforcestats.Snapshot
|
||||
var err error
|
||||
if cached, ok := server.store.(cachedWorkforceStatsLoader); ok {
|
||||
// The 30-day window is already pushed down as a time-bounded, projected
|
||||
// Mongo query; the cached loader deduplicates it for 45s per scope.
|
||||
data, err = cached.LoadWorkforceStatsCached(ctx, requestOrganizationID, q)
|
||||
} else {
|
||||
data, err = server.store.LoadWorkforceStats(ctx, requestOrganizationID, q)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -25,6 +25,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -2125,10 +2126,93 @@ func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string,
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
maybePruneSourceBrowserPreviewCache(cacheDir)
|
||||
cacheKey := sourceBrowserPreviewCacheKey(request)
|
||||
return filepath.Join(cacheDir, cacheKey+".mp4"), nil
|
||||
}
|
||||
|
||||
// The on-disk preview cache is bounded: entries older than the TTL are
|
||||
// deleted outright, then least-recently-written entries are evicted until
|
||||
// the directory is under the size cap. Without this bound the /tmp cache
|
||||
// grows without limit on pods that serve many long videos.
|
||||
const (
|
||||
sourceBrowserPreviewCacheMaxBytes = 2 << 30 // 2 GiB total cap
|
||||
sourceBrowserPreviewCacheMaxAge = 7 * 24 * time.Hour
|
||||
sourceBrowserPreviewCachePruneEvery = 10 * time.Minute
|
||||
)
|
||||
|
||||
var lastSourceBrowserPreviewCachePrune atomic.Int64
|
||||
|
||||
func maybePruneSourceBrowserPreviewCache(dir string) {
|
||||
interval := sourceBrowserPreviewCachePruneEvery.Nanoseconds()
|
||||
now := time.Now().UnixNano()
|
||||
last := lastSourceBrowserPreviewCachePrune.Load()
|
||||
if now-last < interval {
|
||||
return
|
||||
}
|
||||
if !lastSourceBrowserPreviewCachePrune.CompareAndSwap(last, now) {
|
||||
return
|
||||
}
|
||||
go pruneSourceBrowserPreviewCache(dir, time.Now())
|
||||
}
|
||||
|
||||
// pruneSourceBrowserPreviewCache enforces the TTL and size bounds on the
|
||||
// preview cache directory. Best effort: unreadable entries are skipped and
|
||||
// removal failures are ignored (the next prune retries).
|
||||
func pruneSourceBrowserPreviewCache(dir string, now time.Time) {
|
||||
pruneSourceBrowserPreviewCacheWithLimit(dir, now, sourceBrowserPreviewCacheMaxBytes)
|
||||
}
|
||||
|
||||
// pruneSourceBrowserPreviewCacheWithLimit is the testable core of the cache
|
||||
// eviction: delete entries past the TTL, then evict least-recently-written
|
||||
// entries until the directory is under maxBytes.
|
||||
func pruneSourceBrowserPreviewCacheWithLimit(dir string, now time.Time, maxBytes int64) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
type cacheEntry struct {
|
||||
path string
|
||||
modTime time.Time
|
||||
size int64
|
||||
}
|
||||
files := make([]cacheEntry, 0, len(entries))
|
||||
var total int64
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
info, infoErr := entry.Info()
|
||||
if infoErr != nil {
|
||||
continue
|
||||
}
|
||||
if now.Sub(info.ModTime()) > sourceBrowserPreviewCacheMaxAge {
|
||||
_ = os.Remove(filepath.Join(dir, entry.Name()))
|
||||
continue
|
||||
}
|
||||
files = append(files, cacheEntry{
|
||||
path: filepath.Join(dir, entry.Name()),
|
||||
modTime: info.ModTime(),
|
||||
size: info.Size(),
|
||||
})
|
||||
total += info.Size()
|
||||
}
|
||||
if total <= maxBytes {
|
||||
return
|
||||
}
|
||||
sort.Slice(files, func(left int, right int) bool {
|
||||
return files[left].modTime.Before(files[right].modTime)
|
||||
})
|
||||
for _, file := range files {
|
||||
if total <= maxBytes {
|
||||
break
|
||||
}
|
||||
if removeErr := os.Remove(file.path); removeErr == nil {
|
||||
total -= file.size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseBOSObjectPath(sourcePath string) (string, string, bool) {
|
||||
bucket, object, ok := parseBOSPath(sourcePath)
|
||||
if !ok || object == "" {
|
||||
@ -2324,6 +2408,81 @@ func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWr
|
||||
return nil
|
||||
}
|
||||
|
||||
// sourceVideoPreviewSyncMode is the server-side rollback switch for the C5
|
||||
// video direct-link change: setting DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 keeps
|
||||
// every browser-preview request on the legacy blocking ffmpeg stream (no
|
||||
// background pre-transcode, no redirect to the direct storage URL).
|
||||
var sourceVideoPreviewSyncMode = sync.OnceValue(func() bool {
|
||||
return strings.TrimSpace(os.Getenv("DROBOTICFLOW_VIDEO_PREVIEW_SYNC")) == "1"
|
||||
})
|
||||
|
||||
// discardPreviewResponseWriter lets the background pre-transcode reuse
|
||||
// streamSourceBrowserVideoPreview unchanged: its output is dropped and only
|
||||
// the cached artifact on disk matters.
|
||||
type discardPreviewResponseWriter struct{}
|
||||
|
||||
func (discardPreviewResponseWriter) Header() http.Header { return http.Header{} }
|
||||
func (discardPreviewResponseWriter) WriteHeader(int) {}
|
||||
func (discardPreviewResponseWriter) Write(p []byte) (int, error) { return len(p), nil }
|
||||
|
||||
// startSourceBrowserVideoPretranscode warms the preview cache in the
|
||||
// background. The per-key lock inside streamSourceBrowserVideoPreview
|
||||
// deduplicates concurrent warmups, and a warm run that finds the cache
|
||||
// already populated returns immediately without invoking ffmpeg.
|
||||
func startSourceBrowserVideoPretranscode(previewRequest sourceBrowserPreviewRequest) {
|
||||
go func() {
|
||||
// ffmpeg's own rw_timeout (15 minutes without progress) bounds the
|
||||
// pull; streamSourceBrowserVideoPreview deliberately detaches
|
||||
// cancellation so a client disconnect cannot truncate the warmup.
|
||||
// Routed through the writer var so tests can stub the transcode.
|
||||
if err := sourceBrowserVideoPreviewWriter(context.Background(), discardPreviewResponseWriter{}, previewRequest); err != nil {
|
||||
fmt.Printf("[api] background browser video preview transcode failed: %v\n", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// writeBrowserVideoPreviewResponse serves the browser-transcoded preview for
|
||||
// a video entry (query transcode=browser). Order of preference:
|
||||
// 1. serve the cached pre-transcoded mp4 when present (no ffmpeg at all);
|
||||
// 2. start a background pre-transcode and immediately redirect the player
|
||||
// to the direct storage URL (BOS presigned GET link), so the preview
|
||||
// first frame is never blocked behind server-side ffmpeg;
|
||||
// 3. legacy blocking ffmpeg stream as fallback (no direct URL available,
|
||||
// explicit sync=1 request parameter, or the
|
||||
// DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 rollback switch).
|
||||
//
|
||||
// All video responses set Content-Encoding: identity so the global chi
|
||||
// Compress(5) middleware skips gzip: mp4 payloads are already compressed
|
||||
// media and re-compressing them only burned API pod CPU.
|
||||
func writeBrowserVideoPreviewResponse(writer http.ResponseWriter, request *http.Request, previewRequest sourceBrowserPreviewRequest, directURL string) {
|
||||
// Videos are already-compressed media; the identity marker makes chi's
|
||||
// Compress middleware leave the byte stream untouched.
|
||||
writer.Header().Set("Content-Encoding", "identity")
|
||||
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
|
||||
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(previewRequest.FileName, path.Ext(previewRequest.FileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
direct := strings.TrimSpace(directURL)
|
||||
forcedSync := sourceVideoPreviewSyncMode() ||
|
||||
strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("sync")), "1")
|
||||
if direct != "" && strings.HasPrefix(strings.ToLower(direct), "http") && !forcedSync {
|
||||
startSourceBrowserVideoPretranscode(previewRequest)
|
||||
http.Redirect(writer, request, direct, http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "video/mp4")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
if flusher, ok := writer.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
|
||||
fmt.Printf("[api] browser preview stream failed: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
type bestEffortPreviewWriter struct {
|
||||
target io.Writer
|
||||
err *error
|
||||
@ -2700,21 +2859,7 @@ func (server *server) handleSourceCollectionContent(writer http.ResponseWriter,
|
||||
Payload: content,
|
||||
RedirectURL: redirectURL,
|
||||
}
|
||||
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
|
||||
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "video/mp4")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
if flusher, ok := writer.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
|
||||
fmt.Printf("[api] source browser preview stream failed: %v\n", err)
|
||||
}
|
||||
writeBrowserVideoPreviewResponse(writer, request, previewRequest, redirectURL)
|
||||
return
|
||||
}
|
||||
if redirectURL != "" {
|
||||
@ -2887,26 +3032,14 @@ func (server *server) handlePackageContent(writer http.ResponseWriter, request *
|
||||
FileName: fileName,
|
||||
Payload: content,
|
||||
}
|
||||
directURL := ""
|
||||
if filePath != "" {
|
||||
previewRequest.RedirectURL = filePath
|
||||
} else {
|
||||
previewRequest.RedirectURL = redirectURL
|
||||
directURL = redirectURL
|
||||
}
|
||||
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
|
||||
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Content-Type", "video/mp4")
|
||||
writer.Header().Set("Cache-Control", "no-store")
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
if flusher, ok := writer.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
|
||||
fmt.Printf("[api] package browser preview stream failed: %v\n", err)
|
||||
}
|
||||
writeBrowserVideoPreviewResponse(writer, request, previewRequest, directURL)
|
||||
return
|
||||
}
|
||||
if filePath != "" {
|
||||
|
||||
207
internal/api/data_platform_preview_test.go
Normal file
207
internal/api/data_platform_preview_test.go
Normal file
@ -0,0 +1,207 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
func compressTestHandler(inner http.HandlerFunc) http.Handler {
|
||||
return middleware.Compress(5)(inner)
|
||||
}
|
||||
|
||||
// A cache miss with a direct (http) storage URL must redirect the player to
|
||||
// that URL instead of running server-side ffmpeg on the request path, and
|
||||
// the redirect itself must not come back gzip-encoded.
|
||||
func TestBrowserVideoPreviewCacheMissRedirectsToDirectURL(t *testing.T) {
|
||||
originalPath := sourceBrowserVideoPreviewPath
|
||||
originalWriter := sourceBrowserVideoPreviewWriter
|
||||
defer func() {
|
||||
sourceBrowserVideoPreviewPath = originalPath
|
||||
sourceBrowserVideoPreviewWriter = originalWriter
|
||||
}()
|
||||
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
var warmups sync.WaitGroup
|
||||
warmups.Add(1)
|
||||
var warmedRequest sourceBrowserPreviewRequest
|
||||
var warmupMu sync.Mutex
|
||||
sourceBrowserVideoPreviewWriter = func(ctx context.Context, _ http.ResponseWriter, request sourceBrowserPreviewRequest) error {
|
||||
warmupMu.Lock()
|
||||
warmedRequest = request
|
||||
warmupMu.Unlock()
|
||||
warmups.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
direct := "https://bucket.bcebos.com/dir/camera.mp4?authorization=bce-auth-v1%2Fexample"
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/r/c/content?relativePath=v.mp4&transcode=browser", nil)
|
||||
request.Header.Set("Accept-Encoding", "gzip")
|
||||
recorder := httptest.NewRecorder()
|
||||
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
|
||||
FileName: "camera.mp4",
|
||||
RedirectURL: direct,
|
||||
}, direct)
|
||||
}).ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusTemporaryRedirect {
|
||||
t.Fatalf("expected 307 redirect on cache miss, got %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if location := recorder.Header().Get("Location"); location != direct {
|
||||
t.Fatalf("unexpected redirect location: %s", location)
|
||||
}
|
||||
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "" && encoding != "identity" {
|
||||
t.Fatalf("redirect response must not be compressed, Content-Encoding=%q", encoding)
|
||||
}
|
||||
// The background pre-transcode must have been kicked off exactly for the
|
||||
// preview request that missed the cache.
|
||||
warmups.Wait()
|
||||
warmupMu.Lock()
|
||||
defer warmupMu.Unlock()
|
||||
if warmedRequest.FileName != "camera.mp4" || warmedRequest.RedirectURL != direct {
|
||||
t.Fatalf("background warmup ran for the wrong request: %+v", warmedRequest)
|
||||
}
|
||||
}
|
||||
|
||||
// When the cache holds the pre-transcoded artifact, the response must be
|
||||
// served as-is through the global Compress middleware: the identity
|
||||
// Content-Encoding marker makes chi skip gzip, so video bytes reach the
|
||||
// player unchanged.
|
||||
func TestBrowserVideoPreviewCacheHitServesBytesWithoutGzip(t *testing.T) {
|
||||
originalPath := sourceBrowserVideoPreviewPath
|
||||
defer func() { sourceBrowserVideoPreviewPath = originalPath }()
|
||||
|
||||
payload := []byte("fake-mp4-bytes-0123456789")
|
||||
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
|
||||
tempFile, err := os.CreateTemp(t.TempDir(), "preview-*.mp4")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := tempFile.Write(payload); err != nil {
|
||||
tempFile.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := tempFile.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return tempFile.Name(), nil
|
||||
}
|
||||
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser", nil)
|
||||
request.Header.Set("Accept-Encoding", "gzip")
|
||||
recorder := httptest.NewRecorder()
|
||||
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
|
||||
FileName: "v.mp4",
|
||||
}, "")
|
||||
}).ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200 for cached preview, got %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "identity" {
|
||||
t.Fatalf("expected Content-Encoding identity (chi Compress skipped), got %q", encoding)
|
||||
}
|
||||
if !bytes.Equal(recorder.Body.Bytes(), payload) {
|
||||
t.Fatal("video bytes were transformed on the way out; expected the cached artifact verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
// The sync=1 parameter is the per-request rollback switch: it must keep the
|
||||
// legacy blocking stream instead of redirecting to the direct URL.
|
||||
func TestBrowserVideoPreviewSyncParamForcesBlockingStream(t *testing.T) {
|
||||
originalPath := sourceBrowserVideoPreviewPath
|
||||
originalWriter := sourceBrowserVideoPreviewWriter
|
||||
defer func() {
|
||||
sourceBrowserVideoPreviewPath = originalPath
|
||||
sourceBrowserVideoPreviewWriter = originalWriter
|
||||
}()
|
||||
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
|
||||
return "", os.ErrNotExist
|
||||
}
|
||||
var streamed atomic.Bool
|
||||
sourceBrowserVideoPreviewWriter = func(_ context.Context, writer http.ResponseWriter, _ sourceBrowserPreviewRequest) error {
|
||||
_, _ = writer.Write([]byte("streamed"))
|
||||
streamed.Store(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
direct := "https://bucket.bcebos.com/dir/camera.mp4"
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser&sync=1", nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
writeBrowserVideoPreviewResponse(recorder, request, sourceBrowserPreviewRequest{
|
||||
FileName: "camera.mp4",
|
||||
RedirectURL: direct,
|
||||
}, direct)
|
||||
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("sync=1 must stream, got %d", recorder.Code)
|
||||
}
|
||||
if !streamed.Load() {
|
||||
t.Fatal("sync=1 must use the blocking server-side stream")
|
||||
}
|
||||
if recorder.Header().Get("Location") != "" {
|
||||
t.Fatal("sync=1 must not redirect to the direct URL")
|
||||
}
|
||||
}
|
||||
|
||||
// Cache eviction: entries past the TTL are removed regardless of size, and
|
||||
// under a size cap the least-recently-written entries are evicted first.
|
||||
func TestPruneSourceBrowserPreviewCacheTTLAndLRU(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(name string, age time.Duration, size int64) string {
|
||||
filePath := filepath.Join(dir, name)
|
||||
payload := make([]byte, size)
|
||||
if err := os.WriteFile(filePath, payload, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutOff := time.Now().Add(-age)
|
||||
if err := os.Chtimes(filePath, cutOff, cutOff); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return filePath
|
||||
}
|
||||
stale := write("stale.mp4", sourceBrowserPreviewCacheMaxAge+time.Hour, 10)
|
||||
oldest := write("oldest.mp4", 3*time.Hour, 600)
|
||||
middle := write("middle.mp4", 2*time.Hour, 600)
|
||||
newest := write("newest.mp4", time.Hour, 600)
|
||||
|
||||
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1300)
|
||||
|
||||
if _, err := os.Stat(stale); !os.IsNotExist(err) {
|
||||
t.Fatal("entry past the TTL must be deleted")
|
||||
}
|
||||
if _, err := os.Stat(oldest); !os.IsNotExist(err) {
|
||||
t.Fatal("least-recently-written entry must be evicted first under the size cap")
|
||||
}
|
||||
for _, kept := range []string{middle, newest} {
|
||||
if _, err := os.Stat(kept); err != nil {
|
||||
t.Fatalf("entry %s should have survived pruning: %v", kept, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Under the size cap but with nothing past the TTL, the directory must be
|
||||
// left untouched.
|
||||
func TestPruneSourceBrowserPreviewCacheNoopWhenWithinBounds(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
filePath := filepath.Join(dir, "fresh.mp4")
|
||||
if err := os.WriteFile(filePath, bytes.Repeat([]byte("x"), 128), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1<<20)
|
||||
if _, err := os.Stat(filePath); err != nil {
|
||||
t.Fatalf("entry within bounds must not be evicted: %v", err)
|
||||
}
|
||||
}
|
||||
118
internal/api/human_batches_flush_test.go
Normal file
118
internal/api/human_batches_flush_test.go
Normal file
@ -0,0 +1,118 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
// slow-page report S4: GET /api/human-batches must stay a read-only path —
|
||||
// no flushHumanReleasePartitions writes — while the background sweeper still
|
||||
// releases wait-expired batches the write paths no longer cover.
|
||||
func newHumanBatchesFlushServer(t *testing.T) (*server, *store.MemoryStore, domain.HumanBatchRecord) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
organization := domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Name: "Default Organization", Slug: "default", Status: domain.OrganizationStatusActive}
|
||||
if err := dataStore.UpsertOrganization(ctx, organization); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
admin := domain.UserRecord{ID: "admin-1", Name: "admin", Role: domain.UserRoleAdmin, Status: domain.UserStatusActive, OrganizationID: organization.ID}
|
||||
if err := dataStore.UpsertUser(ctx, admin); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
batch := domain.HumanBatchRecord{
|
||||
ID: "batch-sweep", OrganizationID: organization.ID, PluginID: "annotation-qa", ProjectID: "project-1", WorkflowID: "workflow-1", RunID: "run-1", GateNodeID: "gate-1",
|
||||
ReleasePolicy: domain.HumanReleaseAcceptedPartition, PartitionMaxItems: 10, PartitionMaxWaitSeconds: 1, Status: domain.HumanBatchActive,
|
||||
}
|
||||
if err := dataStore.UpsertHumanBatch(ctx, batch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Accepted long enough ago that the PartitionMaxWaitSeconds window has
|
||||
// expired — the exact case the old GET-path flush used to release.
|
||||
item := domain.HumanItemRecord{
|
||||
ID: "item-sweep-1", OrganizationID: organization.ID, PluginID: "annotation-qa", BatchID: batch.ID,
|
||||
ProjectID: batch.ProjectID, WorkflowID: batch.WorkflowID, RunID: batch.RunID, GateNodeID: batch.GateNodeID,
|
||||
Status: domain.HumanItemAccepted, AcceptedAt: nowISO(), Version: 1,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "clip-1"},
|
||||
}
|
||||
if err := dataStore.UpsertHumanItem(ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &server{store: dataStore}, dataStore, batch
|
||||
}
|
||||
|
||||
func TestHumanBatchesListDoesNotFlushReleasePartitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
server, dataStore, batch := newHumanBatchesFlushServer(t)
|
||||
organization, err := dataStore.GetOrganization(ctx, domain.DefaultOrganizationID)
|
||||
if err != nil || organization == nil {
|
||||
t.Fatalf("organization=%#v err=%v", organization, err)
|
||||
}
|
||||
admin, err := dataStore.GetUser(ctx, "admin-1")
|
||||
if err != nil || admin == nil {
|
||||
t.Fatalf("admin=%#v err=%v", admin, err)
|
||||
}
|
||||
principal := &authenticatedPrincipal{User: *admin, Organization: *organization}
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/human-batches", nil).WithContext(
|
||||
context.WithValue(ctx, authContextKey{}, principal),
|
||||
)
|
||||
response := httptest.NewRecorder()
|
||||
server.handleHumanBatchesList(response, request)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
partitions, err := dataStore.ListReleasePartitions(ctx, batch.OrganizationID, batch.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(partitions) != 0 {
|
||||
t.Fatalf("GET must not create release partitions, got %#v", partitions)
|
||||
}
|
||||
stored, err := dataStore.GetHumanItem(ctx, "item-sweep-1")
|
||||
if err != nil || stored == nil {
|
||||
t.Fatalf("item=%#v err=%v", stored, err)
|
||||
}
|
||||
if stored.ReleasePartitionID != "" || stored.Version != 1 {
|
||||
t.Fatalf("GET must not write items: partition=%q version=%d", stored.ReleasePartitionID, stored.Version)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanReleaseSweeperFlushesWaitExpiredBatches(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
server, dataStore, batch := newHumanBatchesFlushServer(t)
|
||||
ids, err := dataStore.ListHumanBatchIDsAwaitingRelease(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ids) != 1 || ids[0] != batch.ID {
|
||||
t.Fatalf("awaiting ids=%#v", ids)
|
||||
}
|
||||
server.sweepHumanReleasePartitions()
|
||||
partitions, err := dataStore.ListReleasePartitions(ctx, batch.OrganizationID, batch.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(partitions) != 1 || len(partitions[0].ItemIDs) != 1 {
|
||||
t.Fatalf("sweeper must release the wait-expired batch: %#v", partitions)
|
||||
}
|
||||
stored, err := dataStore.GetHumanItem(ctx, "item-sweep-1")
|
||||
if err != nil || stored == nil {
|
||||
t.Fatalf("item=%#v err=%v", stored, err)
|
||||
}
|
||||
if stored.ReleasePartitionID == "" {
|
||||
t.Fatal("sweeper must assign the item's release partition")
|
||||
}
|
||||
// Nothing left awaiting: the sweeper's next pass is a no-op.
|
||||
ids, err = dataStore.ListHumanBatchIDsAwaitingRelease(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ids) != 0 {
|
||||
t.Fatalf("no batch should remain awaiting, got %#v", ids)
|
||||
}
|
||||
}
|
||||
258
internal/api/human_items_pagination_test.go
Normal file
258
internal/api/human_items_pagination_test.go
Normal file
@ -0,0 +1,258 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func TestHumanItemsListPaginatesAndProjectsPayload(t *testing.T) {
|
||||
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
|
||||
admin := domain.UserRecord{ID: "admin-pagination", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
|
||||
|
||||
// Five extra items with list-needed metadata in sliceWorkflow plus a heavy
|
||||
// annotationWorkspace draft that must never cross the list wire.
|
||||
for index := 0; index < 5; index++ {
|
||||
item := fixture.item
|
||||
item.ID = domain.NewID("human-item")
|
||||
item.CreatedAt = time.Now().UTC().Add(time.Duration(index) * time.Minute).Format(time.RFC3339Nano)
|
||||
item.UpdatedAt = item.CreatedAt
|
||||
item.AssignedUserID = ""
|
||||
item.Status = domain.HumanItemReady
|
||||
item.Subject.Payload = map[string]any{
|
||||
"sliceWorkflow": map[string]any{"status": "editing", "phase": "annotate"},
|
||||
"annotationWorkspace": map[string]any{"sliceSegments": []any{map[string]any{"id": "seg", "start": 0, "end": 10}}},
|
||||
}
|
||||
if err := fixture.store.UpsertHumanItem(fixture.ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
first := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3", admin, nil)
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("page 1 status=%d body=%s", first.Code, first.Body.String())
|
||||
}
|
||||
var page1 struct {
|
||||
Items []domain.HumanItemRecord `json:"items"`
|
||||
NextCursor string `json:"nextCursor"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
if err := json.Unmarshal(first.Body.Bytes(), &page1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page1.Items) != 3 || !page1.HasMore || page1.NextCursor == "" {
|
||||
t.Fatalf("page 1 items=%d hasMore=%v cursor=%q", len(page1.Items), page1.HasMore, page1.NextCursor)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range page1.Items {
|
||||
seen[item.ID] = true
|
||||
if _, ok := item.Subject.Payload["sliceWorkflow"]; !ok {
|
||||
t.Fatalf("list projection must keep sliceWorkflow for %s", item.ID)
|
||||
}
|
||||
if _, ok := item.Subject.Payload["annotationWorkspace"]; ok {
|
||||
t.Fatalf("list projection must drop annotationWorkspace for %s", item.ID)
|
||||
}
|
||||
}
|
||||
|
||||
second := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3&cursor="+page1.NextCursor, admin, nil)
|
||||
if second.Code != http.StatusOK {
|
||||
t.Fatalf("page 2 status=%d body=%s", second.Code, second.Body.String())
|
||||
}
|
||||
var page2 struct {
|
||||
Items []domain.HumanItemRecord `json:"items"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
if err := json.Unmarshal(second.Body.Bytes(), &page2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page2.Items) != 3 || page2.HasMore {
|
||||
t.Fatalf("page 2 items=%d hasMore=%v", len(page2.Items), page2.HasMore)
|
||||
}
|
||||
for _, item := range page2.Items {
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("cursor page repeated item %s", item.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHumanItemsListBackfillsAcrossFilteredRawPages reproduces the t17 review
|
||||
// finding: the handler-level access filters can swallow an entire raw page
|
||||
// (here via a stage that is absent from the batch definition), and the fetch
|
||||
// loop must advance the raw cursor between rounds. Without that, every round
|
||||
// re-reads the same page and the response silently starves or repeats.
|
||||
func TestHumanItemsListBackfillsAcrossFilteredRawPages(t *testing.T) {
|
||||
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
|
||||
admin := domain.UserRecord{ID: "admin-backfill", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
|
||||
|
||||
// A full raw page (fetch floor is 100) of items whose stage is missing
|
||||
// from the batch definition: the store returns them, the handler drops
|
||||
// every one via humanStage lookup failure.
|
||||
for index := 0; index < 100; index++ {
|
||||
ghost := fixture.item
|
||||
ghost.ID = domain.NewID("human-item")
|
||||
ghost.CurrentStageID = "stage-that-does-not-exist"
|
||||
ghost.CreatedAt = time.Now().UTC().Add(time.Duration(200-index) * time.Second).Format(time.RFC3339Nano)
|
||||
ghost.UpdatedAt = ghost.CreatedAt
|
||||
if err := fixture.store.UpsertHumanItem(fixture.ctx, ghost); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Older visible items: only reachable once the loop skips past the ghost page.
|
||||
visible := make([]domain.HumanItemRecord, 0, 4)
|
||||
for index := 0; index < 4; index++ {
|
||||
item := fixture.item
|
||||
item.ID = fmt.Sprintf("human-item-visible-%d", index)
|
||||
item.CreatedAt = time.Now().UTC().Add(time.Duration(50-index) * time.Second).Format(time.RFC3339Nano)
|
||||
item.UpdatedAt = item.CreatedAt
|
||||
if err := fixture.store.UpsertHumanItem(fixture.ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
visible = append(visible, item)
|
||||
}
|
||||
|
||||
recorder := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3", admin, nil)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var page struct {
|
||||
Items []domain.HumanItemRecord `json:"items"`
|
||||
NextCursor string `json:"nextCursor"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &page); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 3 {
|
||||
t.Fatalf("expected the loop to backfill to the page limit, got %d items", len(page.Items))
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range page.Items {
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("duplicate item %s in a single page", item.ID)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
if item.CurrentStageID == "stage-that-does-not-exist" {
|
||||
t.Fatalf("handler-filtered item leaked into the page: %s", item.ID)
|
||||
}
|
||||
}
|
||||
if !page.HasMore || page.NextCursor == "" {
|
||||
t.Fatalf("continuation metadata missing after backfill: hasMore=%v cursor=%q", page.HasMore, page.NextCursor)
|
||||
}
|
||||
|
||||
// The continuation cursor must point strictly after the last returned
|
||||
// item: the next page contains the remaining visible item and no overlap.
|
||||
next := fixture.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=all&limit=3&cursor="+page.NextCursor, admin, nil)
|
||||
if next.Code != http.StatusOK {
|
||||
t.Fatalf("follow-up status=%d body=%s", next.Code, next.Body.String())
|
||||
}
|
||||
var followUp struct {
|
||||
Items []domain.HumanItemRecord `json:"items"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
if err := json.Unmarshal(next.Body.Bytes(), &followUp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, item := range followUp.Items {
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("continuation page repeated item %s", item.ID)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
}
|
||||
if len(seen) != len(page.Items)+len(followUp.Items) || seen[visible[len(visible)-1].ID] != true {
|
||||
t.Fatalf("pagination lost items: unique=%d", len(seen))
|
||||
}
|
||||
|
||||
// Malformed cursors are rejected like the runs endpoint.
|
||||
bad := fixture.request(http.MethodGet, "/api/human-items?pool=all&cursor=not-a-cursor", admin, nil)
|
||||
if bad.Code != http.StatusBadRequest {
|
||||
t.Fatalf("malformed cursor status=%d body=%s", bad.Code, bad.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanItemsAvailablePoolPushesFiltersIntoStore(t *testing.T) {
|
||||
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
|
||||
|
||||
// An assigned item and a cancelled item must both be excluded by the store
|
||||
// filter itself, not by post-fetch dropping.
|
||||
assigned := fixture.item
|
||||
assigned.ID = domain.NewID("human-item")
|
||||
assigned.AssignedUserID = fixture.annotator.ID
|
||||
assigned.Status = domain.HumanItemClaimed
|
||||
if err := fixture.store.UpsertHumanItem(fixture.ctx, assigned); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancelled := fixture.item
|
||||
cancelled.ID = domain.NewID("human-item")
|
||||
cancelled.AssignedUserID = ""
|
||||
cancelled.Status = domain.HumanItemCancelled
|
||||
if err := fixture.store.UpsertHumanItem(fixture.ctx, cancelled); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
pushed, err := fixture.store.ListHumanItems(fixture.ctx, domain.DefaultOrganizationID, domain.HumanItemQuery{
|
||||
PluginID: "annotation-qa",
|
||||
UnassignedOnly: true,
|
||||
Statuses: []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(pushed) != 1 || pushed[0].ID != fixture.item.ID {
|
||||
t.Fatalf("pushdown query returned %#v", pushed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunsListDefaultsToPaginatedContract(t *testing.T) {
|
||||
fixture := newFanyuV5FourStageFixture(t, domain.FanyuV5StageCutAdjustment, domain.HumanItemReady, "", 0)
|
||||
admin := domain.UserRecord{ID: "admin-runs-default", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin, Status: domain.UserStatusActive}
|
||||
|
||||
for index := 0; index < 60; index++ {
|
||||
run := domain.RunRecord{
|
||||
ID: domain.NewID("run"), OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: "project-v5", WorkflowID: fixture.batch.WorkflowID,
|
||||
Status: domain.RunStatusCompleted,
|
||||
CreatedAt: time.Now().UTC().Add(-time.Duration(index) * time.Minute).Format(time.RFC3339Nano),
|
||||
}
|
||||
if err := fixture.store.UpsertRun(fixture.ctx, run); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
response := fixture.request(http.MethodGet, "/api/runs", admin, nil)
|
||||
if response.Code != http.StatusOK {
|
||||
t.Fatalf("runs status=%d body=%s", response.Code, response.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
Items []domain.RunSummary `json:"items"`
|
||||
NextCursor string `json:"nextCursor"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
Total int `json:"total"`
|
||||
StatusCounts map[string]int `json:"statusCounts"`
|
||||
}
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The no-parameter path keeps the legacy {"items"}-only shape but must
|
||||
// never stream the whole collection again.
|
||||
if len(payload.Items) != 50 {
|
||||
t.Fatalf("default runs page items=%d", len(payload.Items))
|
||||
}
|
||||
|
||||
paged := fixture.request(http.MethodGet, "/api/runs?limit=10", admin, nil)
|
||||
if paged.Code != http.StatusOK {
|
||||
t.Fatalf("paged runs status=%d body=%s", paged.Code, paged.Body.String())
|
||||
}
|
||||
if err := json.Unmarshal(paged.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Items) != 10 || !payload.HasMore || payload.NextCursor == "" {
|
||||
t.Fatalf("paged runs items=%d hasMore=%v", len(payload.Items), payload.HasMore)
|
||||
}
|
||||
if payload.Total != 60 || payload.StatusCounts[string(domain.RunStatusCompleted)] != 60 {
|
||||
t.Fatalf("aggregates missing: total=%d counts=%v", payload.Total, payload.StatusCounts)
|
||||
}
|
||||
}
|
||||
@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
cryptorand "crypto/rand"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
@ -13,6 +15,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -166,13 +169,59 @@ func parseHumanBatchQuery(request *http.Request) domain.HumanBatchQuery {
|
||||
}
|
||||
}
|
||||
|
||||
func parseHumanItemQuery(request *http.Request) domain.HumanItemQuery {
|
||||
query := request.URL.Query()
|
||||
return domain.HumanItemQuery{
|
||||
PluginID: query.Get("pluginId"), BatchID: query.Get("batchId"), ProjectID: query.Get("projectId"),
|
||||
WorkflowID: query.Get("workflowId"), RunID: query.Get("runId"), CurrentStageID: query.Get("stageId"),
|
||||
Status: domain.HumanItemStatus(query.Get("status")),
|
||||
func parseHumanItemQuery(request *http.Request) (domain.HumanItemQuery, *validationIssue) {
|
||||
values := request.URL.Query()
|
||||
query := domain.HumanItemQuery{
|
||||
PluginID: values.Get("pluginId"), BatchID: values.Get("batchId"), ProjectID: values.Get("projectId"),
|
||||
WorkflowID: values.Get("workflowId"), RunID: values.Get("runId"), CurrentStageID: values.Get("stageId"),
|
||||
Status: domain.HumanItemStatus(values.Get("status")),
|
||||
}
|
||||
// Reject malformed cursors like the runs endpoint instead of silently
|
||||
// falling back to the first page.
|
||||
cursor, err := decodeHumanItemCursor(values.Get("cursor"))
|
||||
if err != nil {
|
||||
return query, &validationIssue{Field: "cursor", Message: "cursor must be a valid opaque human item cursor"}
|
||||
}
|
||||
query.Cursor = cursor
|
||||
return query, nil
|
||||
}
|
||||
|
||||
const (
|
||||
humanItemsDefaultLimit = 500
|
||||
humanItemsMaxLimit = 1000
|
||||
// Raw Mongo pages larger than the requested page absorb items dropped by
|
||||
// the in-memory access filters without unbounded over-fetching.
|
||||
humanItemsMaxFetchRounds = 6
|
||||
)
|
||||
|
||||
func decodeHumanItemCursor(raw string) (*domain.HumanItemCursor, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil, nil
|
||||
}
|
||||
payload, err := base64.RawURLEncoding.DecodeString(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cursor must be a valid opaque human item cursor")
|
||||
}
|
||||
var cursor domain.HumanItemCursor
|
||||
if err := json.Unmarshal(payload, &cursor); err != nil {
|
||||
return nil, fmt.Errorf("cursor must be a valid opaque human item cursor")
|
||||
}
|
||||
if strings.TrimSpace(cursor.CreatedAt) == "" || strings.TrimSpace(cursor.ID) == "" {
|
||||
return nil, fmt.Errorf("cursor must include createdAt and id")
|
||||
}
|
||||
return &cursor, nil
|
||||
}
|
||||
|
||||
func encodeHumanItemCursor(cursor *domain.HumanItemCursor) string {
|
||||
if cursor == nil {
|
||||
return ""
|
||||
}
|
||||
payload, err := json.Marshal(cursor)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(payload)
|
||||
}
|
||||
|
||||
func parseHumanRejectQuery(request *http.Request) domain.HumanRejectQuery {
|
||||
@ -189,12 +238,9 @@ func (server *server) handleHumanBatchesList(writer http.ResponseWriter, request
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
for index := range items {
|
||||
if err := server.flushHumanReleasePartitions(request.Context(), &items[index], false); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Read path stays read-only (S4): release partitions are flushed by the
|
||||
// item-acceptance write paths and by the background sweeper started in
|
||||
// NewHandlerWithConfig (startHumanReleasePartitionSweeper).
|
||||
principal, _ := currentPrincipal(request)
|
||||
if !annotationManager(principal) {
|
||||
filtered := make([]domain.HumanBatchRecord, 0, len(items))
|
||||
@ -234,7 +280,11 @@ func (server *server) handleHumanItemsList(writer http.ResponseWriter, request *
|
||||
renderJSON(writer, http.StatusForbidden, map[string]string{"error": "only administrators may list all human items"})
|
||||
return
|
||||
}
|
||||
query := parseHumanItemQuery(request)
|
||||
query, queryIssue := parseHumanItemQuery(request)
|
||||
if queryIssue != nil {
|
||||
renderValidationError(writer, *queryIssue)
|
||||
return
|
||||
}
|
||||
if pool == "reviewed" {
|
||||
if principal == nil || (principal.User.Role != domain.UserRoleQualityReviewer && principal.User.Role != domain.UserRoleAcceptanceReviewer && !annotationManager(principal)) {
|
||||
renderJSON(writer, http.StatusForbidden, map[string]string{"error": "review history requires reviewer access"})
|
||||
@ -251,58 +301,115 @@ func (server *server) handleHumanItemsList(writer http.ResponseWriter, request *
|
||||
if pool == "mine" && principal != nil {
|
||||
query.ActorUserID = principal.User.ID
|
||||
}
|
||||
items, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
limit := humanItemsDefaultLimit
|
||||
if rawLimit := strings.TrimSpace(request.URL.Query().Get("limit")); rawLimit != "" {
|
||||
parsed, issue := parseDatabaseBrowserPageQuery(rawLimit, "limit", humanItemsDefaultLimit, 1, humanItemsMaxLimit)
|
||||
if issue != nil {
|
||||
renderValidationError(writer, *issue)
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
listCtx, listErr := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, items)
|
||||
if listErr != nil {
|
||||
renderError(writer, http.StatusInternalServerError, listErr)
|
||||
return
|
||||
// The handler-level "available" pool rule is pushed into the Mongo filter
|
||||
// so unassigned/claimable pages do not drag assigned rows through the wire.
|
||||
if pool == "available" && query.Status == "" {
|
||||
query.UnassignedOnly = true
|
||||
query.Statuses = []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}
|
||||
}
|
||||
request = request.WithContext(listCtx)
|
||||
filtered := make([]domain.HumanItemRecord, 0, len(items))
|
||||
// List responses never need the workbench draft or disposition histories.
|
||||
query.ListProjection = true
|
||||
fetchSize := limit
|
||||
if fetchSize < 100 {
|
||||
fetchSize = 100
|
||||
}
|
||||
|
||||
filtered := make([]domain.HumanItemRecord, 0, limit)
|
||||
batchByID := map[string]*domain.HumanBatchRecord{}
|
||||
for _, item := range items {
|
||||
batch, loaded := batchByID[item.BatchID]
|
||||
if !loaded {
|
||||
batch, err = server.humanBatchForListRead(request.Context(), item.BatchID)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
var nextCursor *domain.HumanItemCursor
|
||||
hasMore := false
|
||||
exhaustedAll := false
|
||||
for round := 0; round < humanItemsMaxFetchRounds && len(filtered) < limit; round++ {
|
||||
query.Limit = fetchSize
|
||||
items, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
listCtx, listErr := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, items)
|
||||
if listErr != nil {
|
||||
renderError(writer, http.StatusInternalServerError, listErr)
|
||||
return
|
||||
}
|
||||
request = request.WithContext(listCtx)
|
||||
pageExhausted := len(items) < fetchSize
|
||||
pageFull := false
|
||||
for index, item := range items {
|
||||
batch, loaded := batchByID[item.BatchID]
|
||||
if !loaded {
|
||||
batch, err = server.humanBatchForListRead(request.Context(), item.BatchID)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
batchByID[item.BatchID] = batch
|
||||
}
|
||||
batchByID[item.BatchID] = batch
|
||||
}
|
||||
stage, _, found := humanStage(batch, item.CurrentStageID)
|
||||
if randomClaimAnnotator(principal) && item.PluginID == "annotation-qa" && annotationClaimStage(batch, item.CurrentStageID) && item.AssignedUserID != principal.User.ID && item.LatestAuthorUserID != principal.User.ID {
|
||||
continue
|
||||
}
|
||||
if !found || !server.canAccessHumanItem(request.Context(), request, item, "") {
|
||||
continue
|
||||
}
|
||||
isAuthor := principal != nil && item.LatestAuthorUserID == principal.User.ID
|
||||
if pool != "reviewed" && !server.canAccessHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage)) && !isAuthor {
|
||||
continue
|
||||
}
|
||||
if pool == "mine" {
|
||||
isAssigned := principal != nil && item.AssignedUserID == principal.User.ID
|
||||
isMySubmitted := isAuthor && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemSubmitted) && item.CurrentStageID == "quality_review"
|
||||
if !isAssigned && !isMySubmitted {
|
||||
stage, _, found := humanStage(batch, item.CurrentStageID)
|
||||
if randomClaimAnnotator(principal) && item.PluginID == "annotation-qa" && annotationClaimStage(batch, item.CurrentStageID) && item.AssignedUserID != principal.User.ID && item.LatestAuthorUserID != principal.User.ID {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if pool == "available" {
|
||||
availableStatus := item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted
|
||||
if item.AssignedUserID != "" || !availableStatus {
|
||||
if !found || !server.canAccessHumanItem(request.Context(), request, item, "") {
|
||||
continue
|
||||
}
|
||||
isAuthor := principal != nil && item.LatestAuthorUserID == principal.User.ID
|
||||
if pool != "reviewed" && !server.canAccessHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage)) && !isAuthor {
|
||||
continue
|
||||
}
|
||||
if pool == "mine" {
|
||||
isAssigned := principal != nil && item.AssignedUserID == principal.User.ID
|
||||
isMySubmitted := isAuthor && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemSubmitted) && item.CurrentStageID == "quality_review"
|
||||
if !isAssigned && !isMySubmitted {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if pool == "available" {
|
||||
availableStatus := item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted
|
||||
if item.AssignedUserID != "" || !availableStatus {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if principal != nil && principal.User.Role == domain.UserRoleAcceptanceReviewer {
|
||||
claimable := item.AssignedUserID == "" && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted) && server.canClaimHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage))
|
||||
item.CanClaim = &claimable
|
||||
}
|
||||
filtered = append(filtered, item)
|
||||
if len(filtered) >= limit {
|
||||
// Continuation resumes right after the last returned item.
|
||||
nextCursor = &domain.HumanItemCursor{CreatedAt: item.CreatedAt, ID: item.ID}
|
||||
pageFull = true
|
||||
// More rows exist if the raw page still has unconsumed items
|
||||
// or the store itself is not exhausted.
|
||||
hasMore = index < len(items)-1 || !pageExhausted
|
||||
break
|
||||
}
|
||||
}
|
||||
if principal != nil && principal.User.Role == domain.UserRoleAcceptanceReviewer {
|
||||
claimable := item.AssignedUserID == "" && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemRework || item.Status == domain.HumanItemSubmitted) && server.canClaimHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage))
|
||||
item.CanClaim = &claimable
|
||||
if pageFull {
|
||||
break
|
||||
}
|
||||
filtered = append(filtered, item)
|
||||
if pageExhausted {
|
||||
exhaustedAll = true
|
||||
break
|
||||
}
|
||||
last := items[len(items)-1]
|
||||
nextCursor = &domain.HumanItemCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
||||
// Advance the raw fetch cursor so the next round continues strictly
|
||||
// after this page; otherwise the same raw page repeats every round
|
||||
// (t17 review finding) and items can be duplicated or starve the loop.
|
||||
query.Cursor = nextCursor
|
||||
}
|
||||
// Fetch-round cap reached without exhausting the store: let the client
|
||||
// continue from the last raw cursor instead of silently truncating.
|
||||
if nextCursor != nil && !exhaustedAll && !hasMore && len(filtered) < limit {
|
||||
hasMore = true
|
||||
}
|
||||
if pool == "reviewed" {
|
||||
acceptanceIDs := []string{}
|
||||
@ -344,7 +451,14 @@ func (server *server) handleHumanItemsList(writer http.ResponseWriter, request *
|
||||
return
|
||||
}
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": filtered})
|
||||
// Keep the legacy {"items"} body byte-compatible when the page is
|
||||
// exhausted; pagination metadata appears only when a next page exists.
|
||||
response := map[string]any{"items": filtered}
|
||||
if hasMore {
|
||||
response["nextCursor"] = encodeHumanItemCursor(nextCursor)
|
||||
response["hasMore"] = true
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, response)
|
||||
}
|
||||
|
||||
func (server *server) anonymizeV58AcceptanceItem(ctx context.Context, item *domain.HumanItemRecord) {
|
||||
@ -2591,6 +2705,56 @@ func (server *server) flushHumanReleasePartitions(ctx context.Context, batch *do
|
||||
return server.store.UpsertHumanBatch(ctx, *batch)
|
||||
}
|
||||
|
||||
// humanReleaseFlushSweepInterval is how often the background sweeper looks for
|
||||
// batches whose PartitionMaxWaitSeconds wait window expired without a further
|
||||
// item write. The write paths already flush on every item acceptance, so this
|
||||
// interval only adds latency for the wait-expiry case that the old
|
||||
// GET /api/human-batches flush used to cover opportunistically.
|
||||
const humanReleaseFlushSweepInterval = 60 * time.Second
|
||||
|
||||
// One sweeper per process: NewHandlerWithConfig also runs in every test, and a
|
||||
// single loop is enough for the whole API (flushHumanReleasePartitions is
|
||||
// idempotent across replicas, same as it was across concurrent GETs).
|
||||
var humanReleaseFlushSweeperOnce sync.Once
|
||||
|
||||
// startHumanReleasePartitionSweeper moves release-partition flushing off the
|
||||
// GET read path (S4): item-acceptance write paths flush immediately, and this
|
||||
// periodic sweep covers wait-expiry batches within one interval.
|
||||
func (server *server) startHumanReleasePartitionSweeper() {
|
||||
humanReleaseFlushSweeperOnce.Do(func() {
|
||||
go func() {
|
||||
ticker := time.NewTicker(humanReleaseFlushSweepInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
server.sweepHumanReleasePartitions()
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func (server *server) sweepHumanReleasePartitions() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
ids, err := server.store.ListHumanBatchIDsAwaitingRelease(ctx)
|
||||
if err != nil {
|
||||
log.Printf("[human-release-sweeper] list awaiting batches: %v", err)
|
||||
return
|
||||
}
|
||||
for _, id := range ids {
|
||||
batch, err := server.store.GetHumanBatch(ctx, id)
|
||||
if err != nil {
|
||||
log.Printf("[human-release-sweeper] get batch %s: %v", id, err)
|
||||
continue
|
||||
}
|
||||
if batch == nil {
|
||||
continue
|
||||
}
|
||||
if err := server.flushHumanReleasePartitions(ctx, batch, false); err != nil {
|
||||
log.Printf("[human-release-sweeper] flush batch %s: %v", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// History grants read access only; mutation handlers retain current-stage checks.
|
||||
func (server *server) canReadReviewedHumanItem(ctx context.Context, request *http.Request, item domain.HumanItemRecord) bool {
|
||||
principal, ok := currentPrincipal(request)
|
||||
|
||||
@ -13,6 +13,7 @@ import (
|
||||
"mime"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
@ -47,6 +48,7 @@ type validationIssue struct {
|
||||
|
||||
type server struct {
|
||||
humanEvidenceIndexes humanEvidenceCache
|
||||
dashboardResponses dashboardResponseCache
|
||||
store store.Store
|
||||
ai ai.Service
|
||||
operatorRunnerImage string
|
||||
@ -87,6 +89,114 @@ type runJobTerminator interface {
|
||||
|
||||
var storageConnectionBucketsResolver = listStorageBucketsFromConnection
|
||||
var storageConnectionObjectsResolver = listStorageObjectsFromConnection
|
||||
|
||||
// dashboardResponseCache is the C4/S2 30-60s in-process response cache for the
|
||||
// dashboard and production dashboard endpoints.
|
||||
//
|
||||
// Consistency / invalidation policy:
|
||||
// - Entries expire after dashboardResponseCacheTTL (45s). Dashboard and
|
||||
// production counters tolerate that staleness by contract.
|
||||
// - Every non-GET /api request flushes the whole cache (see
|
||||
// invalidateDashboardCachesOnWrite), so task submissions, human-item
|
||||
// transitions, scope edits, user/role changes and logouts take effect on
|
||||
// the next dashboard render even inside the TTL window.
|
||||
// - Keys include organization, tenant mode, principal scope (role, user id,
|
||||
// supplier id) and the raw query string, so per-user workforce sections and
|
||||
// tenant isolation are never served cross-principal.
|
||||
// - Writes done directly against the store outside the API (workers) are
|
||||
// visible after at most the TTL.
|
||||
const dashboardResponseCacheTTL = 45 * time.Second
|
||||
|
||||
type dashboardResponseCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]dashboardResponseCacheEntry
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type dashboardResponseCacheEntry struct {
|
||||
body []byte
|
||||
contentType string
|
||||
status int
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
func (c *dashboardResponseCache) get(key string) *dashboardResponseCacheEntry {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
entry, ok := c.entries[key]
|
||||
if !ok || c.now().After(entry.expires) {
|
||||
return nil
|
||||
}
|
||||
body := make([]byte, len(entry.body))
|
||||
copy(body, entry.body)
|
||||
return &dashboardResponseCacheEntry{body: body, contentType: entry.contentType, status: entry.status, expires: entry.expires}
|
||||
}
|
||||
|
||||
func (c *dashboardResponseCache) set(key string, status int, contentType string, body []byte) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.entries == nil {
|
||||
c.entries = map[string]dashboardResponseCacheEntry{}
|
||||
}
|
||||
if len(c.entries) >= 128 {
|
||||
c.entries = map[string]dashboardResponseCacheEntry{}
|
||||
}
|
||||
stored := make([]byte, len(body))
|
||||
copy(stored, body)
|
||||
c.entries[key] = dashboardResponseCacheEntry{body: stored, contentType: contentType, status: status, expires: c.now().Add(dashboardResponseCacheTTL)}
|
||||
}
|
||||
|
||||
func (c *dashboardResponseCache) invalidateAll() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.entries = map[string]dashboardResponseCacheEntry{}
|
||||
}
|
||||
|
||||
func (server *server) invalidateDashboardCachesOnWrite(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodGet && request.Method != http.MethodHead && strings.HasPrefix(request.URL.Path, "/api/") {
|
||||
server.dashboardResponses.invalidateAll()
|
||||
}
|
||||
next.ServeHTTP(writer, request)
|
||||
})
|
||||
}
|
||||
|
||||
func (server *server) dashboardCacheKey(prefix string, request *http.Request) string {
|
||||
scope := "anonymous"
|
||||
if principal, ok := currentPrincipal(request); ok && principal != nil {
|
||||
scope = fmt.Sprintf("%s|%s|%s", principal.User.Role, principal.User.ID, principal.User.SupplierID)
|
||||
}
|
||||
return fmt.Sprintf("%s|%v|%s|%s|%s", prefix, server.tenantEnforced(request), server.requestOrganizationID(request), scope, request.URL.RawQuery)
|
||||
}
|
||||
|
||||
// withDashboardResponseCache wraps a read-only dashboard handler with the
|
||||
// shared 45s response cache. Cache misses execute the handler against a
|
||||
// recorder and store the exact response bytes for later hits.
|
||||
func (server *server) withDashboardResponseCache(prefix string, next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(writer http.ResponseWriter, request *http.Request) {
|
||||
key := server.dashboardCacheKey(prefix, request)
|
||||
if entry := server.dashboardResponses.get(key); entry != nil {
|
||||
if entry.contentType != "" {
|
||||
writer.Header().Set("Content-Type", entry.contentType)
|
||||
}
|
||||
writer.WriteHeader(entry.status)
|
||||
_, _ = writer.Write(entry.body)
|
||||
return
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
next(recorder, request)
|
||||
body := recorder.Body.Bytes()
|
||||
if recorder.Code == http.StatusOK {
|
||||
server.dashboardResponses.set(key, recorder.Code, recorder.Header().Get("Content-Type"), body)
|
||||
}
|
||||
if contentType := recorder.Header().Get("Content-Type"); contentType != "" {
|
||||
writer.Header().Set("Content-Type", contentType)
|
||||
}
|
||||
writer.WriteHeader(recorder.Code)
|
||||
_, _ = writer.Write(body)
|
||||
}
|
||||
}
|
||||
|
||||
var runJobTerminatorFactory = func(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (runJobTerminator, error) {
|
||||
return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, "", "", "", 0)
|
||||
}
|
||||
@ -1269,6 +1379,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
trustedProxyNets: parseTrustedProxyCIDRs(appConfig.TrustedProxyCIDRs),
|
||||
authLimiter: newRateLimiter(20, time.Minute),
|
||||
externalWriteLimiter: newRateLimiter(120, time.Minute),
|
||||
dashboardResponses: dashboardResponseCache{now: time.Now},
|
||||
k8sNamespace: appConfig.K8sNamespace,
|
||||
k8sImagePullSecretName: appConfig.K8sImagePullSecretName,
|
||||
workspacePVCName: appConfig.WorkspacePVCName,
|
||||
@ -1282,9 +1393,11 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
buildTime: appConfig.BuildTime,
|
||||
image: appConfig.Image,
|
||||
}
|
||||
instance.startHumanReleasePartitionSweeper()
|
||||
router := chi.NewRouter()
|
||||
router.Use(diagnosticTrace)
|
||||
router.Use(middleware.Compress(5))
|
||||
router.Use(instance.invalidateDashboardCachesOnWrite)
|
||||
router.Use(instance.apiTokenAuth)
|
||||
router.Use(instance.sessionAuth)
|
||||
router.Use(instance.csrfProtection)
|
||||
@ -1334,9 +1447,9 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
router.Post("/api/operators/image-archives/import", instance.handleOperatorImageArchiveImport)
|
||||
})
|
||||
router.With(instance.requireAdminMonitoringAccess).Get("/api/acceptance/poc-readiness", instance.handlePocAcceptanceReadiness)
|
||||
router.Get("/api/dashboard", instance.handleDashboard)
|
||||
router.Get("/api/dashboard", instance.withDashboardResponseCache("dashboard", instance.handleDashboard))
|
||||
router.Get("/api/workforce-returns", instance.handleWorkforceReturns)
|
||||
router.Get("/api/dashboard/production", instance.handleProductionDashboard)
|
||||
router.Get("/api/dashboard/production", instance.withDashboardResponseCache("production", instance.handleProductionDashboard))
|
||||
router.Get("/api/dashboard/production/scope", instance.handleProductionScope)
|
||||
router.Put("/api/dashboard/production/scope", instance.handleProductionScope)
|
||||
router.Get("/api/plugins", instance.handlePluginsList)
|
||||
@ -1940,7 +2053,12 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
runSummaries, err := server.listRunSummariesForRequest(request.Context(), request, "")
|
||||
// C4: reuse the cursor-paginated run summary query (default page size 50)
|
||||
// instead of materializing every run in the organization. Org-wide
|
||||
// status totals are restored from the page aggregates below so the
|
||||
// response counters stay exact; only trend/average granularity is
|
||||
// limited to the most recent page.
|
||||
runPage, err := server.listRunSummaryPageForRequest(request.Context(), request, domain.RunSummaryQuery{Limit: 50, IncludeAggregates: true, Mode: domain.RunModeProduction})
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
@ -1955,7 +2073,20 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
summary := store.BuildDashboardSummaryAt(projects, operators, runSummaries, auditTasks, settings.Timezone, time.Now())
|
||||
summary := store.BuildDashboardSummaryAt(projects, operators, runPage.Items, auditTasks, settings.Timezone, time.Now())
|
||||
if runPage.StatusCounts != nil {
|
||||
summary.Tasks.Running = int(runPage.StatusCounts[domain.RunStatusRunning])
|
||||
summary.Tasks.Succeeded = int(runPage.StatusCounts[domain.RunStatusCompleted])
|
||||
summary.Tasks.Failed = int(runPage.StatusCounts[domain.RunStatusFailed])
|
||||
totalFinished := summary.Tasks.Succeeded + summary.Tasks.Failed
|
||||
if totalFinished == 0 {
|
||||
summary.Jobs.SuccessRate = 100
|
||||
} else {
|
||||
summary.Jobs.SuccessRate = float64(summary.Tasks.Succeeded) * 100 / float64(totalFinished)
|
||||
}
|
||||
summary.Jobs.Running = summary.Tasks.Running
|
||||
summary.Jobs.Failed = summary.Tasks.Failed
|
||||
}
|
||||
enrichDashboardOperatorInventory(&summary, operators)
|
||||
server.enrichDashboardHumanWork(request.Context(), request, &summary)
|
||||
if err := server.populateDashboardWorkforce(request.Context(), server.requestOrganizationID(request), principal, from, to, &summary); err != nil {
|
||||
@ -1986,11 +2117,26 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http.
|
||||
renderJSON(writer, http.StatusOK, summary)
|
||||
}
|
||||
|
||||
// dashboardHumanWorkCounter is implemented by both store backends; the Mongo
|
||||
// implementation computes the counters with aggregation pipelines (C4) instead
|
||||
// of materializing human items and delivery packages.
|
||||
type dashboardHumanWorkCounter interface {
|
||||
DashboardHumanWorkStats(ctx context.Context, organizationID string) (store.DashboardHumanWorkStats, error)
|
||||
}
|
||||
|
||||
func (server *server) enrichDashboardHumanWork(ctx context.Context, request *http.Request, summary *domain.DashboardSummary) {
|
||||
if summary == nil {
|
||||
return
|
||||
}
|
||||
organizationID := server.requestOrganizationID(request)
|
||||
if counter, ok := server.store.(dashboardHumanWorkCounter); ok {
|
||||
if stats, err := counter.DashboardHumanWorkStats(ctx, organizationID); err == nil {
|
||||
applyDashboardHumanWorkStats(summary, stats)
|
||||
}
|
||||
return
|
||||
}
|
||||
// Fallback for store implementations without the counting interface:
|
||||
// keep the legacy full-materialization loop (unchanged semantics).
|
||||
items, err := server.store.ListHumanItems(ctx, organizationID, domain.HumanItemQuery{})
|
||||
if err == nil {
|
||||
accepted := 0
|
||||
@ -2045,6 +2191,26 @@ func (server *server) enrichDashboardHumanWork(ctx context.Context, request *htt
|
||||
summary.Tasks.Waiting = summary.Tasks.PendingAnnotation + summary.Tasks.PendingQuality + summary.Tasks.PendingAcceptance
|
||||
}
|
||||
|
||||
// applyDashboardHumanWorkStats mirrors the legacy enrich loop's summary
|
||||
// mutations using precomputed counters, keeping the response shape and the
|
||||
// +=/= semantics identical.
|
||||
func applyDashboardHumanWorkStats(summary *domain.DashboardSummary, stats store.DashboardHumanWorkStats) {
|
||||
summary.Data.CaptureDurationSeconds += stats.CaptureDurationSeconds
|
||||
summary.Tasks.PendingAnnotation += stats.PendingAnnotation
|
||||
summary.Tasks.PendingQuality += stats.PendingQuality
|
||||
summary.Tasks.InAnnotation += stats.InAnnotation
|
||||
summary.Tasks.PendingAcceptance += stats.PendingAcceptanceItems
|
||||
summary.Tasks.Rejected += stats.Rejected
|
||||
summary.Tasks.Escalated += stats.Escalated
|
||||
summary.Data.VideoCount = stats.VideoCount
|
||||
if decided := stats.Accepted + stats.Rejected; decided > 0 {
|
||||
summary.Tasks.QualityPassRate = float64(stats.Accepted) * 100 / float64(decided)
|
||||
}
|
||||
summary.Data.PackageCount = stats.PackageCount
|
||||
summary.Tasks.PendingAcceptance += stats.PackagesPendingAcceptance
|
||||
summary.Tasks.Waiting = summary.Tasks.PendingAnnotation + summary.Tasks.PendingQuality + summary.Tasks.PendingAcceptance
|
||||
}
|
||||
|
||||
func enrichDashboardOperatorInventory(summary *domain.DashboardSummary, operators []domain.OperatorRecord) {
|
||||
if summary == nil {
|
||||
return
|
||||
@ -2888,7 +3054,7 @@ func normalizeDashboardPOCReadinessLists(poc *domain.DashboardPOCReadiness) {
|
||||
}
|
||||
|
||||
func (server *server) dashboardPOCReadiness(ctx context.Context, request *http.Request) domain.DashboardPOCReadiness {
|
||||
runs, err := server.listRunsForRequest(ctx, request, "")
|
||||
runs, err := server.dashboardPOCReadinessCandidateRuns(ctx, request)
|
||||
if err != nil {
|
||||
gpu := server.dashboardGPUCapacityGate(ctx)
|
||||
poc := domain.DashboardPOCReadiness{
|
||||
@ -2991,6 +3157,56 @@ func (server *server) enrichDashboardPOC(ctx context.Context, request *http.Requ
|
||||
summary.POC = server.dashboardPOCReadiness(ctx, request)
|
||||
}
|
||||
|
||||
// dashboardPOCReadinessCandidateLimit bounds each point-query half of
|
||||
// dashboardPOCReadinessCandidateRuns. The latest candidate is always first in
|
||||
// the store's recency order, so a modest bound keeps the scan cheap without
|
||||
// ever excluding the run the selectors would pick.
|
||||
const dashboardPOCReadinessCandidateLimit = 25
|
||||
|
||||
// dashboardPOCReadinessCandidateRuns is the slow-page report S4 point-query
|
||||
// replacement for the former listRunsForRequest full-table scan: it fetches
|
||||
// only runs that can carry a POC acceptance artifact, plus runs on the
|
||||
// POC/live acceptance workflows, and merges the two bounded sets. The
|
||||
// downstream selectors (dashboardLatestAcceptancePayload /
|
||||
// dashboardGPUWorkflowPreflightGate) run unchanged over the merged set, and
|
||||
// every run they could possibly pick matches one of the two store filters, so
|
||||
// the response stays byte-compatible with the full-scan behavior.
|
||||
func (server *server) dashboardPOCReadinessCandidateRuns(ctx context.Context, request *http.Request) ([]domain.RunRecord, error) {
|
||||
organizationID := ""
|
||||
if server.tenantEnforced(request) {
|
||||
organizationID = server.requestOrganizationID(request)
|
||||
}
|
||||
acceptanceRuns, err := server.store.ListRunsAcceptanceCandidates(ctx, organizationID, dashboardPOCReadinessCandidateLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gpuRuns, err := server.store.ListRunsWorkflowCandidates(
|
||||
ctx,
|
||||
organizationID,
|
||||
[]string{pocGPUWorkflowAcceptanceWorkflowName, liveWorkflowAcceptanceWorkflowName},
|
||||
[]string{pocGPUWorkflowAcceptanceProjectName, liveWorkflowAcceptanceProjectName},
|
||||
dashboardPOCReadinessCandidateLimit,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := make(map[string]bool, len(acceptanceRuns)+len(gpuRuns))
|
||||
runs := make([]domain.RunRecord, 0, len(acceptanceRuns)+len(gpuRuns))
|
||||
for _, run := range acceptanceRuns {
|
||||
if !seen[run.ID] {
|
||||
seen[run.ID] = true
|
||||
runs = append(runs, run)
|
||||
}
|
||||
}
|
||||
for _, run := range gpuRuns {
|
||||
if !seen[run.ID] {
|
||||
seen[run.ID] = true
|
||||
runs = append(runs, run)
|
||||
}
|
||||
}
|
||||
return runs, nil
|
||||
}
|
||||
|
||||
func (server *server) populateProjectWorkflowCounts(ctx context.Context, request *http.Request, items []domain.ProjectRecord) ([]domain.ProjectRecord, error) {
|
||||
if len(items) == 0 {
|
||||
return items, nil
|
||||
@ -7004,38 +7220,43 @@ func parseRunSummaryQuery(request *http.Request) (domain.RunSummaryQuery, []vali
|
||||
}
|
||||
|
||||
func (server *server) handleRunsList(writer http.ResponseWriter, request *http.Request) {
|
||||
if runSummaryPaginationRequested(request) {
|
||||
query, issues := parseRunSummaryQuery(request)
|
||||
if len(issues) > 0 {
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
startedAt := time.Now()
|
||||
page, err := server.listRunSummaryPageForRequest(request.Context(), request, query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
writer.Header().Set("Server-Timing", fmt.Sprintf("runs;dur=%.1f", float64(time.Since(startedAt).Microseconds())/1000))
|
||||
response := map[string]any{
|
||||
"items": page.Items,
|
||||
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
|
||||
"hasMore": page.HasMore,
|
||||
}
|
||||
if query.IncludeAggregates {
|
||||
response["total"] = page.Total
|
||||
response["statusCounts"] = page.StatusCounts
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, response)
|
||||
// S1: the legacy no-parameter path used to stream every run summary.
|
||||
// Route every request through the cursor-paginated store query (default
|
||||
// limit 50) and keep the response a backward-compatible superset.
|
||||
explicitlyPaginated := runSummaryPaginationRequested(request)
|
||||
query, issues := parseRunSummaryQuery(request)
|
||||
if len(issues) > 0 {
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
items, err := server.listRunSummariesForRequest(request.Context(), request, request.URL.Query().Get("status"))
|
||||
startedAt := time.Now()
|
||||
page, err := server.listRunSummaryPageForRequest(request.Context(), request, query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
items = server.filterRunSummariesForRequest(request, items)
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
||||
items := page.Items
|
||||
if !explicitlyPaginated {
|
||||
// Preserve the legacy per-project visibility filtering for callers
|
||||
// that never opted into the paginated contract.
|
||||
items = server.filterRunSummariesForRequest(request, items)
|
||||
}
|
||||
writer.Header().Set("Server-Timing", fmt.Sprintf("runs;dur=%.1f", float64(time.Since(startedAt).Microseconds())/1000))
|
||||
if !explicitlyPaginated {
|
||||
// Byte-compatible legacy shape: status-only callers pinned {"items"}.
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
||||
return
|
||||
}
|
||||
response := map[string]any{
|
||||
"items": items,
|
||||
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
|
||||
"hasMore": page.HasMore,
|
||||
}
|
||||
if query.IncludeAggregates {
|
||||
response["total"] = page.Total
|
||||
response["statusCounts"] = page.StatusCounts
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, response)
|
||||
}
|
||||
|
||||
func (server *server) loadRun(ctx context.Context, id string) (*domain.RunRecord, error) {
|
||||
@ -7744,11 +7965,11 @@ func (server *server) serveResolvedRunArtifactContent(writer http.ResponseWriter
|
||||
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
|
||||
if artifactBrowserTranscodeRequested(request, artifact, mimeType) {
|
||||
previewRequest := sourceBrowserPreviewRequest{
|
||||
FileName: fileName,
|
||||
FileName: fileName,
|
||||
RedirectURL: previewURL,
|
||||
}
|
||||
// The presigned URL changes on every request; anchor the transcode
|
||||
// cache on the stable object coordinates instead.
|
||||
// cache on the stable object coordinates instead.
|
||||
if objectBucket, objectKey, ok := artifactBOSLocation(artifact); ok {
|
||||
previewRequest.ObjectBucket, previewRequest.ObjectKey = objectBucket, objectKey
|
||||
}
|
||||
@ -7917,12 +8138,35 @@ func (server *server) handleAuditTaskReject(writer http.ResponseWriter, request
|
||||
renderJSON(writer, http.StatusOK, task)
|
||||
}
|
||||
|
||||
const (
|
||||
usersListDefaultLimit = 200
|
||||
usersListMaxLimit = 1000
|
||||
)
|
||||
|
||||
func (server *server) handleUsersList(writer http.ResponseWriter, request *http.Request) {
|
||||
values := request.URL.Query()
|
||||
limit := usersListDefaultLimit
|
||||
if rawLimit := strings.TrimSpace(values.Get("limit")); rawLimit != "" {
|
||||
parsed, issue := parseDatabaseBrowserPageQuery(rawLimit, "limit", usersListDefaultLimit, 1, usersListMaxLimit)
|
||||
if issue != nil {
|
||||
renderValidationError(writer, *issue)
|
||||
return
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
// S5: search matches the fields the account list renders (name, phone).
|
||||
search := strings.ToLower(strings.TrimSpace(values.Get("q")))
|
||||
if len(search) > 200 {
|
||||
renderValidationError(writer, validationIssue{Field: "q", Message: "q must be at most 200 characters"})
|
||||
return
|
||||
}
|
||||
items, err := server.store.ListUsers(request.Context())
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
// Tenant/role filtering runs before search and pagination so neither new
|
||||
// parameter can widen visibility beyond the unpaginated contract.
|
||||
items = server.filterUsersForRequest(request, items)
|
||||
if request.URL.Query().Get("includeDisabled") != "true" {
|
||||
active := items[:0]
|
||||
@ -7933,7 +8177,26 @@ func (server *server) handleUsersList(writer http.ResponseWriter, request *http.
|
||||
}
|
||||
items = active
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeUsers(items)})
|
||||
if search != "" {
|
||||
matched := items[:0]
|
||||
for _, item := range items {
|
||||
if strings.Contains(strings.ToLower(item.Name), search) || strings.Contains(strings.ToLower(item.Phone), search) {
|
||||
matched = append(matched, item)
|
||||
}
|
||||
}
|
||||
items = matched
|
||||
}
|
||||
hasMore := len(items) > limit
|
||||
if hasMore {
|
||||
items = items[:limit]
|
||||
}
|
||||
// Backward-compatible shape: hasMore appears only when the conservative
|
||||
// default cap truncated the result.
|
||||
response := map[string]any{"items": sanitizeUsers(items)}
|
||||
if hasMore {
|
||||
response["hasMore"] = true
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// validateUserPhoneAvailable backs the account create/edit/login handlers,
|
||||
@ -8215,6 +8478,8 @@ func (server *server) handleUsersUpdate(writer http.ResponseWriter, request *htt
|
||||
renderUserMutationError(writer, err)
|
||||
return
|
||||
}
|
||||
// Role/supplier/status/password changes must evict cached principals now.
|
||||
invalidateAuthCacheForUser(item.ID)
|
||||
if item.Status == domain.UserStatusDisabled || payload.Password != nil {
|
||||
if err := server.store.DeleteSessionsByUser(request.Context(), item.ID); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
@ -8328,6 +8593,8 @@ func (server *server) handleUsersDelete(writer http.ResponseWriter, request *htt
|
||||
renderUserMutationError(writer, err)
|
||||
return
|
||||
}
|
||||
// The deleted account's cached principal (if any) must not survive.
|
||||
invalidateAuthCacheForUser(item.ID)
|
||||
if err := server.store.DeleteSessionsByUser(request.Context(), item.ID); err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
|
||||
@ -683,6 +683,9 @@ func (s *server) executeSupplierDeletion(task domain.SupplierDeletionTask, owner
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
// The records phase deletes employee accounts and their sessions directly
|
||||
// in the store; flush cached principals so revoked access is immediate.
|
||||
invalidateAuthCache()
|
||||
task.Phase = "completed"
|
||||
task.Status = "completed"
|
||||
task.Error = ""
|
||||
|
||||
@ -290,6 +290,13 @@ type HumanBatchQuery struct {
|
||||
Status HumanBatchStatus
|
||||
}
|
||||
|
||||
// HumanItemCursor is the opaque continuation cursor for paginated human item
|
||||
// lists, ordered by createdAt desc, id desc (mirroring RunSummaryCursor).
|
||||
type HumanItemCursor struct {
|
||||
CreatedAt string `json:"createdAt"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type HumanItemQuery struct {
|
||||
// Server-populated review history scope; nil means unrestricted.
|
||||
IDs []string
|
||||
@ -303,6 +310,15 @@ type HumanItemQuery struct {
|
||||
RunID string
|
||||
CurrentStageID string
|
||||
Status HumanItemStatus
|
||||
// List-endpoint pagination knobs. Zero values keep the historical
|
||||
// unpaginated behavior for internal callers (dashboard, batch reads).
|
||||
Limit int
|
||||
Cursor *HumanItemCursor
|
||||
ListProjection bool
|
||||
// Pushdown filters mirroring the work-pool rules; used by the list
|
||||
// endpoint so Mongo skips rows the handler would drop anyway.
|
||||
UnassignedOnly bool
|
||||
Statuses []HumanItemStatus
|
||||
}
|
||||
|
||||
type HumanRejectQuery struct {
|
||||
|
||||
@ -34,7 +34,56 @@ func humanItemMatches(item domain.HumanItemRecord, query domain.HumanItemQuery)
|
||||
(query.WorkflowID == "" || item.WorkflowID == query.WorkflowID) &&
|
||||
(query.RunID == "" || item.RunID == query.RunID) &&
|
||||
(query.CurrentStageID == "" || item.CurrentStageID == query.CurrentStageID) &&
|
||||
(query.Status == "" || item.Status == query.Status)
|
||||
(query.Status == "" || item.Status == query.Status) &&
|
||||
(!query.UnassignedOnly || item.AssignedUserID == "") &&
|
||||
(query.Statuses == nil || humanItemStatusMatched(query.Statuses, item.Status))
|
||||
}
|
||||
|
||||
func humanItemStatusMatched(statuses []domain.HumanItemStatus, status domain.HumanItemStatus) bool {
|
||||
for _, candidate := range statuses {
|
||||
if status == candidate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// humanItemListPayloadKeysRemoved mirrors humanItemListProjection: payload
|
||||
// sub-documents that are only needed on the detail/workbench paths are dropped
|
||||
// from list responses to keep pages small.
|
||||
var humanItemListPayloadKeysRemoved = []string{
|
||||
"annotationWorkspace",
|
||||
"annotationDraft",
|
||||
"sliceDispositionHistory",
|
||||
"sliceResetHistory",
|
||||
"sliceArchiveEdits",
|
||||
"sliceRestoreEvidence",
|
||||
"currentDataPreview",
|
||||
}
|
||||
|
||||
func humanItemListProjectionClone(item domain.HumanItemRecord) domain.HumanItemRecord {
|
||||
if item.Subject.Payload == nil {
|
||||
return item
|
||||
}
|
||||
payload := make(map[string]any, len(item.Subject.Payload))
|
||||
for key, value := range item.Subject.Payload {
|
||||
payload[key] = value
|
||||
}
|
||||
for _, key := range humanItemListPayloadKeysRemoved {
|
||||
delete(payload, key)
|
||||
}
|
||||
item.Subject.Payload = payload
|
||||
return item
|
||||
}
|
||||
|
||||
func humanItemAfterCursor(item domain.HumanItemRecord, cursor *domain.HumanItemCursor) bool {
|
||||
if cursor == nil {
|
||||
return true
|
||||
}
|
||||
if item.CreatedAt != cursor.CreatedAt {
|
||||
return item.CreatedAt < cursor.CreatedAt
|
||||
}
|
||||
return item.ID < cursor.ID
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListHumanBatches(_ context.Context, organizationID string, query domain.HumanBatchQuery) ([]domain.HumanBatchRecord, error) {
|
||||
@ -50,6 +99,30 @@ func (store *MemoryStore) ListHumanBatches(_ context.Context, organizationID str
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ListHumanBatchIDsAwaitingRelease implements
|
||||
// Store.ListHumanBatchIDsAwaitingRelease (see the Mongo version's comment).
|
||||
func (store *MemoryStore) ListHumanBatchIDsAwaitingRelease(_ context.Context) ([]string, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
ids := make([]string, 0)
|
||||
for _, item := range store.humanItems {
|
||||
if item.Status == domain.HumanItemAccepted && strings.TrimSpace(item.ReleasePartitionID) == "" && strings.TrimSpace(item.BatchID) != "" {
|
||||
ids = append(ids, item.BatchID)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
// Distinct: a batch with several awaiting items must appear once.
|
||||
deduped := ids[:0]
|
||||
previous := ""
|
||||
for _, id := range ids {
|
||||
if id != previous {
|
||||
deduped = append(deduped, id)
|
||||
previous = id
|
||||
}
|
||||
}
|
||||
return deduped, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) GetHumanBatch(_ context.Context, id string) (*domain.HumanBatchRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
@ -112,11 +185,24 @@ func (store *MemoryStore) ListHumanItems(_ context.Context, organizationID strin
|
||||
defer store.mu.Unlock()
|
||||
items := make([]domain.HumanItemRecord, 0)
|
||||
for _, item := range store.humanItems {
|
||||
if humanOrganizationMatches(item.OrganizationID, organizationID) && humanItemMatches(item, query) {
|
||||
if humanOrganizationMatches(item.OrganizationID, organizationID) && humanItemMatches(item, query) && humanItemAfterCursor(item, query.Cursor) {
|
||||
items = append(items, clone(item))
|
||||
}
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].CreatedAt > items[j].CreatedAt })
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i].CreatedAt != items[j].CreatedAt {
|
||||
return items[i].CreatedAt > items[j].CreatedAt
|
||||
}
|
||||
return items[i].ID > items[j].ID
|
||||
})
|
||||
if query.Limit > 0 && len(items) > query.Limit {
|
||||
items = items[:query.Limit]
|
||||
}
|
||||
if query.ListProjection {
|
||||
for index := range items {
|
||||
items[index] = humanItemListProjectionClone(items[index])
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
@ -466,13 +552,72 @@ func humanItemMongoFilter(organizationID string, query domain.HumanItemQuery) bs
|
||||
if query.Status != "" {
|
||||
filter["status"] = query.Status
|
||||
}
|
||||
if query.UnassignedOnly {
|
||||
filter["assignedUserId"] = bson.M{"$in": bson.A{"", nil}}
|
||||
}
|
||||
if len(query.Statuses) > 0 {
|
||||
statuses := make(bson.A, 0, len(query.Statuses))
|
||||
for _, status := range query.Statuses {
|
||||
statuses = append(statuses, status)
|
||||
}
|
||||
filter["status"] = bson.M{"$in": statuses}
|
||||
}
|
||||
if query.Cursor != nil {
|
||||
// Copy the base conditions before nesting them under $and: embedding
|
||||
// the same map inside itself makes BSON encoding recurse forever and
|
||||
// crashes the process (t8 blocker v1-blocker-cursor-stack-overflow).
|
||||
base := make(bson.M, len(filter))
|
||||
for key, value := range filter {
|
||||
base[key] = value
|
||||
}
|
||||
filter["$and"] = bson.A{base, bson.M{"$or": bson.A{
|
||||
bson.M{"createdAt": bson.M{"$lt": query.Cursor.CreatedAt}},
|
||||
bson.M{"createdAt": query.Cursor.CreatedAt, "id": bson.M{"$lt": query.Cursor.ID}},
|
||||
}}}
|
||||
}
|
||||
return filter
|
||||
}
|
||||
|
||||
// humanItemListProjection drops payload sub-documents that only the
|
||||
// detail/workbench paths need (annotation drafts, disposition histories,
|
||||
// restore evidence). Everything the list page and its enrichment read —
|
||||
// sliceWorkflow status, acceptance aggregates (items/statistics), source
|
||||
// video/artifact locators, duration — stays in the document.
|
||||
func humanItemListProjection() bson.M {
|
||||
projection := bson.M{}
|
||||
for _, key := range humanItemListPayloadKeysRemoved {
|
||||
projection["subject.payload."+key] = 0
|
||||
}
|
||||
return projection
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListHumanBatches(ctx context.Context, organizationID string, query domain.HumanBatchQuery) ([]domain.HumanBatchRecord, error) {
|
||||
return mongoList[domain.HumanBatchRecord](ctx, store.humanBatches, humanBatchMongoFilter(organizationID, query), bson.D{{Key: "createdAt", Value: -1}})
|
||||
}
|
||||
|
||||
// ListHumanBatchIDsAwaitingRelease implements
|
||||
// Store.ListHumanBatchIDsAwaitingRelease: distinct batch IDs that still have
|
||||
// accepted items without a release partition — exactly the batches the
|
||||
// background release sweeper (formerly the GET /api/human-batches flush) can
|
||||
// act on.
|
||||
func (store *MongoStore) ListHumanBatchIDsAwaitingRelease(ctx context.Context) ([]string, error) {
|
||||
values, err := store.humanItems.Distinct(ctx, "batchId", bson.M{
|
||||
"status": string(domain.HumanItemAccepted),
|
||||
"releasePartitionId": bson.M{"$in": bson.A{nil, ""}},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
if id, ok := value.(string); ok && id != "" {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) GetHumanBatch(ctx context.Context, id string) (*domain.HumanBatchRecord, error) {
|
||||
return mongoGetByID[domain.HumanBatchRecord](ctx, store.humanBatches, id)
|
||||
}
|
||||
@ -483,7 +628,18 @@ func (store *MongoStore) UpsertHumanBatch(ctx context.Context, item domain.Human
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListHumanItems(ctx context.Context, organizationID string, query domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
return mongoList[domain.HumanItemRecord](ctx, store.humanItems, humanItemMongoFilter(organizationID, query), bson.D{{Key: "createdAt", Value: -1}})
|
||||
findOptions := options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}})
|
||||
if query.Limit > 0 {
|
||||
findOptions.SetLimit(int64(query.Limit))
|
||||
}
|
||||
if query.ListProjection {
|
||||
findOptions.SetProjection(humanItemListProjection())
|
||||
}
|
||||
cursor, err := store.humanItems.Find(ctx, humanItemMongoFilter(organizationID, query), findOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.HumanItemRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) GetHumanItem(ctx context.Context, id string) (*domain.HumanItemRecord, error) {
|
||||
|
||||
130
internal/store/human_loop_cursor_mongo_test.go
Normal file
130
internal/store/human_loop_cursor_mongo_test.go
Normal file
@ -0,0 +1,130 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// TestHumanItemCursorFilterEncodesWithoutSelfReference guards the t8 blocker:
|
||||
// the cursor branch used to embed the filter map inside itself ($and), which
|
||||
// made BSON encoding recurse until the process died with a stack overflow.
|
||||
// Marshalling the produced filter is the exact client-side encoding path a
|
||||
// live Mongo command takes, so this fails (fatally) on the old code without
|
||||
// needing a running database.
|
||||
func TestHumanItemCursorFilterEncodesWithoutSelfReference(t *testing.T) {
|
||||
query := domain.HumanItemQuery{
|
||||
PluginID: "annotation-qa",
|
||||
ProjectID: "project-a",
|
||||
UnassignedOnly: true,
|
||||
Statuses: []domain.HumanItemStatus{domain.HumanItemReady, domain.HumanItemSubmitted},
|
||||
Cursor: &domain.HumanItemCursor{CreatedAt: "2026-10-09T00:00:00Z", ID: "human-item-1"},
|
||||
}
|
||||
filter := humanItemMongoFilter(domain.DefaultOrganizationID, query)
|
||||
encoded, err := bson.Marshal(filter)
|
||||
if err != nil {
|
||||
t.Fatalf("cursor filter must encode: %v", err)
|
||||
}
|
||||
var round struct {
|
||||
And []map[string]any `bson:"$and"`
|
||||
}
|
||||
if err := bson.Unmarshal(encoded, &round); err != nil {
|
||||
t.Fatalf("cursor filter must decode: %v", err)
|
||||
}
|
||||
if len(round.And) != 2 {
|
||||
t.Fatalf("cursor filter must AND base conditions with the cursor clause: %v", round.And)
|
||||
}
|
||||
base := round.And[0]
|
||||
if base["organizationId"] != domain.DefaultOrganizationID || base["pluginId"] != "annotation-qa" {
|
||||
t.Fatalf("base conditions must survive inside $and: %v", base)
|
||||
}
|
||||
if _, exists := base["$and"]; exists {
|
||||
t.Fatalf("base filter must not reference itself: %v", base)
|
||||
}
|
||||
// Sanity: the non-cursor filter and the base inside $and agree on the
|
||||
// pushed-down pool conditions.
|
||||
if _, exists := base["assignedUserId"]; !exists {
|
||||
t.Fatalf("pool pushdown lost inside $and: %v", base)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHumanItemsListMongoCursorPagination runs the full MongoStore path
|
||||
// (filter construction, real BSON commands, cursor decode) against an
|
||||
// isolated throwaway database. Opt-in like the other *_mongo_test.go suites
|
||||
// so CI without Mongo stays green; the always-run encoding guard above covers
|
||||
// the crash class on every machine.
|
||||
func TestHumanItemsListMongoCursorPagination(t *testing.T) {
|
||||
if os.Getenv("MONGO_URI") == "" || os.Getenv("HUMAN_ITEM_CURSOR_INTEGRATION") != "1" {
|
||||
t.Skip("opt-in Mongo cursor integration (set MONGO_URI and HUMAN_ITEM_CURSOR_INTEGRATION=1)")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")))
|
||||
if err != nil {
|
||||
t.Fatalf("Mongo connection error: %T", err)
|
||||
}
|
||||
defer client.Disconnect(ctx)
|
||||
dbName := fmt.Sprintf("human_item_cursor_test_%d", time.Now().UnixNano())
|
||||
if !strings.HasPrefix(dbName, "human_item_cursor_test_") || dbName == os.Getenv("MONGO_DB") {
|
||||
t.Fatal("invalid isolated test database")
|
||||
}
|
||||
db := client.Database(dbName)
|
||||
defer func() {
|
||||
if err := db.Drop(context.Background()); err != nil {
|
||||
t.Error("isolated database cleanup failed")
|
||||
}
|
||||
}()
|
||||
|
||||
dataStore := NewMongoStore(db)
|
||||
|
||||
for index := 0; index < 5; index++ {
|
||||
item := domain.HumanItemRecord{
|
||||
ID: fmt.Sprintf("human-item-cursor-%d", index),
|
||||
OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa",
|
||||
BatchID: "batch-cursor", ProjectID: "project-cursor", WorkflowID: "workflow-cursor",
|
||||
CurrentStageID: "split_annotation", Status: domain.HumanItemReady,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "item-key"},
|
||||
Version: 1,
|
||||
CreatedAt: time.Now().UTC().Add(time.Duration(index) * time.Minute).Format(time.RFC3339Nano),
|
||||
}
|
||||
item.UpdatedAt = item.CreatedAt
|
||||
if err := dataStore.UpsertHumanItem(ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
first, err := dataStore.ListHumanItems(ctx, domain.DefaultOrganizationID, domain.HumanItemQuery{
|
||||
PluginID: "annotation-qa", Limit: 3, ListProjection: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(first) != 3 {
|
||||
t.Fatalf("first page size=%d", len(first))
|
||||
}
|
||||
second, err := dataStore.ListHumanItems(ctx, domain.DefaultOrganizationID, domain.HumanItemQuery{
|
||||
PluginID: "annotation-qa", Limit: 3, ListProjection: true,
|
||||
Cursor: &domain.HumanItemCursor{CreatedAt: first[len(first)-1].CreatedAt, ID: first[len(first)-1].ID},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(second) != 2 {
|
||||
t.Fatalf("second page size=%d", len(second))
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range append(append([]domain.HumanItemRecord{}, first...), second...) {
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("cursor page repeated item %s", item.ID)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
}
|
||||
}
|
||||
@ -1335,6 +1335,99 @@ func (store *MemoryStore) ListRunArtifactsForProject(ctx context.Context, organi
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// memoryRunAcceptanceCandidate mirrors the Mongo ListRunsAcceptanceCandidates
|
||||
// $or filter (see the api package's dashboardAcceptancePayloadFromArtifact
|
||||
// detection paths) so both stores return the same candidate sets.
|
||||
func memoryRunAcceptanceCandidate(run domain.RunRecord) bool {
|
||||
for _, artifact := range run.Artifacts {
|
||||
if artifact.Name == "poc-operator-full-acceptance-result.json" {
|
||||
return true
|
||||
}
|
||||
if artifact.Data == nil {
|
||||
continue
|
||||
}
|
||||
if _, ok := artifact.Data["acceptanceScope"]; ok {
|
||||
return true
|
||||
}
|
||||
if value, ok := artifact.Data["value"].(map[string]any); ok {
|
||||
if _, ok := value["acceptanceScope"]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if preview, ok := artifact.Data["preview"].(map[string]any); ok {
|
||||
if value, ok := preview["value"].(map[string]any); ok {
|
||||
if _, ok := value["acceptanceScope"]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ListRunsAcceptanceCandidates implements Store.ListRunsAcceptanceCandidates.
|
||||
// Like the Mongo implementation it keeps the stored (non-hydrated) artifacts,
|
||||
// matching the ListRuns semantics the dashboard scan used before.
|
||||
func (store *MemoryStore) ListRunsAcceptanceCandidates(_ context.Context, organizationID string, limit int) ([]domain.RunRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
items := make([]domain.RunRecord, 0)
|
||||
for _, run := range store.runs {
|
||||
if organizationID != "" && !organizationMatches(run.OrganizationID, organizationID) {
|
||||
continue
|
||||
}
|
||||
if memoryRunAcceptanceCandidate(run) {
|
||||
items = append(items, clone(run))
|
||||
}
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
return runFreshnessKey(items[i]) > runFreshnessKey(items[j])
|
||||
})
|
||||
if limit > 0 && len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ListRunsWorkflowCandidates implements Store.ListRunsWorkflowCandidates.
|
||||
func (store *MemoryStore) ListRunsWorkflowCandidates(_ context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error) {
|
||||
workflowSet := make(map[string]bool, len(workflowNames))
|
||||
for _, name := range workflowNames {
|
||||
workflowSet[name] = true
|
||||
}
|
||||
projectSet := make(map[string]bool, len(projectNames))
|
||||
for _, name := range projectNames {
|
||||
projectSet[name] = true
|
||||
}
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
items := make([]domain.RunRecord, 0)
|
||||
for _, run := range store.runs {
|
||||
if organizationID != "" && !organizationMatches(run.OrganizationID, organizationID) {
|
||||
continue
|
||||
}
|
||||
if workflowSet[run.WorkflowName] || projectSet[run.ProjectName] {
|
||||
items = append(items, clone(run))
|
||||
}
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
return runFreshnessKey(items[i]) > runFreshnessKey(items[j])
|
||||
})
|
||||
if limit > 0 && len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// runFreshnessKey mirrors dashboardLatestAcceptancePayload's recency key:
|
||||
// UpdatedAt when present, CreatedAt otherwise.
|
||||
func runFreshnessKey(run domain.RunRecord) string {
|
||||
if key := strings.TrimSpace(run.UpdatedAt); key != "" {
|
||||
return key
|
||||
}
|
||||
return strings.TrimSpace(run.CreatedAt)
|
||||
}
|
||||
|
||||
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
|
||||
func (store *MemoryStore) ArchiveTerminalRuns(_ context.Context, olderThan time.Time) (int, error) {
|
||||
store.mu.Lock()
|
||||
|
||||
@ -1112,6 +1112,91 @@ func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, org
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// runRecencySort orders candidates exactly like the in-memory selector's
|
||||
// freshness key (dashboardLatestAcceptancePayload / latestPocGPUWorkflowRun:
|
||||
// UpdatedAt when present, CreatedAt otherwise), so even runs that never got
|
||||
// an UpdatedAt written sort the same on both sides of the store boundary.
|
||||
func runRecencySort() bson.D {
|
||||
return bson.D{
|
||||
{Key: "updatedAt", Value: bson.M{"$ifNull": bson.A{"$updatedAt", "$createdAt"}}},
|
||||
{Key: "createdAt", Value: -1},
|
||||
}
|
||||
}
|
||||
|
||||
// ListRunsAcceptanceCandidates implements Store.ListRunsAcceptanceCandidates.
|
||||
// The $or clauses mirror dashboardAcceptancePayloadFromArtifact's four
|
||||
// detection paths (artifact name + data/preview.value/value.acceptanceScope),
|
||||
// so the latest acceptance run is always inside the bounded result set while
|
||||
// the in-memory selector keeps its exact picking semantics.
|
||||
func (store *MongoStore) ListRunsAcceptanceCandidates(ctx context.Context, organizationID string, limit int) ([]domain.RunRecord, error) {
|
||||
filter := bson.M{"$or": bson.A{
|
||||
bson.M{"artifacts.name": "poc-operator-full-acceptance-result.json"},
|
||||
bson.M{"artifacts.data.acceptanceScope": bson.M{"$exists": true}},
|
||||
bson.M{"artifacts.data.value.acceptanceScope": bson.M{"$exists": true}},
|
||||
bson.M{"artifacts.data.preview.value.acceptanceScope": bson.M{"$exists": true}},
|
||||
}}
|
||||
if organizationID != "" {
|
||||
filter = organizationScopedFilter(organizationID, filter)
|
||||
}
|
||||
cursor, err := store.runs.Find(
|
||||
ctx,
|
||||
filter,
|
||||
options.Find().
|
||||
SetProjection(runArtifactProjection()).
|
||||
SetSort(runRecencySort()).
|
||||
SetLimit(int64(limit)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for index := range items {
|
||||
hydrateRunDerivedFields(&items[index])
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ListRunsWorkflowCandidates implements Store.ListRunsWorkflowCandidates.
|
||||
// Candidates need full documents (logs + node executions feed the preflight
|
||||
// gate's log scan), but the name filters keep the result set tiny.
|
||||
func (store *MongoStore) ListRunsWorkflowCandidates(ctx context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error) {
|
||||
or := bson.A{}
|
||||
if len(workflowNames) > 0 {
|
||||
or = append(or, bson.M{"workflowName": bson.M{"$in": workflowNames}})
|
||||
}
|
||||
if len(projectNames) > 0 {
|
||||
or = append(or, bson.M{"projectName": bson.M{"$in": projectNames}})
|
||||
}
|
||||
if len(or) == 0 {
|
||||
return []domain.RunRecord{}, nil
|
||||
}
|
||||
filter := bson.M{"$or": or}
|
||||
if organizationID != "" {
|
||||
filter = organizationScopedFilter(organizationID, filter)
|
||||
}
|
||||
cursor, err := store.runs.Find(
|
||||
ctx,
|
||||
filter,
|
||||
options.Find().
|
||||
SetSort(runRecencySort()).
|
||||
SetLimit(int64(limit)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for index := range items {
|
||||
hydrateRunDerivedFields(&items[index])
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// ArchiveTerminalRuns implements Store.ArchiveTerminalRuns.
|
||||
func (store *MongoStore) ArchiveTerminalRuns(ctx context.Context, olderThan time.Time) (int, error) {
|
||||
filter := bson.M{
|
||||
|
||||
@ -2,12 +2,19 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/productionstats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
var productionStatsCache = &ttlResultCache[productionstats.Snapshot]{now: time.Now}
|
||||
|
||||
func (s *MemoryStore) GetProductionStatsScope(_ context.Context, org string) (productionstats.Scope, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
@ -106,6 +113,25 @@ func (s *MemoryStore) LoadProductionStats(ctx context.Context, org string, proje
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// LoadProductionStatsCached serves the production snapshot from a 45s in-process
|
||||
// cache keyed by (org, scope projects). Consistency policy (S2): counters may
|
||||
// lag by at most the TTL; the API-level response cache is flushed by every
|
||||
// non-GET /api request (e.g. scope edits) on top of that.
|
||||
func (s *MongoStore) LoadProductionStatsCached(ctx context.Context, org string, projects []string) (productionstats.Snapshot, error) {
|
||||
keyProjects := append([]string(nil), projects...)
|
||||
sort.Strings(keyProjects)
|
||||
key := org + "|" + strings.Join(keyProjects, ",")
|
||||
if snapshot, ok := productionStatsCache.get(key, dashboardStatsCacheTTL); ok {
|
||||
return snapshot, nil
|
||||
}
|
||||
snapshot, err := s.LoadProductionStats(ctx, org, projects)
|
||||
if err != nil {
|
||||
return snapshot, err
|
||||
}
|
||||
productionStatsCache.set(key, snapshot, dashboardStatsCacheTTL)
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// Project-scoped, unpaginated and projected reads: do not load video/artifact
|
||||
// payloads or inherit the automation list's 200-record UI limit. No migrations.
|
||||
func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projects []string) (productionstats.Snapshot, error) {
|
||||
@ -115,7 +141,7 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec
|
||||
}
|
||||
filter := organizationScopedFilter(org, bson.M{"projectId": bson.M{"$in": projects}})
|
||||
read := func(c *mongo.Collection, f bson.M, projection bson.M, target any) error {
|
||||
cursor, err := c.Find(ctx, f, options.Find().SetProjection(projection))
|
||||
cursor, err := c.Find(ctx, f, options.Find().SetProjection(projection).SetMaxTime(15*time.Second))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -141,19 +167,24 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec
|
||||
}
|
||||
}
|
||||
runFilter := organizationScopedFilter(org, bson.M{"$or": bson.A{bson.M{"projectId": bson.M{"$in": projects}}, bson.M{"taskId": bson.M{"$in": captureIDs}}, bson.M{"id": bson.M{"$in": runIDs}}}})
|
||||
if err := read(s.runs, runFilter, fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome", "failure", "error"), &d.Runs); err != nil {
|
||||
return d, err
|
||||
}
|
||||
if err := read(s.humanItems, filter, fields("id", "projectId", "supplierId", "runId", "batchId", "legacyTaskId", "currentStageId", "status", "pendingCombinedSubmissionId", "releasePartitionId", "processingJob.status"), &d.Items); err != nil {
|
||||
return d, err
|
||||
}
|
||||
if err := read(s.humanBatches, filter, fields("id", "sourceBatchId", "stages"), &d.Batches); err != nil {
|
||||
return d, err
|
||||
}
|
||||
if err := read(s.annotationTasks, filter, fields("id", "runId", "supplierId", "status"), &d.Tasks); err != nil {
|
||||
return d, err
|
||||
}
|
||||
if err := read(s.annotationWorklogEvents, filter, fields("id", "taskId", "supplierId", "eventType", "mode", "temporary", "createdAt"), &d.Worklogs); err != nil {
|
||||
var g errgroup.Group
|
||||
g.Go(func() error {
|
||||
return read(s.runs, runFilter, fields("id", "projectId", "workflowId", "taskId", "trigger", "mode", "parentRunId", "createdAt", "endedAt", "status", "deliveryOutcome", "failure", "error"), &d.Runs)
|
||||
})
|
||||
g.Go(func() error {
|
||||
return read(s.humanItems, filter, fields("id", "projectId", "supplierId", "runId", "batchId", "legacyTaskId", "currentStageId", "status", "pendingCombinedSubmissionId", "releasePartitionId", "processingJob.status"), &d.Items)
|
||||
})
|
||||
g.Go(func() error { return read(s.humanBatches, filter, fields("id", "sourceBatchId", "stages"), &d.Batches) })
|
||||
g.Go(func() error {
|
||||
return read(s.annotationTasks, filter, fields("id", "runId", "supplierId", "status"), &d.Tasks)
|
||||
})
|
||||
g.Go(func() error {
|
||||
return read(s.annotationWorklogEvents, filter, fields("id", "taskId", "supplierId", "eventType", "mode", "temporary", "createdAt"), &d.Worklogs)
|
||||
})
|
||||
g.Go(func() error {
|
||||
return read(s.deliveryPackages, filter, fields("id", "parentRunId", "childRunId", "supplierId", "mode", "temporary", "finalExportedAt", "status", "itemIds", "releasePartitionId", "acceptanceHumanItemId", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.createdAt", "qualitySummary.prePackageAcceptance"), &d.Packages)
|
||||
})
|
||||
if err := g.Wait(); err != nil {
|
||||
return d, err
|
||||
}
|
||||
ids := make([]string, 0, len(d.Items))
|
||||
@ -165,8 +196,5 @@ func (s *MongoStore) LoadProductionStats(ctx context.Context, org string, projec
|
||||
return d, err
|
||||
}
|
||||
}
|
||||
if err := read(s.deliveryPackages, filter, fields("id", "parentRunId", "childRunId", "supplierId", "mode", "temporary", "finalExportedAt", "status", "itemIds", "releasePartitionId", "acceptanceHumanItemId", "acceptanceReviews.id", "acceptanceReviews.decision", "acceptanceReviews.createdAt", "qualitySummary.prePackageAcceptance"), &d.Packages); err != nil {
|
||||
return d, err
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
@ -102,3 +102,73 @@ func TestProductionStatsMongoPersistenceAndReadonlyProjection(t *testing.T) {
|
||||
t.Fatalf("report mutated source %+v %v", still, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadProductionStatsCachedMatchesDirectAndServesWithinTTL(t *testing.T) {
|
||||
uri := os.Getenv("DASHBOARD_STATS_MONGO_TEST_URI")
|
||||
if uri == "" {
|
||||
t.Skip("isolated Mongo required: DASHBOARD_STATS_MONGO_TEST_URI=mongodb://127.0.0.1:27028")
|
||||
}
|
||||
if uri != "mongodb://127.0.0.1:27028" {
|
||||
t.Fatal("only isolated loopback test server allowed")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("production_cache_test_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
s := NewMongoStore(db)
|
||||
|
||||
seed := func() {
|
||||
if err := s.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "c1", OrganizationID: "o", ProjectID: "p", RunID: "r1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "r1", OrganizationID: "o", ProjectID: "p", Status: domain.RunStatusWaitingForHuman}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "hi1", OrganizationID: "o", ProjectID: "p", RunID: "r1", CurrentStageID: "quality_review", Status: domain.HumanItemReady}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
seed()
|
||||
|
||||
direct, err := s.LoadProductionStats(ctx, "o", []string{"p"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cached, err := s.LoadProductionStatsCached(ctx, "o", []string{"p"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cached.Captures) != len(direct.Captures) || len(cached.Runs) != len(direct.Runs) || len(cached.Items) != len(direct.Items) {
|
||||
t.Fatalf("cached snapshot must match the direct load: cached=%d/%d/%d direct=%d/%d/%d",
|
||||
len(cached.Captures), len(cached.Runs), len(cached.Items), len(direct.Captures), len(direct.Runs), len(direct.Items))
|
||||
}
|
||||
builtDirect := productionstats.Build(direct)
|
||||
builtCached := productionstats.Build(cached)
|
||||
if len(builtDirect.Sources) != len(builtCached.Sources) {
|
||||
t.Fatalf("aggregation parity: built sources differ (%d vs %d)", len(builtDirect.Sources), len(builtCached.Sources))
|
||||
}
|
||||
|
||||
// Within the TTL the cached snapshot is served as-is even after new writes.
|
||||
if err := s.UpsertRun(ctx, domain.RunRecord{ID: "r2", OrganizationID: "o", ProjectID: "p", Status: domain.RunStatusCompleted}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale, err := s.LoadProductionStatsCached(ctx, "o", []string{"p"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(stale.Runs) != len(direct.Runs) {
|
||||
t.Fatalf("expected the cached snapshot within the TTL, got %d runs", len(stale.Runs))
|
||||
}
|
||||
refreshed, err := s.LoadProductionStats(ctx, "o", []string{"p"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(refreshed.Runs) != len(direct.Runs)+1 {
|
||||
t.Fatalf("direct load must bypass the cache, got %d runs", len(refreshed.Runs))
|
||||
}
|
||||
}
|
||||
|
||||
@ -122,6 +122,24 @@ type Store interface {
|
||||
// every run in the organization (see account-phone-login-era perf fix:
|
||||
// large orgs made the unscoped org-wide scan hang for minutes).
|
||||
ListRunArtifactsForProject(ctx context.Context, organizationID string, projectID string, workflowIDs []string) ([]domain.RunRecord, error)
|
||||
// ListRunsAcceptanceCandidates is the bounded point-query replacement for
|
||||
// dashboardPOCReadiness's former full ListRuns scan (slow-page report S4):
|
||||
// it returns only runs that can possibly carry a POC acceptance artifact
|
||||
// (matching dashboardAcceptancePayloadFromArtifact's detection paths), most
|
||||
// recently updated first, capped at limit. organizationID "" means unscoped,
|
||||
// mirroring ListRuns.
|
||||
ListRunsAcceptanceCandidates(ctx context.Context, organizationID string, limit int) ([]domain.RunRecord, error)
|
||||
// ListRunsWorkflowCandidates is the bounded point-query replacement for the
|
||||
// same scan's GPU-workflow-preflight half: runs whose workflowName or
|
||||
// projectName matches one of the POC/live acceptance names, most recently
|
||||
// updated first, capped at limit. organizationID "" means unscoped.
|
||||
ListRunsWorkflowCandidates(ctx context.Context, organizationID string, workflowNames []string, projectNames []string, limit int) ([]domain.RunRecord, error)
|
||||
// ListHumanBatchIDsAwaitingRelease backs the background release-partition
|
||||
// sweeper that replaced the GET /api/human-batches read-path flush (S4):
|
||||
// it returns the distinct batch IDs that still have accepted items without
|
||||
// a release partition, i.e. exactly the batches flushHumanReleasePartitions
|
||||
// could act on.
|
||||
ListHumanBatchIDsAwaitingRelease(ctx context.Context) ([]string, error)
|
||||
// ArchiveTerminalRuns sets ArchivedAt on every not-yet-archived run whose
|
||||
// status is completed/failed/cancelled and whose EndedAt is older than
|
||||
// olderThan. Archiving never deletes the record — it only excludes it
|
||||
|
||||
@ -2,12 +2,17 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/workforcestats"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"time"
|
||||
)
|
||||
|
||||
func workforceTaskIDs(d workforcestats.Snapshot) []string {
|
||||
@ -118,6 +123,266 @@ func (s *MemoryStore) LoadWorkforceStats(_ context.Context, org string, q workfo
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// DashboardHumanWorkStats carries only the counters the dashboard human-work
|
||||
// enrichment needs. It lets the API avoid materializing every human item and
|
||||
// delivery package just to count them (diagnosis report C4).
|
||||
type DashboardHumanWorkStats struct {
|
||||
PendingAnnotation int
|
||||
PendingQuality int
|
||||
InAnnotation int
|
||||
PendingAcceptanceItems int
|
||||
Accepted int
|
||||
Rejected int
|
||||
Escalated int
|
||||
VideoCount int
|
||||
CaptureDurationSeconds int64
|
||||
PackageCount int
|
||||
PackagesPendingAcceptance int
|
||||
}
|
||||
|
||||
func classifyHumanItemForDashboard(status domain.HumanItemStatus, stageID string) string {
|
||||
switch status {
|
||||
case domain.HumanItemReady, domain.HumanItemRework:
|
||||
if stageID == "quality_review" || stageID == domain.FanyuV5StageCutReview || stageID == domain.FanyuV5StageSemanticReview {
|
||||
return "pendingQuality"
|
||||
}
|
||||
return "pendingAnnotation"
|
||||
case domain.HumanItemClaimed, domain.HumanItemInProgress:
|
||||
return "inAnnotation"
|
||||
case domain.HumanItemSubmitted:
|
||||
if stageID == "quality_review" || stageID == domain.FanyuV5StageCutReview || stageID == domain.FanyuV5StageSemanticReview {
|
||||
return "pendingQuality"
|
||||
}
|
||||
if stageID == "acceptance" {
|
||||
return "pendingAcceptance"
|
||||
}
|
||||
case domain.HumanItemAccepted:
|
||||
return "accepted"
|
||||
case domain.HumanItemRejected:
|
||||
return "rejected"
|
||||
case domain.HumanItemEscalated:
|
||||
return "escalated"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MemoryStore implementation mirrors the legacy enrich loop exactly so the
|
||||
// in-memory replica stays a faithful parity oracle for the Mongo pipeline.
|
||||
func (s *MemoryStore) DashboardHumanWorkStats(_ context.Context, organizationID string) (DashboardHumanWorkStats, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
stats := DashboardHumanWorkStats{}
|
||||
seenVideos := map[string]struct{}{}
|
||||
for _, item := range s.humanItems {
|
||||
if !humanOrganizationMatches(item.OrganizationID, organizationID) {
|
||||
continue
|
||||
}
|
||||
if key := strings.TrimSpace(item.Subject.ItemKey); key != "" {
|
||||
seenVideos[key] = struct{}{}
|
||||
}
|
||||
if duration, ok := item.Subject.Payload["durationSeconds"].(float64); ok && duration > 0 {
|
||||
stats.CaptureDurationSeconds += int64(duration)
|
||||
}
|
||||
switch classifyHumanItemForDashboard(item.Status, item.CurrentStageID) {
|
||||
case "pendingQuality":
|
||||
stats.PendingQuality++
|
||||
case "pendingAnnotation":
|
||||
stats.PendingAnnotation++
|
||||
case "inAnnotation":
|
||||
stats.InAnnotation++
|
||||
case "pendingAcceptance":
|
||||
stats.PendingAcceptanceItems++
|
||||
case "accepted":
|
||||
stats.Accepted++
|
||||
case "rejected":
|
||||
stats.Rejected++
|
||||
case "escalated":
|
||||
stats.Escalated++
|
||||
}
|
||||
}
|
||||
stats.VideoCount = len(seenVideos)
|
||||
packageQuery := domain.DeliveryPackageQuery{Mode: domain.RunModeProduction}
|
||||
for _, item := range s.deliveryPackages {
|
||||
if organizationMatches(item.OrganizationID, organizationID) && deliveryPackageMatches(item, packageQuery) {
|
||||
stats.PackageCount++
|
||||
if item.Status == domain.DeliveryPackagePendingAcceptance {
|
||||
stats.PackagesPendingAcceptance++
|
||||
}
|
||||
}
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
type humanWorkAggregateRow struct {
|
||||
ID struct {
|
||||
Status string `bson:"s"`
|
||||
Stage string `bson:"c"`
|
||||
} `bson:"_id"`
|
||||
Count int32 `bson:"n"`
|
||||
}
|
||||
|
||||
// MongoStore implementation pushes the counting into aggregation pipelines so
|
||||
// the dashboard never materializes humanItems/deliveryPackages documents.
|
||||
func (s *MongoStore) DashboardHumanWorkStats(ctx context.Context, organizationID string) (DashboardHumanWorkStats, error) {
|
||||
stats := DashboardHumanWorkStats{}
|
||||
baseFilter := humanItemMongoFilter(organizationID, domain.HumanItemQuery{})
|
||||
rows := []humanWorkAggregateRow{}
|
||||
cursor, err := s.humanItems.Aggregate(ctx, bson.A{
|
||||
bson.M{"$match": baseFilter},
|
||||
bson.M{"$group": bson.M{"_id": bson.M{"s": "$status", "c": "$currentStageId"}, "n": bson.M{"$sum": 1}}},
|
||||
})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := cursor.All(ctx, &rows); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
count := int(row.Count)
|
||||
switch classifyHumanItemForDashboard(domain.HumanItemStatus(row.ID.Status), row.ID.Stage) {
|
||||
case "pendingQuality":
|
||||
stats.PendingQuality += count
|
||||
case "pendingAnnotation":
|
||||
stats.PendingAnnotation += count
|
||||
case "inAnnotation":
|
||||
stats.InAnnotation += count
|
||||
case "pendingAcceptance":
|
||||
stats.PendingAcceptanceItems += count
|
||||
case "accepted":
|
||||
stats.Accepted += count
|
||||
case "rejected":
|
||||
stats.Rejected += count
|
||||
case "escalated":
|
||||
stats.Escalated += count
|
||||
}
|
||||
}
|
||||
videoRows := []struct {
|
||||
Count int32 `bson:"n"`
|
||||
}{}
|
||||
cursor, err = s.humanItems.Aggregate(ctx, bson.A{
|
||||
bson.M{"$match": baseFilter},
|
||||
bson.M{"$group": bson.M{"_id": "$subject.itemKey"}},
|
||||
bson.M{"$match": bson.M{"_id": bson.M{"$nin": bson.A{"", nil}}}},
|
||||
bson.M{"$count": "n"},
|
||||
})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := cursor.All(ctx, &videoRows); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if len(videoRows) > 0 {
|
||||
stats.VideoCount = int(videoRows[0].Count)
|
||||
}
|
||||
durationRows := []struct {
|
||||
Total float64 `bson:"d"`
|
||||
}{}
|
||||
cursor, err = s.humanItems.Aggregate(ctx, bson.A{
|
||||
bson.M{"$match": baseFilter},
|
||||
// Legacy semantics: only positive numeric durationSeconds values count,
|
||||
// each truncated toward zero exactly like the Go int64(duration) cast.
|
||||
bson.M{"$group": bson.M{"_id": nil, "d": bson.M{"$sum": bson.M{"$trunc": bson.M{"$convert": bson.M{"input": bson.M{"$cond": bson.A{bson.M{"$gt": bson.A{"$subject.payload.durationSeconds", 0}}, "$subject.payload.durationSeconds", 0}}, "to": "double", "onError": 0, "onNull": 0}}}}}},
|
||||
})
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := cursor.All(ctx, &durationRows); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if len(durationRows) > 0 {
|
||||
stats.CaptureDurationSeconds = int64(durationRows[0].Total)
|
||||
}
|
||||
total, err := s.deliveryPackages.CountDocuments(ctx, deliveryPackageMongoFilter(organizationID, domain.DeliveryPackageQuery{Mode: domain.RunModeProduction}))
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.PackageCount = int(total)
|
||||
pending, err := s.deliveryPackages.CountDocuments(ctx, deliveryPackageMongoFilter(organizationID, domain.DeliveryPackageQuery{Mode: domain.RunModeProduction, Status: domain.DeliveryPackagePendingAcceptance}))
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.PackagesPendingAcceptance = int(pending)
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// ttlResultCache is a tiny in-process TTL cache. Consistency policy (diagnosis
|
||||
// report C4/S2): dashboard and production counters tolerate short staleness,
|
||||
// so entries simply expire; every non-GET /api request additionally flushes the
|
||||
// API-level response cache (see internal/api). Bounded to avoid growth.
|
||||
type ttlResultCache[T any] struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]ttlResultCacheEntry[T]
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
type ttlResultCacheEntry[T any] struct {
|
||||
value T
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
func (c *ttlResultCache[T]) get(key string, ttl time.Duration) (T, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var zero T
|
||||
if c.entries == nil {
|
||||
return zero, false
|
||||
}
|
||||
entry, ok := c.entries[key]
|
||||
if !ok || c.now().After(entry.expires) {
|
||||
return zero, false
|
||||
}
|
||||
return entry.value, true
|
||||
}
|
||||
|
||||
func (c *ttlResultCache[T]) set(key string, value T, ttl time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.entries == nil {
|
||||
c.entries = map[string]ttlResultCacheEntry[T]{}
|
||||
}
|
||||
if len(c.entries) >= 128 {
|
||||
c.entries = map[string]ttlResultCacheEntry[T]{}
|
||||
}
|
||||
c.entries[key] = ttlResultCacheEntry[T]{value: value, expires: c.now().Add(ttl)}
|
||||
}
|
||||
|
||||
const dashboardStatsCacheTTL = 45 * time.Second
|
||||
|
||||
var workforceStatsCache = &ttlResultCache[workforcestats.Snapshot]{now: time.Now}
|
||||
|
||||
// LoadWorkforceStatsCached caches the 30-day workforce snapshot per (org, query)
|
||||
// for 45s. The window key is quantized to one minute so the default rolling
|
||||
// dashboard range still hits the cache; the cached snapshot may therefore lag
|
||||
// boundary events by up to TTL plus one minute. Falls back to an exact query on
|
||||
// miss. MemoryStore intentionally has no cached variant: in-process tests and
|
||||
// the local replica expect read-after-write freshness.
|
||||
func (s *MongoStore) LoadWorkforceStatsCached(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) {
|
||||
allowed := make([]string, 0, len(q.AllowedProjects))
|
||||
for id, on := range q.AllowedProjects {
|
||||
if on {
|
||||
allowed = append(allowed, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(allowed)
|
||||
fromKey, toKey := int64(0), int64(0)
|
||||
if !q.From.IsZero() {
|
||||
fromKey = q.From.Unix() / 60
|
||||
}
|
||||
if !q.To.IsZero() {
|
||||
toKey = q.To.Unix() / 60
|
||||
}
|
||||
key := fmt.Sprintf("%s|%d|%d|%s|%s|%s|%v|%v|%s", org, fromKey, toKey, q.ProjectID, q.SupplierID, q.UserID, q.MatchUser, q.Details, strings.Join(allowed, ","))
|
||||
if snapshot, ok := workforceStatsCache.get(key, dashboardStatsCacheTTL); ok {
|
||||
return snapshot, nil
|
||||
}
|
||||
snapshot, err := s.LoadWorkforceStats(ctx, org, q)
|
||||
if err != nil {
|
||||
return snapshot, err
|
||||
}
|
||||
workforceStatsCache.set(key, snapshot, dashboardStatsCacheTTL)
|
||||
return snapshot, nil
|
||||
}
|
||||
|
||||
// Read-only, time-bounded event queries and batched joins. No index migrations or
|
||||
// customer data rewrites. History queries project out media and annotation text.
|
||||
func (s *MongoStore) LoadWorkforceStats(ctx context.Context, org string, q workforcestats.Query) (workforcestats.Snapshot, error) {
|
||||
|
||||
@ -137,3 +137,141 @@ func TestWorkforceStatsMongoReplicaParity(t *testing.T) {
|
||||
t.Fatalf("Mongo/Memory mismatch\n%+v\n%+v", actual, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDashboardHumanWorkStatsParityBetweenMemoryAndMongoAggregation(t *testing.T) {
|
||||
uri := os.Getenv("DASHBOARD_STATS_MONGO_TEST_URI")
|
||||
if uri == "" {
|
||||
t.Skip("isolated Mongo required: DASHBOARD_STATS_MONGO_TEST_URI=mongodb://127.0.0.1:27028")
|
||||
}
|
||||
if uri != "mongodb://127.0.0.1:27028" {
|
||||
t.Fatal("only isolated loopback test server allowed")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("dashboard_stats_test_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
|
||||
memory := NewMemoryStore()
|
||||
mongoStore := NewMongoStore(db)
|
||||
|
||||
seed := func(s Store) {
|
||||
helper := &testing.T{}
|
||||
_ = helper
|
||||
items := []domain.HumanItemRecord{
|
||||
{ID: "i1", OrganizationID: "o", CurrentStageID: "annotation", Status: domain.HumanItemReady,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "v-a", Payload: map[string]any{"durationSeconds": 30.5}}},
|
||||
{ID: "i2", OrganizationID: "o", CurrentStageID: "quality_review", Status: domain.HumanItemRework,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "v-a", Payload: map[string]any{"durationSeconds": 12.5}}},
|
||||
{ID: "i3", OrganizationID: "o", CurrentStageID: "annotation", Status: domain.HumanItemInProgress,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "v-b"}},
|
||||
{ID: "i4", OrganizationID: "o", CurrentStageID: domain.FanyuV5StageCutReview, Status: domain.HumanItemSubmitted,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "v-b", Payload: map[string]any{"durationSeconds": 7.5}}},
|
||||
{ID: "i5", OrganizationID: "o", CurrentStageID: "acceptance", Status: domain.HumanItemSubmitted,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "v-c", Payload: map[string]any{"durationSeconds": -3.0}}},
|
||||
{ID: "i6", OrganizationID: "o", CurrentStageID: "done", Status: domain.HumanItemAccepted,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "v-c", Payload: map[string]any{"durationSeconds": 0}}},
|
||||
{ID: "i7", OrganizationID: "o", CurrentStageID: "done", Status: domain.HumanItemRejected,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "", Payload: map[string]any{"durationSeconds": "broken"}}},
|
||||
{ID: "i8", OrganizationID: "o", CurrentStageID: "annotation", Status: domain.HumanItemEscalated,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: ""}},
|
||||
{ID: "i9", OrganizationID: "foreign", CurrentStageID: "annotation", Status: domain.HumanItemReady,
|
||||
Subject: domain.HumanSubjectRef{ItemKey: "v-x", Payload: map[string]any{"durationSeconds": 999.0}}},
|
||||
}
|
||||
for _, item := range items {
|
||||
if err := s.UpsertHumanItem(ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
packages := []domain.DeliveryPackageRecord{
|
||||
{ID: "p1", OrganizationID: "o", ProjectID: "proj", Status: domain.DeliveryPackagePendingAcceptance},
|
||||
{ID: "p2", OrganizationID: "o", ProjectID: "proj", Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction},
|
||||
{ID: "p3", OrganizationID: "o", ProjectID: "proj", Status: domain.DeliveryPackagePendingAcceptance, Mode: domain.RunModeDebug},
|
||||
{ID: "p4", OrganizationID: "foreign", ProjectID: "proj", Status: domain.DeliveryPackagePendingAcceptance},
|
||||
}
|
||||
for _, pkg := range packages {
|
||||
if err := s.UpsertDeliveryPackage(ctx, pkg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
seed(memory)
|
||||
seed(mongoStore)
|
||||
|
||||
fromMemory, err := memory.DashboardHumanWorkStats(ctx, "o")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fromMongo, err := mongoStore.DashboardHumanWorkStats(ctx, "o")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fromMemory != fromMongo {
|
||||
t.Fatalf("memory and mongo counters diverge:\nmemory=%+v\nmongo =%+v", fromMemory, fromMongo)
|
||||
}
|
||||
if fromMemory.PendingAnnotation != 1 || fromMemory.PendingQuality != 2 || fromMemory.InAnnotation != 1 ||
|
||||
fromMemory.PendingAcceptanceItems != 1 || fromMemory.Accepted != 1 || fromMemory.Rejected != 1 ||
|
||||
fromMemory.Escalated != 1 || fromMemory.VideoCount != 3 || fromMemory.CaptureDurationSeconds != 49 ||
|
||||
fromMemory.PackageCount != 2 || fromMemory.PackagesPendingAcceptance != 1 {
|
||||
t.Fatalf("unexpected counters: %+v", fromMemory)
|
||||
}
|
||||
|
||||
// Legacy full-materialization oracle: ListHumanItems + ListDeliveryPackages.
|
||||
items, err := memory.ListHumanItems(ctx, "o", domain.HumanItemQuery{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
packages, err := memory.ListDeliveryPackages(ctx, "o", domain.DeliveryPackageQuery{Mode: domain.RunModeProduction})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(items) != 8 || len(packages) != 2 {
|
||||
t.Fatalf("oracle fixture sizes changed: items=%d packages=%d", len(items), len(packages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadWorkforceStatsCachedServesSnapshotWithinTTL(t *testing.T) {
|
||||
uri := os.Getenv("DASHBOARD_STATS_MONGO_TEST_URI")
|
||||
if uri == "" {
|
||||
t.Skip("isolated Mongo required: DASHBOARD_STATS_MONGO_TEST_URI=mongodb://127.0.0.1:27028")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(uri))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("workforce_cache_test_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
s := NewMongoStore(db)
|
||||
|
||||
from := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
|
||||
to := from.Add(24 * time.Hour)
|
||||
q := workforcestats.Query{From: from, To: to}
|
||||
first, err := s.LoadWorkforceStatsCached(ctx, "cache-org", q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{ID: "new-event", OrganizationID: "cache-org", ProjectID: "p", TaskID: "t", EventType: "annotation_submitted", CreatedAt: "2026-09-01T12:00:00Z"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cached, err := s.LoadWorkforceStatsCached(ctx, "cache-org", q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cached.Events) != len(first.Events) {
|
||||
t.Fatalf("expected the cached snapshot within the TTL, got %d events vs %d", len(cached.Events), len(first.Events))
|
||||
}
|
||||
fresh, err := s.LoadWorkforceStats(ctx, "cache-org", q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fresh.Events) != len(first.Events)+1 {
|
||||
t.Fatalf("direct load must bypass the cache, got %d events", len(fresh.Events))
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,8 +14,13 @@ export function frontendManualChunk(id = "") {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// motion is only reachable from lazily loaded pages (WorkflowEditor,
|
||||
// Monitoring, Audit, Runs). Pinning it into a manual chunk promotes the
|
||||
// package plus hoisted react interop modules into the application entry
|
||||
// chunk via modulepreload. Let Rollup keep it inside the dynamic graph so
|
||||
// it is only fetched when one of those pages loads.
|
||||
if (id.includes("/motion/")) {
|
||||
return "motion-vendor";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (id.includes("lucide-react")) {
|
||||
|
||||
@ -14,6 +14,68 @@ type LazyVideoPreviewProps = {
|
||||
openOriginalLabel?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* API content endpoints that can serve video either directly from object
|
||||
* storage (a presigned redirect, cheap) or through the server-side
|
||||
* browser-transcode proxy (transcode=browser, expensive). Mirrors the URL
|
||||
* shapes recognized by src/lib/video-preview.js.
|
||||
*/
|
||||
function isAPIVideoContentURL(pathname: string): boolean {
|
||||
const isCollectionContent =
|
||||
pathname.startsWith("/api/source-data/collections/") && pathname.endsWith("/content");
|
||||
const isPackageContent = pathname.startsWith("/api/packages/") && pathname.endsWith("/content");
|
||||
const isObjectStorageContent = pathname === "/api/object-storage/content";
|
||||
const isRunArtifactContent =
|
||||
pathname.startsWith("/api/runs/") &&
|
||||
pathname.includes("/artifacts/") &&
|
||||
pathname.endsWith("/content");
|
||||
const isHumanEvidenceContent =
|
||||
(pathname.startsWith("/api/plugins/human-loop/items/") || pathname.startsWith("/api/human-items/")) &&
|
||||
pathname.includes("/evidence/") &&
|
||||
pathname.endsWith("/content");
|
||||
return (
|
||||
isCollectionContent ||
|
||||
isPackageContent ||
|
||||
isObjectStorageContent ||
|
||||
isRunArtifactContent ||
|
||||
isHumanEvidenceContent
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a `transcode=browser` proxy URL, return the same URL without the
|
||||
* transcode parameter. The API then redirects (307) to a presigned object
|
||||
* storage URL so the video bytes stream straight from storage instead of
|
||||
* passing through the API process. Returns null when the src is not one of
|
||||
* the proxyable API content URLs.
|
||||
*/
|
||||
function directVideoURL(src: string): string | null {
|
||||
try {
|
||||
const url = new URL(src, window.location.origin);
|
||||
if (!isAPIVideoContentURL(url.pathname)) return null;
|
||||
if (url.searchParams.get("transcode") !== "browser") return null;
|
||||
url.searchParams.delete("transcode");
|
||||
const query = url.searchParams.toString();
|
||||
return `${url.pathname}${query ? `?${query}` : ""}${url.hash}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Force the server-side transcode path and its blocking stream fallback. */
|
||||
function proxiedVideoURL(src: string): string {
|
||||
try {
|
||||
const url = new URL(src, window.location.origin);
|
||||
if (!isAPIVideoContentURL(url.pathname)) return src;
|
||||
url.searchParams.set("transcode", "browser");
|
||||
url.searchParams.set("sync", "1");
|
||||
const query = url.searchParams.toString();
|
||||
return `${url.pathname}${query ? `?${query}` : ""}${url.hash}`;
|
||||
} catch {
|
||||
return src;
|
||||
}
|
||||
}
|
||||
|
||||
export function LazyVideoPreview({
|
||||
src,
|
||||
originalSrc,
|
||||
@ -26,11 +88,17 @@ export function LazyVideoPreview({
|
||||
}: LazyVideoPreviewProps) {
|
||||
const [activated, setActivated] = React.useState(false);
|
||||
const [mediaError, setMediaError] = React.useState(false);
|
||||
// Direct playback (presigned redirect) first; the server-side transcode
|
||||
// proxy is the retry when the browser cannot decode the original codec.
|
||||
const [useProxy, setUseProxy] = React.useState(false);
|
||||
const directSrc = directVideoURL(src);
|
||||
const playbackSrc = !useProxy && directSrc ? directSrc : proxiedVideoURL(src);
|
||||
const fallbackURL = originalSrc || src;
|
||||
|
||||
React.useEffect(() => {
|
||||
setActivated(false);
|
||||
setMediaError(false);
|
||||
setUseProxy(false);
|
||||
}, [src]);
|
||||
|
||||
if (mediaError) {
|
||||
@ -77,10 +145,17 @@ export function LazyVideoPreview({
|
||||
|
||||
return (
|
||||
<video
|
||||
src={src}
|
||||
key={playbackSrc}
|
||||
src={playbackSrc}
|
||||
controls
|
||||
preload="metadata"
|
||||
onError={() => setMediaError(true)}
|
||||
onError={() => {
|
||||
if (!useProxy && directSrc) {
|
||||
setUseProxy(true);
|
||||
return;
|
||||
}
|
||||
setMediaError(true);
|
||||
}}
|
||||
className={cn("max-h-[420px] w-full rounded-lg bg-black", videoClassName)}
|
||||
/>
|
||||
);
|
||||
|
||||
@ -986,6 +986,12 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
return;
|
||||
}
|
||||
const knownProjects = projects.length > 0 ? projects : await projectsApi.list();
|
||||
// slow-page diagnosis S6: an unscoped manager must not fan out
|
||||
// listMembers+listWorkflows per project (project count × 2 requests,
|
||||
// each paying the auth chain). Managers skip the fan-out entirely;
|
||||
// non-managers keep it because their claim permission falls back to
|
||||
// cross-project membership lookups (canCurrentUserClaimItem).
|
||||
const fanOutProjectScope = !projectId && !isManager;
|
||||
const humanFilters = {
|
||||
pluginId: "annotation-qa",
|
||||
projectId: projectId || undefined,
|
||||
@ -1013,14 +1019,18 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
] = await Promise.all([
|
||||
projectId
|
||||
? projectsApi.listMembers(projectId)
|
||||
: Promise.allSettled(knownProjects.map((p) => projectsApi.listMembers(p.id))).then((results) =>
|
||||
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
|
||||
),
|
||||
: fanOutProjectScope
|
||||
? Promise.allSettled(knownProjects.map((p) => projectsApi.listMembers(p.id))).then((results) =>
|
||||
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
|
||||
)
|
||||
: Promise.resolve([] as ProjectMember[]),
|
||||
projectId
|
||||
? projectsApi.listWorkflows(projectId)
|
||||
: Promise.allSettled(knownProjects.map((p) => projectsApi.listWorkflows(p.id))).then((results) =>
|
||||
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
|
||||
),
|
||||
: fanOutProjectScope
|
||||
? Promise.allSettled(knownProjects.map((p) => projectsApi.listWorkflows(p.id))).then((results) =>
|
||||
results.flatMap((r) => (r.status === "fulfilled" ? r.value : [])),
|
||||
)
|
||||
: Promise.resolve([] as Workflow[]),
|
||||
annotationOperationsApi.listTemplates(projectId || undefined),
|
||||
annotationOperationsApi.listTasks({
|
||||
projectId: projectId || undefined,
|
||||
@ -2098,6 +2108,41 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
};
|
||||
}, [detail?.task.nodeId, selectedMaterialization]);
|
||||
const selectedRequiredRole = assignmentRoleForStage(selectedHumanItem?.currentStageId || annotationStage);
|
||||
// S6 follow-up: with the per-project fan-out skipped for unscoped managers,
|
||||
// reassignment (eligibleAssignees) would lose its member roster. Load it on
|
||||
// demand — exactly one request for the selected human item's project.
|
||||
const scopedMembersProjectRef = useRef("");
|
||||
const scopedMembersProjectId = selectedHumanItem?.projectId || "";
|
||||
useEffect(() => {
|
||||
if (projectId || !isManager || !scopedMembersProjectId) return;
|
||||
if (scopedMembersProjectRef.current === scopedMembersProjectId) return;
|
||||
scopedMembersProjectRef.current = scopedMembersProjectId;
|
||||
let cancelled = false;
|
||||
// Two requests for one project — still no per-project fan-out.
|
||||
projectsApi
|
||||
.listMembers(scopedMembersProjectId)
|
||||
.then((members) => {
|
||||
if (!cancelled) setProjectMembers(members);
|
||||
})
|
||||
.catch(() => {
|
||||
// Roster stays empty; the assignee dropdown shows its empty state.
|
||||
});
|
||||
projectsApi
|
||||
.listWorkflows(scopedMembersProjectId)
|
||||
.then((scopedWorkflows) => {
|
||||
if (!cancelled) {
|
||||
setWorkflows((current) => {
|
||||
// Merge: keep anything the scoped loader already provided.
|
||||
const known = new Set(current.map((workflow) => workflow.id));
|
||||
return [...current, ...scopedWorkflows.filter((workflow) => !known.has(workflow.id))];
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [projectId, isManager, scopedMembersProjectId]);
|
||||
const eligibleAssigneesForScope = useCallback(
|
||||
(supplierId: string | undefined, requiredRole: HumanAssignmentRole, currentAssigneeId = "") => {
|
||||
if (!isManager) return [];
|
||||
|
||||
@ -11,25 +11,59 @@ type HumanItemFilters = {
|
||||
stageId?: string;
|
||||
status?: HumanItemStatus;
|
||||
pool?: "available" | "mine" | "all" | "reviewed";
|
||||
/** Server default is 500 (max 1000); pass explicitly for larger pools. */
|
||||
limit?: number;
|
||||
/** Opaque continuation cursor from a previous page response. */
|
||||
cursor?: string;
|
||||
};
|
||||
|
||||
const queryString = (filters: Record<string, string | undefined>) => {
|
||||
const queryString = (filters: Record<string, string | number | undefined>) => {
|
||||
const params = new URLSearchParams();
|
||||
Object.entries(filters).forEach(([key, value]) => {
|
||||
if (value) params.set(key, value);
|
||||
if (value !== undefined && value !== "") params.set(key, String(value));
|
||||
});
|
||||
const query = params.toString();
|
||||
return query ? `?${query}` : "";
|
||||
};
|
||||
|
||||
type HumanItemPage = {
|
||||
items: HumanItem[];
|
||||
/** Opaque cursor for the next page; absent when there are no more pages. */
|
||||
nextCursor?: string;
|
||||
hasMore?: boolean;
|
||||
};
|
||||
|
||||
const listItemsPage = async (filters: HumanItemFilters = {}) =>
|
||||
apiRequest<HumanItemPage>(`/api/human-items${queryString(filters)}`);
|
||||
|
||||
const listItems = async (filters: HumanItemFilters = {}) =>
|
||||
(await apiRequest<{ items: HumanItem[] }>(`/api/human-items${queryString(filters)}`)).items;
|
||||
(await listItemsPage(filters)).items;
|
||||
|
||||
/**
|
||||
* Fetch every page behind the cursor so pools larger than the server's
|
||||
* default page size (500, max 1000) are not silently truncated.
|
||||
*/
|
||||
async function listAllItems(filters: HumanItemFilters = {}) {
|
||||
const items = new Map<string, HumanItem>();
|
||||
let cursor: string | undefined;
|
||||
// Hard cap: 200 pages × up to 1000 items per page guards against a
|
||||
// server bug looping on the same cursor.
|
||||
for (let page = 0; page < 200; page += 1) {
|
||||
const result = await listItemsPage({ ...filters, cursor });
|
||||
for (const item of result.items) {
|
||||
if (!items.has(item.id) || item.version > items.get(item.id)!.version) items.set(item.id, item);
|
||||
}
|
||||
if (!result.hasMore || !result.nextCursor) break;
|
||||
cursor = result.nextCursor;
|
||||
}
|
||||
return [...items.values()];
|
||||
}
|
||||
|
||||
async function listAcceptancePool(filters: HumanItemFilters) {
|
||||
// Keep the legacy package acceptance stage alongside prepackage acceptance.
|
||||
// Both requests retain the server's actor, project and supplier checks.
|
||||
const pages = await Promise.all(
|
||||
["acceptance", "prepackage_acceptance"].map((stageId) => listItems({ ...filters, stageId })),
|
||||
["acceptance", "prepackage_acceptance"].map((stageId) => listAllItems({ ...filters, stageId })),
|
||||
);
|
||||
const items = new Map<string, HumanItem>();
|
||||
for (const item of pages.flat()) {
|
||||
@ -40,6 +74,8 @@ async function listAcceptancePool(filters: HumanItemFilters) {
|
||||
|
||||
export const humanLoopApi = {
|
||||
listItems,
|
||||
listItemsPage,
|
||||
listAllItems,
|
||||
listAcceptancePools: async (
|
||||
filters: Omit<HumanItemFilters, "stageId" | "pool">,
|
||||
actorId: string,
|
||||
|
||||
@ -14,8 +14,28 @@ export type AccountHierarchyIssue = {
|
||||
export const usersApi = {
|
||||
auditHierarchy: async () =>
|
||||
(await apiRequest<{ items: AccountHierarchyIssue[] }>("/api/users/hierarchy-audit")).items,
|
||||
list: async (includeDisabled = false) =>
|
||||
(await apiRequest<{ items: User[] }>(includeDisabled ? "/api/users?includeDisabled=true" : "/api/users")).items,
|
||||
/**
|
||||
* One page of users. The server caps pages at 1000 items (default 200)
|
||||
* and reports `hasMore: true` when the cap truncated the result.
|
||||
*/
|
||||
listPage: async (options: { includeDisabled?: boolean; limit?: number; q?: string } = {}) => {
|
||||
const params = new URLSearchParams();
|
||||
if (options.includeDisabled) params.set("includeDisabled", "true");
|
||||
if (options.limit) params.set("limit", String(options.limit));
|
||||
if (options.q) params.set("q", options.q);
|
||||
const suffix = params.toString() ? `?${params.toString()}` : "";
|
||||
return apiRequest<{ items: User[]; hasMore?: boolean }>(`/api/users${suffix}`);
|
||||
},
|
||||
list: async (includeDisabled = false) => {
|
||||
// Ask for the server's maximum page so organizations with 200–1000
|
||||
// active users are not silently truncated by the 200 default. Larger
|
||||
// organizations surface an explicit error instead of a partial list.
|
||||
const page = await usersApi.listPage({ includeDisabled, limit: 1000 });
|
||||
if (page.hasMore) {
|
||||
throw new Error("用户数量超过单页上限(1000),列表被截断,请联系管理员分批查看");
|
||||
}
|
||||
return page.items;
|
||||
},
|
||||
create: (
|
||||
payload: Pick<User, "name" | "role"> & {
|
||||
phone: string;
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import tailwindcss from "@tailwindcss/vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
import { createReadStream, existsSync, statSync } from "node:fs";
|
||||
import { createReadStream, existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import path from "path";
|
||||
import { defineConfig } from "vite";
|
||||
import { frontendManualChunk } from "./scripts/lib/frontend-chunking.mjs";
|
||||
@ -9,6 +10,8 @@ import { viteProxyTargets } from "./scripts/lib/service-endpoints.mjs";
|
||||
export default defineConfig(() => {
|
||||
const proxyTargets = viteProxyTargets(process.env);
|
||||
const captureWorkbenchMockVideo = process.env.CAPTURE_WORKBENCH_MOCK_VIDEO?.trim();
|
||||
let resolvedOutDir: string | undefined;
|
||||
const compressedTargets: string[] = [];
|
||||
|
||||
return {
|
||||
plugins: [
|
||||
@ -47,6 +50,41 @@ export default defineConfig(() => {
|
||||
});
|
||||
},
|
||||
},
|
||||
{
|
||||
// Emit precompressed .gz siblings for JS/CSS assets. The Go SPA
|
||||
// handler (cmd/api/main.go) serves them directly to clients that
|
||||
// send Accept-Encoding: gzip, avoiding on-the-fly compression in
|
||||
// the API pod.
|
||||
//
|
||||
// Must run in writeBundle reading the final on-disk files: in
|
||||
// rolldown-vite the import-analysis transform that injects
|
||||
// __vite__mapDeps runs after generateBundle, so compressing
|
||||
// output.code there would produce .gz files that no longer match
|
||||
// the emitted .js and crash gzip clients with ReferenceError.
|
||||
name: "precompress-static-assets",
|
||||
apply: "build",
|
||||
configResolved(config) {
|
||||
resolvedOutDir = config.build.outDir;
|
||||
},
|
||||
generateBundle(_options, bundle) {
|
||||
compressedTargets.length = 0;
|
||||
for (const fileName of Object.keys(bundle)) {
|
||||
if (/\.(js|css)$/.test(fileName)) {
|
||||
compressedTargets.push(fileName);
|
||||
}
|
||||
}
|
||||
},
|
||||
writeBundle() {
|
||||
const outDir = path.resolve(process.cwd(), resolvedOutDir ?? "dist");
|
||||
for (const fileName of compressedTargets) {
|
||||
if (!fileName) {
|
||||
continue;
|
||||
}
|
||||
const target = path.join(outDir, fileName);
|
||||
writeFileSync(`${target}.gz`, gzipSync(readFileSync(target), { level: 9 }));
|
||||
}
|
||||
},
|
||||
},
|
||||
],
|
||||
build: {
|
||||
rollupOptions: {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user