droboticflow-new/internal/api/data_platform_preview_test.go
d-robotics da446d13b5 ⚡ Fix slow page loading: static gzip, list pagination, auth cache, dashboard aggregation, video direct link
- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix
- C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown
- C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation
- C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache
- C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback
- S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed
- Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush
- Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500

Co-authored-by: AgentTeams slow-page-fix
2026-10-09 17:45:27 -07:00

208 lines
7.5 KiB
Go

package api
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/go-chi/chi/v5/middleware"
)
func compressTestHandler(inner http.HandlerFunc) http.Handler {
return middleware.Compress(5)(inner)
}
// A cache miss with a direct (http) storage URL must redirect the player to
// that URL instead of running server-side ffmpeg on the request path, and
// the redirect itself must not come back gzip-encoded.
func TestBrowserVideoPreviewCacheMissRedirectsToDirectURL(t *testing.T) {
originalPath := sourceBrowserVideoPreviewPath
originalWriter := sourceBrowserVideoPreviewWriter
defer func() {
sourceBrowserVideoPreviewPath = originalPath
sourceBrowserVideoPreviewWriter = originalWriter
}()
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
return "", os.ErrNotExist
}
var warmups sync.WaitGroup
warmups.Add(1)
var warmedRequest sourceBrowserPreviewRequest
var warmupMu sync.Mutex
sourceBrowserVideoPreviewWriter = func(ctx context.Context, _ http.ResponseWriter, request sourceBrowserPreviewRequest) error {
warmupMu.Lock()
warmedRequest = request
warmupMu.Unlock()
warmups.Done()
return nil
}
direct := "https://bucket.bcebos.com/dir/camera.mp4?authorization=bce-auth-v1%2Fexample"
request := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/r/c/content?relativePath=v.mp4&transcode=browser", nil)
request.Header.Set("Accept-Encoding", "gzip")
recorder := httptest.NewRecorder()
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
FileName: "camera.mp4",
RedirectURL: direct,
}, direct)
}).ServeHTTP(recorder, request)
if recorder.Code != http.StatusTemporaryRedirect {
t.Fatalf("expected 307 redirect on cache miss, got %d body=%s", recorder.Code, recorder.Body.String())
}
if location := recorder.Header().Get("Location"); location != direct {
t.Fatalf("unexpected redirect location: %s", location)
}
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "" && encoding != "identity" {
t.Fatalf("redirect response must not be compressed, Content-Encoding=%q", encoding)
}
// The background pre-transcode must have been kicked off exactly for the
// preview request that missed the cache.
warmups.Wait()
warmupMu.Lock()
defer warmupMu.Unlock()
if warmedRequest.FileName != "camera.mp4" || warmedRequest.RedirectURL != direct {
t.Fatalf("background warmup ran for the wrong request: %+v", warmedRequest)
}
}
// When the cache holds the pre-transcoded artifact, the response must be
// served as-is through the global Compress middleware: the identity
// Content-Encoding marker makes chi skip gzip, so video bytes reach the
// player unchanged.
func TestBrowserVideoPreviewCacheHitServesBytesWithoutGzip(t *testing.T) {
originalPath := sourceBrowserVideoPreviewPath
defer func() { sourceBrowserVideoPreviewPath = originalPath }()
payload := []byte("fake-mp4-bytes-0123456789")
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
tempFile, err := os.CreateTemp(t.TempDir(), "preview-*.mp4")
if err != nil {
return "", err
}
if _, err := tempFile.Write(payload); err != nil {
tempFile.Close()
return "", err
}
if err := tempFile.Close(); err != nil {
return "", err
}
return tempFile.Name(), nil
}
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser", nil)
request.Header.Set("Accept-Encoding", "gzip")
recorder := httptest.NewRecorder()
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
FileName: "v.mp4",
}, "")
}).ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200 for cached preview, got %d body=%s", recorder.Code, recorder.Body.String())
}
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "identity" {
t.Fatalf("expected Content-Encoding identity (chi Compress skipped), got %q", encoding)
}
if !bytes.Equal(recorder.Body.Bytes(), payload) {
t.Fatal("video bytes were transformed on the way out; expected the cached artifact verbatim")
}
}
// The sync=1 parameter is the per-request rollback switch: it must keep the
// legacy blocking stream instead of redirecting to the direct URL.
func TestBrowserVideoPreviewSyncParamForcesBlockingStream(t *testing.T) {
originalPath := sourceBrowserVideoPreviewPath
originalWriter := sourceBrowserVideoPreviewWriter
defer func() {
sourceBrowserVideoPreviewPath = originalPath
sourceBrowserVideoPreviewWriter = originalWriter
}()
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
return "", os.ErrNotExist
}
var streamed atomic.Bool
sourceBrowserVideoPreviewWriter = func(_ context.Context, writer http.ResponseWriter, _ sourceBrowserPreviewRequest) error {
_, _ = writer.Write([]byte("streamed"))
streamed.Store(true)
return nil
}
direct := "https://bucket.bcebos.com/dir/camera.mp4"
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser&sync=1", nil)
recorder := httptest.NewRecorder()
writeBrowserVideoPreviewResponse(recorder, request, sourceBrowserPreviewRequest{
FileName: "camera.mp4",
RedirectURL: direct,
}, direct)
if recorder.Code != http.StatusOK {
t.Fatalf("sync=1 must stream, got %d", recorder.Code)
}
if !streamed.Load() {
t.Fatal("sync=1 must use the blocking server-side stream")
}
if recorder.Header().Get("Location") != "" {
t.Fatal("sync=1 must not redirect to the direct URL")
}
}
// Cache eviction: entries past the TTL are removed regardless of size, and
// under a size cap the least-recently-written entries are evicted first.
func TestPruneSourceBrowserPreviewCacheTTLAndLRU(t *testing.T) {
dir := t.TempDir()
write := func(name string, age time.Duration, size int64) string {
filePath := filepath.Join(dir, name)
payload := make([]byte, size)
if err := os.WriteFile(filePath, payload, 0o644); err != nil {
t.Fatal(err)
}
cutOff := time.Now().Add(-age)
if err := os.Chtimes(filePath, cutOff, cutOff); err != nil {
t.Fatal(err)
}
return filePath
}
stale := write("stale.mp4", sourceBrowserPreviewCacheMaxAge+time.Hour, 10)
oldest := write("oldest.mp4", 3*time.Hour, 600)
middle := write("middle.mp4", 2*time.Hour, 600)
newest := write("newest.mp4", time.Hour, 600)
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1300)
if _, err := os.Stat(stale); !os.IsNotExist(err) {
t.Fatal("entry past the TTL must be deleted")
}
if _, err := os.Stat(oldest); !os.IsNotExist(err) {
t.Fatal("least-recently-written entry must be evicted first under the size cap")
}
for _, kept := range []string{middle, newest} {
if _, err := os.Stat(kept); err != nil {
t.Fatalf("entry %s should have survived pruning: %v", kept, err)
}
}
}
// Under the size cap but with nothing past the TTL, the directory must be
// left untouched.
func TestPruneSourceBrowserPreviewCacheNoopWhenWithinBounds(t *testing.T) {
dir := t.TempDir()
filePath := filepath.Join(dir, "fresh.mp4")
if err := os.WriteFile(filePath, bytes.Repeat([]byte("x"), 128), 0o644); err != nil {
t.Fatal(err)
}
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1<<20)
if _, err := os.Stat(filePath); err != nil {
t.Fatalf("entry within bounds must not be evicted: %v", err)
}
}