forked from longtao.wu/droboticflow
- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix - C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown - C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation - C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache - C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback - S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed - Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush - Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500 Co-authored-by: AgentTeams slow-page-fix
208 lines
7.5 KiB
Go
208 lines
7.5 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5/middleware"
|
|
)
|
|
|
|
func compressTestHandler(inner http.HandlerFunc) http.Handler {
|
|
return middleware.Compress(5)(inner)
|
|
}
|
|
|
|
// A cache miss with a direct (http) storage URL must redirect the player to
|
|
// that URL instead of running server-side ffmpeg on the request path, and
|
|
// the redirect itself must not come back gzip-encoded.
|
|
func TestBrowserVideoPreviewCacheMissRedirectsToDirectURL(t *testing.T) {
|
|
originalPath := sourceBrowserVideoPreviewPath
|
|
originalWriter := sourceBrowserVideoPreviewWriter
|
|
defer func() {
|
|
sourceBrowserVideoPreviewPath = originalPath
|
|
sourceBrowserVideoPreviewWriter = originalWriter
|
|
}()
|
|
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
|
|
return "", os.ErrNotExist
|
|
}
|
|
var warmups sync.WaitGroup
|
|
warmups.Add(1)
|
|
var warmedRequest sourceBrowserPreviewRequest
|
|
var warmupMu sync.Mutex
|
|
sourceBrowserVideoPreviewWriter = func(ctx context.Context, _ http.ResponseWriter, request sourceBrowserPreviewRequest) error {
|
|
warmupMu.Lock()
|
|
warmedRequest = request
|
|
warmupMu.Unlock()
|
|
warmups.Done()
|
|
return nil
|
|
}
|
|
|
|
direct := "https://bucket.bcebos.com/dir/camera.mp4?authorization=bce-auth-v1%2Fexample"
|
|
request := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/r/c/content?relativePath=v.mp4&transcode=browser", nil)
|
|
request.Header.Set("Accept-Encoding", "gzip")
|
|
recorder := httptest.NewRecorder()
|
|
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
|
|
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
|
|
FileName: "camera.mp4",
|
|
RedirectURL: direct,
|
|
}, direct)
|
|
}).ServeHTTP(recorder, request)
|
|
|
|
if recorder.Code != http.StatusTemporaryRedirect {
|
|
t.Fatalf("expected 307 redirect on cache miss, got %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if location := recorder.Header().Get("Location"); location != direct {
|
|
t.Fatalf("unexpected redirect location: %s", location)
|
|
}
|
|
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "" && encoding != "identity" {
|
|
t.Fatalf("redirect response must not be compressed, Content-Encoding=%q", encoding)
|
|
}
|
|
// The background pre-transcode must have been kicked off exactly for the
|
|
// preview request that missed the cache.
|
|
warmups.Wait()
|
|
warmupMu.Lock()
|
|
defer warmupMu.Unlock()
|
|
if warmedRequest.FileName != "camera.mp4" || warmedRequest.RedirectURL != direct {
|
|
t.Fatalf("background warmup ran for the wrong request: %+v", warmedRequest)
|
|
}
|
|
}
|
|
|
|
// When the cache holds the pre-transcoded artifact, the response must be
|
|
// served as-is through the global Compress middleware: the identity
|
|
// Content-Encoding marker makes chi skip gzip, so video bytes reach the
|
|
// player unchanged.
|
|
func TestBrowserVideoPreviewCacheHitServesBytesWithoutGzip(t *testing.T) {
|
|
originalPath := sourceBrowserVideoPreviewPath
|
|
defer func() { sourceBrowserVideoPreviewPath = originalPath }()
|
|
|
|
payload := []byte("fake-mp4-bytes-0123456789")
|
|
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
|
|
tempFile, err := os.CreateTemp(t.TempDir(), "preview-*.mp4")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if _, err := tempFile.Write(payload); err != nil {
|
|
tempFile.Close()
|
|
return "", err
|
|
}
|
|
if err := tempFile.Close(); err != nil {
|
|
return "", err
|
|
}
|
|
return tempFile.Name(), nil
|
|
}
|
|
|
|
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser", nil)
|
|
request.Header.Set("Accept-Encoding", "gzip")
|
|
recorder := httptest.NewRecorder()
|
|
compressTestHandler(func(writer http.ResponseWriter, request *http.Request) {
|
|
writeBrowserVideoPreviewResponse(writer, request, sourceBrowserPreviewRequest{
|
|
FileName: "v.mp4",
|
|
}, "")
|
|
}).ServeHTTP(recorder, request)
|
|
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 for cached preview, got %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if encoding := recorder.Header().Get("Content-Encoding"); encoding != "identity" {
|
|
t.Fatalf("expected Content-Encoding identity (chi Compress skipped), got %q", encoding)
|
|
}
|
|
if !bytes.Equal(recorder.Body.Bytes(), payload) {
|
|
t.Fatal("video bytes were transformed on the way out; expected the cached artifact verbatim")
|
|
}
|
|
}
|
|
|
|
// The sync=1 parameter is the per-request rollback switch: it must keep the
|
|
// legacy blocking stream instead of redirecting to the direct URL.
|
|
func TestBrowserVideoPreviewSyncParamForcesBlockingStream(t *testing.T) {
|
|
originalPath := sourceBrowserVideoPreviewPath
|
|
originalWriter := sourceBrowserVideoPreviewWriter
|
|
defer func() {
|
|
sourceBrowserVideoPreviewPath = originalPath
|
|
sourceBrowserVideoPreviewWriter = originalWriter
|
|
}()
|
|
sourceBrowserVideoPreviewPath = func(context.Context, sourceBrowserPreviewRequest) (string, error) {
|
|
return "", os.ErrNotExist
|
|
}
|
|
var streamed atomic.Bool
|
|
sourceBrowserVideoPreviewWriter = func(_ context.Context, writer http.ResponseWriter, _ sourceBrowserPreviewRequest) error {
|
|
_, _ = writer.Write([]byte("streamed"))
|
|
streamed.Store(true)
|
|
return nil
|
|
}
|
|
|
|
direct := "https://bucket.bcebos.com/dir/camera.mp4"
|
|
request := httptest.NewRequest(http.MethodGet, "/api/packages/r/p/content?relativePath=v.mp4&transcode=browser&sync=1", nil)
|
|
recorder := httptest.NewRecorder()
|
|
writeBrowserVideoPreviewResponse(recorder, request, sourceBrowserPreviewRequest{
|
|
FileName: "camera.mp4",
|
|
RedirectURL: direct,
|
|
}, direct)
|
|
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("sync=1 must stream, got %d", recorder.Code)
|
|
}
|
|
if !streamed.Load() {
|
|
t.Fatal("sync=1 must use the blocking server-side stream")
|
|
}
|
|
if recorder.Header().Get("Location") != "" {
|
|
t.Fatal("sync=1 must not redirect to the direct URL")
|
|
}
|
|
}
|
|
|
|
// Cache eviction: entries past the TTL are removed regardless of size, and
|
|
// under a size cap the least-recently-written entries are evicted first.
|
|
func TestPruneSourceBrowserPreviewCacheTTLAndLRU(t *testing.T) {
|
|
dir := t.TempDir()
|
|
write := func(name string, age time.Duration, size int64) string {
|
|
filePath := filepath.Join(dir, name)
|
|
payload := make([]byte, size)
|
|
if err := os.WriteFile(filePath, payload, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cutOff := time.Now().Add(-age)
|
|
if err := os.Chtimes(filePath, cutOff, cutOff); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return filePath
|
|
}
|
|
stale := write("stale.mp4", sourceBrowserPreviewCacheMaxAge+time.Hour, 10)
|
|
oldest := write("oldest.mp4", 3*time.Hour, 600)
|
|
middle := write("middle.mp4", 2*time.Hour, 600)
|
|
newest := write("newest.mp4", time.Hour, 600)
|
|
|
|
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1300)
|
|
|
|
if _, err := os.Stat(stale); !os.IsNotExist(err) {
|
|
t.Fatal("entry past the TTL must be deleted")
|
|
}
|
|
if _, err := os.Stat(oldest); !os.IsNotExist(err) {
|
|
t.Fatal("least-recently-written entry must be evicted first under the size cap")
|
|
}
|
|
for _, kept := range []string{middle, newest} {
|
|
if _, err := os.Stat(kept); err != nil {
|
|
t.Fatalf("entry %s should have survived pruning: %v", kept, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Under the size cap but with nothing past the TTL, the directory must be
|
|
// left untouched.
|
|
func TestPruneSourceBrowserPreviewCacheNoopWhenWithinBounds(t *testing.T) {
|
|
dir := t.TempDir()
|
|
filePath := filepath.Join(dir, "fresh.mp4")
|
|
if err := os.WriteFile(filePath, bytes.Repeat([]byte("x"), 128), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pruneSourceBrowserPreviewCacheWithLimit(dir, time.Now(), 1<<20)
|
|
if _, err := os.Stat(filePath); err != nil {
|
|
t.Fatalf("entry within bounds must not be evicted: %v", err)
|
|
}
|
|
}
|