droboticflow/internal/api/object_storage_browser_test.go
eust-w 8780cd598c 🐛 Fix large model downloads and package drobotic-pull 1.0.1
The old shipped v1.0.0 used a 300-second whole-response deadline. Keep active transfers alive, validate ranged retries, filter BOS folder markers, and preserve accurate failure/progress reporting. Verify the Linux package with interrupted 4 GiB transfers, a 307-second stream, and the real 7.47 GiB checkpoint with matching SHA256 receipts.
2026-10-22 14:19:48 +08:00

193 lines
7.8 KiB
Go

package api
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"github.com/baidubce/bce-sdk-go/bce"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
)
func TestObjectStorageListSummaryRecognizesDirectoryMarker(t *testing.T) {
for _, test := range []struct {
key string
kind string
}{
{key: "checkpoint/model/", kind: "directory"},
{key: "checkpoint/config.json/", kind: "directory"},
{key: "checkpoint/empty", kind: "file"},
{key: "checkpoint/config.json", kind: "json"},
} {
t.Run(test.key, func(t *testing.T) {
summary := objectStorageSummaryFromListObject("datastorage", bosapi.ObjectSummaryType{Key: test.key})
if summary.Kind != test.kind || summary.Key != test.key {
t.Fatalf("summary kind=%q key=%q, want kind=%q key=%q", summary.Kind, summary.Key, test.kind, test.key)
}
})
}
}
func TestObjectStorageDirectoryManifestExcludesDirectoryMarkers(t *testing.T) {
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(context.Background(), domain.CreateStorageConnectionInput{
Name: "manifest-test", Provider: domain.StorageProviderBOS, Bucket: "datastorage",
Endpoint: "https://bos.example.test", AccessKeyID: "test-key", SecretAccessKey: "test-secret",
})
if err != nil {
t.Fatalf("create connection: %v", err)
}
browser := newFakeObjectStorageBrowser()
browser.objects = map[string]domain.ObjectStorageObjectSummary{
"datastorage|checkpoint/model/": {
Bucket: "datastorage", Key: "checkpoint/model/", Kind: "file",
},
"datastorage|checkpoint/known-directory": {
Bucket: "datastorage", Key: "checkpoint/known-directory", Kind: "directory",
},
"datastorage|checkpoint/empty": {
Bucket: "datastorage", Key: "checkpoint/empty", Kind: "file",
},
"datastorage|checkpoint/model/model.safetensors": {
Bucket: "datastorage", Key: "checkpoint/model/model.safetensors", Kind: "file", SizeBytes: 1024,
},
}
originalResolver := storageConnectionBrowserResolver
storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
return browser, func() {}, nil
}
t.Cleanup(func() { storageConnectionBrowserResolver = originalResolver })
recorder := httptest.NewRecorder()
newBusinessTestHandler(dataStore, nil).ServeHTTP(recorder,
httptest.NewRequest(http.MethodGet, "/api/object-storage/directory/manifest?connectionId="+connection.ID+"&prefix=checkpoint", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("manifest status=%d body=%s", recorder.Code, recorder.Body.String())
}
var manifest objectStorageDirectoryManifestResponse
if err := json.NewDecoder(recorder.Body).Decode(&manifest); err != nil {
t.Fatalf("decode manifest: %v", err)
}
if manifest.TotalFiles != 2 || len(manifest.Files) != 2 || manifest.TotalBytes != 1024 {
t.Fatalf("manifest includes directory markers or drops empty files: %+v", manifest)
}
for i, relativePath := range []string{"empty", "model/model.safetensors"} {
if manifest.Files[i].RelativePath != relativePath || manifest.Files[i].DownloadURL == "" {
t.Fatalf("manifest file %d = %+v, want downloadable %q", i, manifest.Files[i], relativePath)
}
}
}
func TestExactObjectStorageKeyPreservesTrailingSlash(t *testing.T) {
if got := exactObjectStorageKey(" algo/nested/ "); got != "algo/nested/" {
t.Fatalf("exactObjectStorageKey preserved key = %q, want %q", got, "algo/nested/")
}
}
func TestIsBOSNoSuchKeyRecognizesServiceError(t *testing.T) {
tests := []struct {
name string
err error
want bool
}{
{name: "explicit no such key", err: bce.NewBceServiceError("NoSuchKey", "The specified key does not exist.", "req-1", http.StatusNotFound), want: true},
{name: "empty body head response", err: bce.NewBceServiceError("NotFound", "404 Not Found", "req-2", http.StatusNotFound), want: true},
{name: "missing bucket remains an error", err: bce.NewBceServiceError("NoSuchBucket", "The specified bucket does not exist.", "req-3", http.StatusNotFound), want: false},
{name: "not found code requires 404", err: bce.NewBceServiceError("NotFound", "unexpected", "req-4", http.StatusInternalServerError), want: false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := isBOSNoSuchKey(test.err); got != test.want {
t.Fatalf("isBOSNoSuchKey() = %t, want %t", got, test.want)
}
})
}
}
func TestRetryObjectStorageSummaryRetriesTransientNotFound(t *testing.T) {
attempts := 0
want := domain.ObjectStorageObjectSummary{Bucket: "datastorage", Key: "large/file.bin"}
got, err := retryObjectStorageSummary(context.Background(), 4, 0, func() (domain.ObjectStorageObjectSummary, error) {
attempts++
if attempts < 3 {
return domain.ObjectStorageObjectSummary{}, bce.NewBceServiceError("NotFound", "404 Not Found", "req-retry", http.StatusNotFound)
}
return want, nil
})
if err != nil {
t.Fatalf("retryObjectStorageSummary() error = %v", err)
}
if attempts != 3 || got.Bucket != want.Bucket || got.Key != want.Key {
t.Fatalf("retryObjectStorageSummary() attempts = %d summary = %#v, want attempts=3 summary=%#v", attempts, got, want)
}
}
func TestRetryObjectStorageSummaryDoesNotRetryOtherErrors(t *testing.T) {
attempts := 0
wantErr := bce.NewBceServiceError("NoSuchBucket", "missing bucket", "req-bucket", http.StatusNotFound)
_, err := retryObjectStorageSummary(context.Background(), 4, 0, func() (domain.ObjectStorageObjectSummary, error) {
attempts++
return domain.ObjectStorageObjectSummary{}, wantErr
})
if !errors.Is(err, wantErr) || attempts != 1 {
t.Fatalf("retryObjectStorageSummary() error = %v attempts = %d, want original error and one attempt", err, attempts)
}
}
func TestRenderObjectStorageWriteErrorTreatsBOSNoSuchKeyAsNotFound(t *testing.T) {
recorder := httptest.NewRecorder()
err := bce.NewBceServiceError("NoSuchKey", "The specified key does not exist.", "req-2", http.StatusNotFound)
renderObjectStorageWriteError(recorder, err)
if recorder.Code != http.StatusNotFound {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusNotFound)
}
if !strings.Contains(recorder.Body.String(), "NoSuchKey") {
t.Fatalf("expected response to include NoSuchKey, got %s", recorder.Body.String())
}
}
func TestObjectStorageJSONPutRejectsOversizedBody(t *testing.T) {
body := `{"key":"big.txt","content":"` + strings.Repeat("x", int(objectStorageJSONPutMaxBytes)) + `"}`
handler := newBusinessTestHandler(store.NewMemoryStore(), nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/object-storage/objects", strings.NewReader(body)))
if recorder.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("expected oversized JSON put to return 413, got %d body=%s", recorder.Code, recorder.Body.String())
}
}
func TestSyncReportEndpointsRequireSessionAuth(t *testing.T) {
appCfg := config.AppConfig{}
handler := NewHandlerWithConfig(store.NewMemoryStore(), nil, appCfg)
for _, path := range []string{"/api/object-storage/sync-report"} {
req := httptest.NewRequest(http.MethodGet, path, nil)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Fatalf("expected endpoint %s to require session auth", path)
}
}
}
func TestRetiredPlatformSyncReportRouteIsNotAvailableToSignedInUsers(t *testing.T) {
handler := newBusinessTestHandlerWithConfig(store.NewMemoryStore(), nil, config.AppConfig{})
for _, suffix := range []string{"", "?download=csv&path=/unused", "?download=xlsx"} {
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/object-storage/sync-report"+suffix, nil))
if recorder.Code != http.StatusNotFound {
t.Fatalf("retired report route = %d, want 404", recorder.Code)
}
}
}