forked from longtao.wu/droboticflow
✨ Add concurrent directory uploads
This commit is contained in:
parent
adf3a7637e
commit
d092be1067
@ -624,6 +624,12 @@ Run responses may also include:
|
||||
- `DELETE /api/object-storage/buckets/:name`
|
||||
- `POST /api/object-storage/objects`
|
||||
- `POST /api/object-storage/objects/upload`
|
||||
- `POST /api/object-storage/uploads/presign-put`
|
||||
- `POST /api/object-storage/uploads/batch/prepare`
|
||||
- `POST /api/object-storage/uploads/multipart/initiate`
|
||||
- `POST /api/object-storage/uploads/multipart/part-url`
|
||||
- `POST /api/object-storage/uploads/multipart/complete`
|
||||
- `POST /api/object-storage/uploads/multipart/abort`
|
||||
- `PATCH /api/object-storage/objects`
|
||||
- `DELETE /api/object-storage/objects`
|
||||
|
||||
@ -631,6 +637,8 @@ Run responses may also include:
|
||||
|
||||
`GET` and `HEAD /api/object-storage/content` support browser byte-range preview requests. When a request sends a single `Range: bytes=<start>-<end>` header, the API proxies a ranged BOS read and returns `206 Partial Content`, `Content-Range`, `Accept-Ranges: bytes`, and a bounded `Content-Length` without reading the full object into API memory. Video preview components request only metadata up front and add `?transcode=browser` for browser-compatible MP4 previews when needed.
|
||||
|
||||
`POST /api/object-storage/uploads/batch/prepare?connectionId=<id>` prepares up to 5000 directory-upload entries before the browser transfers bytes. The payload contains `bucket`, optional `prefix`, `conflictPolicy` (`overwrite`, `skip`, or `error`), and `items` with `relativePath`, `kind`, `sizeBytes`, and `contentType`. Relative paths are normalized and traversal segments are rejected. File entries resolve to their destination key; directory entries resolve to `<path>/.keep`. The response gives each item an `upload`, `create_placeholder`, `skip`, or `conflict` action plus aggregate counts. The browser then reuses the existing presigned single-file or multipart upload path with four-file concurrency, so the API does not proxy directory file bodies.
|
||||
|
||||
## Worker API
|
||||
|
||||
Base path: `/worker-api`
|
||||
|
||||
@ -43,6 +43,7 @@ This note tracks the production-facing safeguards that must stay true for Drobod
|
||||
|
||||
- Object-storage JSON object creation is capped at 4 MiB.
|
||||
- Browser multipart uploads stream to BOS and are capped at 2 GiB.
|
||||
- Directory uploads keep browser `webkitRelativePath` values, prepare at most 5000 files/placeholders per batch, and upload four files concurrently through the same presigned direct/multipart path. Conflict policy is explicit (`overwrite`, `skip`, or stop with `error`), failed files can be retried independently, and explicitly listed empty directories are preserved as `.keep` objects because browser directory inputs do not expose truly empty directories.
|
||||
- Large inline object reads use preview or presigned URL behavior instead of loading full objects into API memory.
|
||||
- Browser video previews are lazy-loaded in monitoring, artifact preview, and object-storage browsing surfaces. The UI does not attach the `<video>` source until the user starts the preview, so large MP4 artifacts do not trigger Range requests and decoder work during initial page render or artifact switching.
|
||||
- Run logs, resource samples, and artifacts support strict `limit` and `offset` / `skip` pagination. Empty values use defaults, explicit `limit` must be positive, explicit offsets must be non-negative, and invalid values return validation errors instead of silently falling back to default pages. Logs and resources default to the latest 1000 records for large runs.
|
||||
|
||||
@ -13,6 +13,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
@ -61,15 +62,17 @@ type bosObjectStorageBrowser struct {
|
||||
}
|
||||
|
||||
const (
|
||||
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
|
||||
objectStoragePartialPreviewBytes int64 = 64 * 1024
|
||||
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
|
||||
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
|
||||
objectStoragePresignDefaultExpires = 3600
|
||||
objectStoragePresignMaxExpires = 24 * 3600
|
||||
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
|
||||
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
|
||||
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
|
||||
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
|
||||
objectStoragePartialPreviewBytes int64 = 64 * 1024
|
||||
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
|
||||
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
|
||||
objectStoragePresignDefaultExpires = 3600
|
||||
objectStoragePresignMaxExpires = 24 * 3600
|
||||
objectStorageBatchUploadMaxItems = 5000
|
||||
objectStorageBatchUploadStatConcurrency = 16
|
||||
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
|
||||
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
|
||||
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
|
||||
)
|
||||
|
||||
var storageConnectionBrowserResolver = func(ctx context.Context, connection *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
@ -1151,6 +1154,194 @@ func renderObjectStorageInvalidJSON(writer http.ResponseWriter) {
|
||||
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
|
||||
}
|
||||
|
||||
func normalizeObjectStorageBatchRelativePath(rawValue string) (string, error) {
|
||||
value := strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/")
|
||||
value = strings.TrimSuffix(value, "/")
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("relativePath is required")
|
||||
}
|
||||
if strings.HasPrefix(value, "/") || strings.ContainsRune(value, '\x00') {
|
||||
return "", fmt.Errorf("relativePath must be a relative object path")
|
||||
}
|
||||
parts := strings.Split(value, "/")
|
||||
for _, part := range parts {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return "", fmt.Errorf("relativePath contains an invalid path segment")
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "/"), nil
|
||||
}
|
||||
|
||||
func normalizeObjectStorageBatchPrefix(rawValue string) (string, error) {
|
||||
value := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/"), "/")
|
||||
if value == "" {
|
||||
return "", nil
|
||||
}
|
||||
return normalizeObjectStorageBatchRelativePath(value)
|
||||
}
|
||||
|
||||
func objectStorageBatchItemKey(prefix string, relativePath string, kind string) string {
|
||||
key := relativePath
|
||||
if kind == "directory" {
|
||||
key += "/.keep"
|
||||
}
|
||||
if prefix != "" {
|
||||
key = prefix + "/" + key
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
func prepareObjectStorageBatchItems(input domain.ObjectStorageBatchUploadPrepareInput) ([]domain.ObjectStorageBatchUploadPreparedItem, string, string, *validationIssue) {
|
||||
policy := strings.ToLower(strings.TrimSpace(input.ConflictPolicy))
|
||||
if policy == "" {
|
||||
policy = "overwrite"
|
||||
}
|
||||
if policy != "overwrite" && policy != "skip" && policy != "error" {
|
||||
return nil, "", "", &validationIssue{Field: "conflictPolicy", Message: "conflictPolicy must be overwrite, skip, or error"}
|
||||
}
|
||||
prefix, err := normalizeObjectStorageBatchPrefix(input.Prefix)
|
||||
if err != nil {
|
||||
return nil, "", "", &validationIssue{Field: "prefix", Message: err.Error()}
|
||||
}
|
||||
if len(input.Items) == 0 {
|
||||
return nil, "", "", &validationIssue{Field: "items", Message: "items are required"}
|
||||
}
|
||||
if len(input.Items) > objectStorageBatchUploadMaxItems {
|
||||
return nil, "", "", &validationIssue{Field: "items", Message: fmt.Sprintf("items must contain at most %d entries", objectStorageBatchUploadMaxItems)}
|
||||
}
|
||||
prepared := make([]domain.ObjectStorageBatchUploadPreparedItem, 0, len(input.Items))
|
||||
seenKeys := make(map[string]struct{}, len(input.Items))
|
||||
for index, item := range input.Items {
|
||||
kind := strings.ToLower(strings.TrimSpace(item.Kind))
|
||||
if kind != "file" && kind != "directory" {
|
||||
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].kind", index), Message: "kind must be file or directory"}
|
||||
}
|
||||
if item.SizeBytes < 0 {
|
||||
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].sizeBytes", index), Message: "sizeBytes must not be negative"}
|
||||
}
|
||||
relativePath, pathErr := normalizeObjectStorageBatchRelativePath(item.RelativePath)
|
||||
if pathErr != nil {
|
||||
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: pathErr.Error()}
|
||||
}
|
||||
key := objectStorageBatchItemKey(prefix, relativePath, kind)
|
||||
if _, exists := seenKeys[key]; exists {
|
||||
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: "duplicate destination key"}
|
||||
}
|
||||
seenKeys[key] = struct{}{}
|
||||
prepared = append(prepared, domain.ObjectStorageBatchUploadPreparedItem{
|
||||
RelativePath: relativePath,
|
||||
Key: key,
|
||||
Kind: kind,
|
||||
SizeBytes: item.SizeBytes,
|
||||
ContentType: strings.TrimSpace(item.ContentType),
|
||||
})
|
||||
}
|
||||
return prepared, prefix, policy, nil
|
||||
}
|
||||
|
||||
func inspectObjectStorageBatchConflicts(ctx context.Context, browser objectStorageBrowser, bucket string, items []domain.ObjectStorageBatchUploadPreparedItem) error {
|
||||
workerCount := min(objectStorageBatchUploadStatConcurrency, len(items))
|
||||
indices := make(chan int)
|
||||
var firstErr error
|
||||
var errOnce sync.Once
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(workerCount)
|
||||
for range workerCount {
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for index := range indices {
|
||||
_, err := browser.GetObjectSummary(ctx, bucket, items[index].Key)
|
||||
if err == nil {
|
||||
items[index].Exists = true
|
||||
continue
|
||||
}
|
||||
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
|
||||
continue
|
||||
}
|
||||
errOnce.Do(func() { firstErr = err })
|
||||
}
|
||||
}()
|
||||
}
|
||||
for index := range items {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
indices <- index
|
||||
}
|
||||
close(indices)
|
||||
workers.Wait()
|
||||
if firstErr != nil {
|
||||
return firstErr
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func finalizeObjectStorageBatchPlan(items []domain.ObjectStorageBatchUploadPreparedItem, policy string) domain.ObjectStorageBatchUploadPrepareResult {
|
||||
result := domain.ObjectStorageBatchUploadPrepareResult{ConflictPolicy: policy, Items: items}
|
||||
for index := range result.Items {
|
||||
item := &result.Items[index]
|
||||
if item.Exists && policy == "skip" {
|
||||
item.Action = "skip"
|
||||
item.Message = "destination already exists"
|
||||
result.SkipCount++
|
||||
continue
|
||||
}
|
||||
if item.Exists && policy == "error" {
|
||||
item.Action = "conflict"
|
||||
item.Message = "destination already exists"
|
||||
result.ConflictCount++
|
||||
continue
|
||||
}
|
||||
if item.Kind == "directory" {
|
||||
item.Action = "create_placeholder"
|
||||
result.PlaceholderCount++
|
||||
} else {
|
||||
item.Action = "upload"
|
||||
result.UploadCount++
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (server *server) handleObjectStorageBatchUploadPrepare(writer http.ResponseWriter, request *http.Request) {
|
||||
input, err := decodeObjectStorageJSON[domain.ObjectStorageBatchUploadPrepareInput](request)
|
||||
if err != nil {
|
||||
renderObjectStorageInvalidJSON(writer)
|
||||
return
|
||||
}
|
||||
items, prefix, policy, issue := prepareObjectStorageBatchItems(input)
|
||||
if issue != nil {
|
||||
renderValidationError(writer, *issue)
|
||||
return
|
||||
}
|
||||
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
||||
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
||||
if cleanup != nil {
|
||||
defer cleanup()
|
||||
}
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if browser == nil || connection == nil {
|
||||
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
||||
return
|
||||
}
|
||||
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
|
||||
if err != nil || bucket == "" {
|
||||
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
|
||||
return
|
||||
}
|
||||
if err := inspectObjectStorageBatchConflicts(request.Context(), browser, bucket, items); err != nil {
|
||||
renderObjectStorageWriteError(writer, err)
|
||||
return
|
||||
}
|
||||
result := finalizeObjectStorageBatchPlan(items, policy)
|
||||
result.Bucket = bucket
|
||||
result.Prefix = prefix
|
||||
renderJSON(writer, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (server *server) handleObjectStoragePresignPut(writer http.ResponseWriter, request *http.Request) {
|
||||
input, err := decodeObjectStorageJSON[objectStoragePresignPutInput](request)
|
||||
if err != nil {
|
||||
|
||||
@ -1378,6 +1378,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
router.Post("/api/object-storage/objects", instance.handleObjectStorageObjectCreate)
|
||||
router.Post("/api/object-storage/objects/upload", instance.handleObjectStorageObjectUpload)
|
||||
router.Post("/api/object-storage/uploads/presign-put", instance.handleObjectStoragePresignPut)
|
||||
router.Post("/api/object-storage/uploads/batch/prepare", instance.handleObjectStorageBatchUploadPrepare)
|
||||
router.Post("/api/object-storage/uploads/multipart/initiate", instance.handleObjectStorageMultipartInitiate)
|
||||
router.Post("/api/object-storage/uploads/multipart/part-url", instance.handleObjectStorageMultipartPartURL)
|
||||
router.Post("/api/object-storage/uploads/multipart/complete", instance.handleObjectStorageMultipartComplete)
|
||||
|
||||
@ -9501,6 +9501,109 @@ func TestObjectStorageUploadControlAPISupportsPresignAndMultipart(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageBatchUploadPreparePlansNestedFilesConflictsAndPlaceholders(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
||||
Name: "BOS batch upload",
|
||||
Provider: domain.StorageProviderBOS,
|
||||
Endpoint: "https://bos.example.test",
|
||||
Bucket: "datastorage",
|
||||
AccessKeyID: "ak-storage-browser",
|
||||
SecretAccessKey: "sk-storage-browser",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create storage connection: %v", err)
|
||||
}
|
||||
fakeBrowser := newFakeObjectStorageBrowser()
|
||||
existingKey := "datasets/root/nested/existing.bin"
|
||||
fakeBrowser.objects["datastorage|"+existingKey] = domain.ObjectStorageObjectSummary{
|
||||
Bucket: "datastorage", Key: existingKey, Name: "existing.bin", Kind: "file",
|
||||
}
|
||||
originalResolver := storageConnectionBrowserResolver
|
||||
storageConnectionBrowserResolver = func(_ context.Context, _ *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
return fakeBrowser, func() {}, nil
|
||||
}
|
||||
t.Cleanup(func() { storageConnectionBrowserResolver = originalResolver })
|
||||
handler := NewHandler(dataStore, nil)
|
||||
|
||||
body := `{"bucket":"datastorage","prefix":"datasets","conflictPolicy":"skip","items":[` +
|
||||
`{"relativePath":"root/nested/existing.bin","kind":"file","sizeBytes":10},` +
|
||||
`{"relativePath":"root/new.bin","kind":"file","sizeBytes":20},` +
|
||||
`{"relativePath":"root/empty","kind":"directory"}]}`
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(
|
||||
recorder,
|
||||
httptest.NewRequest(http.MethodPost, "/api/object-storage/uploads/batch/prepare?connectionId="+connection.ID, strings.NewReader(body)),
|
||||
)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("batch prepare status: %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var result domain.ObjectStorageBatchUploadPrepareResult
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil {
|
||||
t.Fatalf("decode batch prepare: %v", err)
|
||||
}
|
||||
if result.UploadCount != 1 || result.PlaceholderCount != 1 || result.SkipCount != 1 || result.ConflictCount != 0 {
|
||||
t.Fatalf("unexpected batch counts: %#v", result)
|
||||
}
|
||||
if result.Items[0].Action != "skip" || result.Items[1].Key != "datasets/root/new.bin" || result.Items[1].Action != "upload" {
|
||||
t.Fatalf("unexpected file plan: %#v", result.Items)
|
||||
}
|
||||
if result.Items[2].Key != "datasets/root/empty/.keep" || result.Items[2].Action != "create_placeholder" {
|
||||
t.Fatalf("unexpected directory plan: %#v", result.Items[2])
|
||||
}
|
||||
|
||||
errorBody := strings.Replace(body, `"conflictPolicy":"skip"`, `"conflictPolicy":"error"`, 1)
|
||||
errorRecorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(
|
||||
errorRecorder,
|
||||
httptest.NewRequest(http.MethodPost, "/api/object-storage/uploads/batch/prepare?connectionId="+connection.ID, strings.NewReader(errorBody)),
|
||||
)
|
||||
if errorRecorder.Code != http.StatusOK {
|
||||
t.Fatalf("batch error policy status: %d body=%s", errorRecorder.Code, errorRecorder.Body.String())
|
||||
}
|
||||
if err := json.Unmarshal(errorRecorder.Body.Bytes(), &result); err != nil {
|
||||
t.Fatalf("decode error plan: %v", err)
|
||||
}
|
||||
if result.ConflictCount != 1 || result.Items[0].Action != "conflict" {
|
||||
t.Fatalf("expected one explicit conflict: %#v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageBatchUploadPrepareRejectsUnsafeAndDuplicatePaths(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
||||
Name: "BOS batch validation", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test",
|
||||
Bucket: "datastorage", AccessKeyID: "ak", SecretAccessKey: "sk",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create storage connection: %v", err)
|
||||
}
|
||||
fakeBrowser := newFakeObjectStorageBrowser()
|
||||
originalResolver := storageConnectionBrowserResolver
|
||||
storageConnectionBrowserResolver = func(_ context.Context, _ *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
return fakeBrowser, func() {}, nil
|
||||
}
|
||||
t.Cleanup(func() { storageConnectionBrowserResolver = originalResolver })
|
||||
handler := NewHandler(dataStore, nil)
|
||||
tests := []string{
|
||||
`{"items":[{"relativePath":"../escape.bin","kind":"file"}]}`,
|
||||
`{"items":[{"relativePath":"same.bin","kind":"file"},{"relativePath":"same.bin","kind":"file"}]}`,
|
||||
`{"conflictPolicy":"rename","items":[{"relativePath":"valid.bin","kind":"file"}]}`,
|
||||
}
|
||||
for _, body := range tests {
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(
|
||||
recorder,
|
||||
httptest.NewRequest(http.MethodPost, "/api/object-storage/uploads/batch/prepare?connectionId="+connection.ID, strings.NewReader(body)),
|
||||
)
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d body=%s for input %s", recorder.Code, recorder.Body.String(), body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageUploadControlAPIRejectsInvalidMultipartInput(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
|
||||
@ -129,3 +129,39 @@ type ObjectStorageMultipartUploadAbortResult struct {
|
||||
UploadID string `json:"uploadId"`
|
||||
Aborted bool `json:"aborted"`
|
||||
}
|
||||
|
||||
type ObjectStorageBatchUploadItemInput struct {
|
||||
RelativePath string `json:"relativePath"`
|
||||
Kind string `json:"kind"`
|
||||
SizeBytes int64 `json:"sizeBytes,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectStorageBatchUploadPrepareInput struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Prefix string `json:"prefix,omitempty"`
|
||||
ConflictPolicy string `json:"conflictPolicy,omitempty"`
|
||||
Items []ObjectStorageBatchUploadItemInput `json:"items"`
|
||||
}
|
||||
|
||||
type ObjectStorageBatchUploadPreparedItem struct {
|
||||
RelativePath string `json:"relativePath"`
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
SizeBytes int64 `json:"sizeBytes,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
Action string `json:"action"`
|
||||
Exists bool `json:"exists"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectStorageBatchUploadPrepareResult struct {
|
||||
Bucket string `json:"bucket"`
|
||||
Prefix string `json:"prefix,omitempty"`
|
||||
ConflictPolicy string `json:"conflictPolicy"`
|
||||
Items []ObjectStorageBatchUploadPreparedItem `json:"items"`
|
||||
UploadCount int `json:"uploadCount"`
|
||||
PlaceholderCount int `json:"placeholderCount"`
|
||||
SkipCount int `json:"skipCount"`
|
||||
ConflictCount int `json:"conflictCount"`
|
||||
}
|
||||
|
||||
338
src/components/object-storage/DirectoryUploadPanel.tsx
Normal file
338
src/components/object-storage/DirectoryUploadPanel.tsx
Normal file
@ -0,0 +1,338 @@
|
||||
import React, { useMemo, useRef, useState } from "react";
|
||||
import { AlertCircle, CheckCircle2, FolderUp, Loader2, RefreshCw, Upload } from "lucide-react";
|
||||
import {
|
||||
DIRECTORY_UPLOAD_CONCURRENCY,
|
||||
directoryUploadBatchItems,
|
||||
directoryUploadEmptyDirectories,
|
||||
directoryUploadFiles,
|
||||
runDirectoryUploadTasks,
|
||||
} from "../../lib/object-storage-directory-upload.js";
|
||||
import { getActionErrorMessage } from "../../lib/action-errors.js";
|
||||
import {
|
||||
objectStorageApi,
|
||||
type ObjectStorageBatchConflictPolicy,
|
||||
type ObjectStorageBatchUploadPreparedItem,
|
||||
} from "../../services/api/object-storage";
|
||||
|
||||
type SelectedDirectoryFile = { relativePath: string; file: File };
|
||||
type UploadState = "pending" | "uploading" | "success" | "skipped" | "conflict" | "failed";
|
||||
type UploadRecord = {
|
||||
item: ObjectStorageBatchUploadPreparedItem;
|
||||
file?: File;
|
||||
state: UploadState;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
const formatBytes = (value: number) => {
|
||||
if (value < 1024) return `${value} B`;
|
||||
const units = ["KB", "MB", "GB", "TB"];
|
||||
let current = value / 1024;
|
||||
let unit = 0;
|
||||
while (current >= 1024 && unit < units.length - 1) {
|
||||
current /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return `${current.toFixed(current >= 10 ? 0 : 1)} ${units[unit]}`;
|
||||
};
|
||||
|
||||
export function DirectoryUploadPanel({
|
||||
connectionId,
|
||||
bucket,
|
||||
prefix,
|
||||
disabled,
|
||||
onUploaded,
|
||||
}: {
|
||||
connectionId: string;
|
||||
bucket: string;
|
||||
prefix: string;
|
||||
disabled?: boolean;
|
||||
onUploaded: () => void | Promise<void>;
|
||||
}) {
|
||||
const [selectedFiles, setSelectedFiles] = useState<SelectedDirectoryFile[]>([]);
|
||||
const [emptyDirectoriesText, setEmptyDirectoriesText] = useState("");
|
||||
const [conflictPolicy, setConflictPolicy] = useState<ObjectStorageBatchConflictPolicy>("overwrite");
|
||||
const [records, setRecords] = useState<UploadRecord[]>([]);
|
||||
const [running, setRunning] = useState(false);
|
||||
const [message, setMessage] = useState("");
|
||||
const [error, setError] = useState("");
|
||||
const [progress, setProgress] = useState({ loadedBytes: 0, totalBytes: 0, completed: 0, total: 0 });
|
||||
const loadedByKeyRef = useRef(new Map<string, number>());
|
||||
const progressFrameRef = useRef<number | null>(null);
|
||||
const pendingProgressRef = useRef<{ completed?: number; total?: number; totalBytes?: number }>({});
|
||||
|
||||
const emptyDirectories = useMemo(() => directoryUploadEmptyDirectories(emptyDirectoriesText), [emptyDirectoriesText]);
|
||||
const selectedBytes = useMemo(
|
||||
() => selectedFiles.reduce((total, item) => total + item.file.size, 0),
|
||||
[selectedFiles],
|
||||
);
|
||||
const failedCount = records.filter((record) => record.state === "failed").length;
|
||||
const successCount = records.filter((record) => record.state === "success").length;
|
||||
const skippedCount = records.filter((record) => record.state === "skipped").length;
|
||||
const conflictCount = records.filter((record) => record.state === "conflict").length;
|
||||
const percent =
|
||||
progress.totalBytes > 0
|
||||
? Math.round((progress.loadedBytes / progress.totalBytes) * 100)
|
||||
: progress.total > 0
|
||||
? Math.round((progress.completed / progress.total) * 100)
|
||||
: 0;
|
||||
|
||||
const flushProgress = (completed?: number, total?: number, totalBytes?: number) => {
|
||||
pendingProgressRef.current = {
|
||||
completed: completed ?? pendingProgressRef.current.completed,
|
||||
total: total ?? pendingProgressRef.current.total,
|
||||
totalBytes: totalBytes ?? pendingProgressRef.current.totalBytes,
|
||||
};
|
||||
if (progressFrameRef.current !== null) return;
|
||||
progressFrameRef.current = window.requestAnimationFrame(() => {
|
||||
progressFrameRef.current = null;
|
||||
const pending = pendingProgressRef.current;
|
||||
pendingProgressRef.current = {};
|
||||
const loadedBytes = (Array.from(loadedByKeyRef.current.values()) as number[]).reduce<number>(
|
||||
(sum, value) => sum + value,
|
||||
0,
|
||||
);
|
||||
setProgress((current) => ({
|
||||
loadedBytes,
|
||||
totalBytes: pending.totalBytes ?? current.totalBytes,
|
||||
completed: pending.completed ?? current.completed,
|
||||
total: pending.total ?? current.total,
|
||||
}));
|
||||
});
|
||||
};
|
||||
|
||||
const executeRecords = async (initialRecords: UploadRecord[]) => {
|
||||
const runnable = initialRecords.filter((record) => record.state === "pending" || record.state === "failed");
|
||||
if (runnable.length === 0) return;
|
||||
setRunning(true);
|
||||
setError("");
|
||||
setMessage("");
|
||||
loadedByKeyRef.current = new Map();
|
||||
const totalBytes = runnable.reduce(
|
||||
(total, record) => total + (record.item.action === "upload" ? record.file?.size || 0 : 0),
|
||||
0,
|
||||
);
|
||||
let completed = 0;
|
||||
setProgress({ loadedBytes: 0, totalBytes, completed: 0, total: runnable.length });
|
||||
const currentRecords = records.length > 0 ? [...records] : [...initialRecords];
|
||||
const updateRecord = (key: string, update: Partial<UploadRecord>) => {
|
||||
const index = currentRecords.findIndex((record) => record.item.key === key);
|
||||
if (index >= 0) currentRecords[index] = { ...currentRecords[index], ...update };
|
||||
setRecords([...currentRecords]);
|
||||
};
|
||||
await runDirectoryUploadTasks(
|
||||
runnable,
|
||||
async (record) => {
|
||||
updateRecord(record.item.key, { state: "uploading", error: undefined });
|
||||
if (record.item.action === "create_placeholder") {
|
||||
await objectStorageApi.putObject(
|
||||
{ bucket, key: record.item.key, content: "", contentType: "application/x-directory" },
|
||||
{ connectionId },
|
||||
);
|
||||
} else {
|
||||
if (!record.file) throw new Error("未找到该条目的本地文件,请重新选择目录。");
|
||||
await objectStorageApi.uploadObject(
|
||||
record.file,
|
||||
{ connectionId, bucket, key: record.item.key, contentType: record.item.contentType },
|
||||
(uploadProgress) => {
|
||||
loadedByKeyRef.current.set(record.item.key, uploadProgress.loadedBytes);
|
||||
flushProgress(undefined, undefined, totalBytes);
|
||||
},
|
||||
);
|
||||
loadedByKeyRef.current.set(record.item.key, record.file.size);
|
||||
}
|
||||
return record.item.key;
|
||||
},
|
||||
{
|
||||
concurrency: DIRECTORY_UPLOAD_CONCURRENCY,
|
||||
onSettled: (result, record) => {
|
||||
completed += 1;
|
||||
if (result.status === "fulfilled") {
|
||||
updateRecord(record.item.key, { state: "success", error: undefined });
|
||||
} else {
|
||||
updateRecord(record.item.key, { state: "failed", error: getActionErrorMessage(result.reason) });
|
||||
}
|
||||
flushProgress(completed, runnable.length, totalBytes);
|
||||
},
|
||||
},
|
||||
);
|
||||
const finalFailed = currentRecords.filter((record) => record.state === "failed").length;
|
||||
setRunning(false);
|
||||
if (finalFailed > 0) {
|
||||
setError(`${finalFailed} 个条目上传失败,可仅重试失败项。`);
|
||||
} else {
|
||||
setMessage("目录上传完成。");
|
||||
await onUploaded();
|
||||
}
|
||||
};
|
||||
|
||||
const prepareAndUpload = async () => {
|
||||
const batchItems = directoryUploadBatchItems(selectedFiles, emptyDirectories);
|
||||
if (batchItems.length === 0) {
|
||||
setError("请先选择一个目录,或填写需要保留的空目录。");
|
||||
return;
|
||||
}
|
||||
setRunning(true);
|
||||
setError("");
|
||||
setMessage("正在检查路径和冲突…");
|
||||
try {
|
||||
const plan = await objectStorageApi.prepareBatchUpload(
|
||||
{ bucket, prefix, conflictPolicy, items: batchItems },
|
||||
{ connectionId },
|
||||
);
|
||||
const filesByPath = new Map<string, File>(selectedFiles.map((item) => [item.relativePath, item.file] as const));
|
||||
const nextRecords: UploadRecord[] = plan.items.map((item) => ({
|
||||
item,
|
||||
file: filesByPath.get(item.relativePath),
|
||||
state: item.action === "skip" ? "skipped" : item.action === "conflict" ? "conflict" : "pending",
|
||||
}));
|
||||
setRecords(nextRecords);
|
||||
if (plan.conflictCount > 0) {
|
||||
setMessage("");
|
||||
setError(`检测到 ${plan.conflictCount} 个冲突。请选择“跳过”或“覆盖”后重新上传。`);
|
||||
return;
|
||||
}
|
||||
setMessage("");
|
||||
await executeRecords(nextRecords);
|
||||
} catch (cause) {
|
||||
setError(getActionErrorMessage(cause));
|
||||
} finally {
|
||||
setRunning(false);
|
||||
}
|
||||
};
|
||||
|
||||
const visibleRecords = records
|
||||
.filter((record) => record.state === "failed" || record.state === "conflict")
|
||||
.slice(0, 50);
|
||||
|
||||
return (
|
||||
<div
|
||||
className="space-y-4 rounded-2xl border border-[#2a2b2f] bg-[#101115] p-4"
|
||||
data-testid="directory-upload-panel"
|
||||
>
|
||||
<div className="flex flex-wrap items-start justify-between gap-3">
|
||||
<div>
|
||||
<div className="flex items-center gap-2 text-sm font-medium text-white">
|
||||
<FolderUp className="h-4 w-4 text-blue-400" />
|
||||
上传目录
|
||||
</div>
|
||||
<p className="mt-1 text-xs leading-5 text-gray-500">
|
||||
保留 webkitRelativePath,4 个文件并发;大文件自动复用分片上传。纯空目录请在下方补充,将写入 .keep。
|
||||
</p>
|
||||
</div>
|
||||
<div className="text-right text-xs text-gray-500">
|
||||
<div>
|
||||
{selectedFiles.length} 个文件 · {formatBytes(selectedBytes)}
|
||||
</div>
|
||||
{emptyDirectories.length > 0 && <div>{emptyDirectories.length} 个空目录</div>}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-3 lg:grid-cols-[1.2fr_0.8fr_0.6fr_auto]">
|
||||
<label className="space-y-2">
|
||||
<span className="block text-xs uppercase tracking-[0.2em] text-gray-500">选择本地目录</span>
|
||||
<input
|
||||
data-testid="directory-upload-input"
|
||||
type="file"
|
||||
multiple
|
||||
{...({ webkitdirectory: "", directory: "" } as React.InputHTMLAttributes<HTMLInputElement>)}
|
||||
onChange={(event) => {
|
||||
setSelectedFiles(directoryUploadFiles(event.target.files) as SelectedDirectoryFile[]);
|
||||
setRecords([]);
|
||||
setError("");
|
||||
setMessage("");
|
||||
}}
|
||||
disabled={disabled || running}
|
||||
className="block w-full rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-[10px] text-sm text-gray-300 file:mr-3 file:rounded-md file:border-0 file:bg-blue-600 file:px-3 file:py-1.5 file:text-xs file:text-white"
|
||||
/>
|
||||
</label>
|
||||
<label className="space-y-2">
|
||||
<span className="block text-xs uppercase tracking-[0.2em] text-gray-500">额外空目录</span>
|
||||
<textarea
|
||||
data-testid="directory-upload-empty-directories"
|
||||
value={emptyDirectoriesText}
|
||||
onChange={(event) => setEmptyDirectoriesText(event.target.value)}
|
||||
disabled={disabled || running}
|
||||
rows={2}
|
||||
className="w-full rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-2 text-xs text-white focus:border-blue-500 focus:outline-none"
|
||||
placeholder={"每行一个相对路径\n例如 dataset/labels-empty"}
|
||||
/>
|
||||
</label>
|
||||
<label className="space-y-2">
|
||||
<span className="block text-xs uppercase tracking-[0.2em] text-gray-500">同名冲突</span>
|
||||
<select
|
||||
data-testid="directory-upload-conflict-policy"
|
||||
value={conflictPolicy}
|
||||
onChange={(event) => setConflictPolicy(event.target.value as ObjectStorageBatchConflictPolicy)}
|
||||
disabled={disabled || running}
|
||||
className="w-full rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-3 text-sm text-white focus:border-blue-500 focus:outline-none"
|
||||
>
|
||||
<option value="overwrite">覆盖已有对象</option>
|
||||
<option value="skip">跳过已有对象</option>
|
||||
<option value="error">发现冲突即停止</option>
|
||||
</select>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
data-testid="directory-upload-button"
|
||||
onClick={prepareAndUpload}
|
||||
disabled={disabled || running || (!selectedFiles.length && !emptyDirectories.length)}
|
||||
className="inline-flex items-center justify-center gap-2 rounded-xl bg-blue-600 px-4 py-3 text-sm font-medium text-white hover:bg-blue-500 disabled:opacity-60 lg:self-end"
|
||||
>
|
||||
{running ? <Loader2 className="h-4 w-4 animate-spin" /> : <Upload className="h-4 w-4" />}
|
||||
{running ? "上传中" : "上传目录"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{(progress.total > 0 || running) && (
|
||||
<div className="space-y-2">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2 text-xs text-gray-400">
|
||||
<span>
|
||||
总体进度 {progress.completed}/{progress.total}
|
||||
</span>
|
||||
<span>
|
||||
{formatBytes(progress.loadedBytes)} / {formatBytes(progress.totalBytes)} · {percent}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-2 overflow-hidden rounded-full bg-[#202126]">
|
||||
<div className="h-full bg-blue-500 transition-all" style={{ width: `${percent}%` }} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(message || error || records.length > 0) && (
|
||||
<div className="flex flex-wrap items-center justify-between gap-3 rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-2 text-xs">
|
||||
<div className="flex items-center gap-2">
|
||||
{error ? (
|
||||
<AlertCircle className="h-4 w-4 text-red-400" />
|
||||
) : (
|
||||
<CheckCircle2 className="h-4 w-4 text-emerald-400" />
|
||||
)}
|
||||
<span className={error ? "text-red-200" : "text-gray-300"}>
|
||||
{error || message || `成功 ${successCount} · 跳过 ${skippedCount} · 冲突 ${conflictCount}`}
|
||||
</span>
|
||||
</div>
|
||||
{failedCount > 0 && !running && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => executeRecords(records)}
|
||||
className="inline-flex items-center gap-1 rounded-lg border border-red-500/30 px-2 py-1 text-red-200 hover:bg-red-500/10"
|
||||
>
|
||||
<RefreshCw className="h-3.5 w-3.5" /> 仅重试失败项
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{visibleRecords.length > 0 && (
|
||||
<div className="max-h-40 space-y-1 overflow-auto text-xs text-red-200">
|
||||
{visibleRecords.map((record) => (
|
||||
<div key={record.item.key} className="rounded-lg bg-red-500/5 px-3 py-2">
|
||||
<div className="font-mono">{record.item.key}</div>
|
||||
<div className="mt-1 text-red-300/80">{record.error || record.item.message || "目标对象冲突"}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
71
src/lib/object-storage-directory-upload.js
Normal file
71
src/lib/object-storage-directory-upload.js
Normal file
@ -0,0 +1,71 @@
|
||||
export const DIRECTORY_UPLOAD_CONCURRENCY = 4;
|
||||
|
||||
export function normalizeDirectoryUploadPath(rawValue) {
|
||||
const value = String(rawValue || "")
|
||||
.trim()
|
||||
.replaceAll("\\", "/")
|
||||
.replace(/^\/+|\/+$/g, "");
|
||||
if (!value) return "";
|
||||
const parts = value.split("/");
|
||||
if (parts.some((part) => !part || part === "." || part === "..")) {
|
||||
return "";
|
||||
}
|
||||
return parts.join("/");
|
||||
}
|
||||
|
||||
export function directoryUploadFiles(fileList) {
|
||||
return Array.from(fileList || []).flatMap((file) => {
|
||||
const relativePath = normalizeDirectoryUploadPath(file.webkitRelativePath || file.name);
|
||||
return relativePath ? [{ relativePath, file }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
export function directoryUploadEmptyDirectories(rawValue) {
|
||||
const seen = new Set();
|
||||
return String(rawValue || "")
|
||||
.split(/\r?\n/)
|
||||
.map(normalizeDirectoryUploadPath)
|
||||
.filter((relativePath) => {
|
||||
if (!relativePath || seen.has(relativePath)) return false;
|
||||
seen.add(relativePath);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export function directoryUploadBatchItems(files, emptyDirectories = []) {
|
||||
const fileItems = files.map(({ relativePath, file }) => ({
|
||||
relativePath,
|
||||
kind: "file",
|
||||
sizeBytes: file.size,
|
||||
contentType: file.type || "application/octet-stream",
|
||||
}));
|
||||
const filePaths = new Set(fileItems.map((item) => item.relativePath));
|
||||
const directoryItems = emptyDirectories
|
||||
.filter((relativePath) => !filePaths.has(`${relativePath}/.keep`))
|
||||
.map((relativePath) => ({ relativePath, kind: "directory", sizeBytes: 0, contentType: "application/x-directory" }));
|
||||
return [...fileItems, ...directoryItems];
|
||||
}
|
||||
|
||||
export async function runDirectoryUploadTasks(tasks, worker, options = {}) {
|
||||
const concurrency = Math.max(
|
||||
1,
|
||||
Math.min(Number(options.concurrency) || DIRECTORY_UPLOAD_CONCURRENCY, tasks.length || 1),
|
||||
);
|
||||
const results = new Array(tasks.length);
|
||||
let nextIndex = 0;
|
||||
const runNext = async () => {
|
||||
while (nextIndex < tasks.length) {
|
||||
const index = nextIndex;
|
||||
nextIndex += 1;
|
||||
try {
|
||||
const value = await worker(tasks[index], index);
|
||||
results[index] = { status: "fulfilled", value };
|
||||
} catch (error) {
|
||||
results[index] = { status: "rejected", reason: error };
|
||||
}
|
||||
options.onSettled?.(results[index], tasks[index], index);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: concurrency }, runNext));
|
||||
return results;
|
||||
}
|
||||
@ -23,6 +23,7 @@ import {
|
||||
} from "lucide-react";
|
||||
import { useTimeZone } from "../context/TimeZoneContext";
|
||||
import { LazyVideoPreview } from "../components/LazyVideoPreview";
|
||||
import { DirectoryUploadPanel } from "../components/object-storage/DirectoryUploadPanel";
|
||||
import { formatCompactDateTime } from "../lib/datetime";
|
||||
import { isObjectStorageEditorLocked, shouldSyncObjectStoragePreviewToEditor } from "../lib/object-storage-preview.js";
|
||||
import { getActionErrorMessage } from "../lib/action-errors.js";
|
||||
@ -1719,6 +1720,18 @@ export default function ObjectStorageBrowser() {
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div key={`${selectedConnectionId}:${selectedBucket}:${currentPrefix}`}>
|
||||
<DirectoryUploadPanel
|
||||
connectionId={selectedConnectionId}
|
||||
bucket={selectedBucket}
|
||||
prefix={currentPrefix}
|
||||
disabled={objectActionLoading || !selectedConnectionId || !selectedBucket}
|
||||
onUploaded={async () => {
|
||||
await reloadObjects("", undefined, "");
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@ -85,6 +85,34 @@ export interface ObjectStorageUploadProgress {
|
||||
partCount?: number;
|
||||
}
|
||||
|
||||
export type ObjectStorageBatchConflictPolicy = "overwrite" | "skip" | "error";
|
||||
export type ObjectStorageBatchPreparedAction = "upload" | "create_placeholder" | "skip" | "conflict";
|
||||
|
||||
export interface ObjectStorageBatchUploadItemInput {
|
||||
relativePath: string;
|
||||
kind: "file" | "directory";
|
||||
sizeBytes?: number;
|
||||
contentType?: string;
|
||||
}
|
||||
|
||||
export interface ObjectStorageBatchUploadPreparedItem extends ObjectStorageBatchUploadItemInput {
|
||||
key: string;
|
||||
action: ObjectStorageBatchPreparedAction;
|
||||
exists: boolean;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface ObjectStorageBatchUploadPrepareResult {
|
||||
bucket: string;
|
||||
prefix?: string;
|
||||
conflictPolicy: ObjectStorageBatchConflictPolicy;
|
||||
items: ObjectStorageBatchUploadPreparedItem[];
|
||||
uploadCount: number;
|
||||
placeholderCount: number;
|
||||
skipCount: number;
|
||||
conflictCount: number;
|
||||
}
|
||||
|
||||
interface ObjectStoragePresignedUploadResult {
|
||||
bucket: string;
|
||||
key: string;
|
||||
@ -297,6 +325,20 @@ export const objectStorageApi = {
|
||||
}
|
||||
return uploadDirectObject(file, options, onProgress);
|
||||
},
|
||||
prepareBatchUpload: (
|
||||
payload: {
|
||||
bucket?: string;
|
||||
prefix?: string;
|
||||
conflictPolicy: ObjectStorageBatchConflictPolicy;
|
||||
items: ObjectStorageBatchUploadItemInput[];
|
||||
},
|
||||
options: { connectionId: string },
|
||||
) =>
|
||||
apiRequest<ObjectStorageBatchUploadPrepareResult>(
|
||||
`/api/object-storage/uploads/batch/prepare${buildObjectStorageParams(options)}`,
|
||||
{ method: "POST" },
|
||||
payload,
|
||||
),
|
||||
uploadObjectWithServerFallback: async (
|
||||
file: File,
|
||||
options: { connectionId: string; bucket?: string; key: string; contentType?: string },
|
||||
|
||||
93
tests/frontend/object-storage-directory-upload.test.mjs
Normal file
93
tests/frontend/object-storage-directory-upload.test.mjs
Normal file
@ -0,0 +1,93 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
import {
|
||||
directoryUploadBatchItems,
|
||||
directoryUploadEmptyDirectories,
|
||||
directoryUploadFiles,
|
||||
normalizeDirectoryUploadPath,
|
||||
runDirectoryUploadTasks,
|
||||
} from "../../src/lib/object-storage-directory-upload.js";
|
||||
|
||||
test("directory upload preserves webkitRelativePath and explicit empty directories", () => {
|
||||
const files = [
|
||||
{ name: "image.png", webkitRelativePath: "dataset/images/image.png", size: 12, type: "image/png" },
|
||||
{ name: "labels.json", webkitRelativePath: "dataset/labels/labels.json", size: 8, type: "" },
|
||||
];
|
||||
const selected = directoryUploadFiles(files);
|
||||
assert.deepEqual(
|
||||
selected.map((item) => item.relativePath),
|
||||
["dataset/images/image.png", "dataset/labels/labels.json"],
|
||||
);
|
||||
|
||||
const emptyDirectories = directoryUploadEmptyDirectories("dataset/empty\ndataset/nested/empty\ndataset/empty");
|
||||
assert.deepEqual(emptyDirectories, ["dataset/empty", "dataset/nested/empty"]);
|
||||
assert.deepEqual(directoryUploadBatchItems(selected, emptyDirectories), [
|
||||
{
|
||||
relativePath: "dataset/images/image.png",
|
||||
kind: "file",
|
||||
sizeBytes: 12,
|
||||
contentType: "image/png",
|
||||
},
|
||||
{
|
||||
relativePath: "dataset/labels/labels.json",
|
||||
kind: "file",
|
||||
sizeBytes: 8,
|
||||
contentType: "application/octet-stream",
|
||||
},
|
||||
{
|
||||
relativePath: "dataset/empty",
|
||||
kind: "directory",
|
||||
sizeBytes: 0,
|
||||
contentType: "application/x-directory",
|
||||
},
|
||||
{
|
||||
relativePath: "dataset/nested/empty",
|
||||
kind: "directory",
|
||||
sizeBytes: 0,
|
||||
contentType: "application/x-directory",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("directory upload rejects traversal-like local paths", () => {
|
||||
assert.equal(normalizeDirectoryUploadPath("../escape.txt"), "");
|
||||
assert.equal(normalizeDirectoryUploadPath("dataset/./file.txt"), "");
|
||||
assert.equal(normalizeDirectoryUploadPath("/dataset/file.txt"), "dataset/file.txt");
|
||||
});
|
||||
|
||||
test("directory upload task runner bounds concurrency and retains individual failures", async () => {
|
||||
const tasks = [1, 2, 3, 4, 5, 6];
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
const settled = [];
|
||||
const results = await runDirectoryUploadTasks(
|
||||
tasks,
|
||||
async (value) => {
|
||||
active += 1;
|
||||
maxActive = Math.max(maxActive, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 2));
|
||||
active -= 1;
|
||||
if (value === 4) throw new Error("upload failed");
|
||||
return value * 2;
|
||||
},
|
||||
{ concurrency: 3, onSettled: (result, value) => settled.push([value, result.status]) },
|
||||
);
|
||||
|
||||
assert.equal(maxActive, 3);
|
||||
assert.equal(results[3].status, "rejected");
|
||||
assert.equal(results[5].value, 12);
|
||||
assert.equal(settled.length, tasks.length);
|
||||
});
|
||||
|
||||
test("directory upload panel exposes directory selection, conflict policy, progress, and failed-only retry", async () => {
|
||||
const source = await readFile(
|
||||
new URL("../../src/components/object-storage/DirectoryUploadPanel.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(source, /webkitdirectory/);
|
||||
assert.match(source, /directory-upload-conflict-policy/);
|
||||
assert.match(source, /总体进度/);
|
||||
assert.match(source, /仅重试失败项/);
|
||||
assert.match(source, /application\/x-directory/);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user