✨ Add concurrent directory uploads

This commit is contained in:
eust-w 2026-07-14 16:14:39 +08:00
parent adf3a7637e
commit d092be1067
11 changed files with 906 additions and 9 deletions

View File

@ -624,6 +624,12 @@ Run responses may also include:
- `DELETE /api/object-storage/buckets/:name`
- `POST /api/object-storage/objects`
- `POST /api/object-storage/objects/upload`
- `POST /api/object-storage/uploads/presign-put`
- `POST /api/object-storage/uploads/batch/prepare`
- `POST /api/object-storage/uploads/multipart/initiate`
- `POST /api/object-storage/uploads/multipart/part-url`
- `POST /api/object-storage/uploads/multipart/complete`
- `POST /api/object-storage/uploads/multipart/abort`
- `PATCH /api/object-storage/objects`
- `DELETE /api/object-storage/objects`
@ -631,6 +637,8 @@ Run responses may also include:
`GET` and `HEAD /api/object-storage/content` support browser byte-range preview requests. When a request sends a single `Range: bytes=<start>-<end>` header, the API proxies a ranged BOS read and returns `206 Partial Content`, `Content-Range`, `Accept-Ranges: bytes`, and a bounded `Content-Length` without reading the full object into API memory. Video preview components request only metadata up front and add `?transcode=browser` for browser-compatible MP4 previews when needed.
`POST /api/object-storage/uploads/batch/prepare?connectionId=<id>` prepares up to 5000 directory-upload entries before the browser transfers bytes. The payload contains `bucket`, optional `prefix`, `conflictPolicy` (`overwrite`, `skip`, or `error`), and `items` with `relativePath`, `kind`, `sizeBytes`, and `contentType`. Relative paths are normalized and traversal segments are rejected. File entries resolve to their destination key; directory entries resolve to `<path>/.keep`. The response gives each item an `upload`, `create_placeholder`, `skip`, or `conflict` action plus aggregate counts. The browser then reuses the existing presigned single-file or multipart upload path with four-file concurrency, so the API does not proxy directory file bodies.
## Worker API
Base path: `/worker-api`

View File

@ -43,6 +43,7 @@ This note tracks the production-facing safeguards that must stay true for Drobod
- Object-storage JSON object creation is capped at 4 MiB.
- Browser multipart uploads stream to BOS and are capped at 2 GiB.
- Directory uploads keep browser `webkitRelativePath` values, prepare at most 5000 files/placeholders per batch, and upload four files concurrently through the same presigned direct/multipart path. Conflict policy is explicit (`overwrite`, `skip`, or stop with `error`), failed files can be retried independently, and explicitly listed empty directories are preserved as `.keep` objects because browser directory inputs do not expose truly empty directories.
- Large inline object reads use preview or presigned URL behavior instead of loading full objects into API memory.
- Browser video previews are lazy-loaded in monitoring, artifact preview, and object-storage browsing surfaces. The UI does not attach the `<video>` source until the user starts the preview, so large MP4 artifacts do not trigger Range requests and decoder work during initial page render or artifact switching.
- Run logs, resource samples, and artifacts support strict `limit` and `offset` / `skip` pagination. Empty values use defaults, explicit `limit` must be positive, explicit offsets must be non-negative, and invalid values return validation errors instead of silently falling back to default pages. Logs and resources default to the latest 1000 records for large runs.

View File

@ -13,6 +13,7 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
@ -61,15 +62,17 @@ type bosObjectStorageBrowser struct {
}
const (
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
objectStoragePartialPreviewBytes int64 = 64 * 1024
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
objectStoragePresignDefaultExpires = 3600
objectStoragePresignMaxExpires = 24 * 3600
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
objectStoragePartialPreviewBytes int64 = 64 * 1024
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
objectStoragePresignDefaultExpires = 3600
objectStoragePresignMaxExpires = 24 * 3600
objectStorageBatchUploadMaxItems = 5000
objectStorageBatchUploadStatConcurrency = 16
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
)
var storageConnectionBrowserResolver = func(ctx context.Context, connection *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
@ -1151,6 +1154,194 @@ func renderObjectStorageInvalidJSON(writer http.ResponseWriter) {
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
}
func normalizeObjectStorageBatchRelativePath(rawValue string) (string, error) {
value := strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/")
value = strings.TrimSuffix(value, "/")
if value == "" {
return "", fmt.Errorf("relativePath is required")
}
if strings.HasPrefix(value, "/") || strings.ContainsRune(value, '\x00') {
return "", fmt.Errorf("relativePath must be a relative object path")
}
parts := strings.Split(value, "/")
for _, part := range parts {
if part == "" || part == "." || part == ".." {
return "", fmt.Errorf("relativePath contains an invalid path segment")
}
}
return strings.Join(parts, "/"), nil
}
func normalizeObjectStorageBatchPrefix(rawValue string) (string, error) {
value := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/"), "/")
if value == "" {
return "", nil
}
return normalizeObjectStorageBatchRelativePath(value)
}
func objectStorageBatchItemKey(prefix string, relativePath string, kind string) string {
key := relativePath
if kind == "directory" {
key += "/.keep"
}
if prefix != "" {
key = prefix + "/" + key
}
return key
}
func prepareObjectStorageBatchItems(input domain.ObjectStorageBatchUploadPrepareInput) ([]domain.ObjectStorageBatchUploadPreparedItem, string, string, *validationIssue) {
policy := strings.ToLower(strings.TrimSpace(input.ConflictPolicy))
if policy == "" {
policy = "overwrite"
}
if policy != "overwrite" && policy != "skip" && policy != "error" {
return nil, "", "", &validationIssue{Field: "conflictPolicy", Message: "conflictPolicy must be overwrite, skip, or error"}
}
prefix, err := normalizeObjectStorageBatchPrefix(input.Prefix)
if err != nil {
return nil, "", "", &validationIssue{Field: "prefix", Message: err.Error()}
}
if len(input.Items) == 0 {
return nil, "", "", &validationIssue{Field: "items", Message: "items are required"}
}
if len(input.Items) > objectStorageBatchUploadMaxItems {
return nil, "", "", &validationIssue{Field: "items", Message: fmt.Sprintf("items must contain at most %d entries", objectStorageBatchUploadMaxItems)}
}
prepared := make([]domain.ObjectStorageBatchUploadPreparedItem, 0, len(input.Items))
seenKeys := make(map[string]struct{}, len(input.Items))
for index, item := range input.Items {
kind := strings.ToLower(strings.TrimSpace(item.Kind))
if kind != "file" && kind != "directory" {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].kind", index), Message: "kind must be file or directory"}
}
if item.SizeBytes < 0 {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].sizeBytes", index), Message: "sizeBytes must not be negative"}
}
relativePath, pathErr := normalizeObjectStorageBatchRelativePath(item.RelativePath)
if pathErr != nil {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: pathErr.Error()}
}
key := objectStorageBatchItemKey(prefix, relativePath, kind)
if _, exists := seenKeys[key]; exists {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: "duplicate destination key"}
}
seenKeys[key] = struct{}{}
prepared = append(prepared, domain.ObjectStorageBatchUploadPreparedItem{
RelativePath: relativePath,
Key: key,
Kind: kind,
SizeBytes: item.SizeBytes,
ContentType: strings.TrimSpace(item.ContentType),
})
}
return prepared, prefix, policy, nil
}
func inspectObjectStorageBatchConflicts(ctx context.Context, browser objectStorageBrowser, bucket string, items []domain.ObjectStorageBatchUploadPreparedItem) error {
workerCount := min(objectStorageBatchUploadStatConcurrency, len(items))
indices := make(chan int)
var firstErr error
var errOnce sync.Once
var workers sync.WaitGroup
workers.Add(workerCount)
for range workerCount {
go func() {
defer workers.Done()
for index := range indices {
_, err := browser.GetObjectSummary(ctx, bucket, items[index].Key)
if err == nil {
items[index].Exists = true
continue
}
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
continue
}
errOnce.Do(func() { firstErr = err })
}
}()
}
for index := range items {
if ctx.Err() != nil {
break
}
indices <- index
}
close(indices)
workers.Wait()
if firstErr != nil {
return firstErr
}
return ctx.Err()
}
func finalizeObjectStorageBatchPlan(items []domain.ObjectStorageBatchUploadPreparedItem, policy string) domain.ObjectStorageBatchUploadPrepareResult {
result := domain.ObjectStorageBatchUploadPrepareResult{ConflictPolicy: policy, Items: items}
for index := range result.Items {
item := &result.Items[index]
if item.Exists && policy == "skip" {
item.Action = "skip"
item.Message = "destination already exists"
result.SkipCount++
continue
}
if item.Exists && policy == "error" {
item.Action = "conflict"
item.Message = "destination already exists"
result.ConflictCount++
continue
}
if item.Kind == "directory" {
item.Action = "create_placeholder"
result.PlaceholderCount++
} else {
item.Action = "upload"
result.UploadCount++
}
}
return result
}
func (server *server) handleObjectStorageBatchUploadPrepare(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[domain.ObjectStorageBatchUploadPrepareInput](request)
if err != nil {
renderObjectStorageInvalidJSON(writer)
return
}
items, prefix, policy, issue := prepareObjectStorageBatchItems(input)
if issue != nil {
renderValidationError(writer, *issue)
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if cleanup != nil {
defer cleanup()
}
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
if err != nil || bucket == "" {
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
return
}
if err := inspectObjectStorageBatchConflicts(request.Context(), browser, bucket, items); err != nil {
renderObjectStorageWriteError(writer, err)
return
}
result := finalizeObjectStorageBatchPlan(items, policy)
result.Bucket = bucket
result.Prefix = prefix
renderJSON(writer, http.StatusOK, result)
}
func (server *server) handleObjectStoragePresignPut(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[objectStoragePresignPutInput](request)
if err != nil {

View File

@ -1378,6 +1378,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
router.Post("/api/object-storage/objects", instance.handleObjectStorageObjectCreate)
router.Post("/api/object-storage/objects/upload", instance.handleObjectStorageObjectUpload)
router.Post("/api/object-storage/uploads/presign-put", instance.handleObjectStoragePresignPut)
router.Post("/api/object-storage/uploads/batch/prepare", instance.handleObjectStorageBatchUploadPrepare)
router.Post("/api/object-storage/uploads/multipart/initiate", instance.handleObjectStorageMultipartInitiate)
router.Post("/api/object-storage/uploads/multipart/part-url", instance.handleObjectStorageMultipartPartURL)
router.Post("/api/object-storage/uploads/multipart/complete", instance.handleObjectStorageMultipartComplete)

View File

@ -9501,6 +9501,109 @@ func TestObjectStorageUploadControlAPISupportsPresignAndMultipart(t *testing.T)
}
}
func TestObjectStorageBatchUploadPreparePlansNestedFilesConflictsAndPlaceholders(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS batch upload",
Provider: domain.StorageProviderBOS,
Endpoint: "https://bos.example.test",
Bucket: "datastorage",
AccessKeyID: "ak-storage-browser",
SecretAccessKey: "sk-storage-browser",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
fakeBrowser := newFakeObjectStorageBrowser()
existingKey := "datasets/root/nested/existing.bin"
fakeBrowser.objects["datastorage|"+existingKey] = domain.ObjectStorageObjectSummary{
Bucket: "datastorage", Key: existingKey, Name: "existing.bin", Kind: "file",
}
originalResolver := storageConnectionBrowserResolver
storageConnectionBrowserResolver = func(_ context.Context, _ *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
return fakeBrowser, func() {}, nil
}
t.Cleanup(func() { storageConnectionBrowserResolver = originalResolver })
handler := NewHandler(dataStore, nil)
body := `{"bucket":"datastorage","prefix":"datasets","conflictPolicy":"skip","items":[` +
`{"relativePath":"root/nested/existing.bin","kind":"file","sizeBytes":10},` +
`{"relativePath":"root/new.bin","kind":"file","sizeBytes":20},` +
`{"relativePath":"root/empty","kind":"directory"}]}`
recorder := httptest.NewRecorder()
handler.ServeHTTP(
recorder,
httptest.NewRequest(http.MethodPost, "/api/object-storage/uploads/batch/prepare?connectionId="+connection.ID, strings.NewReader(body)),
)
if recorder.Code != http.StatusOK {
t.Fatalf("batch prepare status: %d body=%s", recorder.Code, recorder.Body.String())
}
var result domain.ObjectStorageBatchUploadPrepareResult
if err := json.Unmarshal(recorder.Body.Bytes(), &result); err != nil {
t.Fatalf("decode batch prepare: %v", err)
}
if result.UploadCount != 1 || result.PlaceholderCount != 1 || result.SkipCount != 1 || result.ConflictCount != 0 {
t.Fatalf("unexpected batch counts: %#v", result)
}
if result.Items[0].Action != "skip" || result.Items[1].Key != "datasets/root/new.bin" || result.Items[1].Action != "upload" {
t.Fatalf("unexpected file plan: %#v", result.Items)
}
if result.Items[2].Key != "datasets/root/empty/.keep" || result.Items[2].Action != "create_placeholder" {
t.Fatalf("unexpected directory plan: %#v", result.Items[2])
}
errorBody := strings.Replace(body, `"conflictPolicy":"skip"`, `"conflictPolicy":"error"`, 1)
errorRecorder := httptest.NewRecorder()
handler.ServeHTTP(
errorRecorder,
httptest.NewRequest(http.MethodPost, "/api/object-storage/uploads/batch/prepare?connectionId="+connection.ID, strings.NewReader(errorBody)),
)
if errorRecorder.Code != http.StatusOK {
t.Fatalf("batch error policy status: %d body=%s", errorRecorder.Code, errorRecorder.Body.String())
}
if err := json.Unmarshal(errorRecorder.Body.Bytes(), &result); err != nil {
t.Fatalf("decode error plan: %v", err)
}
if result.ConflictCount != 1 || result.Items[0].Action != "conflict" {
t.Fatalf("expected one explicit conflict: %#v", result)
}
}
func TestObjectStorageBatchUploadPrepareRejectsUnsafeAndDuplicatePaths(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS batch validation", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test",
Bucket: "datastorage", AccessKeyID: "ak", SecretAccessKey: "sk",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
fakeBrowser := newFakeObjectStorageBrowser()
originalResolver := storageConnectionBrowserResolver
storageConnectionBrowserResolver = func(_ context.Context, _ *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
return fakeBrowser, func() {}, nil
}
t.Cleanup(func() { storageConnectionBrowserResolver = originalResolver })
handler := NewHandler(dataStore, nil)
tests := []string{
`{"items":[{"relativePath":"../escape.bin","kind":"file"}]}`,
`{"items":[{"relativePath":"same.bin","kind":"file"},{"relativePath":"same.bin","kind":"file"}]}`,
`{"conflictPolicy":"rename","items":[{"relativePath":"valid.bin","kind":"file"}]}`,
}
for _, body := range tests {
recorder := httptest.NewRecorder()
handler.ServeHTTP(
recorder,
httptest.NewRequest(http.MethodPost, "/api/object-storage/uploads/batch/prepare?connectionId="+connection.ID, strings.NewReader(body)),
)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d body=%s for input %s", recorder.Code, recorder.Body.String(), body)
}
}
}
func TestObjectStorageUploadControlAPIRejectsInvalidMultipartInput(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()

View File

@ -129,3 +129,39 @@ type ObjectStorageMultipartUploadAbortResult struct {
UploadID string `json:"uploadId"`
Aborted bool `json:"aborted"`
}
type ObjectStorageBatchUploadItemInput struct {
RelativePath string `json:"relativePath"`
Kind string `json:"kind"`
SizeBytes int64 `json:"sizeBytes,omitempty"`
ContentType string `json:"contentType,omitempty"`
}
type ObjectStorageBatchUploadPrepareInput struct {
Bucket string `json:"bucket"`
Prefix string `json:"prefix,omitempty"`
ConflictPolicy string `json:"conflictPolicy,omitempty"`
Items []ObjectStorageBatchUploadItemInput `json:"items"`
}
type ObjectStorageBatchUploadPreparedItem struct {
RelativePath string `json:"relativePath"`
Key string `json:"key"`
Kind string `json:"kind"`
SizeBytes int64 `json:"sizeBytes,omitempty"`
ContentType string `json:"contentType,omitempty"`
Action string `json:"action"`
Exists bool `json:"exists"`
Message string `json:"message,omitempty"`
}
type ObjectStorageBatchUploadPrepareResult struct {
Bucket string `json:"bucket"`
Prefix string `json:"prefix,omitempty"`
ConflictPolicy string `json:"conflictPolicy"`
Items []ObjectStorageBatchUploadPreparedItem `json:"items"`
UploadCount int `json:"uploadCount"`
PlaceholderCount int `json:"placeholderCount"`
SkipCount int `json:"skipCount"`
ConflictCount int `json:"conflictCount"`
}

View File

@ -0,0 +1,338 @@
import React, { useMemo, useRef, useState } from "react";
import { AlertCircle, CheckCircle2, FolderUp, Loader2, RefreshCw, Upload } from "lucide-react";
import {
DIRECTORY_UPLOAD_CONCURRENCY,
directoryUploadBatchItems,
directoryUploadEmptyDirectories,
directoryUploadFiles,
runDirectoryUploadTasks,
} from "../../lib/object-storage-directory-upload.js";
import { getActionErrorMessage } from "../../lib/action-errors.js";
import {
objectStorageApi,
type ObjectStorageBatchConflictPolicy,
type ObjectStorageBatchUploadPreparedItem,
} from "../../services/api/object-storage";
type SelectedDirectoryFile = { relativePath: string; file: File };
type UploadState = "pending" | "uploading" | "success" | "skipped" | "conflict" | "failed";
type UploadRecord = {
item: ObjectStorageBatchUploadPreparedItem;
file?: File;
state: UploadState;
error?: string;
};
const formatBytes = (value: number) => {
if (value < 1024) return `${value} B`;
const units = ["KB", "MB", "GB", "TB"];
let current = value / 1024;
let unit = 0;
while (current >= 1024 && unit < units.length - 1) {
current /= 1024;
unit += 1;
}
return `${current.toFixed(current >= 10 ? 0 : 1)} ${units[unit]}`;
};
export function DirectoryUploadPanel({
connectionId,
bucket,
prefix,
disabled,
onUploaded,
}: {
connectionId: string;
bucket: string;
prefix: string;
disabled?: boolean;
onUploaded: () => void | Promise<void>;
}) {
const [selectedFiles, setSelectedFiles] = useState<SelectedDirectoryFile[]>([]);
const [emptyDirectoriesText, setEmptyDirectoriesText] = useState("");
const [conflictPolicy, setConflictPolicy] = useState<ObjectStorageBatchConflictPolicy>("overwrite");
const [records, setRecords] = useState<UploadRecord[]>([]);
const [running, setRunning] = useState(false);
const [message, setMessage] = useState("");
const [error, setError] = useState("");
const [progress, setProgress] = useState({ loadedBytes: 0, totalBytes: 0, completed: 0, total: 0 });
const loadedByKeyRef = useRef(new Map<string, number>());
const progressFrameRef = useRef<number | null>(null);
const pendingProgressRef = useRef<{ completed?: number; total?: number; totalBytes?: number }>({});
const emptyDirectories = useMemo(() => directoryUploadEmptyDirectories(emptyDirectoriesText), [emptyDirectoriesText]);
const selectedBytes = useMemo(
() => selectedFiles.reduce((total, item) => total + item.file.size, 0),
[selectedFiles],
);
const failedCount = records.filter((record) => record.state === "failed").length;
const successCount = records.filter((record) => record.state === "success").length;
const skippedCount = records.filter((record) => record.state === "skipped").length;
const conflictCount = records.filter((record) => record.state === "conflict").length;
const percent =
progress.totalBytes > 0
? Math.round((progress.loadedBytes / progress.totalBytes) * 100)
: progress.total > 0
? Math.round((progress.completed / progress.total) * 100)
: 0;
const flushProgress = (completed?: number, total?: number, totalBytes?: number) => {
pendingProgressRef.current = {
completed: completed ?? pendingProgressRef.current.completed,
total: total ?? pendingProgressRef.current.total,
totalBytes: totalBytes ?? pendingProgressRef.current.totalBytes,
};
if (progressFrameRef.current !== null) return;
progressFrameRef.current = window.requestAnimationFrame(() => {
progressFrameRef.current = null;
const pending = pendingProgressRef.current;
pendingProgressRef.current = {};
const loadedBytes = (Array.from(loadedByKeyRef.current.values()) as number[]).reduce<number>(
(sum, value) => sum + value,
0,
);
setProgress((current) => ({
loadedBytes,
totalBytes: pending.totalBytes ?? current.totalBytes,
completed: pending.completed ?? current.completed,
total: pending.total ?? current.total,
}));
});
};
const executeRecords = async (initialRecords: UploadRecord[]) => {
const runnable = initialRecords.filter((record) => record.state === "pending" || record.state === "failed");
if (runnable.length === 0) return;
setRunning(true);
setError("");
setMessage("");
loadedByKeyRef.current = new Map();
const totalBytes = runnable.reduce(
(total, record) => total + (record.item.action === "upload" ? record.file?.size || 0 : 0),
0,
);
let completed = 0;
setProgress({ loadedBytes: 0, totalBytes, completed: 0, total: runnable.length });
const currentRecords = records.length > 0 ? [...records] : [...initialRecords];
const updateRecord = (key: string, update: Partial<UploadRecord>) => {
const index = currentRecords.findIndex((record) => record.item.key === key);
if (index >= 0) currentRecords[index] = { ...currentRecords[index], ...update };
setRecords([...currentRecords]);
};
await runDirectoryUploadTasks(
runnable,
async (record) => {
updateRecord(record.item.key, { state: "uploading", error: undefined });
if (record.item.action === "create_placeholder") {
await objectStorageApi.putObject(
{ bucket, key: record.item.key, content: "", contentType: "application/x-directory" },
{ connectionId },
);
} else {
if (!record.file) throw new Error("未找到该条目的本地文件,请重新选择目录。");
await objectStorageApi.uploadObject(
record.file,
{ connectionId, bucket, key: record.item.key, contentType: record.item.contentType },
(uploadProgress) => {
loadedByKeyRef.current.set(record.item.key, uploadProgress.loadedBytes);
flushProgress(undefined, undefined, totalBytes);
},
);
loadedByKeyRef.current.set(record.item.key, record.file.size);
}
return record.item.key;
},
{
concurrency: DIRECTORY_UPLOAD_CONCURRENCY,
onSettled: (result, record) => {
completed += 1;
if (result.status === "fulfilled") {
updateRecord(record.item.key, { state: "success", error: undefined });
} else {
updateRecord(record.item.key, { state: "failed", error: getActionErrorMessage(result.reason) });
}
flushProgress(completed, runnable.length, totalBytes);
},
},
);
const finalFailed = currentRecords.filter((record) => record.state === "failed").length;
setRunning(false);
if (finalFailed > 0) {
setError(`${finalFailed} 个条目上传失败,可仅重试失败项。`);
} else {
setMessage("目录上传完成。");
await onUploaded();
}
};
const prepareAndUpload = async () => {
const batchItems = directoryUploadBatchItems(selectedFiles, emptyDirectories);
if (batchItems.length === 0) {
setError("请先选择一个目录,或填写需要保留的空目录。");
return;
}
setRunning(true);
setError("");
setMessage("正在检查路径和冲突…");
try {
const plan = await objectStorageApi.prepareBatchUpload(
{ bucket, prefix, conflictPolicy, items: batchItems },
{ connectionId },
);
const filesByPath = new Map<string, File>(selectedFiles.map((item) => [item.relativePath, item.file] as const));
const nextRecords: UploadRecord[] = plan.items.map((item) => ({
item,
file: filesByPath.get(item.relativePath),
state: item.action === "skip" ? "skipped" : item.action === "conflict" ? "conflict" : "pending",
}));
setRecords(nextRecords);
if (plan.conflictCount > 0) {
setMessage("");
setError(`检测到 ${plan.conflictCount} 个冲突。请选择“跳过”或“覆盖”后重新上传。`);
return;
}
setMessage("");
await executeRecords(nextRecords);
} catch (cause) {
setError(getActionErrorMessage(cause));
} finally {
setRunning(false);
}
};
const visibleRecords = records
.filter((record) => record.state === "failed" || record.state === "conflict")
.slice(0, 50);
return (
<div
className="space-y-4 rounded-2xl border border-[#2a2b2f] bg-[#101115] p-4"
data-testid="directory-upload-panel"
>
<div className="flex flex-wrap items-start justify-between gap-3">
<div>
<div className="flex items-center gap-2 text-sm font-medium text-white">
<FolderUp className="h-4 w-4 text-blue-400" />
上传目录
</div>
<p className="mt-1 text-xs leading-5 text-gray-500">
保留 webkitRelativePath,4 个文件并发;大文件自动复用分片上传。纯空目录请在下方补充,将写入 .keep。
</p>
</div>
<div className="text-right text-xs text-gray-500">
<div>
{selectedFiles.length} 个文件 · {formatBytes(selectedBytes)}
</div>
{emptyDirectories.length > 0 && <div>{emptyDirectories.length} 个空目录</div>}
</div>
</div>
<div className="grid gap-3 lg:grid-cols-[1.2fr_0.8fr_0.6fr_auto]">
<label className="space-y-2">
<span className="block text-xs uppercase tracking-[0.2em] text-gray-500">选择本地目录</span>
<input
data-testid="directory-upload-input"
type="file"
multiple
{...({ webkitdirectory: "", directory: "" } as React.InputHTMLAttributes<HTMLInputElement>)}
onChange={(event) => {
setSelectedFiles(directoryUploadFiles(event.target.files) as SelectedDirectoryFile[]);
setRecords([]);
setError("");
setMessage("");
}}
disabled={disabled || running}
className="block w-full rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-[10px] text-sm text-gray-300 file:mr-3 file:rounded-md file:border-0 file:bg-blue-600 file:px-3 file:py-1.5 file:text-xs file:text-white"
/>
</label>
<label className="space-y-2">
<span className="block text-xs uppercase tracking-[0.2em] text-gray-500">额外空目录</span>
<textarea
data-testid="directory-upload-empty-directories"
value={emptyDirectoriesText}
onChange={(event) => setEmptyDirectoriesText(event.target.value)}
disabled={disabled || running}
rows={2}
className="w-full rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-2 text-xs text-white focus:border-blue-500 focus:outline-none"
placeholder={"每行一个相对路径\n例如 dataset/labels-empty"}
/>
</label>
<label className="space-y-2">
<span className="block text-xs uppercase tracking-[0.2em] text-gray-500">同名冲突</span>
<select
data-testid="directory-upload-conflict-policy"
value={conflictPolicy}
onChange={(event) => setConflictPolicy(event.target.value as ObjectStorageBatchConflictPolicy)}
disabled={disabled || running}
className="w-full rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-3 text-sm text-white focus:border-blue-500 focus:outline-none"
>
<option value="overwrite">覆盖已有对象</option>
<option value="skip">跳过已有对象</option>
<option value="error">发现冲突即停止</option>
</select>
</label>
<button
type="button"
data-testid="directory-upload-button"
onClick={prepareAndUpload}
disabled={disabled || running || (!selectedFiles.length && !emptyDirectories.length)}
className="inline-flex items-center justify-center gap-2 rounded-xl bg-blue-600 px-4 py-3 text-sm font-medium text-white hover:bg-blue-500 disabled:opacity-60 lg:self-end"
>
{running ? <Loader2 className="h-4 w-4 animate-spin" /> : <Upload className="h-4 w-4" />}
{running ? "上传中" : "上传目录"}
</button>
</div>
{(progress.total > 0 || running) && (
<div className="space-y-2">
<div className="flex flex-wrap items-center justify-between gap-2 text-xs text-gray-400">
<span>
总体进度 {progress.completed}/{progress.total}
</span>
<span>
{formatBytes(progress.loadedBytes)} / {formatBytes(progress.totalBytes)} · {percent}%
</span>
</div>
<div className="h-2 overflow-hidden rounded-full bg-[#202126]">
<div className="h-full bg-blue-500 transition-all" style={{ width: `${percent}%` }} />
</div>
</div>
)}
{(message || error || records.length > 0) && (
<div className="flex flex-wrap items-center justify-between gap-3 rounded-xl border border-[#2a2b2f] bg-[#0b0c10] px-3 py-2 text-xs">
<div className="flex items-center gap-2">
{error ? (
<AlertCircle className="h-4 w-4 text-red-400" />
) : (
<CheckCircle2 className="h-4 w-4 text-emerald-400" />
)}
<span className={error ? "text-red-200" : "text-gray-300"}>
{error || message || `成功 ${successCount} · 跳过 ${skippedCount} · 冲突 ${conflictCount}`}
</span>
</div>
{failedCount > 0 && !running && (
<button
type="button"
onClick={() => executeRecords(records)}
className="inline-flex items-center gap-1 rounded-lg border border-red-500/30 px-2 py-1 text-red-200 hover:bg-red-500/10"
>
<RefreshCw className="h-3.5 w-3.5" /> 仅重试失败项
</button>
)}
</div>
)}
{visibleRecords.length > 0 && (
<div className="max-h-40 space-y-1 overflow-auto text-xs text-red-200">
{visibleRecords.map((record) => (
<div key={record.item.key} className="rounded-lg bg-red-500/5 px-3 py-2">
<div className="font-mono">{record.item.key}</div>
<div className="mt-1 text-red-300/80">{record.error || record.item.message || "目标对象冲突"}</div>
</div>
))}
</div>
)}
</div>
);
}

View File

@ -0,0 +1,71 @@
export const DIRECTORY_UPLOAD_CONCURRENCY = 4;
export function normalizeDirectoryUploadPath(rawValue) {
const value = String(rawValue || "")
.trim()
.replaceAll("\\", "/")
.replace(/^\/+|\/+$/g, "");
if (!value) return "";
const parts = value.split("/");
if (parts.some((part) => !part || part === "." || part === "..")) {
return "";
}
return parts.join("/");
}
export function directoryUploadFiles(fileList) {
return Array.from(fileList || []).flatMap((file) => {
const relativePath = normalizeDirectoryUploadPath(file.webkitRelativePath || file.name);
return relativePath ? [{ relativePath, file }] : [];
});
}
export function directoryUploadEmptyDirectories(rawValue) {
const seen = new Set();
return String(rawValue || "")
.split(/\r?\n/)
.map(normalizeDirectoryUploadPath)
.filter((relativePath) => {
if (!relativePath || seen.has(relativePath)) return false;
seen.add(relativePath);
return true;
});
}
export function directoryUploadBatchItems(files, emptyDirectories = []) {
const fileItems = files.map(({ relativePath, file }) => ({
relativePath,
kind: "file",
sizeBytes: file.size,
contentType: file.type || "application/octet-stream",
}));
const filePaths = new Set(fileItems.map((item) => item.relativePath));
const directoryItems = emptyDirectories
.filter((relativePath) => !filePaths.has(`${relativePath}/.keep`))
.map((relativePath) => ({ relativePath, kind: "directory", sizeBytes: 0, contentType: "application/x-directory" }));
return [...fileItems, ...directoryItems];
}
export async function runDirectoryUploadTasks(tasks, worker, options = {}) {
const concurrency = Math.max(
1,
Math.min(Number(options.concurrency) || DIRECTORY_UPLOAD_CONCURRENCY, tasks.length || 1),
);
const results = new Array(tasks.length);
let nextIndex = 0;
const runNext = async () => {
while (nextIndex < tasks.length) {
const index = nextIndex;
nextIndex += 1;
try {
const value = await worker(tasks[index], index);
results[index] = { status: "fulfilled", value };
} catch (error) {
results[index] = { status: "rejected", reason: error };
}
options.onSettled?.(results[index], tasks[index], index);
}
};
await Promise.all(Array.from({ length: concurrency }, runNext));
return results;
}

View File

@ -23,6 +23,7 @@ import {
} from "lucide-react";
import { useTimeZone } from "../context/TimeZoneContext";
import { LazyVideoPreview } from "../components/LazyVideoPreview";
import { DirectoryUploadPanel } from "../components/object-storage/DirectoryUploadPanel";
import { formatCompactDateTime } from "../lib/datetime";
import { isObjectStorageEditorLocked, shouldSyncObjectStoragePreviewToEditor } from "../lib/object-storage-preview.js";
import { getActionErrorMessage } from "../lib/action-errors.js";
@ -1719,6 +1720,18 @@ export default function ObjectStorageBrowser() {
</div>
) : null}
</div>
<div key={`${selectedConnectionId}:${selectedBucket}:${currentPrefix}`}>
<DirectoryUploadPanel
connectionId={selectedConnectionId}
bucket={selectedBucket}
prefix={currentPrefix}
disabled={objectActionLoading || !selectedConnectionId || !selectedBucket}
onUploaded={async () => {
await reloadObjects("", undefined, "");
}}
/>
</div>
</div>
</div>
</section>

View File

@ -85,6 +85,34 @@ export interface ObjectStorageUploadProgress {
partCount?: number;
}
export type ObjectStorageBatchConflictPolicy = "overwrite" | "skip" | "error";
export type ObjectStorageBatchPreparedAction = "upload" | "create_placeholder" | "skip" | "conflict";
export interface ObjectStorageBatchUploadItemInput {
relativePath: string;
kind: "file" | "directory";
sizeBytes?: number;
contentType?: string;
}
export interface ObjectStorageBatchUploadPreparedItem extends ObjectStorageBatchUploadItemInput {
key: string;
action: ObjectStorageBatchPreparedAction;
exists: boolean;
message?: string;
}
export interface ObjectStorageBatchUploadPrepareResult {
bucket: string;
prefix?: string;
conflictPolicy: ObjectStorageBatchConflictPolicy;
items: ObjectStorageBatchUploadPreparedItem[];
uploadCount: number;
placeholderCount: number;
skipCount: number;
conflictCount: number;
}
interface ObjectStoragePresignedUploadResult {
bucket: string;
key: string;
@ -297,6 +325,20 @@ export const objectStorageApi = {
}
return uploadDirectObject(file, options, onProgress);
},
prepareBatchUpload: (
payload: {
bucket?: string;
prefix?: string;
conflictPolicy: ObjectStorageBatchConflictPolicy;
items: ObjectStorageBatchUploadItemInput[];
},
options: { connectionId: string },
) =>
apiRequest<ObjectStorageBatchUploadPrepareResult>(
`/api/object-storage/uploads/batch/prepare${buildObjectStorageParams(options)}`,
{ method: "POST" },
payload,
),
uploadObjectWithServerFallback: async (
file: File,
options: { connectionId: string; bucket?: string; key: string; contentType?: string },

View File

@ -0,0 +1,93 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
import {
directoryUploadBatchItems,
directoryUploadEmptyDirectories,
directoryUploadFiles,
normalizeDirectoryUploadPath,
runDirectoryUploadTasks,
} from "../../src/lib/object-storage-directory-upload.js";
test("directory upload preserves webkitRelativePath and explicit empty directories", () => {
const files = [
{ name: "image.png", webkitRelativePath: "dataset/images/image.png", size: 12, type: "image/png" },
{ name: "labels.json", webkitRelativePath: "dataset/labels/labels.json", size: 8, type: "" },
];
const selected = directoryUploadFiles(files);
assert.deepEqual(
selected.map((item) => item.relativePath),
["dataset/images/image.png", "dataset/labels/labels.json"],
);
const emptyDirectories = directoryUploadEmptyDirectories("dataset/empty\ndataset/nested/empty\ndataset/empty");
assert.deepEqual(emptyDirectories, ["dataset/empty", "dataset/nested/empty"]);
assert.deepEqual(directoryUploadBatchItems(selected, emptyDirectories), [
{
relativePath: "dataset/images/image.png",
kind: "file",
sizeBytes: 12,
contentType: "image/png",
},
{
relativePath: "dataset/labels/labels.json",
kind: "file",
sizeBytes: 8,
contentType: "application/octet-stream",
},
{
relativePath: "dataset/empty",
kind: "directory",
sizeBytes: 0,
contentType: "application/x-directory",
},
{
relativePath: "dataset/nested/empty",
kind: "directory",
sizeBytes: 0,
contentType: "application/x-directory",
},
]);
});
test("directory upload rejects traversal-like local paths", () => {
assert.equal(normalizeDirectoryUploadPath("../escape.txt"), "");
assert.equal(normalizeDirectoryUploadPath("dataset/./file.txt"), "");
assert.equal(normalizeDirectoryUploadPath("/dataset/file.txt"), "dataset/file.txt");
});
test("directory upload task runner bounds concurrency and retains individual failures", async () => {
const tasks = [1, 2, 3, 4, 5, 6];
let active = 0;
let maxActive = 0;
const settled = [];
const results = await runDirectoryUploadTasks(
tasks,
async (value) => {
active += 1;
maxActive = Math.max(maxActive, active);
await new Promise((resolve) => setTimeout(resolve, 2));
active -= 1;
if (value === 4) throw new Error("upload failed");
return value * 2;
},
{ concurrency: 3, onSettled: (result, value) => settled.push([value, result.status]) },
);
assert.equal(maxActive, 3);
assert.equal(results[3].status, "rejected");
assert.equal(results[5].value, 12);
assert.equal(settled.length, tasks.length);
});
test("directory upload panel exposes directory selection, conflict policy, progress, and failed-only retry", async () => {
const source = await readFile(
new URL("../../src/components/object-storage/DirectoryUploadPanel.tsx", import.meta.url),
"utf8",
);
assert.match(source, /webkitdirectory/);
assert.match(source, /directory-upload-conflict-policy/);
assert.match(source, /总体进度/);
assert.match(source, /仅重试失败项/);
assert.match(source, /application\/x-directory/);
});