package api import ( "context" "encoding/json" "errors" "fmt" "io" "mime" "net/http" "os" "path" "sort" "strconv" "strings" "sync" "time" "droboticflow/internal/domain" "github.com/baidubce/bce-sdk-go/bce" "github.com/baidubce/bce-sdk-go/services/bos" bosapi "github.com/baidubce/bce-sdk-go/services/bos/api" "golang.org/x/sync/errgroup" ) type objectStorageBrowser interface { ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error) ListObjects(context.Context, string, string) ([]domain.ObjectStorageObjectSummary, error) CreateBucket(context.Context, string) (domain.ObjectStorageBucketSummary, error) DeleteBucket(context.Context, string) error PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error) PutObjectStream(context.Context, string, string, io.Reader, string) (domain.ObjectStorageObjectSummary, error) PresignPutObject(context.Context, string, string, string, int) (domain.ObjectStoragePresignedUploadResult, error) InitiateMultipartUpload(context.Context, string, string, string, int) (domain.ObjectStorageMultipartUploadCreateResult, error) PresignMultipartUploadPart(context.Context, string, string, string, int, int) (domain.ObjectStorageMultipartUploadPartURLResult, error) CompleteMultipartUpload(context.Context, string, string, string, []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) AbortMultipartUpload(context.Context, string, string, string) error RenameObject(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error) MovePrefix(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error) DeleteObject(context.Context, string, string) error DeletePrefix(context.Context, string, string) error GetObjectSummary(context.Context, string, string) (domain.ObjectStorageObjectSummary, error) ReadObject(context.Context, string, string) ([]byte, string, error) ReadObjectRange(context.Context, string, string, int64, int64) (io.ReadCloser, string, error) ReadObjectPreview(context.Context, string, string, int64) ([]byte, string, error) BuildPreviewURL(context.Context, string, string) (string, error) } type objectSyncStorageBrowser interface { objectStorageBrowser PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error) PublishMultipartCopyConditional(context.Context, string, string, string, int64, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error) GetBucketVersioning(context.Context, string) (string, error) } type objectSyncBoundUploadBrowser interface { PresignPutObjectBound(context.Context, string, string, string, string, int64, map[string]string, int) (domain.ObjectStoragePresignedUploadResult, error) InitiateMultipartUploadBound(context.Context, string, string, string, map[string]string, int) (domain.ObjectStorageMultipartUploadCreateResult, error) PresignMultipartUploadPartBound(context.Context, string, string, string, int, string, int64, int) (domain.ObjectStorageMultipartUploadPartURLResult, error) CompleteMultipartUploadBound(context.Context, string, string, string, []domain.ObjectStorageMultipartUploadPart, map[string]string) (domain.ObjectStorageObjectSummary, error) } type objectStoragePage struct { Items []domain.ObjectStorageObjectSummary NextMarker string HasMore bool } type pagedObjectStorageBrowser interface { ListObjectsPage(context.Context, string, string, string, int) (objectStoragePage, error) } type bosObjectStorageBrowser struct { connection *domain.StorageConnectionRecord client *bos.Client } const ( objectStorageInlinePreviewMaxBytes int64 = 256 * 1024 objectStoragePartialPreviewBytes int64 = 64 * 1024 objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024 objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024 objectStoragePresignDefaultExpires = 3600 objectStoragePresignMaxExpires = 24 * 3600 objectStorageBatchUploadMaxItems = 5000 objectStorageBatchUploadStatConcurrency = 16 objectStorageWriteMetadataRetryAttempts = 4 objectStorageWriteMetadataRetryDelay = 100 * time.Millisecond objectStorageKeyMaxUTF8Bytes = 1023 objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated" objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes" objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes" ) var storageConnectionBrowserResolver = func(ctx context.Context, connection *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { if connection == nil { return nil, nil, nil } endpoint := storageEndpointForProvider(connection.Provider, connection.Endpoint) if normalizeStorageProvider(connection.Provider) == domain.StorageProviderOSS { return newOSSObjectStorageBrowser(connection, endpoint) } if normalizeStorageProvider(connection.Provider) == domain.StorageProviderBOS && endpoint == "" { return nil, nil, fmt.Errorf("BOS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name)) } client, err := bos.NewClient( strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint, ) if err != nil { return nil, nil, err } return &bosObjectStorageBrowser{connection: connection, client: client}, func() {}, nil } func objectStorageKindForObject(key string, contentType string) string { contentType = strings.ToLower(strings.TrimSpace(contentType)) if strings.HasPrefix(contentType, "image/") { return "image" } if strings.HasPrefix(contentType, "video/") { return "video" } if strings.HasPrefix(contentType, "audio/") { return "audio" } if strings.HasPrefix(contentType, "application/json") { return "json" } switch strings.ToLower(path.Ext(key)) { case ".json": return "json" case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg": return "image" case ".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv": return "video" case ".mp3", ".wav", ".aac", ".ogg", ".flac", ".m4a": return "audio" } return "file" } func objectStorageSummaryFromListObject(bucket string, item bosapi.ObjectSummaryType) domain.ObjectStorageObjectSummary { key := strings.TrimSpace(item.Key) contentType := detectObjectStorageContentType(key, nil, "") return domain.ObjectStorageObjectSummary{ Bucket: bucket, Key: key, Name: path.Base(strings.TrimRight(key, "/")), Kind: objectStorageKindForObject(key, contentType), SizeBytes: int64(item.Size), ContentType: contentType, LastModified: strings.TrimSpace(item.LastModified), ETag: strings.Trim(item.ETag, "\""), StorageClass: strings.TrimSpace(item.StorageClass), } } func objectStorageSummaryFromPrefix(bucket string, prefix string) domain.ObjectStorageObjectSummary { trimmed := strings.TrimSpace(prefix) return domain.ObjectStorageObjectSummary{ Bucket: bucket, Key: strings.TrimRight(trimmed, "/"), Name: path.Base(strings.TrimRight(trimmed, "/")), Kind: "directory", } } func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObjectMetaResult) domain.ObjectStorageObjectSummary { contentType := "" sizeBytes := int64(0) lastModified := "" etag := "" storageClass := "" versionID := "" metadata := map[string]any{} if meta != nil { contentType = strings.TrimSpace(meta.ContentType) sizeBytes = meta.ContentLength lastModified = strings.TrimSpace(meta.LastModified) etag = strings.Trim(meta.ETag, "\"") storageClass = strings.TrimSpace(meta.StorageClass) versionID = strings.TrimSpace(meta.VersionId) for metaKey, metaValue := range meta.UserMeta { metadata[metaKey] = metaValue } } if len(metadata) == 0 { metadata = nil } return domain.ObjectStorageObjectSummary{ Bucket: bucket, Key: key, Name: path.Base(strings.TrimRight(key, "/")), Kind: objectStorageKindForObject(key, contentType), SizeBytes: sizeBytes, ContentType: contentType, LastModified: lastModified, ETag: etag, VersionID: versionID, StorageClass: storageClass, Metadata: metadata, } } func (browser *bosObjectStorageBrowser) ListBuckets(ctx context.Context) ([]domain.ObjectStorageBucketSummary, error) { result, err := browser.client.ListBucketsWithContext(ctx) if err != nil { return nil, err } items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets)) for _, bucket := range result.Buckets { name := strings.TrimSpace(bucket.Name) if name == "" { continue } items = append(items, domain.ObjectStorageBucketSummary{Name: name, CreatedAt: strings.TrimSpace(bucket.CreationDate)}) } return items, nil } func (browser *bosObjectStorageBrowser) ListObjects(ctx context.Context, bucket string, prefix string) ([]domain.ObjectStorageObjectSummary, error) { items := make([]domain.ObjectStorageObjectSummary, 0) seen := make(map[string]struct{}) marker := "" for { page, err := browser.ListObjectsPage(ctx, bucket, prefix, marker, 1000) if err != nil { return nil, err } for _, item := range page.Items { key := item.Kind + "|" + item.Key if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} items = append(items, item) } if !page.HasMore || page.NextMarker == "" || page.NextMarker == marker { break } marker = page.NextMarker } return items, nil } func (browser *bosObjectStorageBrowser) ListObjectsPage( ctx context.Context, bucket string, prefix string, marker string, limit int, ) (objectStoragePage, error) { normalizedBucket := strings.TrimSpace(bucket) normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/") if limit <= 0 { limit = 100 } if limit > 1000 { limit = 1000 } result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ Prefix: storageObjectRequestPrefix(prefix), Delimiter: "/", MaxKeys: limit, Marker: strings.TrimSpace(marker), }) if err != nil { return objectStoragePage{}, err } if result == nil { return objectStoragePage{Items: []domain.ObjectStorageObjectSummary{}}, nil } items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Contents)) seen := make(map[string]struct{}) for _, item := range result.CommonPrefixes { prefixValue := strings.Trim(strings.TrimSpace(item.Prefix), "/") if prefixValue == "" || prefixValue == normalizedPrefix { continue } key := "dir|" + prefixValue if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} items = append(items, objectStorageSummaryFromPrefix(normalizedBucket, prefixValue)) } for _, item := range result.Contents { keyValue := strings.Trim(strings.TrimSpace(item.Key), "/") if keyValue == "" || keyValue == normalizedPrefix { continue } key := "file|" + keyValue if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} items = append(items, objectStorageSummaryFromListObject(normalizedBucket, item)) } return objectStoragePage{ Items: items, NextMarker: strings.TrimSpace(result.NextMarker), HasMore: result.IsTruncated, }, nil } func (browser *bosObjectStorageBrowser) CreateBucket(_ context.Context, bucket string) (domain.ObjectStorageBucketSummary, error) { if _, err := browser.client.PutBucket(strings.TrimSpace(bucket)); err != nil { return domain.ObjectStorageBucketSummary{}, err } return domain.ObjectStorageBucketSummary{Name: strings.TrimSpace(bucket)}, nil } func (browser *bosObjectStorageBrowser) DeleteBucket(_ context.Context, bucket string) error { return browser.client.DeleteBucket(strings.TrimSpace(bucket)) } func detectObjectStorageContentType(key string, payload []byte, explicit string) string { if trimmed := strings.TrimSpace(explicit); trimmed != "" { return trimmed } if guessed := mime.TypeByExtension(strings.ToLower(path.Ext(key))); guessed != "" { return guessed } if len(payload) > 0 { return http.DetectContentType(payload) } return "application/octet-stream" } func supportsObjectStorageInlinePreview(summary domain.ObjectStorageObjectSummary) bool { contentType := strings.ToLower(strings.TrimSpace(summary.ContentType)) if summary.Kind == "json" { return true } if strings.HasPrefix(contentType, "text/") { return true } switch strings.ToLower(path.Ext(summary.Key)) { case ".txt", ".log", ".md", ".csv", ".tsv", ".yaml", ".yml", ".xml", ".jsonl", ".html", ".htm": return true } return false } func shouldReadObjectStorageInline(summary domain.ObjectStorageObjectSummary) bool { if !supportsObjectStorageInlinePreview(summary) { return false } if summary.SizeBytes <= 0 { return true } return summary.SizeBytes <= objectStorageInlinePreviewMaxBytes } type objectStorageByteRange struct { start int64 end int64 } func parseObjectStorageByteRange(rangeHeader string, sizeBytes int64) (objectStorageByteRange, bool, error) { trimmed := strings.TrimSpace(rangeHeader) if trimmed == "" { return objectStorageByteRange{}, false, nil } if sizeBytes <= 0 { return objectStorageByteRange{}, true, fmt.Errorf("object size is unknown") } if !strings.HasPrefix(strings.ToLower(trimmed), "bytes=") { return objectStorageByteRange{}, true, fmt.Errorf("unsupported range unit") } spec := strings.TrimSpace(trimmed[len("bytes="):]) if spec == "" || strings.Contains(spec, ",") { return objectStorageByteRange{}, true, fmt.Errorf("only a single byte range is supported") } startText, endText, found := strings.Cut(spec, "-") if !found { return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range") } startText = strings.TrimSpace(startText) endText = strings.TrimSpace(endText) if startText == "" && endText == "" { return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range") } if startText == "" { suffixLength, err := strconv.ParseInt(endText, 10, 64) if err != nil || suffixLength <= 0 { return objectStorageByteRange{}, true, fmt.Errorf("invalid suffix byte range") } if suffixLength >= sizeBytes { return objectStorageByteRange{start: 0, end: sizeBytes - 1}, true, nil } return objectStorageByteRange{start: sizeBytes - suffixLength, end: sizeBytes - 1}, true, nil } start, err := strconv.ParseInt(startText, 10, 64) if err != nil || start < 0 { return objectStorageByteRange{}, true, fmt.Errorf("invalid range start") } if start >= sizeBytes { return objectStorageByteRange{}, true, fmt.Errorf("range start is outside object") } end := sizeBytes - 1 if endText != "" { parsedEnd, err := strconv.ParseInt(endText, 10, 64) if err != nil || parsedEnd < start { return objectStorageByteRange{}, true, fmt.Errorf("invalid range end") } if parsedEnd < end { end = parsedEnd } } return objectStorageByteRange{start: start, end: end}, true, nil } func writeObjectStorageRangeHeaders(writer http.ResponseWriter, summary domain.ObjectStorageObjectSummary, contentType string, byteRange objectStorageByteRange, statusCode int) { if contentType == "" { contentType = detectObjectStorageContentType(summary.Key, nil, summary.ContentType) } if contentType != "" { writer.Header().Set("Content-Type", contentType) } writer.Header().Set("Accept-Ranges", "bytes") if summary.ETag != "" { writer.Header().Set("ETag", strconv.Quote(summary.ETag)) } if summary.LastModified != "" { writer.Header().Set("Last-Modified", summary.LastModified) } if statusCode == http.StatusPartialContent { length := byteRange.end - byteRange.start + 1 writer.Header().Set("Content-Length", fmt.Sprintf("%d", length)) writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, summary.SizeBytes)) return } if summary.SizeBytes > 0 { writer.Header().Set("Content-Length", fmt.Sprintf("%d", summary.SizeBytes)) } } func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, payload, contentType)} if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey) } func (browser *bosObjectStorageBrowser) PutObjectConditional( ctx context.Context, bucket string, key string, payload []byte, contentType string, ifMatch string, ifNoneMatch string, userMeta map[string]string, ) (domain.ObjectStorageObjectSummary, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") args := &bosapi.PutObjectArgs{ ContentType: detectObjectStorageContentType(objectKey, payload, contentType), IfMatch: strings.TrimSpace(ifMatch), IfNoneMatch: strings.TrimSpace(ifNoneMatch), UserMeta: userMeta, } if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey) } func (browser *bosObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, nil, contentType)} if _, err := browser.client.PutObjectFromStreamWithContext(ctx, strings.TrimSpace(bucket), objectKey, reader, args); err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey) } func normalizeObjectStoragePresignExpires(seconds int) int { if seconds <= 0 { return objectStoragePresignDefaultExpires } if seconds > objectStoragePresignMaxExpires { return objectStoragePresignMaxExpires } return seconds } func objectStorageExpiresAt(seconds int) string { return time.Now().UTC().Add(time.Duration(seconds) * time.Second).Format(time.RFC3339) } func objectStorageUploadHeaders(contentType string) map[string]string { headers := map[string]string{} if strings.TrimSpace(contentType) != "" { headers["Content-Type"] = strings.TrimSpace(contentType) } return headers } func (browser *bosObjectStorageBrowser) PresignPutObject(_ context.Context, bucket string, key string, contentType string, expiresInSeconds int) (domain.ObjectStoragePresignedUploadResult, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) headers := objectStorageUploadHeaders(contentType) url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, nil) return domain.ObjectStoragePresignedUploadResult{ Bucket: strings.TrimSpace(bucket), Key: objectKey, Method: http.MethodPut, URL: url, Headers: headers, ExpiresAt: objectStorageExpiresAt(expiresInSeconds), }, nil } func (browser *bosObjectStorageBrowser) PresignPutObjectBound( _ context.Context, bucket string, key string, contentType string, contentMD5 string, sizeBytes int64, metadata map[string]string, expiresInSeconds int, ) (domain.ObjectStoragePresignedUploadResult, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) headers := objectStorageUploadHeaders(contentType) headers["Content-MD5"] = strings.TrimSpace(contentMD5) headers["Content-Length"] = strconv.FormatInt(sizeBytes, 10) for metaKey, metaValue := range metadata { headers["x-bce-meta-"+strings.ToLower(strings.TrimSpace(metaKey))] = strings.TrimSpace(metaValue) } url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, nil) return domain.ObjectStoragePresignedUploadResult{ Bucket: strings.TrimSpace(bucket), Key: objectKey, Method: http.MethodPut, URL: url, Headers: headers, ExpiresAt: objectStorageExpiresAt(expiresInSeconds), }, nil } func (browser *bosObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucket string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil) if err != nil { return domain.ObjectStorageMultipartUploadCreateResult{}, err } expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) return domain.ObjectStorageMultipartUploadCreateResult{ Bucket: result.Bucket, Key: result.Key, UploadID: result.UploadId, ExpiresAt: objectStorageExpiresAt(expiresInSeconds), }, nil } func (browser *bosObjectStorageBrowser) InitiateMultipartUploadBound( _ context.Context, bucket string, key string, contentType string, metadata map[string]string, expiresInSeconds int, ) (domain.ObjectStorageMultipartUploadCreateResult, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil) if err != nil { return domain.ObjectStorageMultipartUploadCreateResult{}, err } expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) return domain.ObjectStorageMultipartUploadCreateResult{ Bucket: result.Bucket, Key: result.Key, UploadID: result.UploadId, ExpiresAt: objectStorageExpiresAt(expiresInSeconds), }, nil } func (browser *bosObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucket string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) params := map[string]string{ "uploadId": strings.TrimSpace(uploadID), "partNumber": strconv.Itoa(partNumber), } url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, nil, params) return domain.ObjectStorageMultipartUploadPartURLResult{ Bucket: strings.TrimSpace(bucket), Key: objectKey, UploadID: strings.TrimSpace(uploadID), PartNumber: partNumber, Method: http.MethodPut, URL: url, Headers: map[string]string{}, ExpiresAt: objectStorageExpiresAt(expiresInSeconds), }, nil } func (browser *bosObjectStorageBrowser) PresignMultipartUploadPartBound( _ context.Context, bucket string, key string, uploadID string, partNumber int, contentMD5 string, sizeBytes int64, expiresInSeconds int, ) (domain.ObjectStorageMultipartUploadPartURLResult, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) params := map[string]string{ "uploadId": strings.TrimSpace(uploadID), "partNumber": strconv.Itoa(partNumber), } headers := map[string]string{ "Content-MD5": strings.TrimSpace(contentMD5), "Content-Length": strconv.FormatInt(sizeBytes, 10), } url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, params) return domain.ObjectStorageMultipartUploadPartURLResult{ Bucket: strings.TrimSpace(bucket), Key: objectKey, UploadID: strings.TrimSpace(uploadID), PartNumber: partNumber, Method: http.MethodPut, URL: url, Headers: headers, ExpiresAt: objectStorageExpiresAt(expiresInSeconds), }, nil } func (browser *bosObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) { return browser.CompleteMultipartUploadBound(ctx, bucket, key, uploadID, parts, nil) } func (browser *bosObjectStorageBrowser) CompleteMultipartUploadBound( ctx context.Context, bucket string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart, metadata map[string]string, ) (domain.ObjectStorageObjectSummary, error) { objectKey := strings.Trim(strings.TrimSpace(key), "/") uploadParts := make([]bosapi.UploadInfoType, 0, len(parts)) for _, part := range parts { uploadParts = append(uploadParts, bosapi.UploadInfoType{PartNumber: part.PartNumber, ETag: strings.Trim(part.ETag, `"`)}) } if _, err := browser.client.CompleteMultipartUploadFromStruct(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID), &bosapi.CompleteMultipartUploadArgs{Parts: uploadParts, UserMeta: metadata}); err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey) } func (browser *bosObjectStorageBrowser) AbortMultipartUpload(_ context.Context, bucket string, key string, uploadID string) error { objectKey := strings.Trim(strings.TrimSpace(key), "/") return browser.client.AbortMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID)) } func objectSyncMultipartCopyPartSize(sizeBytes int64) int64 { const ( minimum = int64(256 * 1024 * 1024) maximum = int64(5 * 1024 * 1024 * 1024) parts = int64(1_024) align = int64(1024 * 1024) ) partSize := minimum if sizeBytes > partSize*parts { partSize = ((sizeBytes/parts + align - 1) / align) * align } if partSize > maximum { return maximum } return partSize } func (browser *bosObjectStorageBrowser) PublishMultipartCopyConditional( ctx context.Context, bucket string, sourceKey string, targetKey string, sizeBytes int64, sourceETag string, ifMatch string, ifNoneMatch string, userMeta map[string]string, ) (domain.ObjectStorageObjectSummary, error) { normalizedBucket := strings.TrimSpace(bucket) sourceKey = exactObjectStorageKey(sourceKey) targetKey = exactObjectStorageKey(targetKey) if normalizedBucket == "" || sourceKey == "" || targetKey == "" { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("bucket, source key, and target key are required") } if sizeBytes <= 0 { return browser.PutObjectConditional(ctx, normalizedBucket, targetKey, []byte{}, "application/octet-stream", ifMatch, ifNoneMatch, userMeta) } initiated, err := browser.client.InitiateMultipartUpload(normalizedBucket, targetKey, "", nil) if err != nil { return domain.ObjectStorageObjectSummary{}, err } abort := true defer func() { if abort { _ = browser.client.AbortMultipartUpload(normalizedBucket, targetKey, initiated.UploadId) } }() partSize := objectSyncMultipartCopyPartSize(sizeBytes) partCount := int((sizeBytes + partSize - 1) / partSize) parts := make([]bosapi.UploadInfoType, partCount) copyGroup, copyContext := errgroup.WithContext(ctx) copyGroup.SetLimit(8) for partIndex := range partCount { start := int64(partIndex) * partSize end := start + partSize - 1 if end >= sizeBytes { end = sizeBytes - 1 } partNumber := partIndex + 1 copyGroup.Go(func() error { result, copyErr := browser.client.UploadPartCopyWithContext( copyContext, normalizedBucket, targetKey, normalizedBucket, sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{ SourceRange: fmt.Sprintf("bytes=%d-%d", start, end), IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`), }, ) if copyErr != nil { return copyErr } parts[partIndex] = bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)} return nil }) } if err := copyGroup.Wait(); err != nil { return domain.ObjectStorageObjectSummary{}, err } completeArgs := &bosapi.CompleteMultipartUploadArgs{ Parts: parts, UserMeta: userMeta, IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`), IfNoneMatch: strings.TrimSpace(ifNoneMatch), } completePayload, err := json.Marshal(completeArgs) if err != nil { return domain.ObjectStorageObjectSummary{}, err } body, err := bce.NewBodyFromBytes(completePayload) if err != nil { return domain.ObjectStorageObjectSummary{}, err } bosContext := browser.client.NewBosContext(ctx) if _, err = bosapi.CompleteMultipartUpload(browser.client, normalizedBucket, targetKey, initiated.UploadId, body, completeArgs, bosContext); err != nil { return domain.ObjectStorageObjectSummary{}, err } abort = false return browser.getObjectSummaryAfterWrite(ctx, normalizedBucket, targetKey) } func (browser *bosObjectStorageBrowser) GetBucketVersioning(_ context.Context, bucket string) (string, error) { result, err := browser.client.GetBucketVersioning(strings.TrimSpace(bucket)) if err != nil { return "", err } if result == nil { return "", nil } return strings.ToLower(strings.TrimSpace(result.Status)), nil } // objectStorageSimpleCopyMaxBytes mirrors BOS CopyObject's 5 GB source size limit. // Objects larger than this must use server-side multipart copy (UploadPartCopy). const objectStorageSimpleCopyMaxBytes int64 = 5 * 1024 * 1024 * 1024 // objectStorageMoveProgress reports incremental completion state while a // prefix move copies objects one by one. type objectStorageMoveProgress struct { TotalObjects int CopiedObjects int SkippedObjects int DeletedObjects int CurrentKey string } // etagMatches compares two ETags ignoring surrounding quotes. BOS multipart // copies produce UUID-style ETags that differ from the source etag, so etag // equality is only a resumability hint, never a correctness guarantee. func etagMatches(left string, right string) bool { left = strings.Trim(strings.TrimSpace(left), "\"") right = strings.Trim(strings.TrimSpace(right), "\"") return left != "" && right != "" && left == right } // copyObjectPreservingMetadata copies a single object server-side while keeping // its Content-Type and user metadata. Objects larger than the BOS simple-copy // limit are copied via multipart upload part copy so that moving large media // files (multi-GB recordings) does not fail with a 5 GB limit error. func (browser *bosObjectStorageBrowser) copyObjectPreservingMetadata(ctx context.Context, bucket string, sourceKey string, destinationKey string) (domain.ObjectStorageObjectSummary, error) { sourceKey = exactObjectStorageKey(sourceKey) destinationKey = exactObjectStorageKey(destinationKey) meta, err := browser.client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), sourceKey) if err != nil { return domain.ObjectStorageObjectSummary{}, err } sizeBytes := int64(0) contentType := "" sourceETag := "" var userMeta map[string]string if meta != nil { sizeBytes = meta.ContentLength contentType = strings.TrimSpace(meta.ContentType) sourceETag = strings.Trim(strings.TrimSpace(meta.ETag), "\"") userMeta = meta.UserMeta } if contentType == "" { contentType = detectObjectStorageContentType(destinationKey, nil, "") } if sizeBytes > objectStorageSimpleCopyMaxBytes { return browser.multipartCopyObject(ctx, bucket, sourceKey, destinationKey, sizeBytes, sourceETag, contentType, userMeta) } if _, err := browser.client.CopyObjectWithContext(ctx, strings.TrimSpace(bucket), destinationKey, strings.TrimSpace(bucket), sourceKey, nil); err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.getObjectSummaryAfterWrite(ctx, bucket, destinationKey) } func (browser *bosObjectStorageBrowser) multipartCopyObject(ctx context.Context, bucket string, sourceKey string, destinationKey string, sizeBytes int64, sourceETag string, contentType string, userMeta map[string]string) (domain.ObjectStorageObjectSummary, error) { initiated, err := bosapi.InitiateMultipartUpload(browser.client, strings.TrimSpace(bucket), destinationKey, contentType, nil, browser.client.NewBosContext(ctx)) if err != nil { return domain.ObjectStorageObjectSummary{}, err } abort := true defer func() { if abort { _ = browser.client.AbortMultipartUpload(strings.TrimSpace(bucket), destinationKey, initiated.UploadId) } }() partSize := objectSyncMultipartCopyPartSize(sizeBytes) partCount := int((sizeBytes + partSize - 1) / partSize) parts := make([]bosapi.UploadInfoType, partCount) copyGroup, copyContext := errgroup.WithContext(ctx) copyGroup.SetLimit(8) for partIndex := range partCount { start := int64(partIndex) * partSize end := start + partSize - 1 if end >= sizeBytes { end = sizeBytes - 1 } partNumber := partIndex + 1 copyGroup.Go(func() error { result, copyErr := browser.client.UploadPartCopyWithContext( copyContext, strings.TrimSpace(bucket), destinationKey, strings.TrimSpace(bucket), sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{ SourceRange: fmt.Sprintf("bytes=%d-%d", start, end), IfMatch: sourceETag, }, ) if copyErr != nil { return copyErr } parts[partIndex] = bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, "\"")} return nil }) } if err := copyGroup.Wait(); err != nil { return domain.ObjectStorageObjectSummary{}, err } completeArgs := &bosapi.CompleteMultipartUploadArgs{ Parts: parts, UserMeta: userMeta, } completePayload, err := json.Marshal(completeArgs) if err != nil { return domain.ObjectStorageObjectSummary{}, err } body, err := bce.NewBodyFromBytes(completePayload) if err != nil { return domain.ObjectStorageObjectSummary{}, err } bosContext := browser.client.NewBosContext(ctx) if _, err = bosapi.CompleteMultipartUpload(browser.client, strings.TrimSpace(bucket), destinationKey, initiated.UploadId, body, completeArgs, bosContext); err != nil { return domain.ObjectStorageObjectSummary{}, err } abort = false return browser.getObjectSummaryAfterWrite(ctx, bucket, destinationKey) } func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) { objectKey := exactObjectStorageKey(key) destinationKey := exactObjectStorageKey(newKey) if objectKey == "" || destinationKey == "" { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("source and destination keys are required") } if objectKey == destinationKey { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination key must differ from source key") } source, err := browser.GetObjectSummary(ctx, bucket, objectKey) if err != nil { return domain.ObjectStorageObjectSummary{}, err } destination, targetErr := browser.GetObjectSummary(ctx, bucket, destinationKey) if targetErr == nil { if destination.SizeBytes != source.SizeBytes || !etagMatches(destination.ETag, source.ETag) { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("target already exists with different content: %s", destinationKey) } } else { if !isBOSNoSuchKey(targetErr) { return domain.ObjectStorageObjectSummary{}, targetErr } destination, err = browser.copyObjectPreservingMetadata(ctx, bucket, objectKey, destinationKey) if err != nil { return domain.ObjectStorageObjectSummary{}, err } } if destination.SizeBytes != source.SizeBytes { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("copied object size mismatch; source retained: %s", objectKey) } currentSource, err := browser.GetObjectSummary(ctx, bucket, objectKey) if err != nil { return domain.ObjectStorageObjectSummary{}, err } if currentSource.SizeBytes != source.SizeBytes || !etagMatches(currentSource.ETag, source.ETag) { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("source changed during move; source retained: %s", objectKey) } if err := browser.client.DeleteObject(strings.TrimSpace(bucket), objectKey); err != nil { return domain.ObjectStorageObjectSummary{}, err } return destination, nil } func (browser *bosObjectStorageBrowser) MovePrefix(ctx context.Context, bucket string, prefix string, newPrefix string) (domain.ObjectStorageObjectSummary, error) { return browser.MovePrefixWithProgress(ctx, bucket, prefix, newPrefix, nil) } // objectStorageMoveItem is a single object planned for a prefix move. type objectStorageMoveItem struct { sourceKey string destinationKey string sizeBytes int64 etag string isMarker bool } func (browser *bosObjectStorageBrowser) MovePrefixWithProgress(ctx context.Context, bucket string, prefix string, newPrefix string, progress func(objectStorageMoveProgress)) (domain.ObjectStorageObjectSummary, error) { normalizedBucket := strings.TrimSpace(bucket) sourcePrefix := strings.Trim(strings.TrimSpace(prefix), "/") destinationPrefix := strings.Trim(strings.TrimSpace(newPrefix), "/") if normalizedBucket == "" || sourcePrefix == "" || destinationPrefix == "" { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("bucket, source prefix, and destination prefix are required") } if sourcePrefix == destinationPrefix { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination prefix must differ from source prefix") } if strings.HasPrefix(destinationPrefix, sourcePrefix+"/") { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination prefix cannot be inside the source prefix") } reportProgress := func(state objectStorageMoveProgress) { if progress != nil { progress(state) } } sourceRequestPrefix := sourcePrefix + "/" type sourceObject struct { key string size int64 etag string isMarker bool } objectsToMove := make([]sourceObject, 0) for _, exactKey := range []string{sourcePrefix, sourcePrefix + "/"} { meta, err := browser.client.GetObjectMetaWithContext(ctx, normalizedBucket, exactKey) if err != nil { if isBOSNoSuchKey(err) { continue } return domain.ObjectStorageObjectSummary{}, err } objectsToMove = append(objectsToMove, sourceObject{key: exactKey, size: meta.ContentLength, etag: strings.Trim(meta.ETag, "\""), isMarker: true}) } seen := map[string]struct{}{ sourcePrefix: {}, sourcePrefix + "/": {}, } marker := "" for { result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ Prefix: sourceRequestPrefix, MaxKeys: 1000, Marker: marker, }) if err != nil { return domain.ObjectStorageObjectSummary{}, err } for _, item := range result.Contents { key := exactObjectStorageKey(item.Key) if key == "" { continue } if _, exists := seen[key]; exists { continue } seen[key] = struct{}{} objectsToMove = append(objectsToMove, sourceObject{key: key, size: int64(item.Size), etag: strings.Trim(strings.TrimSpace(item.ETag), "\"")}) } if result == nil || !result.IsTruncated { break } nextMarker := strings.TrimSpace(result.NextMarker) if nextMarker == "" || nextMarker == marker { break } marker = nextMarker } // Snapshot pre-existing destination objects so a failed move can roll back // only the objects this move created, never pre-existing user data. preExistingDestinations := map[string]struct{}{} for _, key := range []string{destinationPrefix, destinationPrefix + "/"} { _, err := browser.client.GetObjectMetaWithContext(ctx, normalizedBucket, key) if err == nil { preExistingDestinations[key] = struct{}{} } else if !isBOSNoSuchKey(err) { return domain.ObjectStorageObjectSummary{}, err } } marker = "" for { result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ Prefix: destinationPrefix + "/", MaxKeys: 1000, Marker: marker, }) if err != nil { return domain.ObjectStorageObjectSummary{}, err } for _, item := range result.Contents { key := exactObjectStorageKey(item.Key) if key != "" { preExistingDestinations[key] = struct{}{} } } if result == nil || !result.IsTruncated { break } nextMarker := strings.TrimSpace(result.NextMarker) if nextMarker == "" || nextMarker == marker { break } marker = nextMarker } total := len(objectsToMove) copied := 0 skipped := 0 createdDestinations := make([]string, 0, len(objectsToMove)) failures := make([]string, 0) for _, item := range objectsToMove { destinationKey := destinationPrefix if strings.HasPrefix(item.key, sourceRequestPrefix) { destinationKey = destinationPrefix + "/" + strings.TrimPrefix(item.key, sourceRequestPrefix) } else if strings.HasSuffix(item.key, "/") { destinationKey = destinationPrefix + "/" } reportProgress(objectStorageMoveProgress{TotalObjects: total, CurrentKey: item.key, CopiedObjects: copied, SkippedObjects: skipped}) _, preExisting := preExistingDestinations[destinationKey] if preExisting { summary, summaryErr := browser.GetObjectSummary(ctx, normalizedBucket, destinationKey) if summaryErr == nil && summary.SizeBytes == item.size && etagMatches(summary.ETag, item.etag) { // Already copied by a previous attempt of this move: skip the // server-side copy but still remove the source below. skipped++ continue } failures = append(failures, "target already exists with different content: "+destinationKey) break } copiedSummary, copyErr := browser.copyObjectPreservingMetadata(ctx, normalizedBucket, item.key, destinationKey) if err := copyErr; err != nil { if isBOSNoSuchKey(err) && item.isMarker { continue } failures = append(failures, fmt.Sprintf("%s: %v", item.key, err)) break } if !preExisting { createdDestinations = append(createdDestinations, destinationKey) } if copiedSummary.SizeBytes != item.size { failures = append(failures, "copied object size mismatch: "+item.key) break } copied++ } if len(failures) > 0 { // Roll back objects this move created so a failed move does not leave // partial duplicates at the destination. Pre-existing destination // objects are never deleted. rollbackFailures := make([]string, 0) for _, destinationKey := range createdDestinations { if err := browser.client.DeleteObject(normalizedBucket, destinationKey); err != nil && !isBOSNoSuchKey(err) { rollbackFailures = append(rollbackFailures, fmt.Sprintf("%s: %v", destinationKey, err)) } } message := fmt.Sprintf("moved %d of %d objects before failing; first error: %s", copied, total, failures[0]) if len(rollbackFailures) > 0 { message += fmt.Sprintf("; rollback removed %d copied objects, %d rollback deletions failed: %s", len(createdDestinations)-len(rollbackFailures), len(rollbackFailures), rollbackFailures[0]) } else if copied > 0 { message += fmt.Sprintf("; rollback removed %d partially copied objects", copied) } return domain.ObjectStorageObjectSummary{}, fmt.Errorf("%s", message) } deleted := 0 for _, item := range objectsToMove { meta, err := browser.client.GetObjectMetaWithContext(ctx, normalizedBucket, item.key) if err != nil { return domain.ObjectStorageObjectSummary{}, err } if meta.ContentLength != item.size || !etagMatches(meta.ETag, item.etag) { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("source changed during move; source retained: %s", item.key) } } for _, item := range objectsToMove { if err := browser.client.DeleteObject(normalizedBucket, item.key); err != nil && !isBOSNoSuchKey(err) { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("copied %d of %d objects but failed to delete source %s: %w", copied, total, item.key, err) } deleted++ } reportProgress(objectStorageMoveProgress{TotalObjects: total, CopiedObjects: copied, SkippedObjects: skipped, DeletedObjects: deleted}) if copied == 0 && skipped == 0 { return domain.ObjectStorageObjectSummary{}, os.ErrNotExist } return objectStorageSummaryFromPrefix(normalizedBucket, destinationPrefix), nil } func objectStorageBaseName(key string) string { normalized := strings.Trim(strings.TrimSpace(key), "/") if normalized == "" { return "" } segments := strings.Split(normalized, "/") return segments[len(segments)-1] } func objectStorageMoveDestination(sourceKey string, destinationPrefix string) string { baseName := objectStorageBaseName(sourceKey) normalizedDestination := strings.Trim(strings.TrimSpace(destinationPrefix), "/") if baseName == "" { return normalizedDestination } if normalizedDestination == "" { return baseName } return normalizedDestination + "/" + baseName } func (browser *bosObjectStorageBrowser) DeleteObject(_ context.Context, bucket string, key string) error { return browser.client.DeleteObject(strings.TrimSpace(bucket), exactObjectStorageKey(key)) } func (browser *bosObjectStorageBrowser) DeletePrefix(ctx context.Context, bucket string, prefix string) error { normalizedBucket := strings.TrimSpace(bucket) normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/") if normalizedBucket == "" || normalizedPrefix == "" { return fmt.Errorf("bucket and prefix are required") } requestPrefix := normalizedPrefix + "/" marker := "" for { result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ Prefix: requestPrefix, MaxKeys: 1000, Marker: marker, }) if err != nil { return err } for _, item := range result.Contents { key := exactObjectStorageKey(item.Key) if key == "" { continue } if err := browser.client.DeleteObject(normalizedBucket, key); err != nil { return err } } if result == nil || !result.IsTruncated { break } nextMarker := strings.TrimSpace(result.NextMarker) if nextMarker == "" || nextMarker == marker { break } marker = nextMarker } for _, markerKey := range []string{normalizedPrefix + "/", normalizedPrefix} { if err := browser.client.DeleteObject(normalizedBucket, markerKey); err != nil && !isBOSNoSuchKey(err) { return err } } return nil } func (browser *bosObjectStorageBrowser) GetObjectSummary(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) { objectKey := exactObjectStorageKey(key) meta, err := browser.client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), objectKey) if err != nil { return domain.ObjectStorageObjectSummary{}, err } return objectStorageSummaryFromMeta(strings.TrimSpace(bucket), objectKey, meta), nil } func (browser *bosObjectStorageBrowser) getObjectSummaryAfterWrite(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) { return retryObjectStorageSummary(ctx, objectStorageWriteMetadataRetryAttempts, objectStorageWriteMetadataRetryDelay, func() (domain.ObjectStorageObjectSummary, error) { return browser.GetObjectSummary(ctx, bucket, key) }) } func retryObjectStorageSummary( ctx context.Context, attempts int, delay time.Duration, lookup func() (domain.ObjectStorageObjectSummary, error), ) (domain.ObjectStorageObjectSummary, error) { if attempts < 1 { attempts = 1 } for attempt := 0; attempt < attempts; attempt++ { summary, err := lookup() if err == nil || !isBOSNoSuchKey(err) || attempt == attempts-1 { return summary, err } timer := time.NewTimer(delay) select { case <-ctx.Done(): if !timer.Stop() { <-timer.C } return domain.ObjectStorageObjectSummary{}, ctx.Err() case <-timer.C: } } return domain.ObjectStorageObjectSummary{}, nil } func (browser *bosObjectStorageBrowser) ReadObject(ctx context.Context, bucket string, key string) ([]byte, string, error) { objectKey := exactObjectStorageKey(key) result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil) if err != nil { return nil, "", err } defer result.Body.Close() payload, err := io.ReadAll(result.Body) if err != nil { return nil, "", err } return payload, strings.TrimSpace(result.ContentType), nil } func (browser *bosObjectStorageBrowser) ReadObjectRange(ctx context.Context, bucket string, key string, start int64, end int64) (io.ReadCloser, string, error) { objectKey := exactObjectStorageKey(key) result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil, start, end) if err != nil { return nil, "", err } return result.Body, strings.TrimSpace(result.ContentType), nil } func (browser *bosObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucket string, key string, maxBytes int64) ([]byte, string, error) { if maxBytes <= 0 { return browser.ReadObject(ctx, bucket, key) } objectKey := exactObjectStorageKey(key) result, err := browser.client.GetObjectWithContext( ctx, strings.TrimSpace(bucket), objectKey, nil, 0, maxBytes-1, ) if err != nil { return nil, "", err } defer result.Body.Close() payload, err := io.ReadAll(io.LimitReader(result.Body, maxBytes)) if err != nil { return nil, "", err } return payload, strings.TrimSpace(result.ContentType), nil } func (browser *bosObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucket string, key string) (string, error) { previewURL := strings.TrimSpace(browser.client.BasicGeneratePresignedUrl(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), 3600)) if previewURL == "" { return "", fmt.Errorf("failed to generate BOS preview url") } return previewURL, nil } func decodeObjectStorageBucketCreateInput(request *http.Request) (domain.ObjectStorageBucketCreateInput, error) { input := domain.ObjectStorageBucketCreateInput{} if err := json.NewDecoder(request.Body).Decode(&input); err != nil { return domain.ObjectStorageBucketCreateInput{}, err } return input, nil } func decodeObjectStorageObjectPutInput(request *http.Request) (domain.ObjectStorageObjectPutInput, error) { input := domain.ObjectStorageObjectPutInput{} if err := json.NewDecoder(request.Body).Decode(&input); err != nil { return domain.ObjectStorageObjectPutInput{}, err } return input, nil } func decodeObjectStorageObjectRenameInput(request *http.Request) (domain.ObjectStorageObjectRenameInput, error) { input := domain.ObjectStorageObjectRenameInput{} if err := json.NewDecoder(request.Body).Decode(&input); err != nil { return domain.ObjectStorageObjectRenameInput{}, err } return input, nil } type objectStoragePresignPutInput struct { Bucket string `json:"bucket"` Key string `json:"key"` ContentType string `json:"contentType,omitempty"` ExpiresInSeconds int `json:"expiresInSeconds,omitempty"` } type objectStorageMultipartInitiateInput struct { Bucket string `json:"bucket"` Key string `json:"key"` ContentType string `json:"contentType,omitempty"` ExpiresInSeconds int `json:"expiresInSeconds,omitempty"` } type objectStorageMultipartPartURLInput struct { Bucket string `json:"bucket"` Key string `json:"key"` UploadID string `json:"uploadId"` PartNumber int `json:"partNumber"` ExpiresInSeconds int `json:"expiresInSeconds,omitempty"` } type objectStorageMultipartCompleteInput struct { Bucket string `json:"bucket"` Key string `json:"key"` UploadID string `json:"uploadId"` Parts []domain.ObjectStorageMultipartUploadPart `json:"parts"` } type objectStorageMultipartAbortInput struct { Bucket string `json:"bucket"` Key string `json:"key"` UploadID string `json:"uploadId"` } func decodeObjectStorageJSON[T any](request *http.Request) (T, error) { var input T if err := json.NewDecoder(request.Body).Decode(&input); err != nil { return input, err } return input, nil } func (server *server) resolveObjectStorageBrowser(ctx context.Context, connectionID string) (objectStorageBrowser, *domain.StorageConnectionRecord, func(), error) { connection, err := server.getStorageConnection(ctx, connectionID) if err != nil && strings.TrimSpace(connectionID) != "" { return nil, nil, nil, err } if connection == nil { return nil, nil, nil, nil } connection = server.storageConnectionWithResolvedEndpoint(connection) browser, cleanup, err := storageConnectionBrowserResolver(ctx, connection) if err != nil { return nil, nil, nil, err } return browser, connection, cleanup, nil } func resolveObjectStorageBucket(connection *domain.StorageConnectionRecord, requested string) (string, error) { requested = strings.Trim(strings.TrimSpace(requested), "/") if connection == nil { return requested, nil } boundBucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/") if boundBucket == "" { return requested, nil } if requested != "" && requested != boundBucket { return "", fmt.Errorf("connection is bound to bucket %s", boundBucket) } return boundBucket, nil } func renderObjectStorageWriteError(writer http.ResponseWriter, err error) { if err == nil { return } if isRequestTooLarge(err) { renderJSON(writer, http.StatusRequestEntityTooLarge, map[string]string{"error": err.Error()}) return } if os.IsNotExist(err) || isBOSNoSuchKey(err) { renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()}) return } renderError(writer, http.StatusInternalServerError, err) } func isRequestTooLarge(err error) bool { if err == nil { return false } var maxBytesErr *http.MaxBytesError return errors.As(err, &maxBytesErr) } func exactObjectStorageKey(key string) string { return strings.TrimSpace(key) } func isBOSNoSuchKey(err error) bool { if err == nil { return false } var serviceErr *bce.BceServiceError if errors.As(err, &serviceErr) { code := strings.TrimSpace(serviceErr.Code) return strings.EqualFold(code, "NoSuchKey") || (serviceErr.StatusCode == http.StatusNotFound && strings.EqualFold(code, "NotFound")) } return strings.Contains(strings.ToLower(err.Error()), "nosuchkey") } func (server *server) handleObjectStorageOverview(writer http.ResponseWriter, request *http.Request) { connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } if browser == nil || connection == nil { renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{ ConnectionID: connectionID, Provider: "BOS", Configured: false, Buckets: []domain.ObjectStorageBucketSummary{}, }) return } bucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/") items := []domain.ObjectStorageBucketSummary{} if bucket != "" { items = append(items, domain.ObjectStorageBucketSummary{Name: bucket}) } renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{ ConnectionID: connection.ID, ConnectionName: connection.Name, Provider: string(connection.Provider), Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint), Bucket: bucket, Configured: true, Buckets: items, }) } func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, request *http.Request) { query := request.URL.Query() connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) if err != nil { renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) return } prefix := strings.Trim(strings.TrimSpace(query.Get("prefix")), "/") marker := strings.TrimSpace(query.Get("marker")) paginationRequested := query.Has("limit") || query.Has("marker") limit := 100 if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" { parsedLimit, parseErr := strconv.Atoi(rawLimit) if parseErr != nil || parsedLimit < 1 || parsedLimit > 1000 { renderValidationError(writer, validationIssue{Field: "limit", Message: "limit must be an integer between 1 and 1000"}) return } limit = parsedLimit } if browser == nil || connection == nil { renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{ ConnectionID: connectionID, Provider: "BOS", Configured: false, Bucket: bucket, Prefix: prefix, Items: []domain.ObjectStorageObjectSummary{}, Limit: limit, }) return } page := objectStoragePage{} if paginationRequested { if pagedBrowser, ok := browser.(pagedObjectStorageBrowser); ok { page, err = pagedBrowser.ListObjectsPage(request.Context(), bucket, prefix, marker, limit) } else { var items []domain.ObjectStorageObjectSummary items, err = browser.ListObjects(request.Context(), bucket, prefix) page = paginateObjectStorageItems(items, marker, limit) } } else { page.Items, err = browser.ListObjects(request.Context(), bucket, prefix) } if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{ ConnectionID: connection.ID, ConnectionName: connection.Name, Provider: string(connection.Provider), Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint), Configured: true, Bucket: bucket, Prefix: prefix, Items: page.Items, NextMarker: page.NextMarker, HasMore: page.HasMore, Limit: limit, }) } func paginateObjectStorageItems(items []domain.ObjectStorageObjectSummary, marker string, limit int) objectStoragePage { start := 0 marker = strings.TrimSpace(marker) if marker != "" { for index, item := range items { if item.Key == marker { start = index + 1 break } } } if start > len(items) { start = len(items) } end := min(start+limit, len(items)) pageItems := append([]domain.ObjectStorageObjectSummary(nil), items[start:end]...) hasMore := end < len(items) nextMarker := "" if hasMore && len(pageItems) > 0 { nextMarker = pageItems[len(pageItems)-1].Key } return objectStoragePage{Items: pageItems, NextMarker: nextMarker, HasMore: hasMore} } func (server *server) handleObjectStorageBucketCreate(writer http.ResponseWriter, request *http.Request) { renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"}) } func (server *server) handleObjectStorageBucketDelete(writer http.ResponseWriter, request *http.Request) { renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"}) } func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter, request *http.Request) { request.Body = http.MaxBytesReader(writer, request.Body, objectStorageJSONPutMaxBytes) input, err := decodeObjectStorageObjectPutInput(request) if err != nil { if isRequestTooLarge(err) { renderObjectStorageWriteError(writer, err) return } renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"}) return } if strings.TrimSpace(input.Key) == "" { renderValidationError(writer, validationIssue{Field: "key", Message: "key is required"}, ) return } connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } bucket, err := resolveObjectStorageBucket(connection, input.Bucket) if err != nil || bucket == "" { renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}) return } item, err := browser.PutObject(request.Context(), bucket, input.Key, []byte(input.Content), input.ContentType) if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item}) } func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter, request *http.Request) { request.Body = http.MaxBytesReader(writer, request.Body, objectStorageMultipartUploadMaxBytes) connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) if err != nil { renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) return } key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/") if bucket == "" || key == "" { renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}, validationIssue{Field: "key", Message: "key is required"}, ) return } if err := request.ParseMultipartForm(32 << 20); err != nil { if isRequestTooLarge(err) { renderObjectStorageWriteError(writer, err) return } renderValidationError(writer, validationIssue{Field: "file", Message: "invalid multipart payload"}) return } file, header, err := request.FormFile("file") if err != nil { renderValidationError(writer, validationIssue{Field: "file", Message: "file is required"}) return } defer file.Close() contentType := strings.TrimSpace(request.URL.Query().Get("contentType")) if contentType == "" && header != nil { contentType = strings.TrimSpace(header.Header.Get("Content-Type")) } item, err := browser.PutObjectStream(request.Context(), bucket, key, file, contentType) if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item}) } func (server *server) resolveObjectStorageUploadTarget(writer http.ResponseWriter, request *http.Request, requestedBucket string, key string) (objectStorageBrowser, string, string, func(), bool) { connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return nil, "", "", cleanup, false } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return nil, "", "", cleanup, false } bucket, err := resolveObjectStorageBucket(connection, requestedBucket) if err != nil || bucket == "" { renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}) return nil, "", "", cleanup, false } objectKey := strings.Trim(strings.TrimSpace(key), "/") if objectKey == "" { renderValidationError(writer, validationIssue{Field: "key", Message: "key is required"}) return nil, "", "", cleanup, false } return browser, bucket, objectKey, cleanup, true } func renderObjectStorageInvalidJSON(writer http.ResponseWriter) { renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"}) } func normalizeObjectStorageBatchRelativePath(rawValue string) (string, error) { value := strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/") value = strings.TrimSuffix(value, "/") if value == "" { return "", fmt.Errorf("relativePath is required") } if strings.HasPrefix(value, "/") || strings.ContainsRune(value, '\x00') { return "", fmt.Errorf("relativePath must be a relative object path") } parts := strings.Split(value, "/") for _, part := range parts { if part == "" || part == "." || part == ".." { return "", fmt.Errorf("relativePath contains an invalid path segment") } } return strings.Join(parts, "/"), nil } func normalizeObjectStorageBatchPrefix(rawValue string) (string, error) { value := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/"), "/") if value == "" { return "", nil } return normalizeObjectStorageBatchRelativePath(value) } func objectStorageBatchItemKey(prefix string, relativePath string, kind string) string { key := relativePath if kind == "directory" { key += "/.keep" } if prefix != "" { key = prefix + "/" + key } return key } func prepareObjectStorageBatchItems(input domain.ObjectStorageBatchUploadPrepareInput) ([]domain.ObjectStorageBatchUploadPreparedItem, string, string, *validationIssue) { policy := strings.ToLower(strings.TrimSpace(input.ConflictPolicy)) if policy == "" { policy = "overwrite" } if policy != "overwrite" && policy != "skip" && policy != "error" { return nil, "", "", &validationIssue{Field: "conflictPolicy", Message: "conflictPolicy must be overwrite, skip, or error"} } prefix, err := normalizeObjectStorageBatchPrefix(input.Prefix) if err != nil { return nil, "", "", &validationIssue{Field: "prefix", Message: err.Error()} } if len(input.Items) == 0 { return nil, "", "", &validationIssue{Field: "items", Message: "items are required"} } if len(input.Items) > objectStorageBatchUploadMaxItems { return nil, "", "", &validationIssue{Field: "items", Message: fmt.Sprintf("items must contain at most %d entries", objectStorageBatchUploadMaxItems)} } prepared := make([]domain.ObjectStorageBatchUploadPreparedItem, 0, len(input.Items)) seenKeys := make(map[string]struct{}, len(input.Items)) for index, item := range input.Items { kind := strings.ToLower(strings.TrimSpace(item.Kind)) if kind != "file" && kind != "directory" { return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].kind", index), Message: "kind must be file or directory"} } if item.SizeBytes < 0 { return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].sizeBytes", index), Message: "sizeBytes must not be negative"} } relativePath, pathErr := normalizeObjectStorageBatchRelativePath(item.RelativePath) if pathErr != nil { return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: pathErr.Error()} } key := objectStorageBatchItemKey(prefix, relativePath, kind) keyBytes := len(key) if keyBytes > objectStorageKeyMaxUTF8Bytes { return nil, "", "", &validationIssue{ Field: fmt.Sprintf("items[%d].relativePath", index), Message: fmt.Sprintf("destination object key must not exceed %d UTF-8 bytes (got %d)", objectStorageKeyMaxUTF8Bytes, keyBytes), } } if _, exists := seenKeys[key]; exists { return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: "duplicate destination key"} } seenKeys[key] = struct{}{} prepared = append(prepared, domain.ObjectStorageBatchUploadPreparedItem{ RelativePath: relativePath, Key: key, Kind: kind, SizeBytes: item.SizeBytes, ContentType: strings.TrimSpace(item.ContentType), }) } return prepared, prefix, policy, nil } func inspectObjectStorageBatchConflicts(ctx context.Context, browser objectStorageBrowser, bucket string, items []domain.ObjectStorageBatchUploadPreparedItem) error { workerCount := min(objectStorageBatchUploadStatConcurrency, len(items)) indices := make(chan int) var firstErr error var errOnce sync.Once var workers sync.WaitGroup workers.Add(workerCount) for range workerCount { go func() { defer workers.Done() for index := range indices { _, err := browser.GetObjectSummary(ctx, bucket, items[index].Key) if err == nil { items[index].Exists = true continue } if os.IsNotExist(err) || isBOSNoSuchKey(err) { continue } errOnce.Do(func() { firstErr = err }) } }() } for index := range items { if ctx.Err() != nil { break } indices <- index } close(indices) workers.Wait() if firstErr != nil { return firstErr } return ctx.Err() } func finalizeObjectStorageBatchPlan(items []domain.ObjectStorageBatchUploadPreparedItem, policy string) domain.ObjectStorageBatchUploadPrepareResult { result := domain.ObjectStorageBatchUploadPrepareResult{ConflictPolicy: policy, Items: items} for index := range result.Items { item := &result.Items[index] if item.Exists && policy == "skip" { item.Action = "skip" item.Message = "destination already exists" result.SkipCount++ continue } if item.Exists && policy == "error" { item.Action = "conflict" item.Message = "destination already exists" result.ConflictCount++ continue } if item.Kind == "directory" { item.Action = "create_placeholder" result.PlaceholderCount++ } else { item.Action = "upload" result.UploadCount++ } } return result } func (server *server) handleObjectStorageBatchUploadPrepare(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageJSON[domain.ObjectStorageBatchUploadPrepareInput](request) if err != nil { renderObjectStorageInvalidJSON(writer) return } items, prefix, policy, issue := prepareObjectStorageBatchItems(input) if issue != nil { renderValidationError(writer, *issue) return } connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if cleanup != nil { defer cleanup() } if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } bucket, err := resolveObjectStorageBucket(connection, input.Bucket) if err != nil || bucket == "" { renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}) return } if err := inspectObjectStorageBatchConflicts(request.Context(), browser, bucket, items); err != nil { renderObjectStorageWriteError(writer, err) return } result := finalizeObjectStorageBatchPlan(items, policy) result.Bucket = bucket result.Prefix = prefix renderJSON(writer, http.StatusOK, result) } func (server *server) handleObjectStoragePresignPut(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageJSON[objectStoragePresignPutInput](request) if err != nil { renderObjectStorageInvalidJSON(writer) return } browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key) if cleanup != nil { defer cleanup() } if !ok { return } result, err := browser.PresignPutObject(request.Context(), bucket, key, input.ContentType, input.ExpiresInSeconds) if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, result) } func (server *server) handleObjectStorageMultipartInitiate(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageJSON[objectStorageMultipartInitiateInput](request) if err != nil { renderObjectStorageInvalidJSON(writer) return } browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key) if cleanup != nil { defer cleanup() } if !ok { return } result, err := browser.InitiateMultipartUpload(request.Context(), bucket, key, input.ContentType, input.ExpiresInSeconds) if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, result) } func (server *server) handleObjectStorageMultipartPartURL(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageJSON[objectStorageMultipartPartURLInput](request) if err != nil { renderObjectStorageInvalidJSON(writer) return } if strings.TrimSpace(input.UploadID) == "" { renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"}) return } if input.PartNumber < 1 || input.PartNumber > 10000 { renderValidationError(writer, validationIssue{Field: "partNumber", Message: "partNumber must be between 1 and 10000"}) return } browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key) if cleanup != nil { defer cleanup() } if !ok { return } result, err := browser.PresignMultipartUploadPart(request.Context(), bucket, key, input.UploadID, input.PartNumber, input.ExpiresInSeconds) if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, result) } func normalizeObjectStorageMultipartParts(parts []domain.ObjectStorageMultipartUploadPart) ([]domain.ObjectStorageMultipartUploadPart, error) { if len(parts) == 0 { return nil, fmt.Errorf("parts are required") } normalized := append([]domain.ObjectStorageMultipartUploadPart(nil), parts...) for index := range normalized { normalized[index].ETag = strings.Trim(strings.TrimSpace(normalized[index].ETag), `"`) if normalized[index].PartNumber < 1 || normalized[index].PartNumber > 10000 { return nil, fmt.Errorf("partNumber must be between 1 and 10000") } if normalized[index].ETag == "" { return nil, fmt.Errorf("part eTag is required") } } sort.Slice(normalized, func(left int, right int) bool { return normalized[left].PartNumber < normalized[right].PartNumber }) for index := 1; index < len(normalized); index++ { if normalized[index].PartNumber == normalized[index-1].PartNumber { return nil, fmt.Errorf("duplicate partNumber %d", normalized[index].PartNumber) } } return normalized, nil } func (server *server) handleObjectStorageMultipartComplete(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageJSON[objectStorageMultipartCompleteInput](request) if err != nil { renderObjectStorageInvalidJSON(writer) return } if strings.TrimSpace(input.UploadID) == "" { renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"}) return } parts, err := normalizeObjectStorageMultipartParts(input.Parts) if err != nil { renderValidationError(writer, validationIssue{Field: "parts", Message: err.Error()}) return } browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key) if cleanup != nil { defer cleanup() } if !ok { return } item, err := browser.CompleteMultipartUpload(request.Context(), bucket, key, input.UploadID, parts) if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item}) } func (server *server) handleObjectStorageMultipartAbort(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageJSON[objectStorageMultipartAbortInput](request) if err != nil { renderObjectStorageInvalidJSON(writer) return } if strings.TrimSpace(input.UploadID) == "" { renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"}) return } browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key) if cleanup != nil { defer cleanup() } if !ok { return } if err := browser.AbortMultipartUpload(request.Context(), bucket, key, input.UploadID); err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, domain.ObjectStorageMultipartUploadAbortResult{Bucket: bucket, Key: key, UploadID: strings.TrimSpace(input.UploadID), Aborted: true}) } func (server *server) handleObjectStorageObjectRename(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageObjectRenameInput(request) if err != nil { renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"}) return } connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } bucket, err := resolveObjectStorageBucket(connection, input.Bucket) if err != nil || bucket == "" { renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}) return } destinationPrefix := strings.Trim(strings.TrimSpace(input.DestinationPrefix), "/") if len(input.Items) > 0 || destinationPrefix != "" { items := input.Items if len(items) == 0 { items = []domain.ObjectStorageObjectMoveItem{{ Key: input.Key, Recursive: input.Recursive, }} } movedObjects := make([]domain.ObjectStorageObjectSummary, 0, len(items)) if issues := objectStorageMovePrecheckIssues(items, destinationPrefix); len(issues) > 0 { renderValidationError(writer, issues...) return } if !input.Force { sourcePrefixes := make([]string, 0, len(items)) for _, moveItem := range items { sourcePrefixes = append(sourcePrefixes, strings.Trim(strings.TrimSpace(moveItem.Key), "/")) } sourcePrefixes = append(sourcePrefixes, destinationPrefix) if conflicts := server.objectStorageCaptureAutomationConflicts(request.Context(), connection.OrganizationID, connectionID, bucket, sourcePrefixes); len(conflicts) > 0 { renderObjectStorageCaptureAutomationConflict(writer, conflicts) return } } for index, moveItem := range items { sourceKey := strings.Trim(strings.TrimSpace(moveItem.Key), "/") if sourceKey == "" { renderValidationError(writer, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "key is required"}) return } destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix) if destinationKey == "" || destinationKey == sourceKey { renderValidationError(writer, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "destination must differ from source"}) return } isDirectory := strings.EqualFold(strings.TrimSpace(moveItem.Kind), "directory") || moveItem.Recursive var item domain.ObjectStorageObjectSummary if isDirectory { item, err = browser.MovePrefix(request.Context(), bucket, sourceKey, destinationKey) } else { item, err = browser.RenameObject(request.Context(), bucket, sourceKey, destinationKey) } if err != nil { renderObjectStorageWriteError(writer, err) return } movedObjects = append(movedObjects, item) } response := domain.ObjectStorageObjectMutationResult{Objects: movedObjects} if len(movedObjects) > 0 { response.Object = movedObjects[0] } renderJSON(writer, http.StatusOK, response) return } if strings.TrimSpace(input.Key) == "" || strings.TrimSpace(input.NewKey) == "" { renderValidationError(writer, validationIssue{Field: "key", Message: "key is required"}, validationIssue{Field: "newKey", Message: "newKey is required"}, ) return } var item domain.ObjectStorageObjectSummary if input.Recursive { item, err = browser.MovePrefix(request.Context(), bucket, input.Key, input.NewKey) } else { item, err = browser.RenameObject(request.Context(), bucket, input.Key, input.NewKey) } if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item}) } func (server *server) handleObjectStorageObjectDelete(writer http.ResponseWriter, request *http.Request) { connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) if err != nil { renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) return } key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/") recursive := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("recursive")), "true") force := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("force")), "true") if bucket == "" || key == "" { renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}, validationIssue{Field: "key", Message: "key is required"}, ) return } if !force { prefixes := []string{key} if recursive { prefixes = append(prefixes, strings.TrimSuffix(key, "/")) } if conflicts := server.objectStorageCaptureAutomationConflicts(request.Context(), connection.OrganizationID, connectionID, bucket, prefixes); len(conflicts) > 0 { renderObjectStorageCaptureAutomationConflict(writer, conflicts) return } } trashKey := ".droboticflow-trash/" + time.Now().UTC().Format("20060102T150405.000000000Z") + "/" + key if recursive { _, err = browser.MovePrefix(request.Context(), bucket, key, trashKey) } else { _, err = browser.RenameObject(request.Context(), bucket, key, trashKey) } if err != nil { if recursive && (os.IsNotExist(err) || isBOSNoSuchKey(err)) { err = nil } } if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, map[string]any{"bucket": bucket, "key": key, "originalKey": key, "trashKey": trashKey, "deleted": true}) } type objectStorageRestoreInput struct { Bucket string `json:"bucket"` TrashKey string `json:"trashKey"` OriginalKey string `json:"originalKey"` Recursive bool `json:"recursive,omitempty"` } func (server *server) handleObjectStorageObjectRestore(writer http.ResponseWriter, request *http.Request) { connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } var payload objectStorageRestoreInput if !decodeJSON(writer, request, &payload) { return } bucket, err := resolveObjectStorageBucket(connection, payload.Bucket) if err != nil { renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) return } trashKey := strings.Trim(strings.TrimSpace(payload.TrashKey), "/") originalKey := strings.Trim(strings.TrimSpace(payload.OriginalKey), "/") if !strings.HasPrefix(trashKey, ".droboticflow-trash/") || originalKey == "" { renderValidationError(writer, validationIssue{Field: "trashKey", Message: "a trash key and originalKey are required"}) return } var restored domain.ObjectStorageObjectSummary if payload.Recursive { restored, err = browser.MovePrefix(request.Context(), bucket, trashKey, originalKey) } else { restored, err = browser.RenameObject(request.Context(), bucket, trashKey, originalKey) } if err != nil { renderObjectStorageWriteError(writer, err) return } renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: restored}) } func (server *server) handleObjectStorageContent(writer http.ResponseWriter, request *http.Request) { connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil && (request.Method == http.MethodGet || request.Method == http.MethodHead) { // Only this content endpoint can use the scoped fallback. All storage // administration/mutation routes keep their tenant-owned connection gate. scoped, scopeErr := server.delegatedStorageRead(request.Context(), server.requestOrganizationID(request), connectionID, request.URL.Query().Get("bucket"), request.URL.Query().Get("key"), false) if scopeErr == nil && scoped != nil { connection = server.storageConnectionWithResolvedEndpoint(scoped) browser, cleanup, err = storageConnectionBrowserResolver(request.Context(), connection) } } if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if cleanup != nil { defer cleanup() } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket")) if err != nil { renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) return } key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/") if bucket == "" || key == "" { renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"}, validationIssue{Field: "key", Message: "key is required"}, ) return } summary, err := browser.GetObjectSummary(request.Context(), bucket, key) if err != nil { renderObjectStorageWriteError(writer, err) return } redirectURL := "" if previewURL, previewErr := browser.BuildPreviewURL(request.Context(), bucket, key); previewErr == nil { redirectURL = previewURL } previewOnly := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("preview")), "true") transcodeBrowser := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && summary.Kind == "video" if transcodeBrowser { previewRequest := sourceBrowserPreviewRequest{ FileName: path.Base(key), RedirectURL: redirectURL, ObjectBucket: bucket, ObjectKey: key, } if previewRequest.RedirectURL == "" { payload, _, err := browser.ReadObject(request.Context(), bucket, key) if err != nil { renderObjectStorageWriteError(writer, err) return } previewRequest.Payload = payload } if previewPath, previewErr := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); previewErr == nil && previewPath != "" { if err := writeSourceContentFile(writer, request, strings.TrimSuffix(path.Base(key), path.Ext(key))+"-browser.mp4", "video/mp4", previewPath); err != nil { renderError(writer, http.StatusInternalServerError, err) } return } writer.Header().Set("Content-Type", "video/mp4") writer.Header().Set("Cache-Control", "no-store") writer.WriteHeader(http.StatusOK) if flusher, ok := writer.(http.Flusher); ok { flusher.Flush() } if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil { fmt.Printf("[api] object storage browser preview stream failed: %v\n", err) } return } if previewOnly && supportsObjectStorageInlinePreview(summary) && summary.SizeBytes > objectStorageInlinePreviewMaxBytes { payload, contentType, err := browser.ReadObjectPreview(request.Context(), bucket, key, objectStoragePartialPreviewBytes) if err != nil { renderObjectStorageWriteError(writer, err) return } writer.Header().Set(objectStoragePreviewTruncatedHeader, "true") writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes)) writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload))) writeSourceContent(writer, request, path.Base(key), contentType, payload) return } requestedRange, hasRange, rangeErr := parseObjectStorageByteRange(request.Header.Get("Range"), summary.SizeBytes) if rangeErr != nil { if summary.SizeBytes > 0 { writer.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", summary.SizeBytes)) } writer.Header().Set("Accept-Ranges", "bytes") renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr) return } // Browser media requests normally include Range, including open-ended // ranges. Redirect those too: proxying them shares the API's public // bandwidth and fixed response deadline across all active viewers. // Authorization and object resolution above still precede URL issuance. if request.Method == http.MethodGet && summary.Kind == "video" && !previewOnly && redirectURL != "" { writer.Header().Set("Cache-Control", "private, no-store") writer.Header().Set("Referrer-Policy", "no-referrer") http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) return } if request.Method == http.MethodHead || hasRange { if !hasRange && summary.SizeBytes > 0 { requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1} } statusCode := http.StatusOK if hasRange { statusCode = http.StatusPartialContent } if request.Method == http.MethodHead { writeObjectStorageRangeHeaders(writer, summary, summary.ContentType, requestedRange, statusCode) writer.WriteHeader(statusCode) return } reader, contentType, err := browser.ReadObjectRange(request.Context(), bucket, key, requestedRange.start, requestedRange.end) if err != nil { if redirectURL != "" { http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) return } renderObjectStorageWriteError(writer, err) return } defer reader.Close() writeObjectStorageRangeHeaders(writer, summary, contentType, requestedRange, statusCode) writer.WriteHeader(statusCode) if _, err := io.Copy(writer, reader); err != nil { fmt.Printf("[api] object storage range stream failed: %v\n", err) } return } if redirectURL != "" && !shouldReadObjectStorageInline(summary) { http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) return } payload, contentType, err := browser.ReadObject(request.Context(), bucket, key) if err != nil { if redirectURL != "" { http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) return } renderObjectStorageWriteError(writer, err) return } if previewOnly { writer.Header().Set(objectStoragePreviewTruncatedHeader, "false") if summary.SizeBytes > 0 { writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes)) } writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload))) } writeSourceContent(writer, request, path.Base(key), contentType, payload) }