droboticflow/internal/api/object_storage_browser.go
2026-08-19 03:08:37 +08:00

2116 lines
77 KiB
Go

package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"path"
"sort"
"strconv"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/services/bos"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
"golang.org/x/sync/errgroup"
)
type objectStorageBrowser interface {
ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error)
ListObjects(context.Context, string, string) ([]domain.ObjectStorageObjectSummary, error)
CreateBucket(context.Context, string) (domain.ObjectStorageBucketSummary, error)
DeleteBucket(context.Context, string) error
PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error)
PutObjectStream(context.Context, string, string, io.Reader, string) (domain.ObjectStorageObjectSummary, error)
PresignPutObject(context.Context, string, string, string, int) (domain.ObjectStoragePresignedUploadResult, error)
InitiateMultipartUpload(context.Context, string, string, string, int) (domain.ObjectStorageMultipartUploadCreateResult, error)
PresignMultipartUploadPart(context.Context, string, string, string, int, int) (domain.ObjectStorageMultipartUploadPartURLResult, error)
CompleteMultipartUpload(context.Context, string, string, string, []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error)
AbortMultipartUpload(context.Context, string, string, string) error
RenameObject(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error)
MovePrefix(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error)
DeleteObject(context.Context, string, string) error
DeletePrefix(context.Context, string, string) error
GetObjectSummary(context.Context, string, string) (domain.ObjectStorageObjectSummary, error)
ReadObject(context.Context, string, string) ([]byte, string, error)
ReadObjectRange(context.Context, string, string, int64, int64) (io.ReadCloser, string, error)
ReadObjectPreview(context.Context, string, string, int64) ([]byte, string, error)
BuildPreviewURL(context.Context, string, string) (string, error)
}
type objectSyncStorageBrowser interface {
objectStorageBrowser
PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
PublishMultipartCopyConditional(context.Context, string, string, string, int64, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
GetBucketVersioning(context.Context, string) (string, error)
}
type objectSyncBoundUploadBrowser interface {
PresignPutObjectBound(context.Context, string, string, string, string, int64, map[string]string, int) (domain.ObjectStoragePresignedUploadResult, error)
InitiateMultipartUploadBound(context.Context, string, string, string, map[string]string, int) (domain.ObjectStorageMultipartUploadCreateResult, error)
PresignMultipartUploadPartBound(context.Context, string, string, string, int, string, int64, int) (domain.ObjectStorageMultipartUploadPartURLResult, error)
CompleteMultipartUploadBound(context.Context, string, string, string, []domain.ObjectStorageMultipartUploadPart, map[string]string) (domain.ObjectStorageObjectSummary, error)
}
type objectStoragePage struct {
Items []domain.ObjectStorageObjectSummary
NextMarker string
HasMore bool
}
type pagedObjectStorageBrowser interface {
ListObjectsPage(context.Context, string, string, string, int) (objectStoragePage, error)
}
type bosObjectStorageBrowser struct {
connection *domain.StorageConnectionRecord
client *bos.Client
}
const (
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
objectStoragePartialPreviewBytes int64 = 64 * 1024
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
objectStoragePresignDefaultExpires = 3600
objectStoragePresignMaxExpires = 24 * 3600
objectStorageBatchUploadMaxItems = 5000
objectStorageBatchUploadStatConcurrency = 16
objectStorageWriteMetadataRetryAttempts = 4
objectStorageWriteMetadataRetryDelay = 100 * time.Millisecond
objectStorageKeyMaxUTF8Bytes = 1023
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
)
var storageConnectionBrowserResolver = func(ctx context.Context, connection *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
if connection == nil {
return nil, nil, nil
}
endpoint := storageEndpointForProvider(connection.Provider, connection.Endpoint)
if normalizeStorageProvider(connection.Provider) == domain.StorageProviderOSS {
return newOSSObjectStorageBrowser(connection, endpoint)
}
if normalizeStorageProvider(connection.Provider) == domain.StorageProviderBOS && endpoint == "" {
return nil, nil, fmt.Errorf("BOS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name))
}
client, err := bos.NewClient(
strings.TrimSpace(connection.AccessKeyID),
strings.TrimSpace(connection.SecretAccessKey),
endpoint,
)
if err != nil {
return nil, nil, err
}
return &bosObjectStorageBrowser{connection: connection, client: client}, func() {}, nil
}
func objectStorageKindForObject(key string, contentType string) string {
contentType = strings.ToLower(strings.TrimSpace(contentType))
if strings.HasPrefix(contentType, "image/") {
return "image"
}
if strings.HasPrefix(contentType, "video/") {
return "video"
}
if strings.HasPrefix(contentType, "audio/") {
return "audio"
}
if strings.HasPrefix(contentType, "application/json") {
return "json"
}
switch strings.ToLower(path.Ext(key)) {
case ".json":
return "json"
case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg":
return "image"
case ".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv":
return "video"
case ".mp3", ".wav", ".aac", ".ogg", ".flac", ".m4a":
return "audio"
}
return "file"
}
func objectStorageSummaryFromListObject(bucket string, item bosapi.ObjectSummaryType) domain.ObjectStorageObjectSummary {
key := strings.TrimSpace(item.Key)
contentType := detectObjectStorageContentType(key, nil, "")
return domain.ObjectStorageObjectSummary{
Bucket: bucket,
Key: key,
Name: path.Base(strings.TrimRight(key, "/")),
Kind: objectStorageKindForObject(key, contentType),
SizeBytes: int64(item.Size),
ContentType: contentType,
LastModified: strings.TrimSpace(item.LastModified),
ETag: strings.Trim(item.ETag, "\""),
StorageClass: strings.TrimSpace(item.StorageClass),
}
}
func objectStorageSummaryFromPrefix(bucket string, prefix string) domain.ObjectStorageObjectSummary {
trimmed := strings.TrimSpace(prefix)
return domain.ObjectStorageObjectSummary{
Bucket: bucket,
Key: strings.TrimRight(trimmed, "/"),
Name: path.Base(strings.TrimRight(trimmed, "/")),
Kind: "directory",
}
}
func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObjectMetaResult) domain.ObjectStorageObjectSummary {
contentType := ""
sizeBytes := int64(0)
lastModified := ""
etag := ""
storageClass := ""
versionID := ""
metadata := map[string]any{}
if meta != nil {
contentType = strings.TrimSpace(meta.ContentType)
sizeBytes = meta.ContentLength
lastModified = strings.TrimSpace(meta.LastModified)
etag = strings.Trim(meta.ETag, "\"")
storageClass = strings.TrimSpace(meta.StorageClass)
versionID = strings.TrimSpace(meta.VersionId)
for metaKey, metaValue := range meta.UserMeta {
metadata[metaKey] = metaValue
}
}
if len(metadata) == 0 {
metadata = nil
}
return domain.ObjectStorageObjectSummary{
Bucket: bucket,
Key: key,
Name: path.Base(strings.TrimRight(key, "/")),
Kind: objectStorageKindForObject(key, contentType),
SizeBytes: sizeBytes,
ContentType: contentType,
LastModified: lastModified,
ETag: etag,
VersionID: versionID,
StorageClass: storageClass,
Metadata: metadata,
}
}
func (browser *bosObjectStorageBrowser) ListBuckets(ctx context.Context) ([]domain.ObjectStorageBucketSummary, error) {
result, err := browser.client.ListBucketsWithContext(ctx)
if err != nil {
return nil, err
}
items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets))
for _, bucket := range result.Buckets {
name := strings.TrimSpace(bucket.Name)
if name == "" {
continue
}
items = append(items, domain.ObjectStorageBucketSummary{Name: name, CreatedAt: strings.TrimSpace(bucket.CreationDate)})
}
return items, nil
}
func (browser *bosObjectStorageBrowser) ListObjects(ctx context.Context, bucket string, prefix string) ([]domain.ObjectStorageObjectSummary, error) {
items := make([]domain.ObjectStorageObjectSummary, 0)
seen := make(map[string]struct{})
marker := ""
for {
page, err := browser.ListObjectsPage(ctx, bucket, prefix, marker, 1000)
if err != nil {
return nil, err
}
for _, item := range page.Items {
key := item.Kind + "|" + item.Key
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
items = append(items, item)
}
if !page.HasMore || page.NextMarker == "" || page.NextMarker == marker {
break
}
marker = page.NextMarker
}
return items, nil
}
func (browser *bosObjectStorageBrowser) ListObjectsPage(
ctx context.Context,
bucket string,
prefix string,
marker string,
limit int,
) (objectStoragePage, error) {
normalizedBucket := strings.TrimSpace(bucket)
normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
if limit <= 0 {
limit = 100
}
if limit > 1000 {
limit = 1000
}
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
Prefix: storageObjectRequestPrefix(prefix),
Delimiter: "/",
MaxKeys: limit,
Marker: strings.TrimSpace(marker),
})
if err != nil {
return objectStoragePage{}, err
}
if result == nil {
return objectStoragePage{Items: []domain.ObjectStorageObjectSummary{}}, nil
}
items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Contents))
seen := make(map[string]struct{})
for _, item := range result.CommonPrefixes {
prefixValue := strings.Trim(strings.TrimSpace(item.Prefix), "/")
if prefixValue == "" || prefixValue == normalizedPrefix {
continue
}
key := "dir|" + prefixValue
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
items = append(items, objectStorageSummaryFromPrefix(normalizedBucket, prefixValue))
}
for _, item := range result.Contents {
keyValue := strings.Trim(strings.TrimSpace(item.Key), "/")
if keyValue == "" || keyValue == normalizedPrefix {
continue
}
key := "file|" + keyValue
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
items = append(items, objectStorageSummaryFromListObject(normalizedBucket, item))
}
return objectStoragePage{
Items: items,
NextMarker: strings.TrimSpace(result.NextMarker),
HasMore: result.IsTruncated,
}, nil
}
func (browser *bosObjectStorageBrowser) CreateBucket(_ context.Context, bucket string) (domain.ObjectStorageBucketSummary, error) {
if _, err := browser.client.PutBucket(strings.TrimSpace(bucket)); err != nil {
return domain.ObjectStorageBucketSummary{}, err
}
return domain.ObjectStorageBucketSummary{Name: strings.TrimSpace(bucket)}, nil
}
func (browser *bosObjectStorageBrowser) DeleteBucket(_ context.Context, bucket string) error {
return browser.client.DeleteBucket(strings.TrimSpace(bucket))
}
func detectObjectStorageContentType(key string, payload []byte, explicit string) string {
if trimmed := strings.TrimSpace(explicit); trimmed != "" {
return trimmed
}
if guessed := mime.TypeByExtension(strings.ToLower(path.Ext(key))); guessed != "" {
return guessed
}
if len(payload) > 0 {
return http.DetectContentType(payload)
}
return "application/octet-stream"
}
func supportsObjectStorageInlinePreview(summary domain.ObjectStorageObjectSummary) bool {
contentType := strings.ToLower(strings.TrimSpace(summary.ContentType))
if summary.Kind == "json" {
return true
}
if strings.HasPrefix(contentType, "text/") {
return true
}
switch strings.ToLower(path.Ext(summary.Key)) {
case ".txt", ".log", ".md", ".csv", ".tsv", ".yaml", ".yml", ".xml", ".jsonl", ".html", ".htm":
return true
}
return false
}
func shouldReadObjectStorageInline(summary domain.ObjectStorageObjectSummary) bool {
if !supportsObjectStorageInlinePreview(summary) {
return false
}
if summary.SizeBytes <= 0 {
return true
}
return summary.SizeBytes <= objectStorageInlinePreviewMaxBytes
}
type objectStorageByteRange struct {
start int64
end int64
}
func parseObjectStorageByteRange(rangeHeader string, sizeBytes int64) (objectStorageByteRange, bool, error) {
trimmed := strings.TrimSpace(rangeHeader)
if trimmed == "" {
return objectStorageByteRange{}, false, nil
}
if sizeBytes <= 0 {
return objectStorageByteRange{}, true, fmt.Errorf("object size is unknown")
}
if !strings.HasPrefix(strings.ToLower(trimmed), "bytes=") {
return objectStorageByteRange{}, true, fmt.Errorf("unsupported range unit")
}
spec := strings.TrimSpace(trimmed[len("bytes="):])
if spec == "" || strings.Contains(spec, ",") {
return objectStorageByteRange{}, true, fmt.Errorf("only a single byte range is supported")
}
startText, endText, found := strings.Cut(spec, "-")
if !found {
return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range")
}
startText = strings.TrimSpace(startText)
endText = strings.TrimSpace(endText)
if startText == "" && endText == "" {
return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range")
}
if startText == "" {
suffixLength, err := strconv.ParseInt(endText, 10, 64)
if err != nil || suffixLength <= 0 {
return objectStorageByteRange{}, true, fmt.Errorf("invalid suffix byte range")
}
if suffixLength >= sizeBytes {
return objectStorageByteRange{start: 0, end: sizeBytes - 1}, true, nil
}
return objectStorageByteRange{start: sizeBytes - suffixLength, end: sizeBytes - 1}, true, nil
}
start, err := strconv.ParseInt(startText, 10, 64)
if err != nil || start < 0 {
return objectStorageByteRange{}, true, fmt.Errorf("invalid range start")
}
if start >= sizeBytes {
return objectStorageByteRange{}, true, fmt.Errorf("range start is outside object")
}
end := sizeBytes - 1
if endText != "" {
parsedEnd, err := strconv.ParseInt(endText, 10, 64)
if err != nil || parsedEnd < start {
return objectStorageByteRange{}, true, fmt.Errorf("invalid range end")
}
if parsedEnd < end {
end = parsedEnd
}
}
return objectStorageByteRange{start: start, end: end}, true, nil
}
func writeObjectStorageRangeHeaders(writer http.ResponseWriter, summary domain.ObjectStorageObjectSummary, contentType string, byteRange objectStorageByteRange, statusCode int) {
if contentType == "" {
contentType = detectObjectStorageContentType(summary.Key, nil, summary.ContentType)
}
if contentType != "" {
writer.Header().Set("Content-Type", contentType)
}
writer.Header().Set("Accept-Ranges", "bytes")
if summary.ETag != "" {
writer.Header().Set("ETag", strconv.Quote(summary.ETag))
}
if summary.LastModified != "" {
writer.Header().Set("Last-Modified", summary.LastModified)
}
if statusCode == http.StatusPartialContent {
length := byteRange.end - byteRange.start + 1
writer.Header().Set("Content-Length", fmt.Sprintf("%d", length))
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, summary.SizeBytes))
return
}
if summary.SizeBytes > 0 {
writer.Header().Set("Content-Length", fmt.Sprintf("%d", summary.SizeBytes))
}
}
func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, payload, contentType)}
if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
}
func (browser *bosObjectStorageBrowser) PutObjectConditional(
ctx context.Context,
bucket string,
key string,
payload []byte,
contentType string,
ifMatch string,
ifNoneMatch string,
userMeta map[string]string,
) (domain.ObjectStorageObjectSummary, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
args := &bosapi.PutObjectArgs{
ContentType: detectObjectStorageContentType(objectKey, payload, contentType),
IfMatch: strings.TrimSpace(ifMatch),
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
UserMeta: userMeta,
}
if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
}
func (browser *bosObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, nil, contentType)}
if _, err := browser.client.PutObjectFromStreamWithContext(ctx, strings.TrimSpace(bucket), objectKey, reader, args); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
}
func normalizeObjectStoragePresignExpires(seconds int) int {
if seconds <= 0 {
return objectStoragePresignDefaultExpires
}
if seconds > objectStoragePresignMaxExpires {
return objectStoragePresignMaxExpires
}
return seconds
}
func objectStorageExpiresAt(seconds int) string {
return time.Now().UTC().Add(time.Duration(seconds) * time.Second).Format(time.RFC3339)
}
func objectStorageUploadHeaders(contentType string) map[string]string {
headers := map[string]string{}
if strings.TrimSpace(contentType) != "" {
headers["Content-Type"] = strings.TrimSpace(contentType)
}
return headers
}
func (browser *bosObjectStorageBrowser) PresignPutObject(_ context.Context, bucket string, key string, contentType string, expiresInSeconds int) (domain.ObjectStoragePresignedUploadResult, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
headers := objectStorageUploadHeaders(contentType)
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, nil)
return domain.ObjectStoragePresignedUploadResult{
Bucket: strings.TrimSpace(bucket),
Key: objectKey,
Method: http.MethodPut,
URL: url,
Headers: headers,
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
}, nil
}
func (browser *bosObjectStorageBrowser) PresignPutObjectBound(
_ context.Context,
bucket string,
key string,
contentType string,
contentMD5 string,
sizeBytes int64,
metadata map[string]string,
expiresInSeconds int,
) (domain.ObjectStoragePresignedUploadResult, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
headers := objectStorageUploadHeaders(contentType)
headers["Content-MD5"] = strings.TrimSpace(contentMD5)
headers["Content-Length"] = strconv.FormatInt(sizeBytes, 10)
for metaKey, metaValue := range metadata {
headers["x-bce-meta-"+strings.ToLower(strings.TrimSpace(metaKey))] = strings.TrimSpace(metaValue)
}
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, nil)
return domain.ObjectStoragePresignedUploadResult{
Bucket: strings.TrimSpace(bucket),
Key: objectKey,
Method: http.MethodPut,
URL: url,
Headers: headers,
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
}, nil
}
func (browser *bosObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucket string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil)
if err != nil {
return domain.ObjectStorageMultipartUploadCreateResult{}, err
}
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
return domain.ObjectStorageMultipartUploadCreateResult{
Bucket: result.Bucket,
Key: result.Key,
UploadID: result.UploadId,
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
}, nil
}
func (browser *bosObjectStorageBrowser) InitiateMultipartUploadBound(
_ context.Context,
bucket string,
key string,
contentType string,
metadata map[string]string,
expiresInSeconds int,
) (domain.ObjectStorageMultipartUploadCreateResult, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil)
if err != nil {
return domain.ObjectStorageMultipartUploadCreateResult{}, err
}
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
return domain.ObjectStorageMultipartUploadCreateResult{
Bucket: result.Bucket,
Key: result.Key,
UploadID: result.UploadId,
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
}, nil
}
func (browser *bosObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucket string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
params := map[string]string{
"uploadId": strings.TrimSpace(uploadID),
"partNumber": strconv.Itoa(partNumber),
}
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, nil, params)
return domain.ObjectStorageMultipartUploadPartURLResult{
Bucket: strings.TrimSpace(bucket),
Key: objectKey,
UploadID: strings.TrimSpace(uploadID),
PartNumber: partNumber,
Method: http.MethodPut,
URL: url,
Headers: map[string]string{},
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
}, nil
}
func (browser *bosObjectStorageBrowser) PresignMultipartUploadPartBound(
_ context.Context,
bucket string,
key string,
uploadID string,
partNumber int,
contentMD5 string,
sizeBytes int64,
expiresInSeconds int,
) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
params := map[string]string{
"uploadId": strings.TrimSpace(uploadID),
"partNumber": strconv.Itoa(partNumber),
}
headers := map[string]string{
"Content-MD5": strings.TrimSpace(contentMD5),
"Content-Length": strconv.FormatInt(sizeBytes, 10),
}
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, params)
return domain.ObjectStorageMultipartUploadPartURLResult{
Bucket: strings.TrimSpace(bucket),
Key: objectKey,
UploadID: strings.TrimSpace(uploadID),
PartNumber: partNumber,
Method: http.MethodPut,
URL: url,
Headers: headers,
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
}, nil
}
func (browser *bosObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) {
return browser.CompleteMultipartUploadBound(ctx, bucket, key, uploadID, parts, nil)
}
func (browser *bosObjectStorageBrowser) CompleteMultipartUploadBound(
ctx context.Context,
bucket string,
key string,
uploadID string,
parts []domain.ObjectStorageMultipartUploadPart,
metadata map[string]string,
) (domain.ObjectStorageObjectSummary, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
uploadParts := make([]bosapi.UploadInfoType, 0, len(parts))
for _, part := range parts {
uploadParts = append(uploadParts, bosapi.UploadInfoType{PartNumber: part.PartNumber, ETag: strings.Trim(part.ETag, `"`)})
}
if _, err := browser.client.CompleteMultipartUploadFromStruct(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID), &bosapi.CompleteMultipartUploadArgs{Parts: uploadParts, UserMeta: metadata}); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
}
func (browser *bosObjectStorageBrowser) AbortMultipartUpload(_ context.Context, bucket string, key string, uploadID string) error {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
return browser.client.AbortMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID))
}
func objectSyncMultipartCopyPartSize(sizeBytes int64) int64 {
const (
minimum = int64(256 * 1024 * 1024)
maximum = int64(5 * 1024 * 1024 * 1024)
parts = int64(1_024)
align = int64(1024 * 1024)
)
partSize := minimum
if sizeBytes > partSize*parts {
partSize = ((sizeBytes/parts + align - 1) / align) * align
}
if partSize > maximum {
return maximum
}
return partSize
}
func (browser *bosObjectStorageBrowser) PublishMultipartCopyConditional(
ctx context.Context,
bucket string,
sourceKey string,
targetKey string,
sizeBytes int64,
sourceETag string,
ifMatch string,
ifNoneMatch string,
userMeta map[string]string,
) (domain.ObjectStorageObjectSummary, error) {
normalizedBucket := strings.TrimSpace(bucket)
sourceKey = exactObjectStorageKey(sourceKey)
targetKey = exactObjectStorageKey(targetKey)
if normalizedBucket == "" || sourceKey == "" || targetKey == "" {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("bucket, source key, and target key are required")
}
if sizeBytes <= 0 {
return browser.PutObjectConditional(ctx, normalizedBucket, targetKey, []byte{}, "application/octet-stream", ifMatch, ifNoneMatch, userMeta)
}
initiated, err := browser.client.InitiateMultipartUpload(normalizedBucket, targetKey, "", nil)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
abort := true
defer func() {
if abort {
_ = browser.client.AbortMultipartUpload(normalizedBucket, targetKey, initiated.UploadId)
}
}()
partSize := objectSyncMultipartCopyPartSize(sizeBytes)
partCount := int((sizeBytes + partSize - 1) / partSize)
parts := make([]bosapi.UploadInfoType, partCount)
copyGroup, copyContext := errgroup.WithContext(ctx)
copyGroup.SetLimit(8)
for partIndex := range partCount {
start := int64(partIndex) * partSize
end := start + partSize - 1
if end >= sizeBytes {
end = sizeBytes - 1
}
partNumber := partIndex + 1
copyGroup.Go(func() error {
result, copyErr := browser.client.UploadPartCopyWithContext(
copyContext,
normalizedBucket,
targetKey,
normalizedBucket,
sourceKey,
initiated.UploadId,
partNumber,
&bosapi.UploadPartCopyArgs{
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
},
)
if copyErr != nil {
return copyErr
}
parts[partIndex] = bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)}
return nil
})
}
if err := copyGroup.Wait(); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
completeArgs := &bosapi.CompleteMultipartUploadArgs{
Parts: parts,
UserMeta: userMeta,
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
}
completePayload, err := json.Marshal(completeArgs)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
body, err := bce.NewBodyFromBytes(completePayload)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
bosContext := browser.client.NewBosContext(ctx)
if _, err = bosapi.CompleteMultipartUpload(browser.client, normalizedBucket, targetKey, initiated.UploadId, body, completeArgs, bosContext); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
abort = false
return browser.getObjectSummaryAfterWrite(ctx, normalizedBucket, targetKey)
}
func (browser *bosObjectStorageBrowser) GetBucketVersioning(_ context.Context, bucket string) (string, error) {
result, err := browser.client.GetBucketVersioning(strings.TrimSpace(bucket))
if err != nil {
return "", err
}
if result == nil {
return "", nil
}
return strings.ToLower(strings.TrimSpace(result.Status)), nil
}
func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
objectKey := exactObjectStorageKey(key)
destinationKey := exactObjectStorageKey(newKey)
if objectKey == "" || destinationKey == "" {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("source and destination keys are required")
}
if objectKey == destinationKey {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination key must differ from source key")
}
if _, err := browser.client.CopyObjectWithContext(ctx, strings.TrimSpace(bucket), destinationKey, strings.TrimSpace(bucket), objectKey, nil); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
if err := browser.client.DeleteObject(strings.TrimSpace(bucket), objectKey); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.getObjectSummaryAfterWrite(ctx, bucket, destinationKey)
}
func (browser *bosObjectStorageBrowser) MovePrefix(ctx context.Context, bucket string, prefix string, newPrefix string) (domain.ObjectStorageObjectSummary, error) {
normalizedBucket := strings.TrimSpace(bucket)
sourcePrefix := strings.Trim(strings.TrimSpace(prefix), "/")
destinationPrefix := strings.Trim(strings.TrimSpace(newPrefix), "/")
if normalizedBucket == "" || sourcePrefix == "" || destinationPrefix == "" {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("bucket, source prefix, and destination prefix are required")
}
if sourcePrefix == destinationPrefix {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination prefix must differ from source prefix")
}
if strings.HasPrefix(destinationPrefix, sourcePrefix+"/") {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination prefix cannot be inside the source prefix")
}
sourceRequestPrefix := sourcePrefix + "/"
objectsToMove := make([]string, 0)
for _, exactKey := range []string{sourcePrefix, sourcePrefix + "/"} {
objectsToMove = append(objectsToMove, exactKey)
}
seen := map[string]struct{}{
sourcePrefix: {},
sourcePrefix + "/": {},
}
marker := ""
for {
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
Prefix: sourceRequestPrefix,
MaxKeys: 1000,
Marker: marker,
})
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
for _, item := range result.Contents {
key := exactObjectStorageKey(item.Key)
if key == "" {
continue
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
objectsToMove = append(objectsToMove, key)
}
if result == nil || !result.IsTruncated {
break
}
nextMarker := strings.TrimSpace(result.NextMarker)
if nextMarker == "" || nextMarker == marker {
break
}
marker = nextMarker
}
copied := make([]string, 0, len(objectsToMove))
for _, sourceKey := range objectsToMove {
destinationKey := destinationPrefix
if strings.HasPrefix(sourceKey, sourceRequestPrefix) {
destinationKey = destinationPrefix + "/" + strings.TrimPrefix(sourceKey, sourceRequestPrefix)
} else if strings.HasSuffix(sourceKey, "/") {
destinationKey = destinationPrefix + "/"
}
if _, err := browser.client.CopyObjectWithContext(ctx, normalizedBucket, destinationKey, normalizedBucket, sourceKey, nil); err != nil {
if isBOSNoSuchKey(err) && (sourceKey == sourcePrefix || sourceKey == sourcePrefix+"/") {
continue
}
return domain.ObjectStorageObjectSummary{}, err
}
copied = append(copied, sourceKey)
}
if len(copied) == 0 {
return domain.ObjectStorageObjectSummary{}, os.ErrNotExist
}
for _, sourceKey := range copied {
if err := browser.client.DeleteObject(normalizedBucket, sourceKey); err != nil && !isBOSNoSuchKey(err) {
return domain.ObjectStorageObjectSummary{}, err
}
}
return objectStorageSummaryFromPrefix(normalizedBucket, destinationPrefix), nil
}
func objectStorageBaseName(key string) string {
normalized := strings.Trim(strings.TrimSpace(key), "/")
if normalized == "" {
return ""
}
segments := strings.Split(normalized, "/")
return segments[len(segments)-1]
}
func objectStorageMoveDestination(sourceKey string, destinationPrefix string) string {
baseName := objectStorageBaseName(sourceKey)
normalizedDestination := strings.Trim(strings.TrimSpace(destinationPrefix), "/")
if baseName == "" {
return normalizedDestination
}
if normalizedDestination == "" {
return baseName
}
return normalizedDestination + "/" + baseName
}
func (browser *bosObjectStorageBrowser) DeleteObject(_ context.Context, bucket string, key string) error {
return browser.client.DeleteObject(strings.TrimSpace(bucket), exactObjectStorageKey(key))
}
func (browser *bosObjectStorageBrowser) DeletePrefix(ctx context.Context, bucket string, prefix string) error {
normalizedBucket := strings.TrimSpace(bucket)
normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
if normalizedBucket == "" || normalizedPrefix == "" {
return fmt.Errorf("bucket and prefix are required")
}
requestPrefix := normalizedPrefix + "/"
marker := ""
for {
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
Prefix: requestPrefix,
MaxKeys: 1000,
Marker: marker,
})
if err != nil {
return err
}
for _, item := range result.Contents {
key := exactObjectStorageKey(item.Key)
if key == "" {
continue
}
if err := browser.client.DeleteObject(normalizedBucket, key); err != nil {
return err
}
}
if result == nil || !result.IsTruncated {
break
}
nextMarker := strings.TrimSpace(result.NextMarker)
if nextMarker == "" || nextMarker == marker {
break
}
marker = nextMarker
}
for _, markerKey := range []string{normalizedPrefix + "/", normalizedPrefix} {
if err := browser.client.DeleteObject(normalizedBucket, markerKey); err != nil && !isBOSNoSuchKey(err) {
return err
}
}
return nil
}
func (browser *bosObjectStorageBrowser) GetObjectSummary(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) {
objectKey := exactObjectStorageKey(key)
meta, err := browser.client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), objectKey)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return objectStorageSummaryFromMeta(strings.TrimSpace(bucket), objectKey, meta), nil
}
func (browser *bosObjectStorageBrowser) getObjectSummaryAfterWrite(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) {
return retryObjectStorageSummary(ctx, objectStorageWriteMetadataRetryAttempts, objectStorageWriteMetadataRetryDelay, func() (domain.ObjectStorageObjectSummary, error) {
return browser.GetObjectSummary(ctx, bucket, key)
})
}
func retryObjectStorageSummary(
ctx context.Context,
attempts int,
delay time.Duration,
lookup func() (domain.ObjectStorageObjectSummary, error),
) (domain.ObjectStorageObjectSummary, error) {
if attempts < 1 {
attempts = 1
}
for attempt := 0; attempt < attempts; attempt++ {
summary, err := lookup()
if err == nil || !isBOSNoSuchKey(err) || attempt == attempts-1 {
return summary, err
}
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
if !timer.Stop() {
<-timer.C
}
return domain.ObjectStorageObjectSummary{}, ctx.Err()
case <-timer.C:
}
}
return domain.ObjectStorageObjectSummary{}, nil
}
func (browser *bosObjectStorageBrowser) ReadObject(ctx context.Context, bucket string, key string) ([]byte, string, error) {
objectKey := exactObjectStorageKey(key)
result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil)
if err != nil {
return nil, "", err
}
defer result.Body.Close()
payload, err := io.ReadAll(result.Body)
if err != nil {
return nil, "", err
}
return payload, strings.TrimSpace(result.ContentType), nil
}
func (browser *bosObjectStorageBrowser) ReadObjectRange(ctx context.Context, bucket string, key string, start int64, end int64) (io.ReadCloser, string, error) {
objectKey := exactObjectStorageKey(key)
result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil, start, end)
if err != nil {
return nil, "", err
}
return result.Body, strings.TrimSpace(result.ContentType), nil
}
func (browser *bosObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucket string, key string, maxBytes int64) ([]byte, string, error) {
if maxBytes <= 0 {
return browser.ReadObject(ctx, bucket, key)
}
objectKey := exactObjectStorageKey(key)
result, err := browser.client.GetObjectWithContext(
ctx,
strings.TrimSpace(bucket),
objectKey,
nil,
0,
maxBytes-1,
)
if err != nil {
return nil, "", err
}
defer result.Body.Close()
payload, err := io.ReadAll(io.LimitReader(result.Body, maxBytes))
if err != nil {
return nil, "", err
}
return payload, strings.TrimSpace(result.ContentType), nil
}
func (browser *bosObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucket string, key string) (string, error) {
previewURL := strings.TrimSpace(browser.client.BasicGeneratePresignedUrl(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), 3600))
if previewURL == "" {
return "", fmt.Errorf("failed to generate BOS preview url")
}
return previewURL, nil
}
func decodeObjectStorageBucketCreateInput(request *http.Request) (domain.ObjectStorageBucketCreateInput, error) {
input := domain.ObjectStorageBucketCreateInput{}
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
return domain.ObjectStorageBucketCreateInput{}, err
}
return input, nil
}
func decodeObjectStorageObjectPutInput(request *http.Request) (domain.ObjectStorageObjectPutInput, error) {
input := domain.ObjectStorageObjectPutInput{}
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
return domain.ObjectStorageObjectPutInput{}, err
}
return input, nil
}
func decodeObjectStorageObjectRenameInput(request *http.Request) (domain.ObjectStorageObjectRenameInput, error) {
input := domain.ObjectStorageObjectRenameInput{}
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
return domain.ObjectStorageObjectRenameInput{}, err
}
return input, nil
}
type objectStoragePresignPutInput struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
ContentType string `json:"contentType,omitempty"`
ExpiresInSeconds int `json:"expiresInSeconds,omitempty"`
}
type objectStorageMultipartInitiateInput struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
ContentType string `json:"contentType,omitempty"`
ExpiresInSeconds int `json:"expiresInSeconds,omitempty"`
}
type objectStorageMultipartPartURLInput struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
UploadID string `json:"uploadId"`
PartNumber int `json:"partNumber"`
ExpiresInSeconds int `json:"expiresInSeconds,omitempty"`
}
type objectStorageMultipartCompleteInput struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
UploadID string `json:"uploadId"`
Parts []domain.ObjectStorageMultipartUploadPart `json:"parts"`
}
type objectStorageMultipartAbortInput struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
UploadID string `json:"uploadId"`
}
func decodeObjectStorageJSON[T any](request *http.Request) (T, error) {
var input T
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
return input, err
}
return input, nil
}
func (server *server) resolveObjectStorageBrowser(ctx context.Context, connectionID string) (objectStorageBrowser, *domain.StorageConnectionRecord, func(), error) {
connection, err := server.getStorageConnection(ctx, connectionID)
if err != nil && strings.TrimSpace(connectionID) != "" {
return nil, nil, nil, err
}
if connection == nil {
return nil, nil, nil, nil
}
connection = server.storageConnectionWithResolvedEndpoint(connection)
browser, cleanup, err := storageConnectionBrowserResolver(ctx, connection)
if err != nil {
return nil, nil, nil, err
}
return browser, connection, cleanup, nil
}
func resolveObjectStorageBucket(connection *domain.StorageConnectionRecord, requested string) (string, error) {
requested = strings.Trim(strings.TrimSpace(requested), "/")
if connection == nil {
return requested, nil
}
boundBucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/")
if boundBucket == "" {
return requested, nil
}
if requested != "" && requested != boundBucket {
return "", fmt.Errorf("connection is bound to bucket %s", boundBucket)
}
return boundBucket, nil
}
func renderObjectStorageWriteError(writer http.ResponseWriter, err error) {
if err == nil {
return
}
if isRequestTooLarge(err) {
renderJSON(writer, http.StatusRequestEntityTooLarge, map[string]string{"error": err.Error()})
return
}
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()})
return
}
renderError(writer, http.StatusInternalServerError, err)
}
func isRequestTooLarge(err error) bool {
if err == nil {
return false
}
var maxBytesErr *http.MaxBytesError
return errors.As(err, &maxBytesErr)
}
func exactObjectStorageKey(key string) string {
return strings.TrimSpace(key)
}
func isBOSNoSuchKey(err error) bool {
if err == nil {
return false
}
var serviceErr *bce.BceServiceError
if errors.As(err, &serviceErr) {
code := strings.TrimSpace(serviceErr.Code)
return strings.EqualFold(code, "NoSuchKey") ||
(serviceErr.StatusCode == http.StatusNotFound && strings.EqualFold(code, "NotFound"))
}
return strings.Contains(strings.ToLower(err.Error()), "nosuchkey")
}
func (server *server) handleObjectStorageOverview(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{
ConnectionID: connectionID,
Provider: "BOS",
Configured: false,
Buckets: []domain.ObjectStorageBucketSummary{},
})
return
}
bucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/")
items := []domain.ObjectStorageBucketSummary{}
if bucket != "" {
items = append(items, domain.ObjectStorageBucketSummary{Name: bucket})
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{
ConnectionID: connection.ID,
ConnectionName: connection.Name,
Provider: string(connection.Provider),
Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint),
Bucket: bucket,
Configured: true,
Buckets: items,
})
}
func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, request *http.Request) {
query := request.URL.Query()
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
prefix := strings.Trim(strings.TrimSpace(query.Get("prefix")), "/")
marker := strings.TrimSpace(query.Get("marker"))
paginationRequested := query.Has("limit") || query.Has("marker")
limit := 100
if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
parsedLimit, parseErr := strconv.Atoi(rawLimit)
if parseErr != nil || parsedLimit < 1 || parsedLimit > 1000 {
renderValidationError(writer, validationIssue{Field: "limit", Message: "limit must be an integer between 1 and 1000"})
return
}
limit = parsedLimit
}
if browser == nil || connection == nil {
renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{
ConnectionID: connectionID,
Provider: "BOS",
Configured: false,
Bucket: bucket,
Prefix: prefix,
Items: []domain.ObjectStorageObjectSummary{},
Limit: limit,
})
return
}
page := objectStoragePage{}
if paginationRequested {
if pagedBrowser, ok := browser.(pagedObjectStorageBrowser); ok {
page, err = pagedBrowser.ListObjectsPage(request.Context(), bucket, prefix, marker, limit)
} else {
var items []domain.ObjectStorageObjectSummary
items, err = browser.ListObjects(request.Context(), bucket, prefix)
page = paginateObjectStorageItems(items, marker, limit)
}
} else {
page.Items, err = browser.ListObjects(request.Context(), bucket, prefix)
}
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{
ConnectionID: connection.ID,
ConnectionName: connection.Name,
Provider: string(connection.Provider),
Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint),
Configured: true,
Bucket: bucket,
Prefix: prefix,
Items: page.Items,
NextMarker: page.NextMarker,
HasMore: page.HasMore,
Limit: limit,
})
}
func paginateObjectStorageItems(items []domain.ObjectStorageObjectSummary, marker string, limit int) objectStoragePage {
start := 0
marker = strings.TrimSpace(marker)
if marker != "" {
for index, item := range items {
if item.Key == marker {
start = index + 1
break
}
}
}
if start > len(items) {
start = len(items)
}
end := min(start+limit, len(items))
pageItems := append([]domain.ObjectStorageObjectSummary(nil), items[start:end]...)
hasMore := end < len(items)
nextMarker := ""
if hasMore && len(pageItems) > 0 {
nextMarker = pageItems[len(pageItems)-1].Key
}
return objectStoragePage{Items: pageItems, NextMarker: nextMarker, HasMore: hasMore}
}
func (server *server) handleObjectStorageBucketCreate(writer http.ResponseWriter, request *http.Request) {
renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"})
}
func (server *server) handleObjectStorageBucketDelete(writer http.ResponseWriter, request *http.Request) {
renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"})
}
func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter, request *http.Request) {
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageJSONPutMaxBytes)
input, err := decodeObjectStorageObjectPutInput(request)
if err != nil {
if isRequestTooLarge(err) {
renderObjectStorageWriteError(writer, err)
return
}
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
return
}
if strings.TrimSpace(input.Key) == "" {
renderValidationError(writer,
validationIssue{Field: "key", Message: "key is required"},
)
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
if err != nil || bucket == "" {
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
return
}
item, err := browser.PutObject(request.Context(), bucket, input.Key, []byte(input.Content), input.ContentType)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
}
func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter, request *http.Request) {
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageMultipartUploadMaxBytes)
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
if bucket == "" || key == "" {
renderValidationError(writer,
validationIssue{Field: "bucket", Message: "bucket is required"},
validationIssue{Field: "key", Message: "key is required"},
)
return
}
if err := request.ParseMultipartForm(32 << 20); err != nil {
if isRequestTooLarge(err) {
renderObjectStorageWriteError(writer, err)
return
}
renderValidationError(writer, validationIssue{Field: "file", Message: "invalid multipart payload"})
return
}
file, header, err := request.FormFile("file")
if err != nil {
renderValidationError(writer, validationIssue{Field: "file", Message: "file is required"})
return
}
defer file.Close()
contentType := strings.TrimSpace(request.URL.Query().Get("contentType"))
if contentType == "" && header != nil {
contentType = strings.TrimSpace(header.Header.Get("Content-Type"))
}
item, err := browser.PutObjectStream(request.Context(), bucket, key, file, contentType)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
}
func (server *server) resolveObjectStorageUploadTarget(writer http.ResponseWriter, request *http.Request, requestedBucket string, key string) (objectStorageBrowser, string, string, func(), bool) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return nil, "", "", cleanup, false
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return nil, "", "", cleanup, false
}
bucket, err := resolveObjectStorageBucket(connection, requestedBucket)
if err != nil || bucket == "" {
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
return nil, "", "", cleanup, false
}
objectKey := strings.Trim(strings.TrimSpace(key), "/")
if objectKey == "" {
renderValidationError(writer, validationIssue{Field: "key", Message: "key is required"})
return nil, "", "", cleanup, false
}
return browser, bucket, objectKey, cleanup, true
}
func renderObjectStorageInvalidJSON(writer http.ResponseWriter) {
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
}
func normalizeObjectStorageBatchRelativePath(rawValue string) (string, error) {
value := strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/")
value = strings.TrimSuffix(value, "/")
if value == "" {
return "", fmt.Errorf("relativePath is required")
}
if strings.HasPrefix(value, "/") || strings.ContainsRune(value, '\x00') {
return "", fmt.Errorf("relativePath must be a relative object path")
}
parts := strings.Split(value, "/")
for _, part := range parts {
if part == "" || part == "." || part == ".." {
return "", fmt.Errorf("relativePath contains an invalid path segment")
}
}
return strings.Join(parts, "/"), nil
}
func normalizeObjectStorageBatchPrefix(rawValue string) (string, error) {
value := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/"), "/")
if value == "" {
return "", nil
}
return normalizeObjectStorageBatchRelativePath(value)
}
func objectStorageBatchItemKey(prefix string, relativePath string, kind string) string {
key := relativePath
if kind == "directory" {
key += "/.keep"
}
if prefix != "" {
key = prefix + "/" + key
}
return key
}
func prepareObjectStorageBatchItems(input domain.ObjectStorageBatchUploadPrepareInput) ([]domain.ObjectStorageBatchUploadPreparedItem, string, string, *validationIssue) {
policy := strings.ToLower(strings.TrimSpace(input.ConflictPolicy))
if policy == "" {
policy = "overwrite"
}
if policy != "overwrite" && policy != "skip" && policy != "error" {
return nil, "", "", &validationIssue{Field: "conflictPolicy", Message: "conflictPolicy must be overwrite, skip, or error"}
}
prefix, err := normalizeObjectStorageBatchPrefix(input.Prefix)
if err != nil {
return nil, "", "", &validationIssue{Field: "prefix", Message: err.Error()}
}
if len(input.Items) == 0 {
return nil, "", "", &validationIssue{Field: "items", Message: "items are required"}
}
if len(input.Items) > objectStorageBatchUploadMaxItems {
return nil, "", "", &validationIssue{Field: "items", Message: fmt.Sprintf("items must contain at most %d entries", objectStorageBatchUploadMaxItems)}
}
prepared := make([]domain.ObjectStorageBatchUploadPreparedItem, 0, len(input.Items))
seenKeys := make(map[string]struct{}, len(input.Items))
for index, item := range input.Items {
kind := strings.ToLower(strings.TrimSpace(item.Kind))
if kind != "file" && kind != "directory" {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].kind", index), Message: "kind must be file or directory"}
}
if item.SizeBytes < 0 {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].sizeBytes", index), Message: "sizeBytes must not be negative"}
}
relativePath, pathErr := normalizeObjectStorageBatchRelativePath(item.RelativePath)
if pathErr != nil {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: pathErr.Error()}
}
key := objectStorageBatchItemKey(prefix, relativePath, kind)
keyBytes := len(key)
if keyBytes > objectStorageKeyMaxUTF8Bytes {
return nil, "", "", &validationIssue{
Field: fmt.Sprintf("items[%d].relativePath", index),
Message: fmt.Sprintf("destination object key must not exceed %d UTF-8 bytes (got %d)", objectStorageKeyMaxUTF8Bytes, keyBytes),
}
}
if _, exists := seenKeys[key]; exists {
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: "duplicate destination key"}
}
seenKeys[key] = struct{}{}
prepared = append(prepared, domain.ObjectStorageBatchUploadPreparedItem{
RelativePath: relativePath,
Key: key,
Kind: kind,
SizeBytes: item.SizeBytes,
ContentType: strings.TrimSpace(item.ContentType),
})
}
return prepared, prefix, policy, nil
}
func inspectObjectStorageBatchConflicts(ctx context.Context, browser objectStorageBrowser, bucket string, items []domain.ObjectStorageBatchUploadPreparedItem) error {
workerCount := min(objectStorageBatchUploadStatConcurrency, len(items))
indices := make(chan int)
var firstErr error
var errOnce sync.Once
var workers sync.WaitGroup
workers.Add(workerCount)
for range workerCount {
go func() {
defer workers.Done()
for index := range indices {
_, err := browser.GetObjectSummary(ctx, bucket, items[index].Key)
if err == nil {
items[index].Exists = true
continue
}
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
continue
}
errOnce.Do(func() { firstErr = err })
}
}()
}
for index := range items {
if ctx.Err() != nil {
break
}
indices <- index
}
close(indices)
workers.Wait()
if firstErr != nil {
return firstErr
}
return ctx.Err()
}
func finalizeObjectStorageBatchPlan(items []domain.ObjectStorageBatchUploadPreparedItem, policy string) domain.ObjectStorageBatchUploadPrepareResult {
result := domain.ObjectStorageBatchUploadPrepareResult{ConflictPolicy: policy, Items: items}
for index := range result.Items {
item := &result.Items[index]
if item.Exists && policy == "skip" {
item.Action = "skip"
item.Message = "destination already exists"
result.SkipCount++
continue
}
if item.Exists && policy == "error" {
item.Action = "conflict"
item.Message = "destination already exists"
result.ConflictCount++
continue
}
if item.Kind == "directory" {
item.Action = "create_placeholder"
result.PlaceholderCount++
} else {
item.Action = "upload"
result.UploadCount++
}
}
return result
}
func (server *server) handleObjectStorageBatchUploadPrepare(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[domain.ObjectStorageBatchUploadPrepareInput](request)
if err != nil {
renderObjectStorageInvalidJSON(writer)
return
}
items, prefix, policy, issue := prepareObjectStorageBatchItems(input)
if issue != nil {
renderValidationError(writer, *issue)
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if cleanup != nil {
defer cleanup()
}
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
if err != nil || bucket == "" {
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
return
}
if err := inspectObjectStorageBatchConflicts(request.Context(), browser, bucket, items); err != nil {
renderObjectStorageWriteError(writer, err)
return
}
result := finalizeObjectStorageBatchPlan(items, policy)
result.Bucket = bucket
result.Prefix = prefix
renderJSON(writer, http.StatusOK, result)
}
func (server *server) handleObjectStoragePresignPut(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[objectStoragePresignPutInput](request)
if err != nil {
renderObjectStorageInvalidJSON(writer)
return
}
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
if cleanup != nil {
defer cleanup()
}
if !ok {
return
}
result, err := browser.PresignPutObject(request.Context(), bucket, key, input.ContentType, input.ExpiresInSeconds)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, result)
}
func (server *server) handleObjectStorageMultipartInitiate(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[objectStorageMultipartInitiateInput](request)
if err != nil {
renderObjectStorageInvalidJSON(writer)
return
}
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
if cleanup != nil {
defer cleanup()
}
if !ok {
return
}
result, err := browser.InitiateMultipartUpload(request.Context(), bucket, key, input.ContentType, input.ExpiresInSeconds)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, result)
}
func (server *server) handleObjectStorageMultipartPartURL(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[objectStorageMultipartPartURLInput](request)
if err != nil {
renderObjectStorageInvalidJSON(writer)
return
}
if strings.TrimSpace(input.UploadID) == "" {
renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"})
return
}
if input.PartNumber < 1 || input.PartNumber > 10000 {
renderValidationError(writer, validationIssue{Field: "partNumber", Message: "partNumber must be between 1 and 10000"})
return
}
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
if cleanup != nil {
defer cleanup()
}
if !ok {
return
}
result, err := browser.PresignMultipartUploadPart(request.Context(), bucket, key, input.UploadID, input.PartNumber, input.ExpiresInSeconds)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, result)
}
func normalizeObjectStorageMultipartParts(parts []domain.ObjectStorageMultipartUploadPart) ([]domain.ObjectStorageMultipartUploadPart, error) {
if len(parts) == 0 {
return nil, fmt.Errorf("parts are required")
}
normalized := append([]domain.ObjectStorageMultipartUploadPart(nil), parts...)
for index := range normalized {
normalized[index].ETag = strings.Trim(strings.TrimSpace(normalized[index].ETag), `"`)
if normalized[index].PartNumber < 1 || normalized[index].PartNumber > 10000 {
return nil, fmt.Errorf("partNumber must be between 1 and 10000")
}
if normalized[index].ETag == "" {
return nil, fmt.Errorf("part eTag is required")
}
}
sort.Slice(normalized, func(left int, right int) bool {
return normalized[left].PartNumber < normalized[right].PartNumber
})
for index := 1; index < len(normalized); index++ {
if normalized[index].PartNumber == normalized[index-1].PartNumber {
return nil, fmt.Errorf("duplicate partNumber %d", normalized[index].PartNumber)
}
}
return normalized, nil
}
func (server *server) handleObjectStorageMultipartComplete(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[objectStorageMultipartCompleteInput](request)
if err != nil {
renderObjectStorageInvalidJSON(writer)
return
}
if strings.TrimSpace(input.UploadID) == "" {
renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"})
return
}
parts, err := normalizeObjectStorageMultipartParts(input.Parts)
if err != nil {
renderValidationError(writer, validationIssue{Field: "parts", Message: err.Error()})
return
}
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
if cleanup != nil {
defer cleanup()
}
if !ok {
return
}
item, err := browser.CompleteMultipartUpload(request.Context(), bucket, key, input.UploadID, parts)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
}
func (server *server) handleObjectStorageMultipartAbort(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageJSON[objectStorageMultipartAbortInput](request)
if err != nil {
renderObjectStorageInvalidJSON(writer)
return
}
if strings.TrimSpace(input.UploadID) == "" {
renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"})
return
}
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
if cleanup != nil {
defer cleanup()
}
if !ok {
return
}
if err := browser.AbortMultipartUpload(request.Context(), bucket, key, input.UploadID); err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageMultipartUploadAbortResult{Bucket: bucket, Key: key, UploadID: strings.TrimSpace(input.UploadID), Aborted: true})
}
func (server *server) handleObjectStorageObjectRename(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageObjectRenameInput(request)
if err != nil {
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
if err != nil || bucket == "" {
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
return
}
destinationPrefix := strings.Trim(strings.TrimSpace(input.DestinationPrefix), "/")
if len(input.Items) > 0 || destinationPrefix != "" {
items := input.Items
if len(items) == 0 {
items = []domain.ObjectStorageObjectMoveItem{{
Key: input.Key,
Recursive: input.Recursive,
}}
}
movedObjects := make([]domain.ObjectStorageObjectSummary, 0, len(items))
for index, moveItem := range items {
sourceKey := strings.Trim(strings.TrimSpace(moveItem.Key), "/")
if sourceKey == "" {
renderValidationError(writer, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "key is required"})
return
}
destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix)
if destinationKey == "" || destinationKey == sourceKey {
renderValidationError(writer, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "destination must differ from source"})
return
}
isDirectory := strings.EqualFold(strings.TrimSpace(moveItem.Kind), "directory") || moveItem.Recursive
var item domain.ObjectStorageObjectSummary
if isDirectory {
item, err = browser.MovePrefix(request.Context(), bucket, sourceKey, destinationKey)
} else {
item, err = browser.RenameObject(request.Context(), bucket, sourceKey, destinationKey)
}
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
movedObjects = append(movedObjects, item)
}
response := domain.ObjectStorageObjectMutationResult{Objects: movedObjects}
if len(movedObjects) > 0 {
response.Object = movedObjects[0]
}
renderJSON(writer, http.StatusOK, response)
return
}
if strings.TrimSpace(input.Key) == "" || strings.TrimSpace(input.NewKey) == "" {
renderValidationError(writer,
validationIssue{Field: "key", Message: "key is required"},
validationIssue{Field: "newKey", Message: "newKey is required"},
)
return
}
var item domain.ObjectStorageObjectSummary
if input.Recursive {
item, err = browser.MovePrefix(request.Context(), bucket, input.Key, input.NewKey)
} else {
item, err = browser.RenameObject(request.Context(), bucket, input.Key, input.NewKey)
}
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
}
func (server *server) handleObjectStorageObjectDelete(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
recursive := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("recursive")), "true")
if bucket == "" || key == "" {
renderValidationError(writer,
validationIssue{Field: "bucket", Message: "bucket is required"},
validationIssue{Field: "key", Message: "key is required"},
)
return
}
trashKey := ".droboticflow-trash/" + time.Now().UTC().Format("20060102T150405.000000000Z") + "/" + key
if recursive {
_, err = browser.MovePrefix(request.Context(), bucket, key, trashKey)
} else {
_, err = browser.RenameObject(request.Context(), bucket, key, trashKey)
}
if err != nil {
if recursive && (os.IsNotExist(err) || isBOSNoSuchKey(err)) {
err = nil
}
}
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"bucket": bucket, "key": key, "originalKey": key, "trashKey": trashKey, "deleted": true})
}
type objectStorageRestoreInput struct {
Bucket string `json:"bucket"`
TrashKey string `json:"trashKey"`
OriginalKey string `json:"originalKey"`
Recursive bool `json:"recursive,omitempty"`
}
func (server *server) handleObjectStorageObjectRestore(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
var payload objectStorageRestoreInput
if !decodeJSON(writer, request, &payload) {
return
}
bucket, err := resolveObjectStorageBucket(connection, payload.Bucket)
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
trashKey := strings.Trim(strings.TrimSpace(payload.TrashKey), "/")
originalKey := strings.Trim(strings.TrimSpace(payload.OriginalKey), "/")
if !strings.HasPrefix(trashKey, ".droboticflow-trash/") || originalKey == "" {
renderValidationError(writer, validationIssue{Field: "trashKey", Message: "a trash key and originalKey are required"})
return
}
var restored domain.ObjectStorageObjectSummary
if payload.Recursive {
restored, err = browser.MovePrefix(request.Context(), bucket, trashKey, originalKey)
} else {
restored, err = browser.RenameObject(request.Context(), bucket, trashKey, originalKey)
}
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: restored})
}
func (server *server) handleObjectStorageContent(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
if bucket == "" || key == "" {
renderValidationError(writer,
validationIssue{Field: "bucket", Message: "bucket is required"},
validationIssue{Field: "key", Message: "key is required"},
)
return
}
summary, err := browser.GetObjectSummary(request.Context(), bucket, key)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
redirectURL := ""
if previewURL, previewErr := browser.BuildPreviewURL(request.Context(), bucket, key); previewErr == nil {
redirectURL = previewURL
}
previewOnly := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("preview")), "true")
transcodeBrowser := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && summary.Kind == "video"
if transcodeBrowser {
previewRequest := sourceBrowserPreviewRequest{
FileName: path.Base(key),
RedirectURL: redirectURL,
}
if previewRequest.RedirectURL == "" {
payload, _, err := browser.ReadObject(request.Context(), bucket, key)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
previewRequest.Payload = payload
}
if previewPath, previewErr := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); previewErr == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(path.Base(key), path.Ext(key))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] object storage browser preview stream failed: %v\n", err)
}
return
}
if previewOnly && supportsObjectStorageInlinePreview(summary) && summary.SizeBytes > objectStorageInlinePreviewMaxBytes {
payload, contentType, err := browser.ReadObjectPreview(request.Context(), bucket, key, objectStoragePartialPreviewBytes)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
writer.Header().Set(objectStoragePreviewTruncatedHeader, "true")
writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes))
writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload)))
writeSourceContent(writer, request, path.Base(key), contentType, payload)
return
}
requestedRange, hasRange, rangeErr := parseObjectStorageByteRange(request.Header.Get("Range"), summary.SizeBytes)
if rangeErr != nil {
if summary.SizeBytes > 0 {
writer.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", summary.SizeBytes))
}
writer.Header().Set("Accept-Ranges", "bytes")
renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr)
return
}
if request.Method == http.MethodHead || hasRange {
if !hasRange && summary.SizeBytes > 0 {
requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1}
}
statusCode := http.StatusOK
if hasRange {
statusCode = http.StatusPartialContent
}
if request.Method == http.MethodHead {
writeObjectStorageRangeHeaders(writer, summary, summary.ContentType, requestedRange, statusCode)
writer.WriteHeader(statusCode)
return
}
reader, contentType, err := browser.ReadObjectRange(request.Context(), bucket, key, requestedRange.start, requestedRange.end)
if err != nil {
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
renderObjectStorageWriteError(writer, err)
return
}
defer reader.Close()
writeObjectStorageRangeHeaders(writer, summary, contentType, requestedRange, statusCode)
writer.WriteHeader(statusCode)
if _, err := io.Copy(writer, reader); err != nil {
fmt.Printf("[api] object storage range stream failed: %v\n", err)
}
return
}
if redirectURL != "" && !shouldReadObjectStorageInline(summary) {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
payload, contentType, err := browser.ReadObject(request.Context(), bucket, key)
if err != nil {
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
renderObjectStorageWriteError(writer, err)
return
}
if previewOnly {
writer.Header().Set(objectStoragePreviewTruncatedHeader, "false")
if summary.SizeBytes > 0 {
writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes))
}
writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload)))
}
writeSourceContent(writer, request, path.Base(key), contentType, payload)
}