2116 lines
77 KiB
Go
2116 lines
77 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
|
|
"github.com/baidubce/bce-sdk-go/bce"
|
|
"github.com/baidubce/bce-sdk-go/services/bos"
|
|
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
type objectStorageBrowser interface {
|
|
ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error)
|
|
ListObjects(context.Context, string, string) ([]domain.ObjectStorageObjectSummary, error)
|
|
CreateBucket(context.Context, string) (domain.ObjectStorageBucketSummary, error)
|
|
DeleteBucket(context.Context, string) error
|
|
PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error)
|
|
PutObjectStream(context.Context, string, string, io.Reader, string) (domain.ObjectStorageObjectSummary, error)
|
|
PresignPutObject(context.Context, string, string, string, int) (domain.ObjectStoragePresignedUploadResult, error)
|
|
InitiateMultipartUpload(context.Context, string, string, string, int) (domain.ObjectStorageMultipartUploadCreateResult, error)
|
|
PresignMultipartUploadPart(context.Context, string, string, string, int, int) (domain.ObjectStorageMultipartUploadPartURLResult, error)
|
|
CompleteMultipartUpload(context.Context, string, string, string, []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error)
|
|
AbortMultipartUpload(context.Context, string, string, string) error
|
|
RenameObject(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error)
|
|
MovePrefix(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error)
|
|
DeleteObject(context.Context, string, string) error
|
|
DeletePrefix(context.Context, string, string) error
|
|
GetObjectSummary(context.Context, string, string) (domain.ObjectStorageObjectSummary, error)
|
|
ReadObject(context.Context, string, string) ([]byte, string, error)
|
|
ReadObjectRange(context.Context, string, string, int64, int64) (io.ReadCloser, string, error)
|
|
ReadObjectPreview(context.Context, string, string, int64) ([]byte, string, error)
|
|
BuildPreviewURL(context.Context, string, string) (string, error)
|
|
}
|
|
|
|
type objectSyncStorageBrowser interface {
|
|
objectStorageBrowser
|
|
PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
|
|
PublishMultipartCopyConditional(context.Context, string, string, string, int64, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
|
|
GetBucketVersioning(context.Context, string) (string, error)
|
|
}
|
|
|
|
type objectSyncBoundUploadBrowser interface {
|
|
PresignPutObjectBound(context.Context, string, string, string, string, int64, map[string]string, int) (domain.ObjectStoragePresignedUploadResult, error)
|
|
InitiateMultipartUploadBound(context.Context, string, string, string, map[string]string, int) (domain.ObjectStorageMultipartUploadCreateResult, error)
|
|
PresignMultipartUploadPartBound(context.Context, string, string, string, int, string, int64, int) (domain.ObjectStorageMultipartUploadPartURLResult, error)
|
|
CompleteMultipartUploadBound(context.Context, string, string, string, []domain.ObjectStorageMultipartUploadPart, map[string]string) (domain.ObjectStorageObjectSummary, error)
|
|
}
|
|
|
|
type objectStoragePage struct {
|
|
Items []domain.ObjectStorageObjectSummary
|
|
NextMarker string
|
|
HasMore bool
|
|
}
|
|
|
|
type pagedObjectStorageBrowser interface {
|
|
ListObjectsPage(context.Context, string, string, string, int) (objectStoragePage, error)
|
|
}
|
|
|
|
type bosObjectStorageBrowser struct {
|
|
connection *domain.StorageConnectionRecord
|
|
client *bos.Client
|
|
}
|
|
|
|
const (
|
|
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
|
|
objectStoragePartialPreviewBytes int64 = 64 * 1024
|
|
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
|
|
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
|
|
objectStoragePresignDefaultExpires = 3600
|
|
objectStoragePresignMaxExpires = 24 * 3600
|
|
objectStorageBatchUploadMaxItems = 5000
|
|
objectStorageBatchUploadStatConcurrency = 16
|
|
objectStorageWriteMetadataRetryAttempts = 4
|
|
objectStorageWriteMetadataRetryDelay = 100 * time.Millisecond
|
|
objectStorageKeyMaxUTF8Bytes = 1023
|
|
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
|
|
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
|
|
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
|
|
)
|
|
|
|
var storageConnectionBrowserResolver = func(ctx context.Context, connection *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
|
if connection == nil {
|
|
return nil, nil, nil
|
|
}
|
|
endpoint := storageEndpointForProvider(connection.Provider, connection.Endpoint)
|
|
if normalizeStorageProvider(connection.Provider) == domain.StorageProviderOSS {
|
|
return newOSSObjectStorageBrowser(connection, endpoint)
|
|
}
|
|
if normalizeStorageProvider(connection.Provider) == domain.StorageProviderBOS && endpoint == "" {
|
|
return nil, nil, fmt.Errorf("BOS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name))
|
|
}
|
|
client, err := bos.NewClient(
|
|
strings.TrimSpace(connection.AccessKeyID),
|
|
strings.TrimSpace(connection.SecretAccessKey),
|
|
endpoint,
|
|
)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return &bosObjectStorageBrowser{connection: connection, client: client}, func() {}, nil
|
|
}
|
|
|
|
func objectStorageKindForObject(key string, contentType string) string {
|
|
contentType = strings.ToLower(strings.TrimSpace(contentType))
|
|
if strings.HasPrefix(contentType, "image/") {
|
|
return "image"
|
|
}
|
|
if strings.HasPrefix(contentType, "video/") {
|
|
return "video"
|
|
}
|
|
if strings.HasPrefix(contentType, "audio/") {
|
|
return "audio"
|
|
}
|
|
if strings.HasPrefix(contentType, "application/json") {
|
|
return "json"
|
|
}
|
|
switch strings.ToLower(path.Ext(key)) {
|
|
case ".json":
|
|
return "json"
|
|
case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg":
|
|
return "image"
|
|
case ".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv":
|
|
return "video"
|
|
case ".mp3", ".wav", ".aac", ".ogg", ".flac", ".m4a":
|
|
return "audio"
|
|
}
|
|
return "file"
|
|
}
|
|
|
|
func objectStorageSummaryFromListObject(bucket string, item bosapi.ObjectSummaryType) domain.ObjectStorageObjectSummary {
|
|
key := strings.TrimSpace(item.Key)
|
|
contentType := detectObjectStorageContentType(key, nil, "")
|
|
return domain.ObjectStorageObjectSummary{
|
|
Bucket: bucket,
|
|
Key: key,
|
|
Name: path.Base(strings.TrimRight(key, "/")),
|
|
Kind: objectStorageKindForObject(key, contentType),
|
|
SizeBytes: int64(item.Size),
|
|
ContentType: contentType,
|
|
LastModified: strings.TrimSpace(item.LastModified),
|
|
ETag: strings.Trim(item.ETag, "\""),
|
|
StorageClass: strings.TrimSpace(item.StorageClass),
|
|
}
|
|
}
|
|
|
|
func objectStorageSummaryFromPrefix(bucket string, prefix string) domain.ObjectStorageObjectSummary {
|
|
trimmed := strings.TrimSpace(prefix)
|
|
return domain.ObjectStorageObjectSummary{
|
|
Bucket: bucket,
|
|
Key: strings.TrimRight(trimmed, "/"),
|
|
Name: path.Base(strings.TrimRight(trimmed, "/")),
|
|
Kind: "directory",
|
|
}
|
|
}
|
|
|
|
func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObjectMetaResult) domain.ObjectStorageObjectSummary {
|
|
contentType := ""
|
|
sizeBytes := int64(0)
|
|
lastModified := ""
|
|
etag := ""
|
|
storageClass := ""
|
|
versionID := ""
|
|
metadata := map[string]any{}
|
|
if meta != nil {
|
|
contentType = strings.TrimSpace(meta.ContentType)
|
|
sizeBytes = meta.ContentLength
|
|
lastModified = strings.TrimSpace(meta.LastModified)
|
|
etag = strings.Trim(meta.ETag, "\"")
|
|
storageClass = strings.TrimSpace(meta.StorageClass)
|
|
versionID = strings.TrimSpace(meta.VersionId)
|
|
for metaKey, metaValue := range meta.UserMeta {
|
|
metadata[metaKey] = metaValue
|
|
}
|
|
}
|
|
if len(metadata) == 0 {
|
|
metadata = nil
|
|
}
|
|
return domain.ObjectStorageObjectSummary{
|
|
Bucket: bucket,
|
|
Key: key,
|
|
Name: path.Base(strings.TrimRight(key, "/")),
|
|
Kind: objectStorageKindForObject(key, contentType),
|
|
SizeBytes: sizeBytes,
|
|
ContentType: contentType,
|
|
LastModified: lastModified,
|
|
ETag: etag,
|
|
VersionID: versionID,
|
|
StorageClass: storageClass,
|
|
Metadata: metadata,
|
|
}
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) ListBuckets(ctx context.Context) ([]domain.ObjectStorageBucketSummary, error) {
|
|
result, err := browser.client.ListBucketsWithContext(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets))
|
|
for _, bucket := range result.Buckets {
|
|
name := strings.TrimSpace(bucket.Name)
|
|
if name == "" {
|
|
continue
|
|
}
|
|
items = append(items, domain.ObjectStorageBucketSummary{Name: name, CreatedAt: strings.TrimSpace(bucket.CreationDate)})
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) ListObjects(ctx context.Context, bucket string, prefix string) ([]domain.ObjectStorageObjectSummary, error) {
|
|
items := make([]domain.ObjectStorageObjectSummary, 0)
|
|
seen := make(map[string]struct{})
|
|
marker := ""
|
|
for {
|
|
page, err := browser.ListObjectsPage(ctx, bucket, prefix, marker, 1000)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, item := range page.Items {
|
|
key := item.Kind + "|" + item.Key
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, item)
|
|
}
|
|
if !page.HasMore || page.NextMarker == "" || page.NextMarker == marker {
|
|
break
|
|
}
|
|
marker = page.NextMarker
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) ListObjectsPage(
|
|
ctx context.Context,
|
|
bucket string,
|
|
prefix string,
|
|
marker string,
|
|
limit int,
|
|
) (objectStoragePage, error) {
|
|
normalizedBucket := strings.TrimSpace(bucket)
|
|
normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
|
|
if limit <= 0 {
|
|
limit = 100
|
|
}
|
|
if limit > 1000 {
|
|
limit = 1000
|
|
}
|
|
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
|
|
Prefix: storageObjectRequestPrefix(prefix),
|
|
Delimiter: "/",
|
|
MaxKeys: limit,
|
|
Marker: strings.TrimSpace(marker),
|
|
})
|
|
if err != nil {
|
|
return objectStoragePage{}, err
|
|
}
|
|
if result == nil {
|
|
return objectStoragePage{Items: []domain.ObjectStorageObjectSummary{}}, nil
|
|
}
|
|
items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Contents))
|
|
seen := make(map[string]struct{})
|
|
for _, item := range result.CommonPrefixes {
|
|
prefixValue := strings.Trim(strings.TrimSpace(item.Prefix), "/")
|
|
if prefixValue == "" || prefixValue == normalizedPrefix {
|
|
continue
|
|
}
|
|
key := "dir|" + prefixValue
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, objectStorageSummaryFromPrefix(normalizedBucket, prefixValue))
|
|
}
|
|
for _, item := range result.Contents {
|
|
keyValue := strings.Trim(strings.TrimSpace(item.Key), "/")
|
|
if keyValue == "" || keyValue == normalizedPrefix {
|
|
continue
|
|
}
|
|
key := "file|" + keyValue
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
items = append(items, objectStorageSummaryFromListObject(normalizedBucket, item))
|
|
}
|
|
return objectStoragePage{
|
|
Items: items,
|
|
NextMarker: strings.TrimSpace(result.NextMarker),
|
|
HasMore: result.IsTruncated,
|
|
}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) CreateBucket(_ context.Context, bucket string) (domain.ObjectStorageBucketSummary, error) {
|
|
if _, err := browser.client.PutBucket(strings.TrimSpace(bucket)); err != nil {
|
|
return domain.ObjectStorageBucketSummary{}, err
|
|
}
|
|
return domain.ObjectStorageBucketSummary{Name: strings.TrimSpace(bucket)}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) DeleteBucket(_ context.Context, bucket string) error {
|
|
return browser.client.DeleteBucket(strings.TrimSpace(bucket))
|
|
}
|
|
|
|
func detectObjectStorageContentType(key string, payload []byte, explicit string) string {
|
|
if trimmed := strings.TrimSpace(explicit); trimmed != "" {
|
|
return trimmed
|
|
}
|
|
if guessed := mime.TypeByExtension(strings.ToLower(path.Ext(key))); guessed != "" {
|
|
return guessed
|
|
}
|
|
if len(payload) > 0 {
|
|
return http.DetectContentType(payload)
|
|
}
|
|
return "application/octet-stream"
|
|
}
|
|
|
|
func supportsObjectStorageInlinePreview(summary domain.ObjectStorageObjectSummary) bool {
|
|
contentType := strings.ToLower(strings.TrimSpace(summary.ContentType))
|
|
if summary.Kind == "json" {
|
|
return true
|
|
}
|
|
if strings.HasPrefix(contentType, "text/") {
|
|
return true
|
|
}
|
|
switch strings.ToLower(path.Ext(summary.Key)) {
|
|
case ".txt", ".log", ".md", ".csv", ".tsv", ".yaml", ".yml", ".xml", ".jsonl", ".html", ".htm":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func shouldReadObjectStorageInline(summary domain.ObjectStorageObjectSummary) bool {
|
|
if !supportsObjectStorageInlinePreview(summary) {
|
|
return false
|
|
}
|
|
if summary.SizeBytes <= 0 {
|
|
return true
|
|
}
|
|
return summary.SizeBytes <= objectStorageInlinePreviewMaxBytes
|
|
}
|
|
|
|
type objectStorageByteRange struct {
|
|
start int64
|
|
end int64
|
|
}
|
|
|
|
func parseObjectStorageByteRange(rangeHeader string, sizeBytes int64) (objectStorageByteRange, bool, error) {
|
|
trimmed := strings.TrimSpace(rangeHeader)
|
|
if trimmed == "" {
|
|
return objectStorageByteRange{}, false, nil
|
|
}
|
|
if sizeBytes <= 0 {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("object size is unknown")
|
|
}
|
|
if !strings.HasPrefix(strings.ToLower(trimmed), "bytes=") {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("unsupported range unit")
|
|
}
|
|
spec := strings.TrimSpace(trimmed[len("bytes="):])
|
|
if spec == "" || strings.Contains(spec, ",") {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("only a single byte range is supported")
|
|
}
|
|
startText, endText, found := strings.Cut(spec, "-")
|
|
if !found {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range")
|
|
}
|
|
startText = strings.TrimSpace(startText)
|
|
endText = strings.TrimSpace(endText)
|
|
if startText == "" && endText == "" {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range")
|
|
}
|
|
|
|
if startText == "" {
|
|
suffixLength, err := strconv.ParseInt(endText, 10, 64)
|
|
if err != nil || suffixLength <= 0 {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("invalid suffix byte range")
|
|
}
|
|
if suffixLength >= sizeBytes {
|
|
return objectStorageByteRange{start: 0, end: sizeBytes - 1}, true, nil
|
|
}
|
|
return objectStorageByteRange{start: sizeBytes - suffixLength, end: sizeBytes - 1}, true, nil
|
|
}
|
|
|
|
start, err := strconv.ParseInt(startText, 10, 64)
|
|
if err != nil || start < 0 {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("invalid range start")
|
|
}
|
|
if start >= sizeBytes {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("range start is outside object")
|
|
}
|
|
end := sizeBytes - 1
|
|
if endText != "" {
|
|
parsedEnd, err := strconv.ParseInt(endText, 10, 64)
|
|
if err != nil || parsedEnd < start {
|
|
return objectStorageByteRange{}, true, fmt.Errorf("invalid range end")
|
|
}
|
|
if parsedEnd < end {
|
|
end = parsedEnd
|
|
}
|
|
}
|
|
return objectStorageByteRange{start: start, end: end}, true, nil
|
|
}
|
|
|
|
func writeObjectStorageRangeHeaders(writer http.ResponseWriter, summary domain.ObjectStorageObjectSummary, contentType string, byteRange objectStorageByteRange, statusCode int) {
|
|
if contentType == "" {
|
|
contentType = detectObjectStorageContentType(summary.Key, nil, summary.ContentType)
|
|
}
|
|
if contentType != "" {
|
|
writer.Header().Set("Content-Type", contentType)
|
|
}
|
|
writer.Header().Set("Accept-Ranges", "bytes")
|
|
if summary.ETag != "" {
|
|
writer.Header().Set("ETag", strconv.Quote(summary.ETag))
|
|
}
|
|
if summary.LastModified != "" {
|
|
writer.Header().Set("Last-Modified", summary.LastModified)
|
|
}
|
|
if statusCode == http.StatusPartialContent {
|
|
length := byteRange.end - byteRange.start + 1
|
|
writer.Header().Set("Content-Length", fmt.Sprintf("%d", length))
|
|
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, summary.SizeBytes))
|
|
return
|
|
}
|
|
if summary.SizeBytes > 0 {
|
|
writer.Header().Set("Content-Length", fmt.Sprintf("%d", summary.SizeBytes))
|
|
}
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, payload, contentType)}
|
|
if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PutObjectConditional(
|
|
ctx context.Context,
|
|
bucket string,
|
|
key string,
|
|
payload []byte,
|
|
contentType string,
|
|
ifMatch string,
|
|
ifNoneMatch string,
|
|
userMeta map[string]string,
|
|
) (domain.ObjectStorageObjectSummary, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
args := &bosapi.PutObjectArgs{
|
|
ContentType: detectObjectStorageContentType(objectKey, payload, contentType),
|
|
IfMatch: strings.TrimSpace(ifMatch),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
UserMeta: userMeta,
|
|
}
|
|
if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, nil, contentType)}
|
|
if _, err := browser.client.PutObjectFromStreamWithContext(ctx, strings.TrimSpace(bucket), objectKey, reader, args); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
|
|
}
|
|
|
|
func normalizeObjectStoragePresignExpires(seconds int) int {
|
|
if seconds <= 0 {
|
|
return objectStoragePresignDefaultExpires
|
|
}
|
|
if seconds > objectStoragePresignMaxExpires {
|
|
return objectStoragePresignMaxExpires
|
|
}
|
|
return seconds
|
|
}
|
|
|
|
func objectStorageExpiresAt(seconds int) string {
|
|
return time.Now().UTC().Add(time.Duration(seconds) * time.Second).Format(time.RFC3339)
|
|
}
|
|
|
|
func objectStorageUploadHeaders(contentType string) map[string]string {
|
|
headers := map[string]string{}
|
|
if strings.TrimSpace(contentType) != "" {
|
|
headers["Content-Type"] = strings.TrimSpace(contentType)
|
|
}
|
|
return headers
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PresignPutObject(_ context.Context, bucket string, key string, contentType string, expiresInSeconds int) (domain.ObjectStoragePresignedUploadResult, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
headers := objectStorageUploadHeaders(contentType)
|
|
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, nil)
|
|
return domain.ObjectStoragePresignedUploadResult{
|
|
Bucket: strings.TrimSpace(bucket),
|
|
Key: objectKey,
|
|
Method: http.MethodPut,
|
|
URL: url,
|
|
Headers: headers,
|
|
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
|
}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PresignPutObjectBound(
|
|
_ context.Context,
|
|
bucket string,
|
|
key string,
|
|
contentType string,
|
|
contentMD5 string,
|
|
sizeBytes int64,
|
|
metadata map[string]string,
|
|
expiresInSeconds int,
|
|
) (domain.ObjectStoragePresignedUploadResult, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
headers := objectStorageUploadHeaders(contentType)
|
|
headers["Content-MD5"] = strings.TrimSpace(contentMD5)
|
|
headers["Content-Length"] = strconv.FormatInt(sizeBytes, 10)
|
|
for metaKey, metaValue := range metadata {
|
|
headers["x-bce-meta-"+strings.ToLower(strings.TrimSpace(metaKey))] = strings.TrimSpace(metaValue)
|
|
}
|
|
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, nil)
|
|
return domain.ObjectStoragePresignedUploadResult{
|
|
Bucket: strings.TrimSpace(bucket),
|
|
Key: objectKey,
|
|
Method: http.MethodPut,
|
|
URL: url,
|
|
Headers: headers,
|
|
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
|
}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucket string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil)
|
|
if err != nil {
|
|
return domain.ObjectStorageMultipartUploadCreateResult{}, err
|
|
}
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
return domain.ObjectStorageMultipartUploadCreateResult{
|
|
Bucket: result.Bucket,
|
|
Key: result.Key,
|
|
UploadID: result.UploadId,
|
|
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
|
}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) InitiateMultipartUploadBound(
|
|
_ context.Context,
|
|
bucket string,
|
|
key string,
|
|
contentType string,
|
|
metadata map[string]string,
|
|
expiresInSeconds int,
|
|
) (domain.ObjectStorageMultipartUploadCreateResult, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil)
|
|
if err != nil {
|
|
return domain.ObjectStorageMultipartUploadCreateResult{}, err
|
|
}
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
return domain.ObjectStorageMultipartUploadCreateResult{
|
|
Bucket: result.Bucket,
|
|
Key: result.Key,
|
|
UploadID: result.UploadId,
|
|
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
|
}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucket string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
params := map[string]string{
|
|
"uploadId": strings.TrimSpace(uploadID),
|
|
"partNumber": strconv.Itoa(partNumber),
|
|
}
|
|
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, nil, params)
|
|
return domain.ObjectStorageMultipartUploadPartURLResult{
|
|
Bucket: strings.TrimSpace(bucket),
|
|
Key: objectKey,
|
|
UploadID: strings.TrimSpace(uploadID),
|
|
PartNumber: partNumber,
|
|
Method: http.MethodPut,
|
|
URL: url,
|
|
Headers: map[string]string{},
|
|
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
|
}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PresignMultipartUploadPartBound(
|
|
_ context.Context,
|
|
bucket string,
|
|
key string,
|
|
uploadID string,
|
|
partNumber int,
|
|
contentMD5 string,
|
|
sizeBytes int64,
|
|
expiresInSeconds int,
|
|
) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
params := map[string]string{
|
|
"uploadId": strings.TrimSpace(uploadID),
|
|
"partNumber": strconv.Itoa(partNumber),
|
|
}
|
|
headers := map[string]string{
|
|
"Content-MD5": strings.TrimSpace(contentMD5),
|
|
"Content-Length": strconv.FormatInt(sizeBytes, 10),
|
|
}
|
|
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, params)
|
|
return domain.ObjectStorageMultipartUploadPartURLResult{
|
|
Bucket: strings.TrimSpace(bucket),
|
|
Key: objectKey,
|
|
UploadID: strings.TrimSpace(uploadID),
|
|
PartNumber: partNumber,
|
|
Method: http.MethodPut,
|
|
URL: url,
|
|
Headers: headers,
|
|
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
|
}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) {
|
|
return browser.CompleteMultipartUploadBound(ctx, bucket, key, uploadID, parts, nil)
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) CompleteMultipartUploadBound(
|
|
ctx context.Context,
|
|
bucket string,
|
|
key string,
|
|
uploadID string,
|
|
parts []domain.ObjectStorageMultipartUploadPart,
|
|
metadata map[string]string,
|
|
) (domain.ObjectStorageObjectSummary, error) {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
uploadParts := make([]bosapi.UploadInfoType, 0, len(parts))
|
|
for _, part := range parts {
|
|
uploadParts = append(uploadParts, bosapi.UploadInfoType{PartNumber: part.PartNumber, ETag: strings.Trim(part.ETag, `"`)})
|
|
}
|
|
if _, err := browser.client.CompleteMultipartUploadFromStruct(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID), &bosapi.CompleteMultipartUploadArgs{Parts: uploadParts, UserMeta: metadata}); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.getObjectSummaryAfterWrite(ctx, bucket, objectKey)
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) AbortMultipartUpload(_ context.Context, bucket string, key string, uploadID string) error {
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
return browser.client.AbortMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID))
|
|
}
|
|
|
|
func objectSyncMultipartCopyPartSize(sizeBytes int64) int64 {
|
|
const (
|
|
minimum = int64(256 * 1024 * 1024)
|
|
maximum = int64(5 * 1024 * 1024 * 1024)
|
|
parts = int64(1_024)
|
|
align = int64(1024 * 1024)
|
|
)
|
|
partSize := minimum
|
|
if sizeBytes > partSize*parts {
|
|
partSize = ((sizeBytes/parts + align - 1) / align) * align
|
|
}
|
|
if partSize > maximum {
|
|
return maximum
|
|
}
|
|
return partSize
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) PublishMultipartCopyConditional(
|
|
ctx context.Context,
|
|
bucket string,
|
|
sourceKey string,
|
|
targetKey string,
|
|
sizeBytes int64,
|
|
sourceETag string,
|
|
ifMatch string,
|
|
ifNoneMatch string,
|
|
userMeta map[string]string,
|
|
) (domain.ObjectStorageObjectSummary, error) {
|
|
normalizedBucket := strings.TrimSpace(bucket)
|
|
sourceKey = exactObjectStorageKey(sourceKey)
|
|
targetKey = exactObjectStorageKey(targetKey)
|
|
if normalizedBucket == "" || sourceKey == "" || targetKey == "" {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("bucket, source key, and target key are required")
|
|
}
|
|
if sizeBytes <= 0 {
|
|
return browser.PutObjectConditional(ctx, normalizedBucket, targetKey, []byte{}, "application/octet-stream", ifMatch, ifNoneMatch, userMeta)
|
|
}
|
|
initiated, err := browser.client.InitiateMultipartUpload(normalizedBucket, targetKey, "", nil)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
abort := true
|
|
defer func() {
|
|
if abort {
|
|
_ = browser.client.AbortMultipartUpload(normalizedBucket, targetKey, initiated.UploadId)
|
|
}
|
|
}()
|
|
partSize := objectSyncMultipartCopyPartSize(sizeBytes)
|
|
partCount := int((sizeBytes + partSize - 1) / partSize)
|
|
parts := make([]bosapi.UploadInfoType, partCount)
|
|
copyGroup, copyContext := errgroup.WithContext(ctx)
|
|
copyGroup.SetLimit(8)
|
|
for partIndex := range partCount {
|
|
start := int64(partIndex) * partSize
|
|
end := start + partSize - 1
|
|
if end >= sizeBytes {
|
|
end = sizeBytes - 1
|
|
}
|
|
partNumber := partIndex + 1
|
|
copyGroup.Go(func() error {
|
|
result, copyErr := browser.client.UploadPartCopyWithContext(
|
|
copyContext,
|
|
normalizedBucket,
|
|
targetKey,
|
|
normalizedBucket,
|
|
sourceKey,
|
|
initiated.UploadId,
|
|
partNumber,
|
|
&bosapi.UploadPartCopyArgs{
|
|
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
|
|
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
|
|
},
|
|
)
|
|
if copyErr != nil {
|
|
return copyErr
|
|
}
|
|
parts[partIndex] = bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)}
|
|
return nil
|
|
})
|
|
}
|
|
if err := copyGroup.Wait(); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
completeArgs := &bosapi.CompleteMultipartUploadArgs{
|
|
Parts: parts,
|
|
UserMeta: userMeta,
|
|
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
}
|
|
completePayload, err := json.Marshal(completeArgs)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
body, err := bce.NewBodyFromBytes(completePayload)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
bosContext := browser.client.NewBosContext(ctx)
|
|
if _, err = bosapi.CompleteMultipartUpload(browser.client, normalizedBucket, targetKey, initiated.UploadId, body, completeArgs, bosContext); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
abort = false
|
|
return browser.getObjectSummaryAfterWrite(ctx, normalizedBucket, targetKey)
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) GetBucketVersioning(_ context.Context, bucket string) (string, error) {
|
|
result, err := browser.client.GetBucketVersioning(strings.TrimSpace(bucket))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if result == nil {
|
|
return "", nil
|
|
}
|
|
return strings.ToLower(strings.TrimSpace(result.Status)), nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
|
|
objectKey := exactObjectStorageKey(key)
|
|
destinationKey := exactObjectStorageKey(newKey)
|
|
if objectKey == "" || destinationKey == "" {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("source and destination keys are required")
|
|
}
|
|
if objectKey == destinationKey {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination key must differ from source key")
|
|
}
|
|
if _, err := browser.client.CopyObjectWithContext(ctx, strings.TrimSpace(bucket), destinationKey, strings.TrimSpace(bucket), objectKey, nil); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
if err := browser.client.DeleteObject(strings.TrimSpace(bucket), objectKey); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.getObjectSummaryAfterWrite(ctx, bucket, destinationKey)
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) MovePrefix(ctx context.Context, bucket string, prefix string, newPrefix string) (domain.ObjectStorageObjectSummary, error) {
|
|
normalizedBucket := strings.TrimSpace(bucket)
|
|
sourcePrefix := strings.Trim(strings.TrimSpace(prefix), "/")
|
|
destinationPrefix := strings.Trim(strings.TrimSpace(newPrefix), "/")
|
|
if normalizedBucket == "" || sourcePrefix == "" || destinationPrefix == "" {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("bucket, source prefix, and destination prefix are required")
|
|
}
|
|
if sourcePrefix == destinationPrefix {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination prefix must differ from source prefix")
|
|
}
|
|
if strings.HasPrefix(destinationPrefix, sourcePrefix+"/") {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("destination prefix cannot be inside the source prefix")
|
|
}
|
|
|
|
sourceRequestPrefix := sourcePrefix + "/"
|
|
objectsToMove := make([]string, 0)
|
|
for _, exactKey := range []string{sourcePrefix, sourcePrefix + "/"} {
|
|
objectsToMove = append(objectsToMove, exactKey)
|
|
}
|
|
seen := map[string]struct{}{
|
|
sourcePrefix: {},
|
|
sourcePrefix + "/": {},
|
|
}
|
|
marker := ""
|
|
for {
|
|
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
|
|
Prefix: sourceRequestPrefix,
|
|
MaxKeys: 1000,
|
|
Marker: marker,
|
|
})
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
for _, item := range result.Contents {
|
|
key := exactObjectStorageKey(item.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
objectsToMove = append(objectsToMove, key)
|
|
}
|
|
if result == nil || !result.IsTruncated {
|
|
break
|
|
}
|
|
nextMarker := strings.TrimSpace(result.NextMarker)
|
|
if nextMarker == "" || nextMarker == marker {
|
|
break
|
|
}
|
|
marker = nextMarker
|
|
}
|
|
|
|
copied := make([]string, 0, len(objectsToMove))
|
|
for _, sourceKey := range objectsToMove {
|
|
destinationKey := destinationPrefix
|
|
if strings.HasPrefix(sourceKey, sourceRequestPrefix) {
|
|
destinationKey = destinationPrefix + "/" + strings.TrimPrefix(sourceKey, sourceRequestPrefix)
|
|
} else if strings.HasSuffix(sourceKey, "/") {
|
|
destinationKey = destinationPrefix + "/"
|
|
}
|
|
if _, err := browser.client.CopyObjectWithContext(ctx, normalizedBucket, destinationKey, normalizedBucket, sourceKey, nil); err != nil {
|
|
if isBOSNoSuchKey(err) && (sourceKey == sourcePrefix || sourceKey == sourcePrefix+"/") {
|
|
continue
|
|
}
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
copied = append(copied, sourceKey)
|
|
}
|
|
if len(copied) == 0 {
|
|
return domain.ObjectStorageObjectSummary{}, os.ErrNotExist
|
|
}
|
|
for _, sourceKey := range copied {
|
|
if err := browser.client.DeleteObject(normalizedBucket, sourceKey); err != nil && !isBOSNoSuchKey(err) {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
}
|
|
return objectStorageSummaryFromPrefix(normalizedBucket, destinationPrefix), nil
|
|
}
|
|
|
|
func objectStorageBaseName(key string) string {
|
|
normalized := strings.Trim(strings.TrimSpace(key), "/")
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
segments := strings.Split(normalized, "/")
|
|
return segments[len(segments)-1]
|
|
}
|
|
|
|
func objectStorageMoveDestination(sourceKey string, destinationPrefix string) string {
|
|
baseName := objectStorageBaseName(sourceKey)
|
|
normalizedDestination := strings.Trim(strings.TrimSpace(destinationPrefix), "/")
|
|
if baseName == "" {
|
|
return normalizedDestination
|
|
}
|
|
if normalizedDestination == "" {
|
|
return baseName
|
|
}
|
|
return normalizedDestination + "/" + baseName
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) DeleteObject(_ context.Context, bucket string, key string) error {
|
|
return browser.client.DeleteObject(strings.TrimSpace(bucket), exactObjectStorageKey(key))
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) DeletePrefix(ctx context.Context, bucket string, prefix string) error {
|
|
normalizedBucket := strings.TrimSpace(bucket)
|
|
normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
|
|
if normalizedBucket == "" || normalizedPrefix == "" {
|
|
return fmt.Errorf("bucket and prefix are required")
|
|
}
|
|
requestPrefix := normalizedPrefix + "/"
|
|
marker := ""
|
|
for {
|
|
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
|
|
Prefix: requestPrefix,
|
|
MaxKeys: 1000,
|
|
Marker: marker,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, item := range result.Contents {
|
|
key := exactObjectStorageKey(item.Key)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if err := browser.client.DeleteObject(normalizedBucket, key); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if result == nil || !result.IsTruncated {
|
|
break
|
|
}
|
|
nextMarker := strings.TrimSpace(result.NextMarker)
|
|
if nextMarker == "" || nextMarker == marker {
|
|
break
|
|
}
|
|
marker = nextMarker
|
|
}
|
|
for _, markerKey := range []string{normalizedPrefix + "/", normalizedPrefix} {
|
|
if err := browser.client.DeleteObject(normalizedBucket, markerKey); err != nil && !isBOSNoSuchKey(err) {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) GetObjectSummary(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) {
|
|
objectKey := exactObjectStorageKey(key)
|
|
meta, err := browser.client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), objectKey)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return objectStorageSummaryFromMeta(strings.TrimSpace(bucket), objectKey, meta), nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) getObjectSummaryAfterWrite(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) {
|
|
return retryObjectStorageSummary(ctx, objectStorageWriteMetadataRetryAttempts, objectStorageWriteMetadataRetryDelay, func() (domain.ObjectStorageObjectSummary, error) {
|
|
return browser.GetObjectSummary(ctx, bucket, key)
|
|
})
|
|
}
|
|
|
|
func retryObjectStorageSummary(
|
|
ctx context.Context,
|
|
attempts int,
|
|
delay time.Duration,
|
|
lookup func() (domain.ObjectStorageObjectSummary, error),
|
|
) (domain.ObjectStorageObjectSummary, error) {
|
|
if attempts < 1 {
|
|
attempts = 1
|
|
}
|
|
for attempt := 0; attempt < attempts; attempt++ {
|
|
summary, err := lookup()
|
|
if err == nil || !isBOSNoSuchKey(err) || attempt == attempts-1 {
|
|
return summary, err
|
|
}
|
|
timer := time.NewTimer(delay)
|
|
select {
|
|
case <-ctx.Done():
|
|
if !timer.Stop() {
|
|
<-timer.C
|
|
}
|
|
return domain.ObjectStorageObjectSummary{}, ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
return domain.ObjectStorageObjectSummary{}, nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) ReadObject(ctx context.Context, bucket string, key string) ([]byte, string, error) {
|
|
objectKey := exactObjectStorageKey(key)
|
|
result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
defer result.Body.Close()
|
|
payload, err := io.ReadAll(result.Body)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
return payload, strings.TrimSpace(result.ContentType), nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) ReadObjectRange(ctx context.Context, bucket string, key string, start int64, end int64) (io.ReadCloser, string, error) {
|
|
objectKey := exactObjectStorageKey(key)
|
|
result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil, start, end)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
return result.Body, strings.TrimSpace(result.ContentType), nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucket string, key string, maxBytes int64) ([]byte, string, error) {
|
|
if maxBytes <= 0 {
|
|
return browser.ReadObject(ctx, bucket, key)
|
|
}
|
|
objectKey := exactObjectStorageKey(key)
|
|
result, err := browser.client.GetObjectWithContext(
|
|
ctx,
|
|
strings.TrimSpace(bucket),
|
|
objectKey,
|
|
nil,
|
|
0,
|
|
maxBytes-1,
|
|
)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
defer result.Body.Close()
|
|
payload, err := io.ReadAll(io.LimitReader(result.Body, maxBytes))
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
return payload, strings.TrimSpace(result.ContentType), nil
|
|
}
|
|
|
|
func (browser *bosObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucket string, key string) (string, error) {
|
|
previewURL := strings.TrimSpace(browser.client.BasicGeneratePresignedUrl(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), 3600))
|
|
if previewURL == "" {
|
|
return "", fmt.Errorf("failed to generate BOS preview url")
|
|
}
|
|
return previewURL, nil
|
|
}
|
|
|
|
func decodeObjectStorageBucketCreateInput(request *http.Request) (domain.ObjectStorageBucketCreateInput, error) {
|
|
input := domain.ObjectStorageBucketCreateInput{}
|
|
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
|
|
return domain.ObjectStorageBucketCreateInput{}, err
|
|
}
|
|
return input, nil
|
|
}
|
|
|
|
func decodeObjectStorageObjectPutInput(request *http.Request) (domain.ObjectStorageObjectPutInput, error) {
|
|
input := domain.ObjectStorageObjectPutInput{}
|
|
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
|
|
return domain.ObjectStorageObjectPutInput{}, err
|
|
}
|
|
return input, nil
|
|
}
|
|
|
|
func decodeObjectStorageObjectRenameInput(request *http.Request) (domain.ObjectStorageObjectRenameInput, error) {
|
|
input := domain.ObjectStorageObjectRenameInput{}
|
|
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
|
|
return domain.ObjectStorageObjectRenameInput{}, err
|
|
}
|
|
return input, nil
|
|
}
|
|
|
|
type objectStoragePresignPutInput struct {
|
|
Bucket string `json:"bucket"`
|
|
Key string `json:"key"`
|
|
ContentType string `json:"contentType,omitempty"`
|
|
ExpiresInSeconds int `json:"expiresInSeconds,omitempty"`
|
|
}
|
|
|
|
type objectStorageMultipartInitiateInput struct {
|
|
Bucket string `json:"bucket"`
|
|
Key string `json:"key"`
|
|
ContentType string `json:"contentType,omitempty"`
|
|
ExpiresInSeconds int `json:"expiresInSeconds,omitempty"`
|
|
}
|
|
|
|
type objectStorageMultipartPartURLInput struct {
|
|
Bucket string `json:"bucket"`
|
|
Key string `json:"key"`
|
|
UploadID string `json:"uploadId"`
|
|
PartNumber int `json:"partNumber"`
|
|
ExpiresInSeconds int `json:"expiresInSeconds,omitempty"`
|
|
}
|
|
|
|
type objectStorageMultipartCompleteInput struct {
|
|
Bucket string `json:"bucket"`
|
|
Key string `json:"key"`
|
|
UploadID string `json:"uploadId"`
|
|
Parts []domain.ObjectStorageMultipartUploadPart `json:"parts"`
|
|
}
|
|
|
|
type objectStorageMultipartAbortInput struct {
|
|
Bucket string `json:"bucket"`
|
|
Key string `json:"key"`
|
|
UploadID string `json:"uploadId"`
|
|
}
|
|
|
|
func decodeObjectStorageJSON[T any](request *http.Request) (T, error) {
|
|
var input T
|
|
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
|
|
return input, err
|
|
}
|
|
return input, nil
|
|
}
|
|
|
|
func (server *server) resolveObjectStorageBrowser(ctx context.Context, connectionID string) (objectStorageBrowser, *domain.StorageConnectionRecord, func(), error) {
|
|
connection, err := server.getStorageConnection(ctx, connectionID)
|
|
if err != nil && strings.TrimSpace(connectionID) != "" {
|
|
return nil, nil, nil, err
|
|
}
|
|
if connection == nil {
|
|
return nil, nil, nil, nil
|
|
}
|
|
connection = server.storageConnectionWithResolvedEndpoint(connection)
|
|
browser, cleanup, err := storageConnectionBrowserResolver(ctx, connection)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
return browser, connection, cleanup, nil
|
|
}
|
|
|
|
func resolveObjectStorageBucket(connection *domain.StorageConnectionRecord, requested string) (string, error) {
|
|
requested = strings.Trim(strings.TrimSpace(requested), "/")
|
|
if connection == nil {
|
|
return requested, nil
|
|
}
|
|
boundBucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/")
|
|
if boundBucket == "" {
|
|
return requested, nil
|
|
}
|
|
if requested != "" && requested != boundBucket {
|
|
return "", fmt.Errorf("connection is bound to bucket %s", boundBucket)
|
|
}
|
|
return boundBucket, nil
|
|
}
|
|
|
|
func renderObjectStorageWriteError(writer http.ResponseWriter, err error) {
|
|
if err == nil {
|
|
return
|
|
}
|
|
if isRequestTooLarge(err) {
|
|
renderJSON(writer, http.StatusRequestEntityTooLarge, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
|
|
renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
}
|
|
|
|
func isRequestTooLarge(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
var maxBytesErr *http.MaxBytesError
|
|
return errors.As(err, &maxBytesErr)
|
|
}
|
|
|
|
func exactObjectStorageKey(key string) string {
|
|
return strings.TrimSpace(key)
|
|
}
|
|
|
|
func isBOSNoSuchKey(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
var serviceErr *bce.BceServiceError
|
|
if errors.As(err, &serviceErr) {
|
|
code := strings.TrimSpace(serviceErr.Code)
|
|
return strings.EqualFold(code, "NoSuchKey") ||
|
|
(serviceErr.StatusCode == http.StatusNotFound && strings.EqualFold(code, "NotFound"))
|
|
}
|
|
return strings.Contains(strings.ToLower(err.Error()), "nosuchkey")
|
|
}
|
|
|
|
func (server *server) handleObjectStorageOverview(writer http.ResponseWriter, request *http.Request) {
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{
|
|
ConnectionID: connectionID,
|
|
Provider: "BOS",
|
|
Configured: false,
|
|
Buckets: []domain.ObjectStorageBucketSummary{},
|
|
})
|
|
return
|
|
}
|
|
bucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/")
|
|
items := []domain.ObjectStorageBucketSummary{}
|
|
if bucket != "" {
|
|
items = append(items, domain.ObjectStorageBucketSummary{Name: bucket})
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{
|
|
ConnectionID: connection.ID,
|
|
ConnectionName: connection.Name,
|
|
Provider: string(connection.Provider),
|
|
Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint),
|
|
Bucket: bucket,
|
|
Configured: true,
|
|
Buckets: items,
|
|
})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, request *http.Request) {
|
|
query := request.URL.Query()
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
|
|
return
|
|
}
|
|
prefix := strings.Trim(strings.TrimSpace(query.Get("prefix")), "/")
|
|
marker := strings.TrimSpace(query.Get("marker"))
|
|
paginationRequested := query.Has("limit") || query.Has("marker")
|
|
limit := 100
|
|
if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" {
|
|
parsedLimit, parseErr := strconv.Atoi(rawLimit)
|
|
if parseErr != nil || parsedLimit < 1 || parsedLimit > 1000 {
|
|
renderValidationError(writer, validationIssue{Field: "limit", Message: "limit must be an integer between 1 and 1000"})
|
|
return
|
|
}
|
|
limit = parsedLimit
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{
|
|
ConnectionID: connectionID,
|
|
Provider: "BOS",
|
|
Configured: false,
|
|
Bucket: bucket,
|
|
Prefix: prefix,
|
|
Items: []domain.ObjectStorageObjectSummary{},
|
|
Limit: limit,
|
|
})
|
|
return
|
|
}
|
|
page := objectStoragePage{}
|
|
if paginationRequested {
|
|
if pagedBrowser, ok := browser.(pagedObjectStorageBrowser); ok {
|
|
page, err = pagedBrowser.ListObjectsPage(request.Context(), bucket, prefix, marker, limit)
|
|
} else {
|
|
var items []domain.ObjectStorageObjectSummary
|
|
items, err = browser.ListObjects(request.Context(), bucket, prefix)
|
|
page = paginateObjectStorageItems(items, marker, limit)
|
|
}
|
|
} else {
|
|
page.Items, err = browser.ListObjects(request.Context(), bucket, prefix)
|
|
}
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{
|
|
ConnectionID: connection.ID,
|
|
ConnectionName: connection.Name,
|
|
Provider: string(connection.Provider),
|
|
Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint),
|
|
Configured: true,
|
|
Bucket: bucket,
|
|
Prefix: prefix,
|
|
Items: page.Items,
|
|
NextMarker: page.NextMarker,
|
|
HasMore: page.HasMore,
|
|
Limit: limit,
|
|
})
|
|
}
|
|
|
|
func paginateObjectStorageItems(items []domain.ObjectStorageObjectSummary, marker string, limit int) objectStoragePage {
|
|
start := 0
|
|
marker = strings.TrimSpace(marker)
|
|
if marker != "" {
|
|
for index, item := range items {
|
|
if item.Key == marker {
|
|
start = index + 1
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if start > len(items) {
|
|
start = len(items)
|
|
}
|
|
end := min(start+limit, len(items))
|
|
pageItems := append([]domain.ObjectStorageObjectSummary(nil), items[start:end]...)
|
|
hasMore := end < len(items)
|
|
nextMarker := ""
|
|
if hasMore && len(pageItems) > 0 {
|
|
nextMarker = pageItems[len(pageItems)-1].Key
|
|
}
|
|
return objectStoragePage{Items: pageItems, NextMarker: nextMarker, HasMore: hasMore}
|
|
}
|
|
|
|
func (server *server) handleObjectStorageBucketCreate(writer http.ResponseWriter, request *http.Request) {
|
|
renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageBucketDelete(writer http.ResponseWriter, request *http.Request) {
|
|
renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter, request *http.Request) {
|
|
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageJSONPutMaxBytes)
|
|
input, err := decodeObjectStorageObjectPutInput(request)
|
|
if err != nil {
|
|
if isRequestTooLarge(err) {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
|
|
return
|
|
}
|
|
if strings.TrimSpace(input.Key) == "" {
|
|
renderValidationError(writer,
|
|
validationIssue{Field: "key", Message: "key is required"},
|
|
)
|
|
return
|
|
}
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
|
|
if err != nil || bucket == "" {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
|
|
return
|
|
}
|
|
item, err := browser.PutObject(request.Context(), bucket, input.Key, []byte(input.Content), input.ContentType)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter, request *http.Request) {
|
|
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageMultipartUploadMaxBytes)
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
|
|
return
|
|
}
|
|
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
|
|
if bucket == "" || key == "" {
|
|
renderValidationError(writer,
|
|
validationIssue{Field: "bucket", Message: "bucket is required"},
|
|
validationIssue{Field: "key", Message: "key is required"},
|
|
)
|
|
return
|
|
}
|
|
if err := request.ParseMultipartForm(32 << 20); err != nil {
|
|
if isRequestTooLarge(err) {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderValidationError(writer, validationIssue{Field: "file", Message: "invalid multipart payload"})
|
|
return
|
|
}
|
|
file, header, err := request.FormFile("file")
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "file", Message: "file is required"})
|
|
return
|
|
}
|
|
defer file.Close()
|
|
contentType := strings.TrimSpace(request.URL.Query().Get("contentType"))
|
|
if contentType == "" && header != nil {
|
|
contentType = strings.TrimSpace(header.Header.Get("Content-Type"))
|
|
}
|
|
item, err := browser.PutObjectStream(request.Context(), bucket, key, file, contentType)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
|
|
}
|
|
|
|
func (server *server) resolveObjectStorageUploadTarget(writer http.ResponseWriter, request *http.Request, requestedBucket string, key string) (objectStorageBrowser, string, string, func(), bool) {
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return nil, "", "", cleanup, false
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return nil, "", "", cleanup, false
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, requestedBucket)
|
|
if err != nil || bucket == "" {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
|
|
return nil, "", "", cleanup, false
|
|
}
|
|
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
|
if objectKey == "" {
|
|
renderValidationError(writer, validationIssue{Field: "key", Message: "key is required"})
|
|
return nil, "", "", cleanup, false
|
|
}
|
|
return browser, bucket, objectKey, cleanup, true
|
|
}
|
|
|
|
func renderObjectStorageInvalidJSON(writer http.ResponseWriter) {
|
|
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
|
|
}
|
|
|
|
func normalizeObjectStorageBatchRelativePath(rawValue string) (string, error) {
|
|
value := strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/")
|
|
value = strings.TrimSuffix(value, "/")
|
|
if value == "" {
|
|
return "", fmt.Errorf("relativePath is required")
|
|
}
|
|
if strings.HasPrefix(value, "/") || strings.ContainsRune(value, '\x00') {
|
|
return "", fmt.Errorf("relativePath must be a relative object path")
|
|
}
|
|
parts := strings.Split(value, "/")
|
|
for _, part := range parts {
|
|
if part == "" || part == "." || part == ".." {
|
|
return "", fmt.Errorf("relativePath contains an invalid path segment")
|
|
}
|
|
}
|
|
return strings.Join(parts, "/"), nil
|
|
}
|
|
|
|
func normalizeObjectStorageBatchPrefix(rawValue string) (string, error) {
|
|
value := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rawValue), "\\", "/"), "/")
|
|
if value == "" {
|
|
return "", nil
|
|
}
|
|
return normalizeObjectStorageBatchRelativePath(value)
|
|
}
|
|
|
|
func objectStorageBatchItemKey(prefix string, relativePath string, kind string) string {
|
|
key := relativePath
|
|
if kind == "directory" {
|
|
key += "/.keep"
|
|
}
|
|
if prefix != "" {
|
|
key = prefix + "/" + key
|
|
}
|
|
return key
|
|
}
|
|
|
|
func prepareObjectStorageBatchItems(input domain.ObjectStorageBatchUploadPrepareInput) ([]domain.ObjectStorageBatchUploadPreparedItem, string, string, *validationIssue) {
|
|
policy := strings.ToLower(strings.TrimSpace(input.ConflictPolicy))
|
|
if policy == "" {
|
|
policy = "overwrite"
|
|
}
|
|
if policy != "overwrite" && policy != "skip" && policy != "error" {
|
|
return nil, "", "", &validationIssue{Field: "conflictPolicy", Message: "conflictPolicy must be overwrite, skip, or error"}
|
|
}
|
|
prefix, err := normalizeObjectStorageBatchPrefix(input.Prefix)
|
|
if err != nil {
|
|
return nil, "", "", &validationIssue{Field: "prefix", Message: err.Error()}
|
|
}
|
|
if len(input.Items) == 0 {
|
|
return nil, "", "", &validationIssue{Field: "items", Message: "items are required"}
|
|
}
|
|
if len(input.Items) > objectStorageBatchUploadMaxItems {
|
|
return nil, "", "", &validationIssue{Field: "items", Message: fmt.Sprintf("items must contain at most %d entries", objectStorageBatchUploadMaxItems)}
|
|
}
|
|
prepared := make([]domain.ObjectStorageBatchUploadPreparedItem, 0, len(input.Items))
|
|
seenKeys := make(map[string]struct{}, len(input.Items))
|
|
for index, item := range input.Items {
|
|
kind := strings.ToLower(strings.TrimSpace(item.Kind))
|
|
if kind != "file" && kind != "directory" {
|
|
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].kind", index), Message: "kind must be file or directory"}
|
|
}
|
|
if item.SizeBytes < 0 {
|
|
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].sizeBytes", index), Message: "sizeBytes must not be negative"}
|
|
}
|
|
relativePath, pathErr := normalizeObjectStorageBatchRelativePath(item.RelativePath)
|
|
if pathErr != nil {
|
|
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: pathErr.Error()}
|
|
}
|
|
key := objectStorageBatchItemKey(prefix, relativePath, kind)
|
|
keyBytes := len(key)
|
|
if keyBytes > objectStorageKeyMaxUTF8Bytes {
|
|
return nil, "", "", &validationIssue{
|
|
Field: fmt.Sprintf("items[%d].relativePath", index),
|
|
Message: fmt.Sprintf("destination object key must not exceed %d UTF-8 bytes (got %d)", objectStorageKeyMaxUTF8Bytes, keyBytes),
|
|
}
|
|
}
|
|
if _, exists := seenKeys[key]; exists {
|
|
return nil, "", "", &validationIssue{Field: fmt.Sprintf("items[%d].relativePath", index), Message: "duplicate destination key"}
|
|
}
|
|
seenKeys[key] = struct{}{}
|
|
prepared = append(prepared, domain.ObjectStorageBatchUploadPreparedItem{
|
|
RelativePath: relativePath,
|
|
Key: key,
|
|
Kind: kind,
|
|
SizeBytes: item.SizeBytes,
|
|
ContentType: strings.TrimSpace(item.ContentType),
|
|
})
|
|
}
|
|
return prepared, prefix, policy, nil
|
|
}
|
|
|
|
func inspectObjectStorageBatchConflicts(ctx context.Context, browser objectStorageBrowser, bucket string, items []domain.ObjectStorageBatchUploadPreparedItem) error {
|
|
workerCount := min(objectStorageBatchUploadStatConcurrency, len(items))
|
|
indices := make(chan int)
|
|
var firstErr error
|
|
var errOnce sync.Once
|
|
var workers sync.WaitGroup
|
|
workers.Add(workerCount)
|
|
for range workerCount {
|
|
go func() {
|
|
defer workers.Done()
|
|
for index := range indices {
|
|
_, err := browser.GetObjectSummary(ctx, bucket, items[index].Key)
|
|
if err == nil {
|
|
items[index].Exists = true
|
|
continue
|
|
}
|
|
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
|
|
continue
|
|
}
|
|
errOnce.Do(func() { firstErr = err })
|
|
}
|
|
}()
|
|
}
|
|
for index := range items {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
indices <- index
|
|
}
|
|
close(indices)
|
|
workers.Wait()
|
|
if firstErr != nil {
|
|
return firstErr
|
|
}
|
|
return ctx.Err()
|
|
}
|
|
|
|
func finalizeObjectStorageBatchPlan(items []domain.ObjectStorageBatchUploadPreparedItem, policy string) domain.ObjectStorageBatchUploadPrepareResult {
|
|
result := domain.ObjectStorageBatchUploadPrepareResult{ConflictPolicy: policy, Items: items}
|
|
for index := range result.Items {
|
|
item := &result.Items[index]
|
|
if item.Exists && policy == "skip" {
|
|
item.Action = "skip"
|
|
item.Message = "destination already exists"
|
|
result.SkipCount++
|
|
continue
|
|
}
|
|
if item.Exists && policy == "error" {
|
|
item.Action = "conflict"
|
|
item.Message = "destination already exists"
|
|
result.ConflictCount++
|
|
continue
|
|
}
|
|
if item.Kind == "directory" {
|
|
item.Action = "create_placeholder"
|
|
result.PlaceholderCount++
|
|
} else {
|
|
item.Action = "upload"
|
|
result.UploadCount++
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (server *server) handleObjectStorageBatchUploadPrepare(writer http.ResponseWriter, request *http.Request) {
|
|
input, err := decodeObjectStorageJSON[domain.ObjectStorageBatchUploadPrepareInput](request)
|
|
if err != nil {
|
|
renderObjectStorageInvalidJSON(writer)
|
|
return
|
|
}
|
|
items, prefix, policy, issue := prepareObjectStorageBatchItems(input)
|
|
if issue != nil {
|
|
renderValidationError(writer, *issue)
|
|
return
|
|
}
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
|
|
if err != nil || bucket == "" {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
|
|
return
|
|
}
|
|
if err := inspectObjectStorageBatchConflicts(request.Context(), browser, bucket, items); err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
result := finalizeObjectStorageBatchPlan(items, policy)
|
|
result.Bucket = bucket
|
|
result.Prefix = prefix
|
|
renderJSON(writer, http.StatusOK, result)
|
|
}
|
|
|
|
func (server *server) handleObjectStoragePresignPut(writer http.ResponseWriter, request *http.Request) {
|
|
input, err := decodeObjectStorageJSON[objectStoragePresignPutInput](request)
|
|
if err != nil {
|
|
renderObjectStorageInvalidJSON(writer)
|
|
return
|
|
}
|
|
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if !ok {
|
|
return
|
|
}
|
|
result, err := browser.PresignPutObject(request.Context(), bucket, key, input.ContentType, input.ExpiresInSeconds)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, result)
|
|
}
|
|
|
|
func (server *server) handleObjectStorageMultipartInitiate(writer http.ResponseWriter, request *http.Request) {
|
|
input, err := decodeObjectStorageJSON[objectStorageMultipartInitiateInput](request)
|
|
if err != nil {
|
|
renderObjectStorageInvalidJSON(writer)
|
|
return
|
|
}
|
|
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if !ok {
|
|
return
|
|
}
|
|
result, err := browser.InitiateMultipartUpload(request.Context(), bucket, key, input.ContentType, input.ExpiresInSeconds)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, result)
|
|
}
|
|
|
|
func (server *server) handleObjectStorageMultipartPartURL(writer http.ResponseWriter, request *http.Request) {
|
|
input, err := decodeObjectStorageJSON[objectStorageMultipartPartURLInput](request)
|
|
if err != nil {
|
|
renderObjectStorageInvalidJSON(writer)
|
|
return
|
|
}
|
|
if strings.TrimSpace(input.UploadID) == "" {
|
|
renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"})
|
|
return
|
|
}
|
|
if input.PartNumber < 1 || input.PartNumber > 10000 {
|
|
renderValidationError(writer, validationIssue{Field: "partNumber", Message: "partNumber must be between 1 and 10000"})
|
|
return
|
|
}
|
|
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if !ok {
|
|
return
|
|
}
|
|
result, err := browser.PresignMultipartUploadPart(request.Context(), bucket, key, input.UploadID, input.PartNumber, input.ExpiresInSeconds)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, result)
|
|
}
|
|
|
|
func normalizeObjectStorageMultipartParts(parts []domain.ObjectStorageMultipartUploadPart) ([]domain.ObjectStorageMultipartUploadPart, error) {
|
|
if len(parts) == 0 {
|
|
return nil, fmt.Errorf("parts are required")
|
|
}
|
|
normalized := append([]domain.ObjectStorageMultipartUploadPart(nil), parts...)
|
|
for index := range normalized {
|
|
normalized[index].ETag = strings.Trim(strings.TrimSpace(normalized[index].ETag), `"`)
|
|
if normalized[index].PartNumber < 1 || normalized[index].PartNumber > 10000 {
|
|
return nil, fmt.Errorf("partNumber must be between 1 and 10000")
|
|
}
|
|
if normalized[index].ETag == "" {
|
|
return nil, fmt.Errorf("part eTag is required")
|
|
}
|
|
}
|
|
sort.Slice(normalized, func(left int, right int) bool {
|
|
return normalized[left].PartNumber < normalized[right].PartNumber
|
|
})
|
|
for index := 1; index < len(normalized); index++ {
|
|
if normalized[index].PartNumber == normalized[index-1].PartNumber {
|
|
return nil, fmt.Errorf("duplicate partNumber %d", normalized[index].PartNumber)
|
|
}
|
|
}
|
|
return normalized, nil
|
|
}
|
|
|
|
func (server *server) handleObjectStorageMultipartComplete(writer http.ResponseWriter, request *http.Request) {
|
|
input, err := decodeObjectStorageJSON[objectStorageMultipartCompleteInput](request)
|
|
if err != nil {
|
|
renderObjectStorageInvalidJSON(writer)
|
|
return
|
|
}
|
|
if strings.TrimSpace(input.UploadID) == "" {
|
|
renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"})
|
|
return
|
|
}
|
|
parts, err := normalizeObjectStorageMultipartParts(input.Parts)
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "parts", Message: err.Error()})
|
|
return
|
|
}
|
|
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if !ok {
|
|
return
|
|
}
|
|
item, err := browser.CompleteMultipartUpload(request.Context(), bucket, key, input.UploadID, parts)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageMultipartAbort(writer http.ResponseWriter, request *http.Request) {
|
|
input, err := decodeObjectStorageJSON[objectStorageMultipartAbortInput](request)
|
|
if err != nil {
|
|
renderObjectStorageInvalidJSON(writer)
|
|
return
|
|
}
|
|
if strings.TrimSpace(input.UploadID) == "" {
|
|
renderValidationError(writer, validationIssue{Field: "uploadId", Message: "uploadId is required"})
|
|
return
|
|
}
|
|
browser, bucket, key, cleanup, ok := server.resolveObjectStorageUploadTarget(writer, request, input.Bucket, input.Key)
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := browser.AbortMultipartUpload(request.Context(), bucket, key, input.UploadID); err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageMultipartUploadAbortResult{Bucket: bucket, Key: key, UploadID: strings.TrimSpace(input.UploadID), Aborted: true})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageObjectRename(writer http.ResponseWriter, request *http.Request) {
|
|
input, err := decodeObjectStorageObjectRenameInput(request)
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
|
|
return
|
|
}
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
|
|
if err != nil || bucket == "" {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
|
|
return
|
|
}
|
|
destinationPrefix := strings.Trim(strings.TrimSpace(input.DestinationPrefix), "/")
|
|
if len(input.Items) > 0 || destinationPrefix != "" {
|
|
items := input.Items
|
|
if len(items) == 0 {
|
|
items = []domain.ObjectStorageObjectMoveItem{{
|
|
Key: input.Key,
|
|
Recursive: input.Recursive,
|
|
}}
|
|
}
|
|
movedObjects := make([]domain.ObjectStorageObjectSummary, 0, len(items))
|
|
for index, moveItem := range items {
|
|
sourceKey := strings.Trim(strings.TrimSpace(moveItem.Key), "/")
|
|
if sourceKey == "" {
|
|
renderValidationError(writer, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "key is required"})
|
|
return
|
|
}
|
|
destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix)
|
|
if destinationKey == "" || destinationKey == sourceKey {
|
|
renderValidationError(writer, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "destination must differ from source"})
|
|
return
|
|
}
|
|
isDirectory := strings.EqualFold(strings.TrimSpace(moveItem.Kind), "directory") || moveItem.Recursive
|
|
var item domain.ObjectStorageObjectSummary
|
|
if isDirectory {
|
|
item, err = browser.MovePrefix(request.Context(), bucket, sourceKey, destinationKey)
|
|
} else {
|
|
item, err = browser.RenameObject(request.Context(), bucket, sourceKey, destinationKey)
|
|
}
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
movedObjects = append(movedObjects, item)
|
|
}
|
|
response := domain.ObjectStorageObjectMutationResult{Objects: movedObjects}
|
|
if len(movedObjects) > 0 {
|
|
response.Object = movedObjects[0]
|
|
}
|
|
renderJSON(writer, http.StatusOK, response)
|
|
return
|
|
}
|
|
if strings.TrimSpace(input.Key) == "" || strings.TrimSpace(input.NewKey) == "" {
|
|
renderValidationError(writer,
|
|
validationIssue{Field: "key", Message: "key is required"},
|
|
validationIssue{Field: "newKey", Message: "newKey is required"},
|
|
)
|
|
return
|
|
}
|
|
var item domain.ObjectStorageObjectSummary
|
|
if input.Recursive {
|
|
item, err = browser.MovePrefix(request.Context(), bucket, input.Key, input.NewKey)
|
|
} else {
|
|
item, err = browser.RenameObject(request.Context(), bucket, input.Key, input.NewKey)
|
|
}
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageObjectDelete(writer http.ResponseWriter, request *http.Request) {
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
|
|
return
|
|
}
|
|
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
|
|
recursive := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("recursive")), "true")
|
|
if bucket == "" || key == "" {
|
|
renderValidationError(writer,
|
|
validationIssue{Field: "bucket", Message: "bucket is required"},
|
|
validationIssue{Field: "key", Message: "key is required"},
|
|
)
|
|
return
|
|
}
|
|
trashKey := ".droboticflow-trash/" + time.Now().UTC().Format("20060102T150405.000000000Z") + "/" + key
|
|
if recursive {
|
|
_, err = browser.MovePrefix(request.Context(), bucket, key, trashKey)
|
|
} else {
|
|
_, err = browser.RenameObject(request.Context(), bucket, key, trashKey)
|
|
}
|
|
if err != nil {
|
|
if recursive && (os.IsNotExist(err) || isBOSNoSuchKey(err)) {
|
|
err = nil
|
|
}
|
|
}
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, map[string]any{"bucket": bucket, "key": key, "originalKey": key, "trashKey": trashKey, "deleted": true})
|
|
}
|
|
|
|
type objectStorageRestoreInput struct {
|
|
Bucket string `json:"bucket"`
|
|
TrashKey string `json:"trashKey"`
|
|
OriginalKey string `json:"originalKey"`
|
|
Recursive bool `json:"recursive,omitempty"`
|
|
}
|
|
|
|
func (server *server) handleObjectStorageObjectRestore(writer http.ResponseWriter, request *http.Request) {
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return
|
|
}
|
|
var payload objectStorageRestoreInput
|
|
if !decodeJSON(writer, request, &payload) {
|
|
return
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, payload.Bucket)
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
|
|
return
|
|
}
|
|
trashKey := strings.Trim(strings.TrimSpace(payload.TrashKey), "/")
|
|
originalKey := strings.Trim(strings.TrimSpace(payload.OriginalKey), "/")
|
|
if !strings.HasPrefix(trashKey, ".droboticflow-trash/") || originalKey == "" {
|
|
renderValidationError(writer, validationIssue{Field: "trashKey", Message: "a trash key and originalKey are required"})
|
|
return
|
|
}
|
|
var restored domain.ObjectStorageObjectSummary
|
|
if payload.Recursive {
|
|
restored, err = browser.MovePrefix(request.Context(), bucket, trashKey, originalKey)
|
|
} else {
|
|
restored, err = browser.RenameObject(request.Context(), bucket, trashKey, originalKey)
|
|
}
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: restored})
|
|
}
|
|
|
|
func (server *server) handleObjectStorageContent(writer http.ResponseWriter, request *http.Request) {
|
|
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
|
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || connection == nil {
|
|
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
|
return
|
|
}
|
|
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
|
|
return
|
|
}
|
|
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
|
|
if bucket == "" || key == "" {
|
|
renderValidationError(writer,
|
|
validationIssue{Field: "bucket", Message: "bucket is required"},
|
|
validationIssue{Field: "key", Message: "key is required"},
|
|
)
|
|
return
|
|
}
|
|
summary, err := browser.GetObjectSummary(request.Context(), bucket, key)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
redirectURL := ""
|
|
if previewURL, previewErr := browser.BuildPreviewURL(request.Context(), bucket, key); previewErr == nil {
|
|
redirectURL = previewURL
|
|
}
|
|
previewOnly := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("preview")), "true")
|
|
transcodeBrowser := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && summary.Kind == "video"
|
|
if transcodeBrowser {
|
|
previewRequest := sourceBrowserPreviewRequest{
|
|
FileName: path.Base(key),
|
|
RedirectURL: redirectURL,
|
|
}
|
|
if previewRequest.RedirectURL == "" {
|
|
payload, _, err := browser.ReadObject(request.Context(), bucket, key)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
previewRequest.Payload = payload
|
|
}
|
|
if previewPath, previewErr := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); previewErr == nil && previewPath != "" {
|
|
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(path.Base(key), path.Ext(key))+"-browser.mp4", "video/mp4", previewPath); err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
}
|
|
return
|
|
}
|
|
writer.Header().Set("Content-Type", "video/mp4")
|
|
writer.Header().Set("Cache-Control", "no-store")
|
|
writer.WriteHeader(http.StatusOK)
|
|
if flusher, ok := writer.(http.Flusher); ok {
|
|
flusher.Flush()
|
|
}
|
|
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
|
|
fmt.Printf("[api] object storage browser preview stream failed: %v\n", err)
|
|
}
|
|
return
|
|
}
|
|
if previewOnly && supportsObjectStorageInlinePreview(summary) && summary.SizeBytes > objectStorageInlinePreviewMaxBytes {
|
|
payload, contentType, err := browser.ReadObjectPreview(request.Context(), bucket, key, objectStoragePartialPreviewBytes)
|
|
if err != nil {
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
writer.Header().Set(objectStoragePreviewTruncatedHeader, "true")
|
|
writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes))
|
|
writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload)))
|
|
writeSourceContent(writer, request, path.Base(key), contentType, payload)
|
|
return
|
|
}
|
|
requestedRange, hasRange, rangeErr := parseObjectStorageByteRange(request.Header.Get("Range"), summary.SizeBytes)
|
|
if rangeErr != nil {
|
|
if summary.SizeBytes > 0 {
|
|
writer.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", summary.SizeBytes))
|
|
}
|
|
writer.Header().Set("Accept-Ranges", "bytes")
|
|
renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr)
|
|
return
|
|
}
|
|
if request.Method == http.MethodHead || hasRange {
|
|
if !hasRange && summary.SizeBytes > 0 {
|
|
requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1}
|
|
}
|
|
statusCode := http.StatusOK
|
|
if hasRange {
|
|
statusCode = http.StatusPartialContent
|
|
}
|
|
if request.Method == http.MethodHead {
|
|
writeObjectStorageRangeHeaders(writer, summary, summary.ContentType, requestedRange, statusCode)
|
|
writer.WriteHeader(statusCode)
|
|
return
|
|
}
|
|
reader, contentType, err := browser.ReadObjectRange(request.Context(), bucket, key, requestedRange.start, requestedRange.end)
|
|
if err != nil {
|
|
if redirectURL != "" {
|
|
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
|
|
return
|
|
}
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
defer reader.Close()
|
|
writeObjectStorageRangeHeaders(writer, summary, contentType, requestedRange, statusCode)
|
|
writer.WriteHeader(statusCode)
|
|
if _, err := io.Copy(writer, reader); err != nil {
|
|
fmt.Printf("[api] object storage range stream failed: %v\n", err)
|
|
}
|
|
return
|
|
}
|
|
if redirectURL != "" && !shouldReadObjectStorageInline(summary) {
|
|
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
|
|
return
|
|
}
|
|
payload, contentType, err := browser.ReadObject(request.Context(), bucket, key)
|
|
if err != nil {
|
|
if redirectURL != "" {
|
|
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
|
|
return
|
|
}
|
|
renderObjectStorageWriteError(writer, err)
|
|
return
|
|
}
|
|
if previewOnly {
|
|
writer.Header().Set(objectStoragePreviewTruncatedHeader, "false")
|
|
if summary.SizeBytes > 0 {
|
|
writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes))
|
|
}
|
|
writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload)))
|
|
}
|
|
writeSourceContent(writer, request, path.Base(key), contentType, payload)
|
|
}
|