droboticflow/internal/api/object_storage_browser.go
2026-06-26 02:22:43 +08:00

1000 lines
36 KiB
Go

package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"path"
"strconv"
"strings"
"droboticflow/internal/domain"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/services/bos"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
)
type objectStorageBrowser interface {
ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error)
ListObjects(context.Context, string, string) ([]domain.ObjectStorageObjectSummary, error)
CreateBucket(context.Context, string) (domain.ObjectStorageBucketSummary, error)
DeleteBucket(context.Context, string) error
PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error)
PutObjectStream(context.Context, string, string, io.Reader, string) (domain.ObjectStorageObjectSummary, error)
RenameObject(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error)
DeleteObject(context.Context, string, string) error
DeletePrefix(context.Context, string, string) error
GetObjectSummary(context.Context, string, string) (domain.ObjectStorageObjectSummary, error)
ReadObject(context.Context, string, string) ([]byte, string, error)
ReadObjectRange(context.Context, string, string, int64, int64) (io.ReadCloser, string, error)
ReadObjectPreview(context.Context, string, string, int64) ([]byte, string, error)
BuildPreviewURL(context.Context, string, string) (string, error)
}
type bosObjectStorageBrowser struct {
connection *domain.StorageConnectionRecord
client *bos.Client
}
const (
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
objectStoragePartialPreviewBytes int64 = 64 * 1024
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
)
var storageConnectionBrowserResolver = func(ctx context.Context, connection *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
if connection == nil {
return nil, nil, nil
}
endpoint := storageEndpointForProvider(connection.Provider, connection.Endpoint)
if normalizeStorageProvider(connection.Provider) == domain.StorageProviderBOS && endpoint == "" {
return nil, nil, fmt.Errorf("BOS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name))
}
client, err := bos.NewClient(
strings.TrimSpace(connection.AccessKeyID),
strings.TrimSpace(connection.SecretAccessKey),
endpoint,
)
if err != nil {
return nil, nil, err
}
return &bosObjectStorageBrowser{connection: connection, client: client}, func() {}, nil
}
func objectStorageKindForObject(key string, contentType string) string {
contentType = strings.ToLower(strings.TrimSpace(contentType))
if strings.HasPrefix(contentType, "image/") {
return "image"
}
if strings.HasPrefix(contentType, "video/") {
return "video"
}
if strings.HasPrefix(contentType, "audio/") {
return "audio"
}
if strings.HasPrefix(contentType, "application/json") {
return "json"
}
switch strings.ToLower(path.Ext(key)) {
case ".json":
return "json"
case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg":
return "image"
case ".mp4", ".mov", ".m4v", ".webm", ".avi", ".mkv":
return "video"
case ".mp3", ".wav", ".aac", ".ogg", ".flac", ".m4a":
return "audio"
}
return "file"
}
func objectStorageSummaryFromListObject(bucket string, item bosapi.ObjectSummaryType) domain.ObjectStorageObjectSummary {
key := strings.TrimSpace(item.Key)
contentType := detectObjectStorageContentType(key, nil, "")
return domain.ObjectStorageObjectSummary{
Bucket: bucket,
Key: key,
Name: path.Base(strings.TrimRight(key, "/")),
Kind: objectStorageKindForObject(key, contentType),
SizeBytes: int64(item.Size),
ContentType: contentType,
LastModified: strings.TrimSpace(item.LastModified),
ETag: strings.Trim(item.ETag, "\""),
StorageClass: strings.TrimSpace(item.StorageClass),
}
}
func objectStorageSummaryFromPrefix(bucket string, prefix string) domain.ObjectStorageObjectSummary {
trimmed := strings.TrimSpace(prefix)
return domain.ObjectStorageObjectSummary{
Bucket: bucket,
Key: strings.TrimRight(trimmed, "/"),
Name: path.Base(strings.TrimRight(trimmed, "/")),
Kind: "directory",
}
}
func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObjectMetaResult) domain.ObjectStorageObjectSummary {
contentType := ""
sizeBytes := int64(0)
lastModified := ""
etag := ""
storageClass := ""
metadata := map[string]any{}
if meta != nil {
contentType = strings.TrimSpace(meta.ContentType)
sizeBytes = meta.ContentLength
lastModified = strings.TrimSpace(meta.LastModified)
etag = strings.Trim(meta.ETag, "\"")
storageClass = strings.TrimSpace(meta.StorageClass)
for metaKey, metaValue := range meta.UserMeta {
metadata[metaKey] = metaValue
}
}
if len(metadata) == 0 {
metadata = nil
}
return domain.ObjectStorageObjectSummary{
Bucket: bucket,
Key: key,
Name: path.Base(strings.TrimRight(key, "/")),
Kind: objectStorageKindForObject(key, contentType),
SizeBytes: sizeBytes,
ContentType: contentType,
LastModified: lastModified,
ETag: etag,
StorageClass: storageClass,
Metadata: metadata,
}
}
func (browser *bosObjectStorageBrowser) ListBuckets(ctx context.Context) ([]domain.ObjectStorageBucketSummary, error) {
result, err := browser.client.ListBucketsWithContext(ctx)
if err != nil {
return nil, err
}
items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets))
for _, bucket := range result.Buckets {
name := strings.TrimSpace(bucket.Name)
if name == "" {
continue
}
items = append(items, domain.ObjectStorageBucketSummary{Name: name, CreatedAt: strings.TrimSpace(bucket.CreationDate)})
}
return items, nil
}
func (browser *bosObjectStorageBrowser) ListObjects(ctx context.Context, bucket string, prefix string) ([]domain.ObjectStorageObjectSummary, error) {
normalizedBucket := strings.TrimSpace(bucket)
requestPrefix := storageObjectRequestPrefix(prefix)
items := make([]domain.ObjectStorageObjectSummary, 0)
seen := make(map[string]struct{})
marker := ""
for {
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
Prefix: requestPrefix,
Delimiter: "/",
MaxKeys: 1000,
Marker: marker,
})
if err != nil {
return nil, err
}
for _, item := range result.CommonPrefixes {
prefixValue := strings.Trim(strings.TrimSpace(item.Prefix), "/")
if prefixValue == "" || prefixValue == strings.Trim(strings.TrimSpace(prefix), "/") {
continue
}
if _, exists := seen["dir|"+prefixValue]; exists {
continue
}
seen["dir|"+prefixValue] = struct{}{}
items = append(items, objectStorageSummaryFromPrefix(normalizedBucket, prefixValue))
}
for _, item := range result.Contents {
key := strings.Trim(strings.TrimSpace(item.Key), "/")
if key == "" || key == strings.Trim(strings.TrimSpace(prefix), "/") {
continue
}
if _, exists := seen["file|"+key]; exists {
continue
}
seen["file|"+key] = struct{}{}
items = append(items, objectStorageSummaryFromListObject(normalizedBucket, item))
}
if result == nil || !result.IsTruncated {
break
}
nextMarker := strings.TrimSpace(result.NextMarker)
if nextMarker == "" || nextMarker == marker {
break
}
marker = nextMarker
}
return items, nil
}
func (browser *bosObjectStorageBrowser) CreateBucket(_ context.Context, bucket string) (domain.ObjectStorageBucketSummary, error) {
if _, err := browser.client.PutBucket(strings.TrimSpace(bucket)); err != nil {
return domain.ObjectStorageBucketSummary{}, err
}
return domain.ObjectStorageBucketSummary{Name: strings.TrimSpace(bucket)}, nil
}
func (browser *bosObjectStorageBrowser) DeleteBucket(_ context.Context, bucket string) error {
return browser.client.DeleteBucket(strings.TrimSpace(bucket))
}
func detectObjectStorageContentType(key string, payload []byte, explicit string) string {
if trimmed := strings.TrimSpace(explicit); trimmed != "" {
return trimmed
}
if guessed := mime.TypeByExtension(strings.ToLower(path.Ext(key))); guessed != "" {
return guessed
}
if len(payload) > 0 {
return http.DetectContentType(payload)
}
return "application/octet-stream"
}
func supportsObjectStorageInlinePreview(summary domain.ObjectStorageObjectSummary) bool {
contentType := strings.ToLower(strings.TrimSpace(summary.ContentType))
if summary.Kind == "json" {
return true
}
if strings.HasPrefix(contentType, "text/") {
return true
}
switch strings.ToLower(path.Ext(summary.Key)) {
case ".txt", ".log", ".md", ".csv", ".tsv", ".yaml", ".yml", ".xml", ".jsonl":
return true
}
return false
}
func shouldReadObjectStorageInline(summary domain.ObjectStorageObjectSummary) bool {
if !supportsObjectStorageInlinePreview(summary) {
return false
}
if summary.SizeBytes <= 0 {
return true
}
return summary.SizeBytes <= objectStorageInlinePreviewMaxBytes
}
type objectStorageByteRange struct {
start int64
end int64
}
func parseObjectStorageByteRange(rangeHeader string, sizeBytes int64) (objectStorageByteRange, bool, error) {
trimmed := strings.TrimSpace(rangeHeader)
if trimmed == "" {
return objectStorageByteRange{}, false, nil
}
if sizeBytes <= 0 {
return objectStorageByteRange{}, true, fmt.Errorf("object size is unknown")
}
if !strings.HasPrefix(strings.ToLower(trimmed), "bytes=") {
return objectStorageByteRange{}, true, fmt.Errorf("unsupported range unit")
}
spec := strings.TrimSpace(trimmed[len("bytes="):])
if spec == "" || strings.Contains(spec, ",") {
return objectStorageByteRange{}, true, fmt.Errorf("only a single byte range is supported")
}
startText, endText, found := strings.Cut(spec, "-")
if !found {
return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range")
}
startText = strings.TrimSpace(startText)
endText = strings.TrimSpace(endText)
if startText == "" && endText == "" {
return objectStorageByteRange{}, true, fmt.Errorf("invalid byte range")
}
if startText == "" {
suffixLength, err := strconv.ParseInt(endText, 10, 64)
if err != nil || suffixLength <= 0 {
return objectStorageByteRange{}, true, fmt.Errorf("invalid suffix byte range")
}
if suffixLength >= sizeBytes {
return objectStorageByteRange{start: 0, end: sizeBytes - 1}, true, nil
}
return objectStorageByteRange{start: sizeBytes - suffixLength, end: sizeBytes - 1}, true, nil
}
start, err := strconv.ParseInt(startText, 10, 64)
if err != nil || start < 0 {
return objectStorageByteRange{}, true, fmt.Errorf("invalid range start")
}
if start >= sizeBytes {
return objectStorageByteRange{}, true, fmt.Errorf("range start is outside object")
}
end := sizeBytes - 1
if endText != "" {
parsedEnd, err := strconv.ParseInt(endText, 10, 64)
if err != nil || parsedEnd < start {
return objectStorageByteRange{}, true, fmt.Errorf("invalid range end")
}
if parsedEnd < end {
end = parsedEnd
}
}
return objectStorageByteRange{start: start, end: end}, true, nil
}
func writeObjectStorageRangeHeaders(writer http.ResponseWriter, summary domain.ObjectStorageObjectSummary, contentType string, byteRange objectStorageByteRange, statusCode int) {
if contentType == "" {
contentType = detectObjectStorageContentType(summary.Key, nil, summary.ContentType)
}
if contentType != "" {
writer.Header().Set("Content-Type", contentType)
}
writer.Header().Set("Accept-Ranges", "bytes")
if summary.ETag != "" {
writer.Header().Set("ETag", strconv.Quote(summary.ETag))
}
if summary.LastModified != "" {
writer.Header().Set("Last-Modified", summary.LastModified)
}
if statusCode == http.StatusPartialContent {
length := byteRange.end - byteRange.start + 1
writer.Header().Set("Content-Length", fmt.Sprintf("%d", length))
writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, summary.SizeBytes))
return
}
if summary.SizeBytes > 0 {
writer.Header().Set("Content-Length", fmt.Sprintf("%d", summary.SizeBytes))
}
}
func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, payload, contentType)}
if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.GetObjectSummary(ctx, bucket, objectKey)
}
func (browser *bosObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, nil, contentType)}
if _, err := browser.client.PutObjectFromStreamWithContext(ctx, strings.TrimSpace(bucket), objectKey, reader, args); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.GetObjectSummary(ctx, bucket, objectKey)
}
func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
objectKey := exactObjectStorageKey(key)
destinationKey := exactObjectStorageKey(newKey)
if _, err := browser.client.CopyObjectWithContext(ctx, strings.TrimSpace(bucket), destinationKey, strings.TrimSpace(bucket), objectKey, nil); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
if err := browser.client.DeleteObject(strings.TrimSpace(bucket), objectKey); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.GetObjectSummary(ctx, bucket, destinationKey)
}
func (browser *bosObjectStorageBrowser) DeleteObject(_ context.Context, bucket string, key string) error {
return browser.client.DeleteObject(strings.TrimSpace(bucket), exactObjectStorageKey(key))
}
func (browser *bosObjectStorageBrowser) DeletePrefix(ctx context.Context, bucket string, prefix string) error {
normalizedBucket := strings.TrimSpace(bucket)
normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
if normalizedBucket == "" || normalizedPrefix == "" {
return fmt.Errorf("bucket and prefix are required")
}
requestPrefix := normalizedPrefix + "/"
marker := ""
for {
result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
Prefix: requestPrefix,
MaxKeys: 1000,
Marker: marker,
})
if err != nil {
return err
}
for _, item := range result.Contents {
key := exactObjectStorageKey(item.Key)
if key == "" {
continue
}
if err := browser.client.DeleteObject(normalizedBucket, key); err != nil {
return err
}
}
if result == nil || !result.IsTruncated {
break
}
nextMarker := strings.TrimSpace(result.NextMarker)
if nextMarker == "" || nextMarker == marker {
break
}
marker = nextMarker
}
for _, markerKey := range []string{normalizedPrefix + "/", normalizedPrefix} {
if err := browser.client.DeleteObject(normalizedBucket, markerKey); err != nil && !isBOSNoSuchKey(err) {
return err
}
}
return nil
}
func (browser *bosObjectStorageBrowser) GetObjectSummary(ctx context.Context, bucket string, key string) (domain.ObjectStorageObjectSummary, error) {
objectKey := exactObjectStorageKey(key)
meta, err := browser.client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), objectKey)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return objectStorageSummaryFromMeta(strings.TrimSpace(bucket), objectKey, meta), nil
}
func (browser *bosObjectStorageBrowser) ReadObject(ctx context.Context, bucket string, key string) ([]byte, string, error) {
objectKey := exactObjectStorageKey(key)
result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil)
if err != nil {
return nil, "", err
}
defer result.Body.Close()
payload, err := io.ReadAll(result.Body)
if err != nil {
return nil, "", err
}
return payload, strings.TrimSpace(result.ContentType), nil
}
func (browser *bosObjectStorageBrowser) ReadObjectRange(ctx context.Context, bucket string, key string, start int64, end int64) (io.ReadCloser, string, error) {
objectKey := exactObjectStorageKey(key)
result, err := browser.client.GetObjectWithContext(ctx, strings.TrimSpace(bucket), objectKey, nil, start, end)
if err != nil {
return nil, "", err
}
return result.Body, strings.TrimSpace(result.ContentType), nil
}
func (browser *bosObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucket string, key string, maxBytes int64) ([]byte, string, error) {
if maxBytes <= 0 {
return browser.ReadObject(ctx, bucket, key)
}
objectKey := exactObjectStorageKey(key)
result, err := browser.client.GetObjectWithContext(
ctx,
strings.TrimSpace(bucket),
objectKey,
nil,
0,
maxBytes-1,
)
if err != nil {
return nil, "", err
}
defer result.Body.Close()
payload, err := io.ReadAll(io.LimitReader(result.Body, maxBytes))
if err != nil {
return nil, "", err
}
return payload, strings.TrimSpace(result.ContentType), nil
}
func (browser *bosObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucket string, key string) (string, error) {
previewURL := strings.TrimSpace(browser.client.BasicGeneratePresignedUrl(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), 3600))
if previewURL == "" {
return "", fmt.Errorf("failed to generate BOS preview url")
}
return previewURL, nil
}
func decodeObjectStorageBucketCreateInput(request *http.Request) (domain.ObjectStorageBucketCreateInput, error) {
input := domain.ObjectStorageBucketCreateInput{}
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
return domain.ObjectStorageBucketCreateInput{}, err
}
return input, nil
}
func decodeObjectStorageObjectPutInput(request *http.Request) (domain.ObjectStorageObjectPutInput, error) {
input := domain.ObjectStorageObjectPutInput{}
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
return domain.ObjectStorageObjectPutInput{}, err
}
return input, nil
}
func decodeObjectStorageObjectRenameInput(request *http.Request) (domain.ObjectStorageObjectRenameInput, error) {
input := domain.ObjectStorageObjectRenameInput{}
if err := json.NewDecoder(request.Body).Decode(&input); err != nil {
return domain.ObjectStorageObjectRenameInput{}, err
}
return input, nil
}
func (server *server) resolveObjectStorageBrowser(ctx context.Context, connectionID string) (objectStorageBrowser, *domain.StorageConnectionRecord, func(), error) {
connection, err := server.getStorageConnection(ctx, connectionID)
if err != nil && strings.TrimSpace(connectionID) != "" {
return nil, nil, nil, err
}
if connection == nil {
return nil, nil, nil, nil
}
connection = server.storageConnectionWithResolvedEndpoint(connection)
browser, cleanup, err := storageConnectionBrowserResolver(ctx, connection)
if err != nil {
return nil, nil, nil, err
}
return browser, connection, cleanup, nil
}
func resolveObjectStorageBucket(connection *domain.StorageConnectionRecord, requested string) (string, error) {
requested = strings.Trim(strings.TrimSpace(requested), "/")
if connection == nil {
return requested, nil
}
boundBucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/")
if boundBucket == "" {
return requested, nil
}
if requested != "" && requested != boundBucket {
return "", fmt.Errorf("connection is bound to bucket %s", boundBucket)
}
return boundBucket, nil
}
func renderObjectStorageWriteError(writer http.ResponseWriter, err error) {
if err == nil {
return
}
if isRequestTooLarge(err) {
renderJSON(writer, http.StatusRequestEntityTooLarge, map[string]string{"error": err.Error()})
return
}
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()})
return
}
renderError(writer, http.StatusInternalServerError, err)
}
func isRequestTooLarge(err error) bool {
if err == nil {
return false
}
var maxBytesErr *http.MaxBytesError
return errors.As(err, &maxBytesErr)
}
func exactObjectStorageKey(key string) string {
return strings.TrimSpace(key)
}
func isBOSNoSuchKey(err error) bool {
if err == nil {
return false
}
var serviceErr *bce.BceServiceError
if errors.As(err, &serviceErr) {
return strings.EqualFold(strings.TrimSpace(serviceErr.Code), "NoSuchKey")
}
return strings.Contains(strings.ToLower(err.Error()), "nosuchkey")
}
func (server *server) handleObjectStorageOverview(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{
ConnectionID: connectionID,
Provider: "BOS",
Configured: false,
Buckets: []domain.ObjectStorageBucketSummary{},
})
return
}
bucket := strings.Trim(strings.TrimSpace(connection.Bucket), "/")
items := []domain.ObjectStorageBucketSummary{}
if bucket != "" {
items = append(items, domain.ObjectStorageBucketSummary{Name: bucket})
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageOverview{
ConnectionID: connection.ID,
ConnectionName: connection.Name,
Provider: string(connection.Provider),
Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint),
Bucket: bucket,
Configured: true,
Buckets: items,
})
}
func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
prefix := strings.Trim(strings.TrimSpace(request.URL.Query().Get("prefix")), "/")
if browser == nil || connection == nil {
renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{
ConnectionID: connectionID,
Provider: "BOS",
Configured: false,
Bucket: bucket,
Prefix: prefix,
Items: []domain.ObjectStorageObjectSummary{},
})
return
}
items, err := browser.ListObjects(request.Context(), bucket, prefix)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{
ConnectionID: connection.ID,
ConnectionName: connection.Name,
Provider: string(connection.Provider),
Endpoint: server.storageEndpointForProvider(connection.Provider, connection.Endpoint),
Configured: true,
Bucket: bucket,
Prefix: prefix,
Items: items,
})
}
func (server *server) handleObjectStorageBucketCreate(writer http.ResponseWriter, request *http.Request) {
renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"})
}
func (server *server) handleObjectStorageBucketDelete(writer http.ResponseWriter, request *http.Request) {
renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"})
}
func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter, request *http.Request) {
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageJSONPutMaxBytes)
input, err := decodeObjectStorageObjectPutInput(request)
if err != nil {
if isRequestTooLarge(err) {
renderObjectStorageWriteError(writer, err)
return
}
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
return
}
if strings.TrimSpace(input.Key) == "" {
renderValidationError(writer,
validationIssue{Field: "key", Message: "key is required"},
)
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
if err != nil || bucket == "" {
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
return
}
item, err := browser.PutObject(request.Context(), bucket, input.Key, []byte(input.Content), input.ContentType)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
}
func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter, request *http.Request) {
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageMultipartUploadMaxBytes)
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
if bucket == "" || key == "" {
renderValidationError(writer,
validationIssue{Field: "bucket", Message: "bucket is required"},
validationIssue{Field: "key", Message: "key is required"},
)
return
}
if err := request.ParseMultipartForm(32 << 20); err != nil {
if isRequestTooLarge(err) {
renderObjectStorageWriteError(writer, err)
return
}
renderValidationError(writer, validationIssue{Field: "file", Message: "invalid multipart payload"})
return
}
file, header, err := request.FormFile("file")
if err != nil {
renderValidationError(writer, validationIssue{Field: "file", Message: "file is required"})
return
}
defer file.Close()
contentType := strings.TrimSpace(request.URL.Query().Get("contentType"))
if contentType == "" && header != nil {
contentType = strings.TrimSpace(header.Header.Get("Content-Type"))
}
item, err := browser.PutObjectStream(request.Context(), bucket, key, file, contentType)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
}
func (server *server) handleObjectStorageObjectRename(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageObjectRenameInput(request)
if err != nil {
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
return
}
if strings.TrimSpace(input.Key) == "" || strings.TrimSpace(input.NewKey) == "" {
renderValidationError(writer,
validationIssue{Field: "key", Message: "key is required"},
validationIssue{Field: "newKey", Message: "newKey is required"},
)
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
if err != nil || bucket == "" {
renderValidationError(writer, validationIssue{Field: "bucket", Message: "bucket is required"})
return
}
item, err := browser.RenameObject(request.Context(), bucket, input.Key, input.NewKey)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectMutationResult{Object: item})
}
func (server *server) handleObjectStorageObjectDelete(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
recursive := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("recursive")), "true")
if bucket == "" || key == "" {
renderValidationError(writer,
validationIssue{Field: "bucket", Message: "bucket is required"},
validationIssue{Field: "key", Message: "key is required"},
)
return
}
if recursive {
err = browser.DeletePrefix(request.Context(), bucket, key)
} else {
err = browser.DeleteObject(request.Context(), bucket, key)
}
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
renderJSON(writer, http.StatusOK, domain.ObjectStorageObjectDeleteResult{Bucket: bucket, Key: key, Deleted: true})
}
func (server *server) handleObjectStorageContent(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
bucket, err := resolveObjectStorageBucket(connection, request.URL.Query().Get("bucket"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
key := strings.Trim(strings.TrimSpace(request.URL.Query().Get("key")), "/")
if bucket == "" || key == "" {
renderValidationError(writer,
validationIssue{Field: "bucket", Message: "bucket is required"},
validationIssue{Field: "key", Message: "key is required"},
)
return
}
summary, err := browser.GetObjectSummary(request.Context(), bucket, key)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
redirectURL := ""
if previewURL, previewErr := browser.BuildPreviewURL(request.Context(), bucket, key); previewErr == nil {
redirectURL = previewURL
}
previewOnly := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("preview")), "true")
transcodeBrowser := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && summary.Kind == "video"
if transcodeBrowser {
previewRequest := sourceBrowserPreviewRequest{
FileName: path.Base(key),
RedirectURL: redirectURL,
}
if previewRequest.RedirectURL == "" {
payload, _, err := browser.ReadObject(request.Context(), bucket, key)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
previewRequest.Payload = payload
}
if previewPath, previewErr := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); previewErr == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(path.Base(key), path.Ext(key))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] object storage browser preview stream failed: %v\n", err)
}
return
}
if previewOnly && supportsObjectStorageInlinePreview(summary) && summary.SizeBytes > objectStorageInlinePreviewMaxBytes {
payload, contentType, err := browser.ReadObjectPreview(request.Context(), bucket, key, objectStoragePartialPreviewBytes)
if err != nil {
renderObjectStorageWriteError(writer, err)
return
}
writer.Header().Set(objectStoragePreviewTruncatedHeader, "true")
writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes))
writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload)))
writeSourceContent(writer, request, path.Base(key), contentType, payload)
return
}
requestedRange, hasRange, rangeErr := parseObjectStorageByteRange(request.Header.Get("Range"), summary.SizeBytes)
if rangeErr != nil {
if summary.SizeBytes > 0 {
writer.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", summary.SizeBytes))
}
writer.Header().Set("Accept-Ranges", "bytes")
renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr)
return
}
if request.Method == http.MethodHead || hasRange {
if !hasRange && summary.SizeBytes > 0 {
requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1}
}
statusCode := http.StatusOK
if hasRange {
statusCode = http.StatusPartialContent
}
if request.Method == http.MethodHead {
writeObjectStorageRangeHeaders(writer, summary, summary.ContentType, requestedRange, statusCode)
writer.WriteHeader(statusCode)
return
}
reader, contentType, err := browser.ReadObjectRange(request.Context(), bucket, key, requestedRange.start, requestedRange.end)
if err != nil {
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
renderObjectStorageWriteError(writer, err)
return
}
defer reader.Close()
writeObjectStorageRangeHeaders(writer, summary, contentType, requestedRange, statusCode)
writer.WriteHeader(statusCode)
if _, err := io.Copy(writer, reader); err != nil {
fmt.Printf("[api] object storage range stream failed: %v\n", err)
}
return
}
if redirectURL != "" && !shouldReadObjectStorageInline(summary) {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
payload, contentType, err := browser.ReadObject(request.Context(), bucket, key)
if err != nil {
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
renderObjectStorageWriteError(writer, err)
return
}
if previewOnly {
writer.Header().Set(objectStoragePreviewTruncatedHeader, "false")
if summary.SizeBytes > 0 {
writer.Header().Set(objectStoragePreviewOriginalSizeHeader, fmt.Sprintf("%d", summary.SizeBytes))
}
writer.Header().Set(objectStoragePreviewReturnedSizeHeader, fmt.Sprintf("%d", len(payload)))
}
writeSourceContent(writer, request, path.Base(key), contentType, payload)
}