droboticflow-new/internal/api/oss_object_storage_browser.go
2026-08-19 03:08:37 +08:00

365 lines
15 KiB
Go

package api
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"path"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
)
type ossObjectStorageBrowser struct {
connection *domain.StorageConnectionRecord
client *oss.Client
}
var _ objectStorageBrowser = (*ossObjectStorageBrowser)(nil)
func newOSSObjectStorageBrowser(connection *domain.StorageConnectionRecord, endpoint string) (objectStorageBrowser, func(), error) {
if connection == nil {
return nil, nil, nil
}
endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/")
if endpoint == "" {
return nil, nil, fmt.Errorf("OSS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name))
}
client, err := oss.New(endpoint, strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey))
if err != nil {
return nil, nil, err
}
return &ossObjectStorageBrowser{connection: connection, client: client}, func() {}, nil
}
func (browser *ossObjectStorageBrowser) bucket(name string) (*oss.Bucket, error) {
return browser.client.Bucket(strings.TrimSpace(name))
}
func (browser *ossObjectStorageBrowser) ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error) {
result, err := browser.client.ListBuckets()
if err != nil {
return nil, err
}
items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets))
for _, item := range result.Buckets {
items = append(items, domain.ObjectStorageBucketSummary{Name: item.Name, CreatedAt: item.CreationDate.UTC().Format(time.RFC3339)})
}
return items, nil
}
func ossObjectSummary(bucketName string, item oss.ObjectProperties) domain.ObjectStorageObjectSummary {
contentType := detectObjectStorageContentType(item.Key, nil, "")
return domain.ObjectStorageObjectSummary{
Bucket: bucketName, Key: item.Key, Name: path.Base(strings.TrimRight(item.Key, "/")),
Kind: objectStorageKindForObject(item.Key, contentType), SizeBytes: item.Size, ContentType: contentType,
ETag: strings.Trim(item.ETag, `"`), LastModified: item.LastModified.UTC().Format(time.RFC3339), StorageClass: item.StorageClass,
}
}
func (browser *ossObjectStorageBrowser) ListObjects(ctx context.Context, bucketName string, prefix string) ([]domain.ObjectStorageObjectSummary, error) {
page, err := browser.ListObjectsPage(ctx, bucketName, prefix, "", 1000)
return page.Items, err
}
func (browser *ossObjectStorageBrowser) ListObjectsPage(_ context.Context, bucketName string, prefix string, marker string, limit int) (objectStoragePage, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return objectStoragePage{}, err
}
if limit <= 0 || limit > 1000 {
limit = 1000
}
result, err := bucket.ListObjects(oss.Prefix(strings.TrimSpace(prefix)), oss.Delimiter("/"), oss.Marker(strings.TrimSpace(marker)), oss.MaxKeys(limit))
if err != nil {
return objectStoragePage{}, err
}
items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Objects))
for _, commonPrefix := range result.CommonPrefixes {
items = append(items, objectStorageSummaryFromPrefix(bucketName, commonPrefix))
}
for _, object := range result.Objects {
items = append(items, ossObjectSummary(bucketName, object))
}
return objectStoragePage{Items: items, NextMarker: result.NextMarker, HasMore: result.IsTruncated}, nil
}
func (browser *ossObjectStorageBrowser) CreateBucket(_ context.Context, bucketName string) (domain.ObjectStorageBucketSummary, error) {
bucketName = strings.TrimSpace(bucketName)
if err := browser.client.CreateBucket(bucketName); err != nil {
return domain.ObjectStorageBucketSummary{}, err
}
return domain.ObjectStorageBucketSummary{Name: bucketName, CreatedAt: nowISO()}, nil
}
func (browser *ossObjectStorageBrowser) DeleteBucket(_ context.Context, bucketName string) error {
return browser.client.DeleteBucket(strings.TrimSpace(bucketName))
}
func (browser *ossObjectStorageBrowser) PutObject(ctx context.Context, bucketName string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) {
return browser.PutObjectStream(ctx, bucketName, key, bytes.NewReader(payload), contentType)
}
func (browser *ossObjectStorageBrowser) PutObjectStream(ctx context.Context, bucketName string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
options := make([]oss.Option, 0, 1)
if strings.TrimSpace(contentType) != "" {
options = append(options, oss.ContentType(strings.TrimSpace(contentType)))
}
if err := bucket.PutObject(exactObjectStorageKey(key), reader, options...); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.GetObjectSummary(ctx, bucketName, key)
}
func (browser *ossObjectStorageBrowser) PresignPutObject(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStoragePresignedUploadResult, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStoragePresignedUploadResult{}, err
}
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
options := make([]oss.Option, 0, 1)
headers := objectStorageUploadHeaders(contentType)
if strings.TrimSpace(contentType) != "" {
options = append(options, oss.ContentType(strings.TrimSpace(contentType)))
}
url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), options...)
if err != nil {
return domain.ObjectStoragePresignedUploadResult{}, err
}
return domain.ObjectStoragePresignedUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), Method: http.MethodPut, URL: url, Headers: headers, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil
}
func (browser *ossObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStorageMultipartUploadCreateResult{}, err
}
options := make([]oss.Option, 0, 1)
if strings.TrimSpace(contentType) != "" {
options = append(options, oss.ContentType(strings.TrimSpace(contentType)))
}
result, err := bucket.InitiateMultipartUpload(exactObjectStorageKey(key), options...)
if err != nil {
return domain.ObjectStorageMultipartUploadCreateResult{}, err
}
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
return domain.ObjectStorageMultipartUploadCreateResult{Bucket: result.Bucket, Key: result.Key, UploadID: result.UploadID, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil
}
func (browser *ossObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucketName string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStorageMultipartUploadPartURLResult{}, err
}
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), oss.AddParam("uploadId", strings.TrimSpace(uploadID)), oss.AddParam("partNumber", partNumber))
if err != nil {
return domain.ObjectStorageMultipartUploadPartURLResult{}, err
}
return domain.ObjectStorageMultipartUploadPartURLResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID), PartNumber: partNumber, Method: http.MethodPut, URL: url, Headers: map[string]string{}, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil
}
func (browser *ossObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucketName string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
completed := make([]oss.UploadPart, 0, len(parts))
for _, item := range parts {
completed = append(completed, oss.UploadPart{PartNumber: item.PartNumber, ETag: strings.Trim(item.ETag, `"`)})
}
sort.Slice(completed, func(i, j int) bool { return completed[i].PartNumber < completed[j].PartNumber })
_, err = bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)}, completed)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.GetObjectSummary(ctx, bucketName, key)
}
func (browser *ossObjectStorageBrowser) AbortMultipartUpload(_ context.Context, bucketName string, key string, uploadID string) error {
bucket, err := browser.bucket(bucketName)
if err != nil {
return err
}
return bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)})
}
func (browser *ossObjectStorageBrowser) RenameObject(ctx context.Context, bucketName string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
key, newKey = exactObjectStorageKey(key), exactObjectStorageKey(newKey)
if key == "" || newKey == "" || key == newKey {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("distinct source and destination keys are required")
}
if _, err := bucket.CopyObject(key, newKey); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
if err := bucket.DeleteObject(key); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.GetObjectSummary(ctx, bucketName, newKey)
}
func (browser *ossObjectStorageBrowser) MovePrefix(ctx context.Context, bucketName string, prefix string, newPrefix string) (domain.ObjectStorageObjectSummary, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
prefix, newPrefix = strings.Trim(strings.TrimSpace(prefix), "/"), strings.Trim(strings.TrimSpace(newPrefix), "/")
if prefix == "" || newPrefix == "" || prefix == newPrefix || strings.HasPrefix(newPrefix, prefix+"/") {
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("valid distinct source and destination prefixes are required")
}
marker := ""
for {
result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000))
if listErr != nil {
return domain.ObjectStorageObjectSummary{}, listErr
}
for _, item := range result.Objects {
destination := newPrefix + "/" + strings.TrimPrefix(item.Key, prefix+"/")
if _, err := bucket.CopyObject(item.Key, destination); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
if err := bucket.DeleteObject(item.Key); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
}
if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker {
break
}
marker = result.NextMarker
}
return objectStorageSummaryFromPrefix(strings.TrimSpace(bucketName), newPrefix), nil
}
func (browser *ossObjectStorageBrowser) DeleteObject(_ context.Context, bucketName string, key string) error {
bucket, err := browser.bucket(bucketName)
if err != nil {
return err
}
return bucket.DeleteObject(exactObjectStorageKey(key))
}
func (browser *ossObjectStorageBrowser) DeletePrefix(_ context.Context, bucketName string, prefix string) error {
bucket, err := browser.bucket(bucketName)
if err != nil {
return err
}
prefix = strings.Trim(strings.TrimSpace(prefix), "/")
marker := ""
for {
result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000))
if listErr != nil {
return listErr
}
keys := make([]string, 0, len(result.Objects))
for _, item := range result.Objects {
keys = append(keys, item.Key)
}
if len(keys) > 0 {
if _, deleteErr := bucket.DeleteObjects(keys); deleteErr != nil {
return deleteErr
}
}
if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker {
break
}
marker = result.NextMarker
}
_ = bucket.DeleteObject(prefix)
return bucket.DeleteObject(prefix + "/")
}
func ossSummaryFromHeader(bucketName string, key string, header http.Header) domain.ObjectStorageObjectSummary {
size, _ := strconv.ParseInt(strings.TrimSpace(header.Get("Content-Length")), 10, 64)
contentType := strings.TrimSpace(header.Get("Content-Type"))
metadata := map[string]any{}
for name, values := range header {
if strings.HasPrefix(strings.ToLower(name), "x-oss-meta-") {
metadata[strings.TrimPrefix(strings.ToLower(name), "x-oss-meta-")] = strings.Join(values, ",")
}
}
if len(metadata) == 0 {
metadata = nil
}
return domain.ObjectStorageObjectSummary{Bucket: strings.TrimSpace(bucketName), Key: key, Name: path.Base(strings.TrimRight(key, "/")), Kind: objectStorageKindForObject(key, contentType), SizeBytes: size, ContentType: contentType, ETag: strings.Trim(header.Get("ETag"), `"`), LastModified: header.Get("Last-Modified"), StorageClass: header.Get("x-oss-storage-class"), VersionID: header.Get("x-oss-version-id"), Metadata: metadata}
}
func (browser *ossObjectStorageBrowser) GetObjectSummary(_ context.Context, bucketName string, key string) (domain.ObjectStorageObjectSummary, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
key = exactObjectStorageKey(key)
header, err := bucket.GetObjectDetailedMeta(key)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return ossSummaryFromHeader(bucketName, key, header), nil
}
func (browser *ossObjectStorageBrowser) ReadObject(_ context.Context, bucketName string, key string) ([]byte, string, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return nil, "", err
}
body, err := bucket.GetObject(exactObjectStorageKey(key))
if err != nil {
return nil, "", err
}
defer body.Close()
payload, err := io.ReadAll(body)
if err != nil {
return nil, "", err
}
summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key)
return payload, summary.ContentType, nil
}
func (browser *ossObjectStorageBrowser) ReadObjectRange(_ context.Context, bucketName string, key string, start int64, end int64) (io.ReadCloser, string, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return nil, "", err
}
body, err := bucket.GetObject(exactObjectStorageKey(key), oss.Range(start, end))
if err != nil {
return nil, "", err
}
summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key)
return body, summary.ContentType, nil
}
func (browser *ossObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucketName string, key string, maxBytes int64) ([]byte, string, error) {
if maxBytes <= 0 {
return browser.ReadObject(ctx, bucketName, key)
}
body, contentType, err := browser.ReadObjectRange(ctx, bucketName, key, 0, maxBytes-1)
if err != nil {
return nil, "", err
}
defer body.Close()
payload, err := io.ReadAll(io.LimitReader(body, maxBytes))
return payload, contentType, err
}
func (browser *ossObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucketName string, key string) (string, error) {
bucket, err := browser.bucket(bucketName)
if err != nil {
return "", err
}
return bucket.SignURL(exactObjectStorageKey(key), oss.HTTPGet, 3600)
}