365 lines
15 KiB
Go
365 lines
15 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"path"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
|
|
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
|
)
|
|
|
|
type ossObjectStorageBrowser struct {
|
|
connection *domain.StorageConnectionRecord
|
|
client *oss.Client
|
|
}
|
|
|
|
var _ objectStorageBrowser = (*ossObjectStorageBrowser)(nil)
|
|
|
|
func newOSSObjectStorageBrowser(connection *domain.StorageConnectionRecord, endpoint string) (objectStorageBrowser, func(), error) {
|
|
if connection == nil {
|
|
return nil, nil, nil
|
|
}
|
|
endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/")
|
|
if endpoint == "" {
|
|
return nil, nil, fmt.Errorf("OSS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name))
|
|
}
|
|
client, err := oss.New(endpoint, strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey))
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return &ossObjectStorageBrowser{connection: connection, client: client}, func() {}, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) bucket(name string) (*oss.Bucket, error) {
|
|
return browser.client.Bucket(strings.TrimSpace(name))
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error) {
|
|
result, err := browser.client.ListBuckets()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets))
|
|
for _, item := range result.Buckets {
|
|
items = append(items, domain.ObjectStorageBucketSummary{Name: item.Name, CreatedAt: item.CreationDate.UTC().Format(time.RFC3339)})
|
|
}
|
|
return items, nil
|
|
}
|
|
|
|
func ossObjectSummary(bucketName string, item oss.ObjectProperties) domain.ObjectStorageObjectSummary {
|
|
contentType := detectObjectStorageContentType(item.Key, nil, "")
|
|
return domain.ObjectStorageObjectSummary{
|
|
Bucket: bucketName, Key: item.Key, Name: path.Base(strings.TrimRight(item.Key, "/")),
|
|
Kind: objectStorageKindForObject(item.Key, contentType), SizeBytes: item.Size, ContentType: contentType,
|
|
ETag: strings.Trim(item.ETag, `"`), LastModified: item.LastModified.UTC().Format(time.RFC3339), StorageClass: item.StorageClass,
|
|
}
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) ListObjects(ctx context.Context, bucketName string, prefix string) ([]domain.ObjectStorageObjectSummary, error) {
|
|
page, err := browser.ListObjectsPage(ctx, bucketName, prefix, "", 1000)
|
|
return page.Items, err
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) ListObjectsPage(_ context.Context, bucketName string, prefix string, marker string, limit int) (objectStoragePage, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return objectStoragePage{}, err
|
|
}
|
|
if limit <= 0 || limit > 1000 {
|
|
limit = 1000
|
|
}
|
|
result, err := bucket.ListObjects(oss.Prefix(strings.TrimSpace(prefix)), oss.Delimiter("/"), oss.Marker(strings.TrimSpace(marker)), oss.MaxKeys(limit))
|
|
if err != nil {
|
|
return objectStoragePage{}, err
|
|
}
|
|
items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Objects))
|
|
for _, commonPrefix := range result.CommonPrefixes {
|
|
items = append(items, objectStorageSummaryFromPrefix(bucketName, commonPrefix))
|
|
}
|
|
for _, object := range result.Objects {
|
|
items = append(items, ossObjectSummary(bucketName, object))
|
|
}
|
|
return objectStoragePage{Items: items, NextMarker: result.NextMarker, HasMore: result.IsTruncated}, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) CreateBucket(_ context.Context, bucketName string) (domain.ObjectStorageBucketSummary, error) {
|
|
bucketName = strings.TrimSpace(bucketName)
|
|
if err := browser.client.CreateBucket(bucketName); err != nil {
|
|
return domain.ObjectStorageBucketSummary{}, err
|
|
}
|
|
return domain.ObjectStorageBucketSummary{Name: bucketName, CreatedAt: nowISO()}, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) DeleteBucket(_ context.Context, bucketName string) error {
|
|
return browser.client.DeleteBucket(strings.TrimSpace(bucketName))
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) PutObject(ctx context.Context, bucketName string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) {
|
|
return browser.PutObjectStream(ctx, bucketName, key, bytes.NewReader(payload), contentType)
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) PutObjectStream(ctx context.Context, bucketName string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
options := make([]oss.Option, 0, 1)
|
|
if strings.TrimSpace(contentType) != "" {
|
|
options = append(options, oss.ContentType(strings.TrimSpace(contentType)))
|
|
}
|
|
if err := bucket.PutObject(exactObjectStorageKey(key), reader, options...); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.GetObjectSummary(ctx, bucketName, key)
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) PresignPutObject(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStoragePresignedUploadResult, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStoragePresignedUploadResult{}, err
|
|
}
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
options := make([]oss.Option, 0, 1)
|
|
headers := objectStorageUploadHeaders(contentType)
|
|
if strings.TrimSpace(contentType) != "" {
|
|
options = append(options, oss.ContentType(strings.TrimSpace(contentType)))
|
|
}
|
|
url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), options...)
|
|
if err != nil {
|
|
return domain.ObjectStoragePresignedUploadResult{}, err
|
|
}
|
|
return domain.ObjectStoragePresignedUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), Method: http.MethodPut, URL: url, Headers: headers, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStorageMultipartUploadCreateResult{}, err
|
|
}
|
|
options := make([]oss.Option, 0, 1)
|
|
if strings.TrimSpace(contentType) != "" {
|
|
options = append(options, oss.ContentType(strings.TrimSpace(contentType)))
|
|
}
|
|
result, err := bucket.InitiateMultipartUpload(exactObjectStorageKey(key), options...)
|
|
if err != nil {
|
|
return domain.ObjectStorageMultipartUploadCreateResult{}, err
|
|
}
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
return domain.ObjectStorageMultipartUploadCreateResult{Bucket: result.Bucket, Key: result.Key, UploadID: result.UploadID, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucketName string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStorageMultipartUploadPartURLResult{}, err
|
|
}
|
|
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
|
url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), oss.AddParam("uploadId", strings.TrimSpace(uploadID)), oss.AddParam("partNumber", partNumber))
|
|
if err != nil {
|
|
return domain.ObjectStorageMultipartUploadPartURLResult{}, err
|
|
}
|
|
return domain.ObjectStorageMultipartUploadPartURLResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID), PartNumber: partNumber, Method: http.MethodPut, URL: url, Headers: map[string]string{}, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucketName string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
completed := make([]oss.UploadPart, 0, len(parts))
|
|
for _, item := range parts {
|
|
completed = append(completed, oss.UploadPart{PartNumber: item.PartNumber, ETag: strings.Trim(item.ETag, `"`)})
|
|
}
|
|
sort.Slice(completed, func(i, j int) bool { return completed[i].PartNumber < completed[j].PartNumber })
|
|
_, err = bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)}, completed)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.GetObjectSummary(ctx, bucketName, key)
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) AbortMultipartUpload(_ context.Context, bucketName string, key string, uploadID string) error {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)})
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) RenameObject(ctx context.Context, bucketName string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
key, newKey = exactObjectStorageKey(key), exactObjectStorageKey(newKey)
|
|
if key == "" || newKey == "" || key == newKey {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("distinct source and destination keys are required")
|
|
}
|
|
if _, err := bucket.CopyObject(key, newKey); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
if err := bucket.DeleteObject(key); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return browser.GetObjectSummary(ctx, bucketName, newKey)
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) MovePrefix(ctx context.Context, bucketName string, prefix string, newPrefix string) (domain.ObjectStorageObjectSummary, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
prefix, newPrefix = strings.Trim(strings.TrimSpace(prefix), "/"), strings.Trim(strings.TrimSpace(newPrefix), "/")
|
|
if prefix == "" || newPrefix == "" || prefix == newPrefix || strings.HasPrefix(newPrefix, prefix+"/") {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("valid distinct source and destination prefixes are required")
|
|
}
|
|
marker := ""
|
|
for {
|
|
result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000))
|
|
if listErr != nil {
|
|
return domain.ObjectStorageObjectSummary{}, listErr
|
|
}
|
|
for _, item := range result.Objects {
|
|
destination := newPrefix + "/" + strings.TrimPrefix(item.Key, prefix+"/")
|
|
if _, err := bucket.CopyObject(item.Key, destination); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
if err := bucket.DeleteObject(item.Key); err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
}
|
|
if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker {
|
|
break
|
|
}
|
|
marker = result.NextMarker
|
|
}
|
|
return objectStorageSummaryFromPrefix(strings.TrimSpace(bucketName), newPrefix), nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) DeleteObject(_ context.Context, bucketName string, key string) error {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bucket.DeleteObject(exactObjectStorageKey(key))
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) DeletePrefix(_ context.Context, bucketName string, prefix string) error {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
prefix = strings.Trim(strings.TrimSpace(prefix), "/")
|
|
marker := ""
|
|
for {
|
|
result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000))
|
|
if listErr != nil {
|
|
return listErr
|
|
}
|
|
keys := make([]string, 0, len(result.Objects))
|
|
for _, item := range result.Objects {
|
|
keys = append(keys, item.Key)
|
|
}
|
|
if len(keys) > 0 {
|
|
if _, deleteErr := bucket.DeleteObjects(keys); deleteErr != nil {
|
|
return deleteErr
|
|
}
|
|
}
|
|
if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker {
|
|
break
|
|
}
|
|
marker = result.NextMarker
|
|
}
|
|
_ = bucket.DeleteObject(prefix)
|
|
return bucket.DeleteObject(prefix + "/")
|
|
}
|
|
|
|
func ossSummaryFromHeader(bucketName string, key string, header http.Header) domain.ObjectStorageObjectSummary {
|
|
size, _ := strconv.ParseInt(strings.TrimSpace(header.Get("Content-Length")), 10, 64)
|
|
contentType := strings.TrimSpace(header.Get("Content-Type"))
|
|
metadata := map[string]any{}
|
|
for name, values := range header {
|
|
if strings.HasPrefix(strings.ToLower(name), "x-oss-meta-") {
|
|
metadata[strings.TrimPrefix(strings.ToLower(name), "x-oss-meta-")] = strings.Join(values, ",")
|
|
}
|
|
}
|
|
if len(metadata) == 0 {
|
|
metadata = nil
|
|
}
|
|
return domain.ObjectStorageObjectSummary{Bucket: strings.TrimSpace(bucketName), Key: key, Name: path.Base(strings.TrimRight(key, "/")), Kind: objectStorageKindForObject(key, contentType), SizeBytes: size, ContentType: contentType, ETag: strings.Trim(header.Get("ETag"), `"`), LastModified: header.Get("Last-Modified"), StorageClass: header.Get("x-oss-storage-class"), VersionID: header.Get("x-oss-version-id"), Metadata: metadata}
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) GetObjectSummary(_ context.Context, bucketName string, key string) (domain.ObjectStorageObjectSummary, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
key = exactObjectStorageKey(key)
|
|
header, err := bucket.GetObjectDetailedMeta(key)
|
|
if err != nil {
|
|
return domain.ObjectStorageObjectSummary{}, err
|
|
}
|
|
return ossSummaryFromHeader(bucketName, key, header), nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) ReadObject(_ context.Context, bucketName string, key string) ([]byte, string, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
body, err := bucket.GetObject(exactObjectStorageKey(key))
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
defer body.Close()
|
|
payload, err := io.ReadAll(body)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key)
|
|
return payload, summary.ContentType, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) ReadObjectRange(_ context.Context, bucketName string, key string, start int64, end int64) (io.ReadCloser, string, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
body, err := bucket.GetObject(exactObjectStorageKey(key), oss.Range(start, end))
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key)
|
|
return body, summary.ContentType, nil
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucketName string, key string, maxBytes int64) ([]byte, string, error) {
|
|
if maxBytes <= 0 {
|
|
return browser.ReadObject(ctx, bucketName, key)
|
|
}
|
|
body, contentType, err := browser.ReadObjectRange(ctx, bucketName, key, 0, maxBytes-1)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
defer body.Close()
|
|
payload, err := io.ReadAll(io.LimitReader(body, maxBytes))
|
|
return payload, contentType, err
|
|
}
|
|
|
|
func (browser *ossObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucketName string, key string) (string, error) {
|
|
bucket, err := browser.bucket(bucketName)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return bucket.SignURL(exactObjectStorageKey(key), oss.HTTPGet, 3600)
|
|
}
|