package api import ( "bytes" "context" "fmt" "io" "net/http" "path" "sort" "strconv" "strings" "time" "droboticflow/internal/domain" "github.com/aliyun/aliyun-oss-go-sdk/oss" ) type ossObjectStorageBrowser struct { connection *domain.StorageConnectionRecord client *oss.Client } var _ objectStorageBrowser = (*ossObjectStorageBrowser)(nil) func newOSSObjectStorageBrowser(connection *domain.StorageConnectionRecord, endpoint string) (objectStorageBrowser, func(), error) { if connection == nil { return nil, nil, nil } endpoint = strings.TrimRight(strings.TrimSpace(endpoint), "/") if endpoint == "" { return nil, nil, fmt.Errorf("OSS endpoint is required for storage connection %s", strings.TrimSpace(connection.Name)) } client, err := oss.New(endpoint, strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey)) if err != nil { return nil, nil, err } return &ossObjectStorageBrowser{connection: connection, client: client}, func() {}, nil } func (browser *ossObjectStorageBrowser) bucket(name string) (*oss.Bucket, error) { return browser.client.Bucket(strings.TrimSpace(name)) } func (browser *ossObjectStorageBrowser) ListBuckets(context.Context) ([]domain.ObjectStorageBucketSummary, error) { result, err := browser.client.ListBuckets() if err != nil { return nil, err } items := make([]domain.ObjectStorageBucketSummary, 0, len(result.Buckets)) for _, item := range result.Buckets { items = append(items, domain.ObjectStorageBucketSummary{Name: item.Name, CreatedAt: item.CreationDate.UTC().Format(time.RFC3339)}) } return items, nil } func ossObjectSummary(bucketName string, item oss.ObjectProperties) domain.ObjectStorageObjectSummary { contentType := detectObjectStorageContentType(item.Key, nil, "") return domain.ObjectStorageObjectSummary{ Bucket: bucketName, Key: item.Key, Name: path.Base(strings.TrimRight(item.Key, "/")), Kind: objectStorageKindForObject(item.Key, contentType), SizeBytes: item.Size, ContentType: contentType, ETag: strings.Trim(item.ETag, `"`), LastModified: item.LastModified.UTC().Format(time.RFC3339), StorageClass: item.StorageClass, } } func (browser *ossObjectStorageBrowser) ListObjects(ctx context.Context, bucketName string, prefix string) ([]domain.ObjectStorageObjectSummary, error) { page, err := browser.ListObjectsPage(ctx, bucketName, prefix, "", 1000) return page.Items, err } func (browser *ossObjectStorageBrowser) ListObjectsPage(_ context.Context, bucketName string, prefix string, marker string, limit int) (objectStoragePage, error) { bucket, err := browser.bucket(bucketName) if err != nil { return objectStoragePage{}, err } if limit <= 0 || limit > 1000 { limit = 1000 } result, err := bucket.ListObjects(oss.Prefix(strings.TrimSpace(prefix)), oss.Delimiter("/"), oss.Marker(strings.TrimSpace(marker)), oss.MaxKeys(limit)) if err != nil { return objectStoragePage{}, err } items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Objects)) for _, commonPrefix := range result.CommonPrefixes { items = append(items, objectStorageSummaryFromPrefix(bucketName, commonPrefix)) } for _, object := range result.Objects { items = append(items, ossObjectSummary(bucketName, object)) } return objectStoragePage{Items: items, NextMarker: result.NextMarker, HasMore: result.IsTruncated}, nil } func (browser *ossObjectStorageBrowser) CreateBucket(_ context.Context, bucketName string) (domain.ObjectStorageBucketSummary, error) { bucketName = strings.TrimSpace(bucketName) if err := browser.client.CreateBucket(bucketName); err != nil { return domain.ObjectStorageBucketSummary{}, err } return domain.ObjectStorageBucketSummary{Name: bucketName, CreatedAt: nowISO()}, nil } func (browser *ossObjectStorageBrowser) DeleteBucket(_ context.Context, bucketName string) error { return browser.client.DeleteBucket(strings.TrimSpace(bucketName)) } func (browser *ossObjectStorageBrowser) PutObject(ctx context.Context, bucketName string, key string, payload []byte, contentType string) (domain.ObjectStorageObjectSummary, error) { return browser.PutObjectStream(ctx, bucketName, key, bytes.NewReader(payload), contentType) } func (browser *ossObjectStorageBrowser) PutObjectStream(ctx context.Context, bucketName string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStorageObjectSummary{}, err } options := make([]oss.Option, 0, 1) if strings.TrimSpace(contentType) != "" { options = append(options, oss.ContentType(strings.TrimSpace(contentType))) } if err := bucket.PutObject(exactObjectStorageKey(key), reader, options...); err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.GetObjectSummary(ctx, bucketName, key) } func (browser *ossObjectStorageBrowser) PresignPutObject(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStoragePresignedUploadResult, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStoragePresignedUploadResult{}, err } expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) options := make([]oss.Option, 0, 1) headers := objectStorageUploadHeaders(contentType) if strings.TrimSpace(contentType) != "" { options = append(options, oss.ContentType(strings.TrimSpace(contentType))) } url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), options...) if err != nil { return domain.ObjectStoragePresignedUploadResult{}, err } return domain.ObjectStoragePresignedUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), Method: http.MethodPut, URL: url, Headers: headers, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil } func (browser *ossObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucketName string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStorageMultipartUploadCreateResult{}, err } options := make([]oss.Option, 0, 1) if strings.TrimSpace(contentType) != "" { options = append(options, oss.ContentType(strings.TrimSpace(contentType))) } result, err := bucket.InitiateMultipartUpload(exactObjectStorageKey(key), options...) if err != nil { return domain.ObjectStorageMultipartUploadCreateResult{}, err } expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) return domain.ObjectStorageMultipartUploadCreateResult{Bucket: result.Bucket, Key: result.Key, UploadID: result.UploadID, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil } func (browser *ossObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucketName string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStorageMultipartUploadPartURLResult{}, err } expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds) url, err := bucket.SignURL(exactObjectStorageKey(key), oss.HTTPPut, int64(expiresInSeconds), oss.AddParam("uploadId", strings.TrimSpace(uploadID)), oss.AddParam("partNumber", partNumber)) if err != nil { return domain.ObjectStorageMultipartUploadPartURLResult{}, err } return domain.ObjectStorageMultipartUploadPartURLResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID), PartNumber: partNumber, Method: http.MethodPut, URL: url, Headers: map[string]string{}, ExpiresAt: objectStorageExpiresAt(expiresInSeconds)}, nil } func (browser *ossObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucketName string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStorageObjectSummary{}, err } completed := make([]oss.UploadPart, 0, len(parts)) for _, item := range parts { completed = append(completed, oss.UploadPart{PartNumber: item.PartNumber, ETag: strings.Trim(item.ETag, `"`)}) } sort.Slice(completed, func(i, j int) bool { return completed[i].PartNumber < completed[j].PartNumber }) _, err = bucket.CompleteMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)}, completed) if err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.GetObjectSummary(ctx, bucketName, key) } func (browser *ossObjectStorageBrowser) AbortMultipartUpload(_ context.Context, bucketName string, key string, uploadID string) error { bucket, err := browser.bucket(bucketName) if err != nil { return err } return bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{Bucket: strings.TrimSpace(bucketName), Key: exactObjectStorageKey(key), UploadID: strings.TrimSpace(uploadID)}) } func (browser *ossObjectStorageBrowser) RenameObject(ctx context.Context, bucketName string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStorageObjectSummary{}, err } key, newKey = exactObjectStorageKey(key), exactObjectStorageKey(newKey) if key == "" || newKey == "" || key == newKey { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("distinct source and destination keys are required") } if _, err := bucket.CopyObject(key, newKey); err != nil { return domain.ObjectStorageObjectSummary{}, err } if err := bucket.DeleteObject(key); err != nil { return domain.ObjectStorageObjectSummary{}, err } return browser.GetObjectSummary(ctx, bucketName, newKey) } func (browser *ossObjectStorageBrowser) MovePrefix(ctx context.Context, bucketName string, prefix string, newPrefix string) (domain.ObjectStorageObjectSummary, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStorageObjectSummary{}, err } prefix, newPrefix = strings.Trim(strings.TrimSpace(prefix), "/"), strings.Trim(strings.TrimSpace(newPrefix), "/") if prefix == "" || newPrefix == "" || prefix == newPrefix || strings.HasPrefix(newPrefix, prefix+"/") { return domain.ObjectStorageObjectSummary{}, fmt.Errorf("valid distinct source and destination prefixes are required") } marker := "" for { result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000)) if listErr != nil { return domain.ObjectStorageObjectSummary{}, listErr } for _, item := range result.Objects { destination := newPrefix + "/" + strings.TrimPrefix(item.Key, prefix+"/") if _, err := bucket.CopyObject(item.Key, destination); err != nil { return domain.ObjectStorageObjectSummary{}, err } if err := bucket.DeleteObject(item.Key); err != nil { return domain.ObjectStorageObjectSummary{}, err } } if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker { break } marker = result.NextMarker } return objectStorageSummaryFromPrefix(strings.TrimSpace(bucketName), newPrefix), nil } func (browser *ossObjectStorageBrowser) DeleteObject(_ context.Context, bucketName string, key string) error { bucket, err := browser.bucket(bucketName) if err != nil { return err } return bucket.DeleteObject(exactObjectStorageKey(key)) } func (browser *ossObjectStorageBrowser) DeletePrefix(_ context.Context, bucketName string, prefix string) error { bucket, err := browser.bucket(bucketName) if err != nil { return err } prefix = strings.Trim(strings.TrimSpace(prefix), "/") marker := "" for { result, listErr := bucket.ListObjects(oss.Prefix(prefix+"/"), oss.Marker(marker), oss.MaxKeys(1000)) if listErr != nil { return listErr } keys := make([]string, 0, len(result.Objects)) for _, item := range result.Objects { keys = append(keys, item.Key) } if len(keys) > 0 { if _, deleteErr := bucket.DeleteObjects(keys); deleteErr != nil { return deleteErr } } if !result.IsTruncated || result.NextMarker == "" || result.NextMarker == marker { break } marker = result.NextMarker } _ = bucket.DeleteObject(prefix) return bucket.DeleteObject(prefix + "/") } func ossSummaryFromHeader(bucketName string, key string, header http.Header) domain.ObjectStorageObjectSummary { size, _ := strconv.ParseInt(strings.TrimSpace(header.Get("Content-Length")), 10, 64) contentType := strings.TrimSpace(header.Get("Content-Type")) metadata := map[string]any{} for name, values := range header { if strings.HasPrefix(strings.ToLower(name), "x-oss-meta-") { metadata[strings.TrimPrefix(strings.ToLower(name), "x-oss-meta-")] = strings.Join(values, ",") } } if len(metadata) == 0 { metadata = nil } return domain.ObjectStorageObjectSummary{Bucket: strings.TrimSpace(bucketName), Key: key, Name: path.Base(strings.TrimRight(key, "/")), Kind: objectStorageKindForObject(key, contentType), SizeBytes: size, ContentType: contentType, ETag: strings.Trim(header.Get("ETag"), `"`), LastModified: header.Get("Last-Modified"), StorageClass: header.Get("x-oss-storage-class"), VersionID: header.Get("x-oss-version-id"), Metadata: metadata} } func (browser *ossObjectStorageBrowser) GetObjectSummary(_ context.Context, bucketName string, key string) (domain.ObjectStorageObjectSummary, error) { bucket, err := browser.bucket(bucketName) if err != nil { return domain.ObjectStorageObjectSummary{}, err } key = exactObjectStorageKey(key) header, err := bucket.GetObjectDetailedMeta(key) if err != nil { return domain.ObjectStorageObjectSummary{}, err } return ossSummaryFromHeader(bucketName, key, header), nil } func (browser *ossObjectStorageBrowser) ReadObject(_ context.Context, bucketName string, key string) ([]byte, string, error) { bucket, err := browser.bucket(bucketName) if err != nil { return nil, "", err } body, err := bucket.GetObject(exactObjectStorageKey(key)) if err != nil { return nil, "", err } defer body.Close() payload, err := io.ReadAll(body) if err != nil { return nil, "", err } summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key) return payload, summary.ContentType, nil } func (browser *ossObjectStorageBrowser) ReadObjectRange(_ context.Context, bucketName string, key string, start int64, end int64) (io.ReadCloser, string, error) { bucket, err := browser.bucket(bucketName) if err != nil { return nil, "", err } body, err := bucket.GetObject(exactObjectStorageKey(key), oss.Range(start, end)) if err != nil { return nil, "", err } summary, _ := browser.GetObjectSummary(context.Background(), bucketName, key) return body, summary.ContentType, nil } func (browser *ossObjectStorageBrowser) ReadObjectPreview(ctx context.Context, bucketName string, key string, maxBytes int64) ([]byte, string, error) { if maxBytes <= 0 { return browser.ReadObject(ctx, bucketName, key) } body, contentType, err := browser.ReadObjectRange(ctx, bucketName, key, 0, maxBytes-1) if err != nil { return nil, "", err } defer body.Close() payload, err := io.ReadAll(io.LimitReader(body, maxBytes)) return payload, contentType, err } func (browser *ossObjectStorageBrowser) BuildPreviewURL(_ context.Context, bucketName string, key string) (string, error) { bucket, err := browser.bucket(bucketName) if err != nil { return "", err } return bucket.SignURL(exactObjectStorageKey(key), oss.HTTPGet, 3600) }