forked from longtao.wu/droboticflow
274 lines
8.2 KiB
Go
274 lines
8.2 KiB
Go
package objectsync
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
|
|
"github.com/baidubce/bce-sdk-go/bce"
|
|
"github.com/baidubce/bce-sdk-go/services/bos"
|
|
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
type ObjectMetadata struct {
|
|
UserMeta map[string]string
|
|
Exists bool
|
|
ETag string
|
|
VersionID string
|
|
SizeBytes int64
|
|
LastModified string
|
|
}
|
|
|
|
type Publisher interface {
|
|
Head(context.Context, domain.StorageConnectionRecord, string, string) (ObjectMetadata, error)
|
|
PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (ObjectMetadata, error)
|
|
PutInternal(context.Context, domain.StorageConnectionRecord, string, string, io.Reader, int64, string, map[string]string) (ObjectMetadata, error)
|
|
AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error
|
|
Delete(context.Context, domain.StorageConnectionRecord, string, string) error
|
|
}
|
|
|
|
type BOSPublisher struct {
|
|
GlobalEndpoint string
|
|
}
|
|
|
|
func (publisher BOSPublisher) client(connection domain.StorageConnectionRecord) (*bos.Client, error) {
|
|
endpoint := strings.TrimSpace(connection.Endpoint)
|
|
if endpoint == "" {
|
|
endpoint = strings.TrimSpace(publisher.GlobalEndpoint)
|
|
}
|
|
if endpoint == "" {
|
|
return nil, fmt.Errorf("BOS endpoint is required")
|
|
}
|
|
return bos.NewClient(strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint)
|
|
}
|
|
|
|
func IsPreconditionFailure(err error) bool {
|
|
var serviceError *bce.BceServiceError
|
|
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusPreconditionFailed
|
|
}
|
|
|
|
func isNotFound(err error) bool {
|
|
var serviceError *bce.BceServiceError
|
|
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusNotFound
|
|
}
|
|
|
|
func (publisher BOSPublisher) Head(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
result, err := client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
|
if isNotFound(err) {
|
|
return ObjectMetadata{Exists: false}, nil
|
|
}
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
return ObjectMetadata{
|
|
UserMeta: result.UserMeta,
|
|
Exists: true,
|
|
ETag: strings.Trim(result.ETag, `"`),
|
|
VersionID: strings.TrimSpace(result.VersionId),
|
|
SizeBytes: result.ContentLength,
|
|
LastModified: strings.TrimSpace(result.LastModified),
|
|
}, nil
|
|
}
|
|
|
|
func (publisher BOSPublisher) ListPrefix(ctx context.Context, connection domain.StorageConnectionRecord, bucket, prefix string) ([]string, error) {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
keys := []string{}
|
|
marker := ""
|
|
for {
|
|
result, e := client.ListObjectsWithContext(ctx, bucket, &bosapi.ListObjectsArgs{Prefix: prefix, Marker: marker, MaxKeys: 1000})
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
for _, item := range result.Contents {
|
|
keys = append(keys, item.Key)
|
|
}
|
|
if !result.IsTruncated {
|
|
return keys, nil
|
|
}
|
|
if result.NextMarker == "" || result.NextMarker == marker {
|
|
return nil, fmt.Errorf("incomplete listing")
|
|
}
|
|
marker = result.NextMarker
|
|
}
|
|
}
|
|
|
|
func multipartCopyPartSize(sizeBytes int64) int64 {
|
|
const (
|
|
minimum = int64(256 * 1024 * 1024)
|
|
maximum = int64(5 * 1024 * 1024 * 1024)
|
|
parts = int64(1_024)
|
|
align = int64(1024 * 1024)
|
|
)
|
|
partSize := minimum
|
|
if sizeBytes > partSize*parts {
|
|
partSize = ((sizeBytes/parts + align - 1) / align) * align
|
|
}
|
|
if partSize > maximum {
|
|
return maximum
|
|
}
|
|
return partSize
|
|
}
|
|
|
|
func (publisher BOSPublisher) PublishConditional(
|
|
ctx context.Context,
|
|
connection domain.StorageConnectionRecord,
|
|
bucket string,
|
|
sourceKey string,
|
|
targetKey string,
|
|
sizeBytes int64,
|
|
sourceETag string,
|
|
ifMatch string,
|
|
ifNoneMatch string,
|
|
userMeta map[string]string,
|
|
) (ObjectMetadata, error) {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
bucket = strings.TrimSpace(bucket)
|
|
sourceKey = strings.Trim(strings.TrimSpace(sourceKey), "/")
|
|
targetKey = strings.Trim(strings.TrimSpace(targetKey), "/")
|
|
if sizeBytes == 0 {
|
|
body, bodyErr := bce.NewBodyFromBytes([]byte{})
|
|
if bodyErr != nil {
|
|
return ObjectMetadata{}, bodyErr
|
|
}
|
|
_, err = client.PutObjectWithContext(ctx, bucket, targetKey, body, &bosapi.PutObjectArgs{
|
|
ContentType: "application/octet-stream",
|
|
UserMeta: userMeta,
|
|
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
})
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
return publisher.Head(ctx, connection, bucket, targetKey)
|
|
}
|
|
initiated, err := client.InitiateMultipartUpload(bucket, targetKey, "", nil)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
abort := true
|
|
defer func() {
|
|
if abort {
|
|
_ = client.AbortMultipartUpload(bucket, targetKey, initiated.UploadId)
|
|
}
|
|
}()
|
|
partSize := multipartCopyPartSize(sizeBytes)
|
|
partCount := int((sizeBytes + partSize - 1) / partSize)
|
|
parts := make([]bosapi.UploadInfoType, partCount)
|
|
copyGroup, copyContext := errgroup.WithContext(ctx)
|
|
copyGroup.SetLimit(8)
|
|
for partIndex := range partCount {
|
|
start := int64(partIndex) * partSize
|
|
end := start + partSize - 1
|
|
if end >= sizeBytes {
|
|
end = sizeBytes - 1
|
|
}
|
|
partNumber := partIndex + 1
|
|
copyGroup.Go(func() error {
|
|
result, copyErr := client.UploadPartCopyWithContext(copyContext, bucket, targetKey, bucket, sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{
|
|
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
|
|
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
|
|
})
|
|
if copyErr != nil {
|
|
return copyErr
|
|
}
|
|
parts[partIndex] = bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)}
|
|
return nil
|
|
})
|
|
}
|
|
if err := copyGroup.Wait(); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
args := &bosapi.CompleteMultipartUploadArgs{
|
|
Parts: parts,
|
|
UserMeta: userMeta,
|
|
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
}
|
|
payload, err := json.Marshal(args)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
body, err := bce.NewBodyFromBytes(payload)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
bosContext := client.NewBosContext(ctx)
|
|
if _, err = bosapi.CompleteMultipartUpload(client, bucket, targetKey, initiated.UploadId, body, args, bosContext); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
abort = false
|
|
return publisher.Head(ctx, connection, bucket, targetKey)
|
|
}
|
|
|
|
func (publisher BOSPublisher) PutInternal(
|
|
ctx context.Context,
|
|
connection domain.StorageConnectionRecord,
|
|
bucket string,
|
|
key string,
|
|
reader io.Reader,
|
|
size int64,
|
|
contentType string,
|
|
userMeta map[string]string,
|
|
) (ObjectMetadata, error) {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
body, err := bce.NewBodyFromSizedReader(reader, size)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
_, err = client.PutObjectWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), body, &bosapi.PutObjectArgs{
|
|
ContentType: strings.TrimSpace(contentType),
|
|
UserMeta: userMeta,
|
|
IfNoneMatch: "*",
|
|
})
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
return publisher.Head(ctx, connection, bucket, key)
|
|
}
|
|
|
|
func (publisher BOSPublisher) Delete(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) error {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
default:
|
|
return client.DeleteObject(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
|
}
|
|
}
|
|
|
|
func (publisher BOSPublisher) AbortMultipart(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, uploadID string) error {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
default:
|
|
return client.AbortMultipartUpload(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), strings.TrimSpace(uploadID))
|
|
}
|
|
}
|