forked from longtao.wu/droboticflow
277 lines
8.9 KiB
Go
277 lines
8.9 KiB
Go
package objectsync
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"droboticflow/internal/bosclient"
|
|
"droboticflow/internal/domain"
|
|
|
|
"github.com/baidubce/bce-sdk-go/bce"
|
|
"github.com/baidubce/bce-sdk-go/services/bos"
|
|
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
type ObjectMetadata struct {
|
|
Exists bool
|
|
ETag string
|
|
VersionID string
|
|
SizeBytes int64
|
|
LastModified string
|
|
UserMeta map[string]string
|
|
}
|
|
|
|
type Publisher interface {
|
|
Head(context.Context, domain.StorageConnectionRecord, string, string) (ObjectMetadata, error)
|
|
PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (ObjectMetadata, error)
|
|
PutInternal(context.Context, domain.StorageConnectionRecord, string, string, io.Reader, int64, string, map[string]string) (ObjectMetadata, error)
|
|
AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error
|
|
Delete(context.Context, domain.StorageConnectionRecord, string, string) error
|
|
}
|
|
|
|
type BOSPublisher struct {
|
|
GlobalEndpoint string
|
|
}
|
|
|
|
func (publisher BOSPublisher) client(connection domain.StorageConnectionRecord) (*bos.Client, error) {
|
|
endpoint := strings.TrimSpace(connection.Endpoint)
|
|
if endpoint == "" {
|
|
endpoint = strings.TrimSpace(publisher.GlobalEndpoint)
|
|
}
|
|
if endpoint == "" {
|
|
return nil, fmt.Errorf("BOS endpoint is required")
|
|
}
|
|
return bosclient.New(strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint)
|
|
}
|
|
|
|
func IsPreconditionFailure(err error) bool {
|
|
var serviceError *bce.BceServiceError
|
|
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusPreconditionFailed
|
|
}
|
|
|
|
func isNotFound(err error) bool {
|
|
var serviceError *bce.BceServiceError
|
|
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusNotFound
|
|
}
|
|
|
|
func (publisher BOSPublisher) Head(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) {
|
|
if err := connection.CheckStoragePath(bucket, key, false); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, bosclient.ControlTimeout)
|
|
defer cancel()
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
result, err := client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
|
if isNotFound(err) {
|
|
return ObjectMetadata{Exists: false}, nil
|
|
}
|
|
if err != nil {
|
|
return ObjectMetadata{}, bosclient.Error(ctx, "head", err)
|
|
}
|
|
// HTTP header names are case-insensitive; the SDK returns canonicalized
|
|
// metadata suffixes while publication identities use lowercase keys.
|
|
userMeta := make(map[string]string, len(result.UserMeta))
|
|
for key, value := range result.UserMeta {
|
|
userMeta[strings.ToLower(strings.TrimSpace(key))] = value
|
|
}
|
|
return ObjectMetadata{
|
|
Exists: true,
|
|
ETag: strings.Trim(result.ETag, `"`),
|
|
VersionID: strings.TrimSpace(result.VersionId),
|
|
SizeBytes: result.ContentLength,
|
|
LastModified: strings.TrimSpace(result.LastModified),
|
|
UserMeta: userMeta,
|
|
}, nil
|
|
}
|
|
|
|
func multipartCopyPartSize(sizeBytes int64) int64 {
|
|
const (
|
|
minimum = int64(256 * 1024 * 1024)
|
|
maximum = int64(5 * 1024 * 1024 * 1024)
|
|
parts = int64(1_024)
|
|
align = int64(1024 * 1024)
|
|
)
|
|
partSize := minimum
|
|
if sizeBytes > partSize*parts {
|
|
partSize = ((sizeBytes/parts + align - 1) / align) * align
|
|
}
|
|
if partSize > maximum {
|
|
return maximum
|
|
}
|
|
return partSize
|
|
}
|
|
|
|
func (publisher BOSPublisher) PublishConditional(
|
|
ctx context.Context,
|
|
connection domain.StorageConnectionRecord,
|
|
bucket string,
|
|
sourceKey string,
|
|
targetKey string,
|
|
sizeBytes int64,
|
|
sourceETag string,
|
|
ifMatch string,
|
|
ifNoneMatch string,
|
|
userMeta map[string]string,
|
|
) (result ObjectMetadata, resultErr error) {
|
|
if err := connection.CheckStoragePath(bucket, sourceKey, false); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
if err := connection.CheckStoragePath(bucket, targetKey, false); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, bosclient.PublishTimeout)
|
|
defer cancel()
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
bucket = strings.TrimSpace(bucket)
|
|
sourceKey = strings.Trim(strings.TrimSpace(sourceKey), "/")
|
|
targetKey = strings.Trim(strings.TrimSpace(targetKey), "/")
|
|
if sizeBytes == 0 {
|
|
body, bodyErr := bce.NewBodyFromBytes([]byte{})
|
|
if bodyErr != nil {
|
|
return ObjectMetadata{}, bodyErr
|
|
}
|
|
_, err = client.PutObjectWithContext(ctx, bucket, targetKey, body, &bosapi.PutObjectArgs{
|
|
ContentType: "application/octet-stream",
|
|
UserMeta: userMeta,
|
|
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
})
|
|
if err != nil {
|
|
return ObjectMetadata{}, bosclient.Error(ctx, "put empty object", err)
|
|
}
|
|
return publisher.Head(ctx, connection, bucket, targetKey)
|
|
}
|
|
initiated, err := bosclient.Initiate(ctx, client, bucket, targetKey, "")
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
abort := true
|
|
defer func() {
|
|
if abort {
|
|
if cleanupErr := bosclient.CleanupAbort(ctx, client, bucket, targetKey, initiated.UploadId); cleanupErr != nil {
|
|
resultErr = errors.Join(resultErr, fmt.Errorf("cleanup upload %s: %w", initiated.UploadId, cleanupErr))
|
|
}
|
|
}
|
|
}()
|
|
partSize := multipartCopyPartSize(sizeBytes)
|
|
partCount := int((sizeBytes + partSize - 1) / partSize)
|
|
parts := make([]bosapi.UploadInfoType, partCount)
|
|
copyGroup, copyContext := errgroup.WithContext(ctx)
|
|
copyGroup.SetLimit(8)
|
|
for partIndex := range partCount {
|
|
start := int64(partIndex) * partSize
|
|
end := start + partSize - 1
|
|
if end >= sizeBytes {
|
|
end = sizeBytes - 1
|
|
}
|
|
partNumber := partIndex + 1
|
|
if copyContext.Err() != nil {
|
|
break
|
|
}
|
|
copyGroup.Go(func() error {
|
|
partCtx, cancel := context.WithTimeout(copyContext, bosclient.DataTimeout)
|
|
defer cancel()
|
|
result, copyErr := client.UploadPartCopyWithContext(partCtx, bucket, targetKey, bucket, sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{
|
|
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
|
|
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
|
|
})
|
|
if copyErr != nil {
|
|
return bosclient.Error(partCtx, "copy part", copyErr)
|
|
}
|
|
parts[partIndex] = bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)}
|
|
return nil
|
|
})
|
|
}
|
|
if err := copyGroup.Wait(); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
if err := ctx.Err(); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
args := &bosapi.CompleteMultipartUploadArgs{
|
|
Parts: parts,
|
|
UserMeta: userMeta,
|
|
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
}
|
|
if err = bosclient.Complete(ctx, client, bucket, targetKey, initiated.UploadId, args); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
abort = false
|
|
return publisher.Head(ctx, connection, bucket, targetKey)
|
|
}
|
|
|
|
func (publisher BOSPublisher) PutInternal(
|
|
ctx context.Context,
|
|
connection domain.StorageConnectionRecord,
|
|
bucket string,
|
|
key string,
|
|
reader io.Reader,
|
|
size int64,
|
|
contentType string,
|
|
userMeta map[string]string,
|
|
) (ObjectMetadata, error) {
|
|
if err := connection.CheckStoragePath(bucket, key, false); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, bosclient.DataTimeout)
|
|
defer cancel()
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
body, checksum, cleanup, err := bosclient.UploadBody(ctx, reader, size)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
defer cleanup()
|
|
_, err = client.PutObjectWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), body, &bosapi.PutObjectArgs{
|
|
ContentType: strings.TrimSpace(contentType),
|
|
ContentMD5: checksum,
|
|
UserMeta: userMeta,
|
|
IfNoneMatch: "*",
|
|
})
|
|
if err != nil {
|
|
return ObjectMetadata{}, bosclient.Error(ctx, "put internal object", err)
|
|
}
|
|
return publisher.Head(ctx, connection, bucket, key)
|
|
}
|
|
|
|
func (publisher BOSPublisher) Delete(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) error {
|
|
if err := connection.CheckStoragePath(bucket, key, false); err != nil {
|
|
return err
|
|
}
|
|
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bosclient.Delete(ctx, client, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
|
}
|
|
|
|
func (publisher BOSPublisher) AbortMultipart(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, uploadID string) error {
|
|
if err := connection.CheckStoragePath(bucket, key, false); err != nil {
|
|
return err
|
|
}
|
|
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return bosclient.Abort(ctx, client, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), strings.TrimSpace(uploadID))
|
|
}
|