235 lines
7.2 KiB
Go
235 lines
7.2 KiB
Go
package objectsync
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
|
|
"github.com/baidubce/bce-sdk-go/bce"
|
|
"github.com/baidubce/bce-sdk-go/services/bos"
|
|
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
|
)
|
|
|
|
type ObjectMetadata struct {
|
|
Exists bool
|
|
ETag string
|
|
VersionID string
|
|
SizeBytes int64
|
|
LastModified string
|
|
}
|
|
|
|
type Publisher interface {
|
|
Head(context.Context, domain.StorageConnectionRecord, string, string) (ObjectMetadata, error)
|
|
PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (ObjectMetadata, error)
|
|
PutInternal(context.Context, domain.StorageConnectionRecord, string, string, io.Reader, int64, string, map[string]string) (ObjectMetadata, error)
|
|
AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error
|
|
Delete(context.Context, domain.StorageConnectionRecord, string, string) error
|
|
}
|
|
|
|
type BOSPublisher struct {
|
|
GlobalEndpoint string
|
|
}
|
|
|
|
func (publisher BOSPublisher) client(connection domain.StorageConnectionRecord) (*bos.Client, error) {
|
|
endpoint := strings.TrimSpace(connection.Endpoint)
|
|
if endpoint == "" {
|
|
endpoint = strings.TrimSpace(publisher.GlobalEndpoint)
|
|
}
|
|
if endpoint == "" {
|
|
return nil, fmt.Errorf("BOS endpoint is required")
|
|
}
|
|
return bos.NewClient(strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint)
|
|
}
|
|
|
|
func IsPreconditionFailure(err error) bool {
|
|
var serviceError *bce.BceServiceError
|
|
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusPreconditionFailed
|
|
}
|
|
|
|
func isNotFound(err error) bool {
|
|
var serviceError *bce.BceServiceError
|
|
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusNotFound
|
|
}
|
|
|
|
func (publisher BOSPublisher) Head(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
result, err := client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
|
if isNotFound(err) {
|
|
return ObjectMetadata{Exists: false}, nil
|
|
}
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
return ObjectMetadata{
|
|
Exists: true,
|
|
ETag: strings.Trim(result.ETag, `"`),
|
|
VersionID: strings.TrimSpace(result.VersionId),
|
|
SizeBytes: result.ContentLength,
|
|
LastModified: strings.TrimSpace(result.LastModified),
|
|
}, nil
|
|
}
|
|
|
|
func multipartCopyPartSize(sizeBytes int64) int64 {
|
|
const (
|
|
minimum = int64(64 * 1024 * 1024)
|
|
maximum = int64(5 * 1024 * 1024 * 1024)
|
|
parts = int64(10_000)
|
|
align = int64(1024 * 1024)
|
|
)
|
|
partSize := minimum
|
|
if sizeBytes > partSize*parts {
|
|
partSize = ((sizeBytes/parts + align - 1) / align) * align
|
|
}
|
|
if partSize > maximum {
|
|
return maximum
|
|
}
|
|
return partSize
|
|
}
|
|
|
|
func (publisher BOSPublisher) PublishConditional(
|
|
ctx context.Context,
|
|
connection domain.StorageConnectionRecord,
|
|
bucket string,
|
|
sourceKey string,
|
|
targetKey string,
|
|
sizeBytes int64,
|
|
sourceETag string,
|
|
ifMatch string,
|
|
ifNoneMatch string,
|
|
userMeta map[string]string,
|
|
) (ObjectMetadata, error) {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
bucket = strings.TrimSpace(bucket)
|
|
sourceKey = strings.Trim(strings.TrimSpace(sourceKey), "/")
|
|
targetKey = strings.Trim(strings.TrimSpace(targetKey), "/")
|
|
if sizeBytes == 0 {
|
|
body, bodyErr := bce.NewBodyFromBytes([]byte{})
|
|
if bodyErr != nil {
|
|
return ObjectMetadata{}, bodyErr
|
|
}
|
|
_, err = client.PutObjectWithContext(ctx, bucket, targetKey, body, &bosapi.PutObjectArgs{
|
|
ContentType: "application/octet-stream",
|
|
UserMeta: userMeta,
|
|
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
})
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
return publisher.Head(ctx, connection, bucket, targetKey)
|
|
}
|
|
initiated, err := client.InitiateMultipartUpload(bucket, targetKey, "", nil)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
abort := true
|
|
defer func() {
|
|
if abort {
|
|
_ = client.AbortMultipartUpload(bucket, targetKey, initiated.UploadId)
|
|
}
|
|
}()
|
|
partSize := multipartCopyPartSize(sizeBytes)
|
|
parts := make([]bosapi.UploadInfoType, 0, int((sizeBytes+partSize-1)/partSize))
|
|
for start, partNumber := int64(0), 1; start < sizeBytes; start, partNumber = start+partSize, partNumber+1 {
|
|
end := start + partSize - 1
|
|
if end >= sizeBytes {
|
|
end = sizeBytes - 1
|
|
}
|
|
result, copyErr := client.UploadPartCopyWithContext(ctx, bucket, targetKey, bucket, sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{
|
|
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
|
|
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
|
|
})
|
|
if copyErr != nil {
|
|
return ObjectMetadata{}, copyErr
|
|
}
|
|
parts = append(parts, bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)})
|
|
}
|
|
args := &bosapi.CompleteMultipartUploadArgs{
|
|
Parts: parts,
|
|
UserMeta: userMeta,
|
|
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
|
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
|
}
|
|
payload, err := json.Marshal(args)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
body, err := bce.NewBodyFromBytes(payload)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
bosContext := client.NewBosContext(ctx)
|
|
if _, err = bosapi.CompleteMultipartUpload(client, bucket, targetKey, initiated.UploadId, body, args, bosContext); err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
abort = false
|
|
return publisher.Head(ctx, connection, bucket, targetKey)
|
|
}
|
|
|
|
func (publisher BOSPublisher) PutInternal(
|
|
ctx context.Context,
|
|
connection domain.StorageConnectionRecord,
|
|
bucket string,
|
|
key string,
|
|
reader io.Reader,
|
|
size int64,
|
|
contentType string,
|
|
userMeta map[string]string,
|
|
) (ObjectMetadata, error) {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
body, err := bce.NewBodyFromSizedReader(reader, size)
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
_, err = client.PutObjectWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), body, &bosapi.PutObjectArgs{
|
|
ContentType: strings.TrimSpace(contentType),
|
|
UserMeta: userMeta,
|
|
IfNoneMatch: "*",
|
|
})
|
|
if err != nil {
|
|
return ObjectMetadata{}, err
|
|
}
|
|
return publisher.Head(ctx, connection, bucket, key)
|
|
}
|
|
|
|
func (publisher BOSPublisher) Delete(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) error {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
default:
|
|
return client.DeleteObject(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
|
}
|
|
}
|
|
|
|
func (publisher BOSPublisher) AbortMultipart(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, uploadID string) error {
|
|
client, err := publisher.client(connection)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
default:
|
|
return client.AbortMultipartUpload(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), strings.TrimSpace(uploadID))
|
|
}
|
|
}
|