2026-07-28 20:43:46 +08:00

235 lines
7.2 KiB
Go

package objectsync
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"droboticflow/internal/domain"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/services/bos"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
)
type ObjectMetadata struct {
Exists bool
ETag string
VersionID string
SizeBytes int64
LastModified string
}
type Publisher interface {
Head(context.Context, domain.StorageConnectionRecord, string, string) (ObjectMetadata, error)
PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (ObjectMetadata, error)
PutInternal(context.Context, domain.StorageConnectionRecord, string, string, io.Reader, int64, string, map[string]string) (ObjectMetadata, error)
AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error
Delete(context.Context, domain.StorageConnectionRecord, string, string) error
}
type BOSPublisher struct {
GlobalEndpoint string
}
func (publisher BOSPublisher) client(connection domain.StorageConnectionRecord) (*bos.Client, error) {
endpoint := strings.TrimSpace(connection.Endpoint)
if endpoint == "" {
endpoint = strings.TrimSpace(publisher.GlobalEndpoint)
}
if endpoint == "" {
return nil, fmt.Errorf("BOS endpoint is required")
}
return bos.NewClient(strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint)
}
func IsPreconditionFailure(err error) bool {
var serviceError *bce.BceServiceError
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusPreconditionFailed
}
func isNotFound(err error) bool {
var serviceError *bce.BceServiceError
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusNotFound
}
func (publisher BOSPublisher) Head(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) {
client, err := publisher.client(connection)
if err != nil {
return ObjectMetadata{}, err
}
result, err := client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
if isNotFound(err) {
return ObjectMetadata{Exists: false}, nil
}
if err != nil {
return ObjectMetadata{}, err
}
return ObjectMetadata{
Exists: true,
ETag: strings.Trim(result.ETag, `"`),
VersionID: strings.TrimSpace(result.VersionId),
SizeBytes: result.ContentLength,
LastModified: strings.TrimSpace(result.LastModified),
}, nil
}
func multipartCopyPartSize(sizeBytes int64) int64 {
const (
minimum = int64(64 * 1024 * 1024)
maximum = int64(5 * 1024 * 1024 * 1024)
parts = int64(10_000)
align = int64(1024 * 1024)
)
partSize := minimum
if sizeBytes > partSize*parts {
partSize = ((sizeBytes/parts + align - 1) / align) * align
}
if partSize > maximum {
return maximum
}
return partSize
}
func (publisher BOSPublisher) PublishConditional(
ctx context.Context,
connection domain.StorageConnectionRecord,
bucket string,
sourceKey string,
targetKey string,
sizeBytes int64,
sourceETag string,
ifMatch string,
ifNoneMatch string,
userMeta map[string]string,
) (ObjectMetadata, error) {
client, err := publisher.client(connection)
if err != nil {
return ObjectMetadata{}, err
}
bucket = strings.TrimSpace(bucket)
sourceKey = strings.Trim(strings.TrimSpace(sourceKey), "/")
targetKey = strings.Trim(strings.TrimSpace(targetKey), "/")
if sizeBytes == 0 {
body, bodyErr := bce.NewBodyFromBytes([]byte{})
if bodyErr != nil {
return ObjectMetadata{}, bodyErr
}
_, err = client.PutObjectWithContext(ctx, bucket, targetKey, body, &bosapi.PutObjectArgs{
ContentType: "application/octet-stream",
UserMeta: userMeta,
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
})
if err != nil {
return ObjectMetadata{}, err
}
return publisher.Head(ctx, connection, bucket, targetKey)
}
initiated, err := client.InitiateMultipartUpload(bucket, targetKey, "", nil)
if err != nil {
return ObjectMetadata{}, err
}
abort := true
defer func() {
if abort {
_ = client.AbortMultipartUpload(bucket, targetKey, initiated.UploadId)
}
}()
partSize := multipartCopyPartSize(sizeBytes)
parts := make([]bosapi.UploadInfoType, 0, int((sizeBytes+partSize-1)/partSize))
for start, partNumber := int64(0), 1; start < sizeBytes; start, partNumber = start+partSize, partNumber+1 {
end := start + partSize - 1
if end >= sizeBytes {
end = sizeBytes - 1
}
result, copyErr := client.UploadPartCopyWithContext(ctx, bucket, targetKey, bucket, sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
})
if copyErr != nil {
return ObjectMetadata{}, copyErr
}
parts = append(parts, bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)})
}
args := &bosapi.CompleteMultipartUploadArgs{
Parts: parts,
UserMeta: userMeta,
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
}
payload, err := json.Marshal(args)
if err != nil {
return ObjectMetadata{}, err
}
body, err := bce.NewBodyFromBytes(payload)
if err != nil {
return ObjectMetadata{}, err
}
bosContext := client.NewBosContext(ctx)
if _, err = bosapi.CompleteMultipartUpload(client, bucket, targetKey, initiated.UploadId, body, args, bosContext); err != nil {
return ObjectMetadata{}, err
}
abort = false
return publisher.Head(ctx, connection, bucket, targetKey)
}
func (publisher BOSPublisher) PutInternal(
ctx context.Context,
connection domain.StorageConnectionRecord,
bucket string,
key string,
reader io.Reader,
size int64,
contentType string,
userMeta map[string]string,
) (ObjectMetadata, error) {
client, err := publisher.client(connection)
if err != nil {
return ObjectMetadata{}, err
}
body, err := bce.NewBodyFromSizedReader(reader, size)
if err != nil {
return ObjectMetadata{}, err
}
_, err = client.PutObjectWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), body, &bosapi.PutObjectArgs{
ContentType: strings.TrimSpace(contentType),
UserMeta: userMeta,
IfNoneMatch: "*",
})
if err != nil {
return ObjectMetadata{}, err
}
return publisher.Head(ctx, connection, bucket, key)
}
func (publisher BOSPublisher) Delete(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) error {
client, err := publisher.client(connection)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
return client.DeleteObject(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
}
}
func (publisher BOSPublisher) AbortMultipart(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, uploadID string) error {
client, err := publisher.client(connection)
if err != nil {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
return client.AbortMultipartUpload(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), strings.TrimSpace(uploadID))
}
}