package objectsync import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "strings" "droboticflow/internal/domain" "github.com/baidubce/bce-sdk-go/bce" "github.com/baidubce/bce-sdk-go/services/bos" bosapi "github.com/baidubce/bce-sdk-go/services/bos/api" "golang.org/x/sync/errgroup" ) type ObjectMetadata struct { UserMeta map[string]string Exists bool ETag string VersionID string SizeBytes int64 LastModified string } type Publisher interface { Head(context.Context, domain.StorageConnectionRecord, string, string) (ObjectMetadata, error) PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (ObjectMetadata, error) PutInternal(context.Context, domain.StorageConnectionRecord, string, string, io.Reader, int64, string, map[string]string) (ObjectMetadata, error) AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error Delete(context.Context, domain.StorageConnectionRecord, string, string) error } type BOSPublisher struct { GlobalEndpoint string } func (publisher BOSPublisher) client(connection domain.StorageConnectionRecord) (*bos.Client, error) { endpoint := strings.TrimSpace(connection.Endpoint) if endpoint == "" { endpoint = strings.TrimSpace(publisher.GlobalEndpoint) } if endpoint == "" { return nil, fmt.Errorf("BOS endpoint is required") } return bos.NewClient(strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint) } func IsPreconditionFailure(err error) bool { var serviceError *bce.BceServiceError return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusPreconditionFailed } func isNotFound(err error) bool { var serviceError *bce.BceServiceError return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusNotFound } func (publisher BOSPublisher) Head(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) { client, err := publisher.client(connection) if err != nil { return ObjectMetadata{}, err } result, err := client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/")) if isNotFound(err) { return ObjectMetadata{Exists: false}, nil } if err != nil { return ObjectMetadata{}, err } return ObjectMetadata{ UserMeta: result.UserMeta, Exists: true, ETag: strings.Trim(result.ETag, `"`), VersionID: strings.TrimSpace(result.VersionId), SizeBytes: result.ContentLength, LastModified: strings.TrimSpace(result.LastModified), }, nil } func (publisher BOSPublisher) ListPrefix(ctx context.Context, connection domain.StorageConnectionRecord, bucket, prefix string) ([]string, error) { client, err := publisher.client(connection) if err != nil { return nil, err } keys := []string{} marker := "" for { result, e := client.ListObjectsWithContext(ctx, bucket, &bosapi.ListObjectsArgs{Prefix: prefix, Marker: marker, MaxKeys: 1000}) if e != nil { return nil, e } for _, item := range result.Contents { keys = append(keys, item.Key) } if !result.IsTruncated { return keys, nil } if result.NextMarker == "" || result.NextMarker == marker { return nil, fmt.Errorf("incomplete listing") } marker = result.NextMarker } } func multipartCopyPartSize(sizeBytes int64) int64 { const ( minimum = int64(256 * 1024 * 1024) maximum = int64(5 * 1024 * 1024 * 1024) parts = int64(1_024) align = int64(1024 * 1024) ) partSize := minimum if sizeBytes > partSize*parts { partSize = ((sizeBytes/parts + align - 1) / align) * align } if partSize > maximum { return maximum } return partSize } func (publisher BOSPublisher) PublishConditional( ctx context.Context, connection domain.StorageConnectionRecord, bucket string, sourceKey string, targetKey string, sizeBytes int64, sourceETag string, ifMatch string, ifNoneMatch string, userMeta map[string]string, ) (ObjectMetadata, error) { client, err := publisher.client(connection) if err != nil { return ObjectMetadata{}, err } bucket = strings.TrimSpace(bucket) sourceKey = strings.Trim(strings.TrimSpace(sourceKey), "/") targetKey = strings.Trim(strings.TrimSpace(targetKey), "/") if sizeBytes == 0 { body, bodyErr := bce.NewBodyFromBytes([]byte{}) if bodyErr != nil { return ObjectMetadata{}, bodyErr } _, err = client.PutObjectWithContext(ctx, bucket, targetKey, body, &bosapi.PutObjectArgs{ ContentType: "application/octet-stream", UserMeta: userMeta, IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`), IfNoneMatch: strings.TrimSpace(ifNoneMatch), }) if err != nil { return ObjectMetadata{}, err } return publisher.Head(ctx, connection, bucket, targetKey) } initiated, err := client.InitiateMultipartUpload(bucket, targetKey, "", nil) if err != nil { return ObjectMetadata{}, err } abort := true defer func() { if abort { _ = client.AbortMultipartUpload(bucket, targetKey, initiated.UploadId) } }() partSize := multipartCopyPartSize(sizeBytes) partCount := int((sizeBytes + partSize - 1) / partSize) parts := make([]bosapi.UploadInfoType, partCount) copyGroup, copyContext := errgroup.WithContext(ctx) copyGroup.SetLimit(8) for partIndex := range partCount { start := int64(partIndex) * partSize end := start + partSize - 1 if end >= sizeBytes { end = sizeBytes - 1 } partNumber := partIndex + 1 copyGroup.Go(func() error { result, copyErr := client.UploadPartCopyWithContext(copyContext, bucket, targetKey, bucket, sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{ SourceRange: fmt.Sprintf("bytes=%d-%d", start, end), IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`), }) if copyErr != nil { return copyErr } parts[partIndex] = bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)} return nil }) } if err := copyGroup.Wait(); err != nil { return ObjectMetadata{}, err } args := &bosapi.CompleteMultipartUploadArgs{ Parts: parts, UserMeta: userMeta, IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`), IfNoneMatch: strings.TrimSpace(ifNoneMatch), } payload, err := json.Marshal(args) if err != nil { return ObjectMetadata{}, err } body, err := bce.NewBodyFromBytes(payload) if err != nil { return ObjectMetadata{}, err } bosContext := client.NewBosContext(ctx) if _, err = bosapi.CompleteMultipartUpload(client, bucket, targetKey, initiated.UploadId, body, args, bosContext); err != nil { return ObjectMetadata{}, err } abort = false return publisher.Head(ctx, connection, bucket, targetKey) } func (publisher BOSPublisher) PutInternal( ctx context.Context, connection domain.StorageConnectionRecord, bucket string, key string, reader io.Reader, size int64, contentType string, userMeta map[string]string, ) (ObjectMetadata, error) { client, err := publisher.client(connection) if err != nil { return ObjectMetadata{}, err } body, err := bce.NewBodyFromSizedReader(reader, size) if err != nil { return ObjectMetadata{}, err } _, err = client.PutObjectWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), body, &bosapi.PutObjectArgs{ ContentType: strings.TrimSpace(contentType), UserMeta: userMeta, IfNoneMatch: "*", }) if err != nil { return ObjectMetadata{}, err } return publisher.Head(ctx, connection, bucket, key) } func (publisher BOSPublisher) Delete(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) error { client, err := publisher.client(connection) if err != nil { return err } select { case <-ctx.Done(): return ctx.Err() default: return client.DeleteObject(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/")) } } func (publisher BOSPublisher) AbortMultipart(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, uploadID string) error { client, err := publisher.client(connection) if err != nil { return err } select { case <-ctx.Done(): return ctx.Err() default: return client.AbortMultipartUpload(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), strings.TrimSpace(uploadID)) } }