2026-10-08 15:08:01 +08:00

358 lines
13 KiB
Go

package objectsync
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/bosclient"
"droboticflow/internal/domain"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
)
func bosFixtureConnection(server *httptest.Server) domain.StorageConnectionRecord {
return domain.StorageConnectionRecord{Endpoint: server.URL, AccessKeyID: "fixture-key", SecretAccessKey: "fixture-secret"}
}
func bosFixtureResponse(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && r.URL.Query().Has("uploads"):
_, _ = io.WriteString(w, `{"bucket":"fixture","key":"target","uploadId":"fixture-upload"}`)
case r.Method == http.MethodPut && r.URL.Query().Has("partNumber"):
_, _ = io.WriteString(w, `{"eTag":"part-etag"}`)
case r.Method == http.MethodPost:
_, _ = io.WriteString(w, `{"bucket":"fixture","key":"target","eTag":"final-etag"}`)
case r.Method == http.MethodHead:
w.Header().Set("Content-Length", "4")
w.Header().Set("ETag", `"final-etag"`)
case r.Method == http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
default:
w.Header().Set("ETag", `"final-etag"`)
}
}
// These requests exercise the real SDK and HTTP transport, not a Publisher mock.
func TestBOSRequestsHonorParentCancellation(t *testing.T) {
for _, stage := range []string{"initiate", "copy", "complete", "head", "delete", "abort", "put"} {
t.Run(stage, func(t *testing.T) {
entered := make(chan struct{}, 1)
unblock := make(chan struct{})
var cleanup atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
matched := (stage == "initiate" && r.URL.Query().Has("uploads")) ||
(stage == "copy" && r.URL.Query().Has("partNumber")) ||
(stage == "complete" && r.Method == http.MethodPost && r.URL.Query().Has("uploadId")) ||
(stage == "head" && r.Method == http.MethodHead) ||
(stage == "delete" && r.Method == http.MethodDelete && !r.URL.Query().Has("uploadId")) ||
(stage == "abort" && r.Method == http.MethodDelete) ||
(stage == "put" && r.Method == http.MethodPut)
if matched {
_, _ = io.Copy(io.Discard, r.Body)
entered <- struct{}{}
select {
case <-r.Context().Done():
case <-unblock:
}
return
}
if r.Method == http.MethodDelete {
cleanup.Add(1)
}
bosFixtureResponse(w, r)
}))
defer server.Close()
defer close(unblock)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
publisher := BOSPublisher{}
connection := bosFixtureConnection(server)
done := make(chan error, 1)
go func() {
var err error
switch stage {
case "head":
_, err = publisher.Head(ctx, connection, "fixture", "target")
case "delete":
err = publisher.Delete(ctx, connection, "fixture", "target")
case "abort":
err = publisher.AbortMultipart(ctx, connection, "fixture", "target", "fixture-upload")
case "put":
_, err = publisher.PutInternal(ctx, connection, "fixture", "target", strings.NewReader("data"), 4, "application/octet-stream", nil)
default:
_, err = publisher.PublishConditional(ctx, connection, "fixture", "source", "target", 4, "source-etag", "", "*", nil)
}
done <- err
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("BOS request did not arrive")
}
cancel()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("expected context cancellation, got %v", err)
}
case <-time.After(time.Second):
t.Fatal("SDK ignored parent cancellation")
}
if (stage == "copy" || stage == "complete") && cleanup.Load() != 1 {
t.Fatalf("cleanup was not attempted after cancellation: %d", cleanup.Load())
}
})
}
}
func TestBOSInitiateHonorsDeadline(t *testing.T) {
unblock := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-unblock:
}
}))
defer server.Close()
defer close(unblock)
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
_, err := (BOSPublisher{}).PublishConditional(ctx, bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("deadline identity lost: %v", err)
}
}
func TestBOSClaimLossCancelsRealSDKInitiation(t *testing.T) {
entered := make(chan struct{})
unblock := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(entered)
select {
case <-r.Context().Done():
case <-unblock:
}
}))
defer server.Close()
defer close(unblock)
coordinator := &Coordinator{renewInterval: 5 * time.Millisecond}
done := make(chan error, 1)
go func() {
done <- coordinator.withRenewingClaim(t.Context(), func(context.Context) (bool, error) {
select {
case <-entered:
return false, nil
default:
return true, nil
}
}, func(ctx context.Context) error {
_, err := (BOSPublisher{}).PublishConditional(ctx, bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
return err
})
}()
select {
case err := <-done:
if !errors.Is(err, errClaimLost) {
t.Fatalf("lost claim was not returned: %v", err)
}
case <-time.After(time.Second):
t.Fatal("claim loss did not interrupt the real SDK request")
}
}
func TestBOSConditionalPublicationPreservesSignedConstraints(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
if !strings.HasPrefix(r.Header.Get("Authorization"), "bce-auth-v1/fixture-key/") {
t.Error("request lost SDK authentication")
}
if r.Method == http.MethodPut {
if r.Header.Get("x-bce-copy-source") != "/fixture/source" || r.Header.Get("x-bce-copy-source-range") != "bytes=0-3" || r.Header.Get("x-bce-copy-source-if-match") != "source-etag" {
t.Errorf("copy constraints changed: %v", r.Header)
}
}
if r.Method == http.MethodPost && r.URL.Query().Has("uploadId") {
if r.Header.Get("if-match") != "previous-etag" || r.Header.Get("x-bce-meta-sha256") != "digest" {
t.Errorf("conditional completion lost metadata/precondition: %v", r.Header)
}
var args bosapi.CompleteMultipartUploadArgs
if err := json.NewDecoder(r.Body).Decode(&args); err != nil || len(args.Parts) != 1 || args.Parts[0].ETag != "part-etag" {
t.Errorf("incorrect completion manifest: %+v %v", args, err)
}
}
bosFixtureResponse(w, r)
}))
defer server.Close()
meta, err := (BOSPublisher{}).PublishConditional(t.Context(), bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "previous-etag", "", map[string]string{"sha256": "digest"})
if err != nil || !meta.Exists || meta.ETag != "final-etag" || meta.SizeBytes != 4 || calls.Load() != 4 {
t.Fatalf("publication failed: %+v %v calls=%d", meta, err, calls.Load())
}
}
func TestBOSCleanupFailureRetainsOriginalPreconditionError(t *testing.T) {
var cleanup atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.Method == http.MethodPut {
w.WriteHeader(http.StatusPreconditionFailed)
_, _ = io.WriteString(w, `{"code":"PreconditionFailed","message":"source changed"}`)
return
}
if r.Method == http.MethodDelete {
cleanup.Add(1)
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = io.WriteString(w, `{"code":"ServiceUnavailable","message":"cleanup unavailable"}`)
return
}
bosFixtureResponse(w, r)
}))
defer server.Close()
_, err := (BOSPublisher{}).PublishConditional(t.Context(), bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
if !IsPreconditionFailure(err) || !strings.Contains(err.Error(), "cleanup upload fixture-upload") || cleanup.Load() != 1 {
t.Fatalf("original failure/cleanup result lost or retried: %v calls=%d", err, cleanup.Load())
}
}
func TestBOSCleanupHasIndependentFiniteBudget(t *testing.T) {
unblock := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPut {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusPreconditionFailed)
_, _ = io.WriteString(w, `{"code":"PreconditionFailed","message":"source changed"}`)
return
}
if r.Method == http.MethodDelete {
select {
case <-r.Context().Done():
case <-unblock:
}
return
}
bosFixtureResponse(w, r)
}))
defer server.Close()
defer close(unblock)
started := time.Now()
_, err := (BOSPublisher{}).PublishConditional(t.Context(), bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
elapsed := time.Since(started)
if !IsPreconditionFailure(err) || !errors.Is(err, context.DeadlineExceeded) || elapsed < bosclient.CleanupTimeout || elapsed > bosclient.CleanupTimeout+2*time.Second {
t.Fatalf("cleanup did not enforce its budget: elapsed=%s error=%v", elapsed, err)
}
}
func TestBOSPutInternalSourceReadHonorsCancellationBeforeHTTP(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); bosFixtureResponse(w, r) }))
defer server.Close()
reader, writer := io.Pipe()
defer reader.Close()
defer writer.Close()
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
go func() {
_, err := (BOSPublisher{}).PutInternal(ctx, bosFixtureConnection(server), "fixture", "manifest", reader, 4, "application/x-ndjson", nil)
done <- err
}()
time.Sleep(10 * time.Millisecond)
cancel()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("wrong cancel error: %v", err)
}
case <-time.After(time.Second):
t.Fatal("blocked source read ignored cancellation")
}
if requests.Load() != 0 {
t.Fatal("incomplete source reached BOS")
}
}
func TestBOSPutInternalStreamsWithMD5LengthAndConditionalCreate(t *testing.T) {
const payload = "preserve upload integrity"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPut {
bytes, err := io.ReadAll(r.Body)
if err != nil || string(bytes) != payload || r.ContentLength != int64(len(payload)) || r.Header.Get("Content-MD5") != "7FEqMDz8juRNUIFs0JaHFw==" || r.Header.Get("If-None-Match") != "*" {
t.Errorf("wire length/checksum/conditional integrity lost: body=%q headers=%v err=%v", bytes, r.Header, err)
}
}
bosFixtureResponse(w, r)
}))
defer server.Close()
_, err := (BOSPublisher{}).PutInternal(t.Context(), bosFixtureConnection(server), "fixture", "manifest", strings.NewReader(payload), int64(len(payload)), "application/x-ndjson", nil)
if err != nil {
t.Fatal(err)
}
}
type stalledReplayReader struct {
input *strings.Reader
replay bool
entered chan struct{}
closed chan struct{}
closeOnce sync.Once
enterOnce sync.Once
}
func (reader *stalledReplayReader) Read(buffer []byte) (int, error) {
if reader.replay {
reader.enterOnce.Do(func() { close(reader.entered) })
<-reader.closed
return 0, io.ErrClosedPipe
}
return reader.input.Read(buffer)
}
func (reader *stalledReplayReader) Seek(offset int64, whence int) (int64, error) {
if whence == io.SeekStart {
reader.replay = true
}
return reader.input.Seek(offset, whence)
}
func (reader *stalledReplayReader) Close() error {
reader.closeOnce.Do(func() { close(reader.closed) })
return nil
}
func TestBOSPutInternalCancelsBlockedStreamingReplay(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.Copy(io.Discard, r.Body)
bosFixtureResponse(w, r)
}))
defer server.Close()
reader := &stalledReplayReader{input: strings.NewReader("data"), entered: make(chan struct{}), closed: make(chan struct{})}
defer reader.Close()
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
go func() {
_, err := (BOSPublisher{}).PutInternal(ctx, bosFixtureConnection(server), "fixture", "manifest", reader, 4, "application/x-ndjson", nil)
done <- err
}()
select {
case <-reader.entered:
case <-time.After(time.Second):
t.Fatal("upload never reached second-pass source read")
}
cancel()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("wrong cancellation: %v", err)
}
case <-time.After(time.Second):
t.Fatal("HTTP upload body read survived cancellation")
}
}