forked from longtao.wu/droboticflow
358 lines
13 KiB
Go
358 lines
13 KiB
Go
package objectsync
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/bosclient"
|
|
"droboticflow/internal/domain"
|
|
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
|
)
|
|
|
|
func bosFixtureConnection(server *httptest.Server) domain.StorageConnectionRecord {
|
|
return domain.StorageConnectionRecord{Endpoint: server.URL, AccessKeyID: "fixture-key", SecretAccessKey: "fixture-secret"}
|
|
}
|
|
|
|
func bosFixtureResponse(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
switch {
|
|
case r.Method == http.MethodPost && r.URL.Query().Has("uploads"):
|
|
_, _ = io.WriteString(w, `{"bucket":"fixture","key":"target","uploadId":"fixture-upload"}`)
|
|
case r.Method == http.MethodPut && r.URL.Query().Has("partNumber"):
|
|
_, _ = io.WriteString(w, `{"eTag":"part-etag"}`)
|
|
case r.Method == http.MethodPost:
|
|
_, _ = io.WriteString(w, `{"bucket":"fixture","key":"target","eTag":"final-etag"}`)
|
|
case r.Method == http.MethodHead:
|
|
w.Header().Set("Content-Length", "4")
|
|
w.Header().Set("ETag", `"final-etag"`)
|
|
case r.Method == http.MethodDelete:
|
|
w.WriteHeader(http.StatusNoContent)
|
|
default:
|
|
w.Header().Set("ETag", `"final-etag"`)
|
|
}
|
|
}
|
|
|
|
// These requests exercise the real SDK and HTTP transport, not a Publisher mock.
|
|
func TestBOSRequestsHonorParentCancellation(t *testing.T) {
|
|
for _, stage := range []string{"initiate", "copy", "complete", "head", "delete", "abort", "put"} {
|
|
t.Run(stage, func(t *testing.T) {
|
|
entered := make(chan struct{}, 1)
|
|
unblock := make(chan struct{})
|
|
var cleanup atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
matched := (stage == "initiate" && r.URL.Query().Has("uploads")) ||
|
|
(stage == "copy" && r.URL.Query().Has("partNumber")) ||
|
|
(stage == "complete" && r.Method == http.MethodPost && r.URL.Query().Has("uploadId")) ||
|
|
(stage == "head" && r.Method == http.MethodHead) ||
|
|
(stage == "delete" && r.Method == http.MethodDelete && !r.URL.Query().Has("uploadId")) ||
|
|
(stage == "abort" && r.Method == http.MethodDelete) ||
|
|
(stage == "put" && r.Method == http.MethodPut)
|
|
if matched {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
entered <- struct{}{}
|
|
select {
|
|
case <-r.Context().Done():
|
|
case <-unblock:
|
|
}
|
|
return
|
|
}
|
|
if r.Method == http.MethodDelete {
|
|
cleanup.Add(1)
|
|
}
|
|
bosFixtureResponse(w, r)
|
|
}))
|
|
defer server.Close()
|
|
defer close(unblock)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
publisher := BOSPublisher{}
|
|
connection := bosFixtureConnection(server)
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
var err error
|
|
switch stage {
|
|
case "head":
|
|
_, err = publisher.Head(ctx, connection, "fixture", "target")
|
|
case "delete":
|
|
err = publisher.Delete(ctx, connection, "fixture", "target")
|
|
case "abort":
|
|
err = publisher.AbortMultipart(ctx, connection, "fixture", "target", "fixture-upload")
|
|
case "put":
|
|
_, err = publisher.PutInternal(ctx, connection, "fixture", "target", strings.NewReader("data"), 4, "application/octet-stream", nil)
|
|
default:
|
|
_, err = publisher.PublishConditional(ctx, connection, "fixture", "source", "target", 4, "source-etag", "", "*", nil)
|
|
}
|
|
done <- err
|
|
}()
|
|
select {
|
|
case <-entered:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("BOS request did not arrive")
|
|
}
|
|
cancel()
|
|
select {
|
|
case err := <-done:
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("expected context cancellation, got %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("SDK ignored parent cancellation")
|
|
}
|
|
if (stage == "copy" || stage == "complete") && cleanup.Load() != 1 {
|
|
t.Fatalf("cleanup was not attempted after cancellation: %d", cleanup.Load())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBOSInitiateHonorsDeadline(t *testing.T) {
|
|
unblock := make(chan struct{})
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
select {
|
|
case <-r.Context().Done():
|
|
case <-unblock:
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
defer close(unblock)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
defer cancel()
|
|
_, err := (BOSPublisher{}).PublishConditional(ctx, bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
|
|
if !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("deadline identity lost: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestBOSClaimLossCancelsRealSDKInitiation(t *testing.T) {
|
|
entered := make(chan struct{})
|
|
unblock := make(chan struct{})
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
close(entered)
|
|
select {
|
|
case <-r.Context().Done():
|
|
case <-unblock:
|
|
}
|
|
}))
|
|
defer server.Close()
|
|
defer close(unblock)
|
|
coordinator := &Coordinator{renewInterval: 5 * time.Millisecond}
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- coordinator.withRenewingClaim(t.Context(), func(context.Context) (bool, error) {
|
|
select {
|
|
case <-entered:
|
|
return false, nil
|
|
default:
|
|
return true, nil
|
|
}
|
|
}, func(ctx context.Context) error {
|
|
_, err := (BOSPublisher{}).PublishConditional(ctx, bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
|
|
return err
|
|
})
|
|
}()
|
|
select {
|
|
case err := <-done:
|
|
if !errors.Is(err, errClaimLost) {
|
|
t.Fatalf("lost claim was not returned: %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("claim loss did not interrupt the real SDK request")
|
|
}
|
|
}
|
|
|
|
func TestBOSConditionalPublicationPreservesSignedConstraints(t *testing.T) {
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
if !strings.HasPrefix(r.Header.Get("Authorization"), "bce-auth-v1/fixture-key/") {
|
|
t.Error("request lost SDK authentication")
|
|
}
|
|
if r.Method == http.MethodPut {
|
|
if r.Header.Get("x-bce-copy-source") != "/fixture/source" || r.Header.Get("x-bce-copy-source-range") != "bytes=0-3" || r.Header.Get("x-bce-copy-source-if-match") != "source-etag" {
|
|
t.Errorf("copy constraints changed: %v", r.Header)
|
|
}
|
|
}
|
|
if r.Method == http.MethodPost && r.URL.Query().Has("uploadId") {
|
|
if r.Header.Get("if-match") != "previous-etag" || r.Header.Get("x-bce-meta-sha256") != "digest" {
|
|
t.Errorf("conditional completion lost metadata/precondition: %v", r.Header)
|
|
}
|
|
var args bosapi.CompleteMultipartUploadArgs
|
|
if err := json.NewDecoder(r.Body).Decode(&args); err != nil || len(args.Parts) != 1 || args.Parts[0].ETag != "part-etag" {
|
|
t.Errorf("incorrect completion manifest: %+v %v", args, err)
|
|
}
|
|
}
|
|
bosFixtureResponse(w, r)
|
|
}))
|
|
defer server.Close()
|
|
meta, err := (BOSPublisher{}).PublishConditional(t.Context(), bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "previous-etag", "", map[string]string{"sha256": "digest"})
|
|
if err != nil || !meta.Exists || meta.ETag != "final-etag" || meta.SizeBytes != 4 || calls.Load() != 4 {
|
|
t.Fatalf("publication failed: %+v %v calls=%d", meta, err, calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestBOSCleanupFailureRetainsOriginalPreconditionError(t *testing.T) {
|
|
var cleanup atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if r.Method == http.MethodPut {
|
|
w.WriteHeader(http.StatusPreconditionFailed)
|
|
_, _ = io.WriteString(w, `{"code":"PreconditionFailed","message":"source changed"}`)
|
|
return
|
|
}
|
|
if r.Method == http.MethodDelete {
|
|
cleanup.Add(1)
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
_, _ = io.WriteString(w, `{"code":"ServiceUnavailable","message":"cleanup unavailable"}`)
|
|
return
|
|
}
|
|
bosFixtureResponse(w, r)
|
|
}))
|
|
defer server.Close()
|
|
_, err := (BOSPublisher{}).PublishConditional(t.Context(), bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
|
|
if !IsPreconditionFailure(err) || !strings.Contains(err.Error(), "cleanup upload fixture-upload") || cleanup.Load() != 1 {
|
|
t.Fatalf("original failure/cleanup result lost or retried: %v calls=%d", err, cleanup.Load())
|
|
}
|
|
}
|
|
|
|
func TestBOSCleanupHasIndependentFiniteBudget(t *testing.T) {
|
|
unblock := make(chan struct{})
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == http.MethodPut {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusPreconditionFailed)
|
|
_, _ = io.WriteString(w, `{"code":"PreconditionFailed","message":"source changed"}`)
|
|
return
|
|
}
|
|
if r.Method == http.MethodDelete {
|
|
select {
|
|
case <-r.Context().Done():
|
|
case <-unblock:
|
|
}
|
|
return
|
|
}
|
|
bosFixtureResponse(w, r)
|
|
}))
|
|
defer server.Close()
|
|
defer close(unblock)
|
|
started := time.Now()
|
|
_, err := (BOSPublisher{}).PublishConditional(t.Context(), bosFixtureConnection(server), "fixture", "source", "target", 4, "source-etag", "", "*", nil)
|
|
elapsed := time.Since(started)
|
|
if !IsPreconditionFailure(err) || !errors.Is(err, context.DeadlineExceeded) || elapsed < bosclient.CleanupTimeout || elapsed > bosclient.CleanupTimeout+2*time.Second {
|
|
t.Fatalf("cleanup did not enforce its budget: elapsed=%s error=%v", elapsed, err)
|
|
}
|
|
}
|
|
|
|
func TestBOSPutInternalSourceReadHonorsCancellationBeforeHTTP(t *testing.T) {
|
|
var requests atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); bosFixtureResponse(w, r) }))
|
|
defer server.Close()
|
|
reader, writer := io.Pipe()
|
|
defer reader.Close()
|
|
defer writer.Close()
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := (BOSPublisher{}).PutInternal(ctx, bosFixtureConnection(server), "fixture", "manifest", reader, 4, "application/x-ndjson", nil)
|
|
done <- err
|
|
}()
|
|
time.Sleep(10 * time.Millisecond)
|
|
cancel()
|
|
select {
|
|
case err := <-done:
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("wrong cancel error: %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("blocked source read ignored cancellation")
|
|
}
|
|
if requests.Load() != 0 {
|
|
t.Fatal("incomplete source reached BOS")
|
|
}
|
|
}
|
|
|
|
func TestBOSPutInternalStreamsWithMD5LengthAndConditionalCreate(t *testing.T) {
|
|
const payload = "preserve upload integrity"
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == http.MethodPut {
|
|
bytes, err := io.ReadAll(r.Body)
|
|
if err != nil || string(bytes) != payload || r.ContentLength != int64(len(payload)) || r.Header.Get("Content-MD5") != "7FEqMDz8juRNUIFs0JaHFw==" || r.Header.Get("If-None-Match") != "*" {
|
|
t.Errorf("wire length/checksum/conditional integrity lost: body=%q headers=%v err=%v", bytes, r.Header, err)
|
|
}
|
|
}
|
|
bosFixtureResponse(w, r)
|
|
}))
|
|
defer server.Close()
|
|
_, err := (BOSPublisher{}).PutInternal(t.Context(), bosFixtureConnection(server), "fixture", "manifest", strings.NewReader(payload), int64(len(payload)), "application/x-ndjson", nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
type stalledReplayReader struct {
|
|
input *strings.Reader
|
|
replay bool
|
|
entered chan struct{}
|
|
closed chan struct{}
|
|
closeOnce sync.Once
|
|
enterOnce sync.Once
|
|
}
|
|
|
|
func (reader *stalledReplayReader) Read(buffer []byte) (int, error) {
|
|
if reader.replay {
|
|
reader.enterOnce.Do(func() { close(reader.entered) })
|
|
<-reader.closed
|
|
return 0, io.ErrClosedPipe
|
|
}
|
|
return reader.input.Read(buffer)
|
|
}
|
|
func (reader *stalledReplayReader) Seek(offset int64, whence int) (int64, error) {
|
|
if whence == io.SeekStart {
|
|
reader.replay = true
|
|
}
|
|
return reader.input.Seek(offset, whence)
|
|
}
|
|
func (reader *stalledReplayReader) Close() error {
|
|
reader.closeOnce.Do(func() { close(reader.closed) })
|
|
return nil
|
|
}
|
|
|
|
func TestBOSPutInternalCancelsBlockedStreamingReplay(t *testing.T) {
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
_, _ = io.Copy(io.Discard, r.Body)
|
|
bosFixtureResponse(w, r)
|
|
}))
|
|
defer server.Close()
|
|
reader := &stalledReplayReader{input: strings.NewReader("data"), entered: make(chan struct{}), closed: make(chan struct{})}
|
|
defer reader.Close()
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
defer cancel()
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := (BOSPublisher{}).PutInternal(ctx, bosFixtureConnection(server), "fixture", "manifest", reader, 4, "application/x-ndjson", nil)
|
|
done <- err
|
|
}()
|
|
select {
|
|
case <-reader.entered:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("upload never reached second-pass source read")
|
|
}
|
|
cancel()
|
|
select {
|
|
case err := <-done:
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("wrong cancellation: %v", err)
|
|
}
|
|
case <-time.After(time.Second):
|
|
t.Fatal("HTTP upload body read survived cancellation")
|
|
}
|
|
}
|