forked from longtao.wu/droboticflow
208 lines
8.6 KiB
Go
208 lines
8.6 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/objectsync"
|
|
"droboticflow/internal/store"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
type deliveryPublisher struct {
|
|
memoryArtifactPublisher
|
|
metadata map[string]map[string]string
|
|
published []string
|
|
fail string
|
|
}
|
|
|
|
func (p *deliveryPublisher) PutInternal(ctx context.Context, c domain.StorageConnectionRecord, b, k string, body io.Reader, size int64, contentType string, meta map[string]string) (objectsync.ObjectMetadata, error) {
|
|
if _, exists := p.objects[b+"/"+k]; exists {
|
|
return objectsync.ObjectMetadata{}, fmt.Errorf("staging conditional create failed")
|
|
}
|
|
return p.memoryArtifactPublisher.PutInternal(ctx, c, b, k, body, size, contentType, meta)
|
|
}
|
|
|
|
func (p *deliveryPublisher) Head(ctx context.Context, c domain.StorageConnectionRecord, b, k string) (objectsync.ObjectMetadata, error) {
|
|
m, e := p.memoryArtifactPublisher.Head(ctx, c, b, k)
|
|
m.ETag = "test"
|
|
m.UserMeta = p.metadata[b+"/"+k]
|
|
return m, e
|
|
}
|
|
func (p *deliveryPublisher) ListPrefix(_ context.Context, _ domain.StorageConnectionRecord, b, k string) ([]string, error) {
|
|
result := []string{}
|
|
for key := range p.objects {
|
|
if strings.HasPrefix(key, b+"/"+k) {
|
|
result = append(result, strings.TrimPrefix(key, b+"/"))
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
func (p *deliveryPublisher) PublishConditional(ctx context.Context, c domain.StorageConnectionRecord, b, src, target string, size int64, etag, match, none string, meta map[string]string) (objectsync.ObjectMetadata, error) {
|
|
if none != "*" {
|
|
return objectsync.ObjectMetadata{}, fmt.Errorf("non-conditional write")
|
|
}
|
|
if target == p.fail {
|
|
return objectsync.ObjectMetadata{}, fmt.Errorf("injected interruption")
|
|
}
|
|
key := b + "/" + target
|
|
if _, ok := p.objects[key]; ok {
|
|
return objectsync.ObjectMetadata{}, fmt.Errorf("conflict")
|
|
}
|
|
p.objects[key] = append([]byte(nil), p.objects[b+"/"+src]...)
|
|
if p.metadata == nil {
|
|
p.metadata = map[string]map[string]string{}
|
|
}
|
|
p.metadata[key] = meta
|
|
p.published = append(p.published, target)
|
|
return p.Head(ctx, c, b, target)
|
|
}
|
|
|
|
func TestCaptureDeliveryPublishesChosenDirectoryAndRetries(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
p := &deliveryPublisher{}
|
|
runtime := &Runtime{store: db, artifactPublisher: p}
|
|
c := domain.StorageConnectionRecord{ID: "c", OrganizationID: "org", Provider: domain.StorageProviderBOS, Bucket: "b"}
|
|
if e := db.UpsertStorageConnection(ctx, c); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
prefix := "deliveries/测试__project/capture"
|
|
record := domain.CaptureRecord{ID: "capture", OrganizationID: "org", RunID: "run-1", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: prefix}}
|
|
if e := db.UpsertCaptureRecord(ctx, record); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
root := t.TempDir()
|
|
if e := os.Mkdir(filepath.Join(root, "clips"), 0755); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e := os.WriteFile(filepath.Join(root, "clips", "left.mp4"), []byte("test-video"), 0600); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e := os.WriteFile(filepath.Join(root, "packages-manifest.json"), []byte(`{"items":[]}`), 0600); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Path: root, Type: domain.PortTypeFile}
|
|
run := &domain.RunRecord{ID: "run-1", OrganizationID: "org"}
|
|
p.fail = prefix + "/clips/left.mp4"
|
|
if _, e := runtime.publishCaptureDelivery(ctx, run, "package", artifact); e == nil {
|
|
t.Fatal("failure swallowed")
|
|
}
|
|
if _, ok := p.objects["b/"+prefix+"/manifest.json"]; ok {
|
|
t.Fatal("partial delivery marked complete")
|
|
}
|
|
p.fail = ""
|
|
got, e := runtime.publishCaptureDelivery(ctx, run, "package", artifact)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if got.Path != "bos://b/"+prefix || got.Payload.Ref.Prefix != prefix {
|
|
t.Fatal(got)
|
|
}
|
|
if p.published[len(p.published)-1] != prefix+"/manifest.json" {
|
|
t.Fatal("completion manifest not last")
|
|
}
|
|
count := len(p.published)
|
|
if _, e = runtime.publishCaptureDelivery(ctx, run, "package", artifact); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if len(p.published) != count {
|
|
t.Fatal("retry rewrote completed objects")
|
|
}
|
|
// A capture can point at its release child after reconciliation. Reusing
|
|
// the same accepted source must not become a conflicting new owner.
|
|
record.RunID = "release-child"
|
|
if e = db.UpsertCaptureRecord(ctx, record); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
db.UpsertRun(ctx, domain.RunRecord{ID: "release-child", OrganizationID: "org", TaskID: record.ID, ParentRunID: "run-1", WorkflowID: record.WorkflowID, Trigger: domain.RunTriggerHumanRelease})
|
|
if _, e = runtime.publishCaptureDelivery(ctx, &domain.RunRecord{ID: "release-child", OrganizationID: "org", ParentRunID: "run-1", Trigger: domain.RunTriggerHumanRelease}, "package", artifact); e != nil {
|
|
t.Fatalf("release child changed delivery owner: %v", e)
|
|
}
|
|
record.RunID = "run-2"
|
|
if e = db.UpsertCaptureRecord(ctx, record); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if _, e = runtime.publishCaptureDelivery(ctx, &domain.RunRecord{ID: "run-2", OrganizationID: "org"}, "package", artifact); e == nil {
|
|
t.Fatal("new attempt overwrote prior delivery")
|
|
}
|
|
if len(p.published) != count {
|
|
t.Fatal("conflict mutated output")
|
|
}
|
|
}
|
|
|
|
func TestCaptureDeliveryMissingConfigurationFailsClosed(t *testing.T) {
|
|
r := &Runtime{store: store.NewMemoryStore()}
|
|
if _, e := r.publishCaptureDelivery(context.Background(), &domain.RunRecord{ID: "old"}, "package", domain.ExecutionArtifact{}); e == nil {
|
|
t.Fatal("legacy task silently used current config")
|
|
}
|
|
}
|
|
|
|
func TestDeliveryMetadataAcceptsSDKHeaderCaseWithoutWeakeningIntegrity(t *testing.T) {
|
|
meta := objectsync.ObjectMetadata{Exists: true, SizeBytes: 89, UserMeta: map[string]string{"Delivery-Owner": "owner", "Sha256": "digest"}}
|
|
if !deliveryObjectMatches(meta, "owner", "digest", 89) {
|
|
t.Fatal("SDK canonical header case rejected")
|
|
}
|
|
for _, wrong := range []objectsync.ObjectMetadata{{Exists: true, SizeBytes: 89, UserMeta: map[string]string{"Delivery-Owner": "other", "Sha256": "digest"}}, {Exists: true, SizeBytes: 89, UserMeta: map[string]string{"Delivery-Owner": "owner", "Sha256": "wrong"}}, {Exists: true, SizeBytes: 88, UserMeta: meta.UserMeta}, {Exists: true, SizeBytes: 89, UserMeta: map[string]string{"Delivery-Owner": "owner", "Sha256": "digest", "sha256": "other"}}} {
|
|
if deliveryObjectMatches(wrong, "owner", "digest", 89) {
|
|
t.Fatal("wrong owner/hash/size or ambiguous case accepted")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCaptureDeliveryPublishesPartsWithoutCollision(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
p := &deliveryPublisher{}
|
|
runtime := &Runtime{store: db, artifactPublisher: p}
|
|
if e := db.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "c", OrganizationID: "org", Provider: domain.StorageProviderBOS, Bucket: "b"}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
config := domain.CaptureAutomationConfig{DeliveryPrefix: "deliveries", DeliveryProjectDirectory: "project"}
|
|
for _, part := range []string{"part0001", "part0002"} {
|
|
prefix, e := domain.CaptureDeliveryPrefix(config, "capture/"+part)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
record := domain.CaptureRecord{ID: part, OrganizationID: "org", RunID: "run-" + part, Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: prefix}}
|
|
if e = db.UpsertCaptureRecord(ctx, record); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
root := t.TempDir()
|
|
if e = os.WriteFile(filepath.Join(root, "packages-manifest.json"), []byte(part), 0600); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
got, e := runtime.publishCaptureDelivery(ctx, &domain.RunRecord{ID: record.RunID, OrganizationID: "org"}, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Path: root, Type: domain.PortTypeFile})
|
|
if e != nil || got.Path != "bos://b/"+prefix {
|
|
t.Fatal(got, e)
|
|
}
|
|
}
|
|
for _, part := range []string{"part0001", "part0002"} {
|
|
if string(p.objects["b/deliveries/project/capture/"+part+"/packages-manifest.json"]) != part {
|
|
t.Fatal("part overwritten")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCaptureDeliveryRejectsLegacyParentOverlap(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
runtime := &Runtime{store: db}
|
|
for _, record := range []domain.CaptureRecord{
|
|
{ID: "old", OrganizationID: "org", RunID: "old-run", Delivery: &domain.CaptureDeliveryTarget{Bucket: "b", Prefix: "deliveries/project/capture"}},
|
|
{ID: "new", OrganizationID: "org", RunID: "new-run", Delivery: &domain.CaptureDeliveryTarget{Bucket: "b", Prefix: "deliveries/project/capture/part0002"}},
|
|
} {
|
|
if e := db.UpsertCaptureRecord(ctx, record); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
_, e := runtime.publishCaptureDelivery(ctx, &domain.RunRecord{ID: "new-run", OrganizationID: "org"}, "package", domain.ExecutionArtifact{})
|
|
if e == nil || !strings.Contains(e.Error(), "重叠") {
|
|
t.Fatal(e)
|
|
}
|
|
}
|