droboticflow/internal/worker/artifact_storage_test.go
2026-08-27 05:58:49 +08:00

293 lines
13 KiB
Go

package worker
import (
"context"
"encoding/json"
"io"
"os"
"path/filepath"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/objectsync"
"droboticflow/internal/store"
)
type memoryArtifactPublisher struct {
objects map[string][]byte
puts []string
}
func (publisher *memoryArtifactPublisher) objectID(bucket string, key string) string {
return bucket + "/" + key
}
func (publisher *memoryArtifactPublisher) Head(_ context.Context, _ domain.StorageConnectionRecord, bucket string, key string) (objectsync.ObjectMetadata, error) {
payload, ok := publisher.objects[publisher.objectID(bucket, key)]
return objectsync.ObjectMetadata{Exists: ok, SizeBytes: int64(len(payload))}, nil
}
func (publisher *memoryArtifactPublisher) PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (objectsync.ObjectMetadata, error) {
return objectsync.ObjectMetadata{}, nil
}
func (publisher *memoryArtifactPublisher) PutInternal(_ context.Context, _ domain.StorageConnectionRecord, bucket string, key string, reader io.Reader, size int64, _ string, _ map[string]string) (objectsync.ObjectMetadata, error) {
payload, err := io.ReadAll(reader)
if err != nil {
return objectsync.ObjectMetadata{}, err
}
if int64(len(payload)) != size {
return objectsync.ObjectMetadata{}, io.ErrUnexpectedEOF
}
if publisher.objects == nil {
publisher.objects = map[string][]byte{}
}
publisher.objects[publisher.objectID(bucket, key)] = payload
publisher.puts = append(publisher.puts, key)
return objectsync.ObjectMetadata{Exists: true, ETag: "test-etag", VersionID: "test-version", SizeBytes: size}, nil
}
func (publisher *memoryArtifactPublisher) AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error {
return nil
}
func (publisher *memoryArtifactPublisher) Delete(_ context.Context, _ domain.StorageConnectionRecord, bucket string, key string) error {
delete(publisher.objects, publisher.objectID(bucket, key))
return nil
}
func TestGatherNodeInputsIncludesWorkflowPayloadByTargetPort(t *testing.T) {
run := &domain.RunRecord{ID: "run-1", ResolvedInputBindings: []domain.ResolvedInputBinding{{
WorkflowInputID: "config", NodeID: "source", TargetPortID: "in-config",
Payload: domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: map[string]any{"ok": true}},
}}}
workflow := domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{ID: "source"}}}
artifacts, lineage, bindings := gatherNodeInputs(workflow, run, nil, "source")
if len(lineage) != 0 || len(artifacts) != 1 || len(bindings) != 1 {
t.Fatalf("unexpected root inputs: artifacts=%#v lineage=%#v bindings=%#v", artifacts, lineage, bindings)
}
if bindings[0].TargetPortID != "in-config" || artifacts[0].Payload == nil || artifacts[0].Payload.Mode != domain.ArtifactTransferModeInline {
t.Fatalf("workflow payload was not bound to target port: %#v %#v", bindings[0], artifacts[0])
}
}
func TestCompactArtifactDataPreservesPrivacyTransformChain(t *testing.T) {
chain := []any{
map[string]any{"kind": "opencv-undistort", "applied": true, "schemaVersion": "droboticflow.orbbec.rectify/v2"},
map[string]any{"kind": "face-anonymization", "applied": true, "modelSha256": "abc"},
}
compact := compactArtifactData(map[string]any{
"transformChain": chain,
"privacyTransform": map[string]any{"method": "yunet"},
"rectification": map[string]any{"distortionModel": "kb"},
"internalPath": "/workspace/private/output.mp4",
})
if compact == nil || len(compact) != 3 {
t.Fatalf("unexpected compact privacy data: %#v", compact)
}
if _, ok := compact["transformChain"]; !ok {
t.Fatalf("transformChain was dropped: %#v", compact)
}
if _, ok := compact["internalPath"]; ok {
t.Fatalf("internal path leaked into compact data: %#v", compact)
}
}
func TestNormalizeOutputArtifactsInlinesSmallJSONWithoutArtifactStorage(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("new runtime: %v", err)
}
path := filepath.Join(t.TempDir(), "result.json")
if err := os.WriteFile(path, []byte(`{"ok":true}`), 0o644); err != nil {
t.Fatalf("write json: %v", err)
}
run := &domain.RunRecord{ID: "run-1", WorkflowID: "workflow-1"}
artifacts, err := runtime.normalizeOutputArtifacts(context.Background(), &domain.WorkflowRecord{}, run, "node-1", []domain.ExecutionArtifact{{
ID: "artifact-1", RunID: run.ID, NodeID: "node-1", Name: "result.json", Type: domain.PortTypeJSON, Path: path,
}})
if err != nil {
t.Fatalf("normalize artifacts: %v", err)
}
if len(artifacts) != 1 || artifacts[0].Payload == nil || artifacts[0].Payload.Mode != domain.ArtifactTransferModeInline {
t.Fatalf("expected inline json payload, got %#v", artifacts)
}
}
func TestStructuredJSONSchemaVersionDoesNotForceExternalReference(t *testing.T) {
payload := map[string]any{
"schemaVersion": "droboticflow.rough-cut-windows/v1",
"windowCount": 1,
"windows": []any{map[string]any{
"startTimestampUs": int64(1_000_000),
"endTimestampUs": int64(2_000_000),
}},
}
if structuredValueHasExternalReference(payload) {
t.Fatalf("schema-only control payload was classified as an external reference: %#v", payload)
}
payload["storagePath"] = "bos://bucket/key"
if !structuredValueHasExternalReference(payload) {
t.Fatalf("storage locator was not classified as an external reference: %#v", payload)
}
}
func TestNormalizeOutputArtifactsExternalizesSmallStructuredJSONReferences(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
publisher := &memoryArtifactPublisher{}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("new runtime: %v", err)
}
runtime.artifactPublisher = publisher
connection := domain.StorageConnectionRecord{ID: "artifact-bos", OrganizationID: "org-artifacts", Provider: domain.StorageProviderBOS, Bucket: "artifact-bucket", Endpoint: "https://bos.example.test"}
if err := dataStore.UpsertStorageConnection(ctx, connection); err != nil {
t.Fatalf("upsert storage connection: %v", err)
}
workflow := &domain.WorkflowRecord{ID: "workflow-artifacts", OrganizationID: "org-artifacts", ArtifactStorage: &domain.ArtifactStoragePolicy{ConnectionID: connection.ID, Bucket: connection.Bucket, PrefixTemplate: "artifacts/{runId}/{nodeId}"}}
run := &domain.RunRecord{ID: "run-artifacts", OrganizationID: workflow.OrganizationID, WorkflowID: workflow.ID}
cases := []struct {
name string
artifactName string
payload string
data map[string]any
}{
{
name: "dataset dataRef",
artifactName: "dataset.json",
payload: `{"ok":true}`,
data: map[string]any{"dataRef": map[string]any{"payloadPath": "/private/source.json", "datasetType": "orbbec-ego"}},
},
{
name: "manifest member external path",
artifactName: "orbbec-normalized-manifest.json",
payload: `{"schemaVersion":"droboticflow.orbbec.normalized/v1","items":[{"filename":"left.mp4","path":"bos://private-bucket/private/left.mp4"}]}`,
data: map[string]any{"schemaVersion": "droboticflow.orbbec.normalized/v1"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), tc.artifactName)
if err := os.WriteFile(path, []byte(tc.payload), 0o644); err != nil {
t.Fatal(err)
}
artifacts, err := runtime.normalizeOutputArtifacts(ctx, workflow, run, "node-1", []domain.ExecutionArtifact{{
ID: "artifact-" + artifactSlug(tc.name), RunID: run.ID, NodeID: "node-1", Name: tc.artifactName, Type: domain.PortTypeJSON, Path: path, Data: tc.data,
}})
if err != nil {
t.Fatalf("normalize artifacts: %v", err)
}
artifact := artifacts[0]
if artifact.Payload == nil || artifact.Payload.Mode != domain.ArtifactTransferModeRef || artifact.Payload.Ref == nil || artifact.Payload.Ref.Kind != domain.ArtifactRefKindObject {
t.Fatalf("structured json must be external ref, got %#v", artifact.Payload)
}
if artifact.Payload.Inline != nil || artifact.Payload.Ref.Key == "" || len(publisher.puts) == 0 {
t.Fatalf("structured json externalization lost object ref or retained inline data: %#v puts=%#v", artifact.Payload, publisher.puts)
}
replayed, err := runtime.normalizeOutputArtifacts(ctx, workflow, run, "node-1", []domain.ExecutionArtifact{artifact})
if err != nil || replayed[0].Payload == nil || replayed[0].Payload.Mode != domain.ArtifactTransferModeRef {
t.Fatalf("ref manifest replay must validate via Head without returning inline data: artifacts=%#v err=%v", replayed, err)
}
})
}
}
func TestNormalizeOutputArtifactsReusesDirectoryMemberObjectRefs(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
publisher := &memoryArtifactPublisher{}
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("new runtime: %v", err)
}
runtime.artifactPublisher = publisher
connection := domain.StorageConnectionRecord{ID: "artifact-bos", OrganizationID: "org-artifacts", Provider: domain.StorageProviderBOS, Bucket: "artifact-bucket", Endpoint: "https://bos.example.test"}
if err := dataStore.UpsertStorageConnection(ctx, connection); err != nil {
t.Fatalf("upsert storage connection: %v", err)
}
workflow := &domain.WorkflowRecord{ID: "workflow-artifacts", OrganizationID: "org-artifacts", ArtifactStorage: &domain.ArtifactStoragePolicy{ConnectionID: connection.ID, Bucket: connection.Bucket, PrefixTemplate: "artifacts/{runId}/{nodeId}"}}
run := &domain.RunRecord{ID: "run-artifacts", OrganizationID: workflow.OrganizationID, WorkflowID: workflow.ID}
root := t.TempDir()
videoPath := filepath.Join(root, "final.mp4")
semanticPath := filepath.Join(root, "semantic.json")
if err := os.WriteFile(videoPath, []byte("real-mp4-bytes"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(semanticPath, []byte(`{"actions":[]}`), 0o644); err != nil {
t.Fatal(err)
}
artifacts, err := runtime.normalizeOutputArtifacts(ctx, workflow, run, "package", []domain.ExecutionArtifact{
{ID: "package-directory", RunID: run.ID, NodeID: "package", Name: "package-directory", Type: domain.PortTypeFile, Path: root},
{ID: "final-video", RunID: run.ID, NodeID: "package", Name: "final.mp4", Type: domain.PortTypeVideo, Path: videoPath},
{ID: "semantic-file", RunID: run.ID, NodeID: "package", Name: "semantic.json", Type: domain.PortTypeFile, Path: semanticPath},
})
if err != nil {
t.Fatalf("normalize artifacts: %v", err)
}
if len(publisher.puts) != 3 {
t.Fatalf("directory files plus one manifest should upload once each, puts=%#v", publisher.puts)
}
directoryRef := artifacts[0].Payload.Ref
if directoryRef == nil || directoryRef.Kind != domain.ArtifactRefKindDirectory {
t.Fatalf("missing directory ref: %#v", artifacts[0].Payload)
}
var manifest directoryManifest
if err := json.Unmarshal(publisher.objects[publisher.objectID(connection.Bucket, directoryRef.ManifestKey)], &manifest); err != nil {
t.Fatalf("decode directory manifest: %v", err)
}
keys := map[string]string{}
for _, entry := range manifest.Entries {
keys[entry.Path] = entry.Key
}
if artifacts[1].Payload == nil || artifacts[1].Payload.Ref == nil || artifacts[1].Payload.Ref.Key != keys["final.mp4"] {
t.Fatalf("video must reuse directory object ref: artifact=%#v keys=%#v", artifacts[1], keys)
}
if artifacts[2].Payload == nil || artifacts[2].Payload.Ref == nil || artifacts[2].Payload.Ref.Key != keys["semantic.json"] {
t.Fatalf("semantic file must reuse directory object ref: artifact=%#v keys=%#v", artifacts[2], keys)
}
}
func TestArtifactStoragePolicyUsesWorkflowOrganizationDefault(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime, err := newTestRuntime(t, dataStore)
if err != nil {
t.Fatalf("new runtime: %v", err)
}
settings := domain.ArtifactStorageSettings{
ConnectionID: "storage-tenant-a",
Bucket: "tenant-a",
PrefixTemplate: "artifacts/{workflowId}/{runId}/{nodeId}",
}
if err := dataStore.UpsertArtifactStorageSettings(context.Background(), "org-tenant-a", settings); err != nil {
t.Fatalf("upsert tenant artifact settings: %v", err)
}
policy, err := runtime.artifactStoragePolicy(context.Background(), &domain.WorkflowRecord{OrganizationID: "org-tenant-a"})
if err != nil {
t.Fatalf("resolve tenant artifact policy: %v", err)
}
if policy == nil || policy.ConnectionID != settings.ConnectionID || policy.Bucket != settings.Bucket {
t.Fatalf("unexpected tenant artifact policy: %#v", policy)
}
missing, err := runtime.artifactStoragePolicy(context.Background(), &domain.WorkflowRecord{OrganizationID: "org-tenant-b"})
if err != nil {
t.Fatalf("resolve other tenant artifact policy: %v", err)
}
if missing != nil {
t.Fatalf("expected no artifact policy for other tenant, got %#v", missing)
}
}
func TestValidateRequiredNodeInputsRejectsMissingArtifact(t *testing.T) {
node := domain.WorkflowNode{ID: "sink", Data: domain.WorkflowNodeData{Inputs: []domain.PortDefinition{{ID: "in", Type: domain.PortTypeJSON, Required: true}}}}
if err := validateRequiredNodeInputs(node, nil); err == nil {
t.Fatal("expected required input failure")
}
if err := validateRequiredNodeInputs(node, []domain.NodeInputBinding{{TargetPortID: "in", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeInline, Type: domain.PortTypeJSON, Inline: true}}}); err != nil {
t.Fatalf("expected payload binding to satisfy required input: %v", err)
}
}