droboticflow/internal/worker/capture_v4_source_test.go

243 lines
9.9 KiB
Go

package worker
import (
"context"
"reflect"
"strings"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func captureV4Workflow(enabled bool) domain.WorkflowRecord {
value := "false"
if enabled {
value = "true"
}
return domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{ID: "package", Data: domain.WorkflowNodeData{
OperatorID: "operator-builtin-package", Hyperparams: []domain.HyperParam{{Key: "DELIVERY_JSON_V4", Value: value}},
}}}}
}
func captureV4Objects() []captureObject {
objects := append(calibrationObjects(), partObjects("part0001")...)
return append(objects, captureObject{Key: "incoming/" + layoutCapture + "/meta_" + layoutCapture + ".json", Size: 20, ETag: "metadata-version"})
}
func TestDiscoverCaptureGroupsForWorkflowV4RequiresCaptureMetadata(t *testing.T) {
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
objects := append(calibrationObjects(), partObjects("part0001")...)
for _, enabled := range []bool{false, true} {
groups, err := discoverCaptureGroupsForWorkflow(objects, config, captureV4Workflow(enabled))
if err != nil || len(groups) != 1 {
t.Fatalf("groups: %+v, %v", groups, err)
}
if enabled {
if !strings.Contains(strings.Join(groups[0].Missing, " "), "meta_"+layoutCapture+".json") {
t.Fatal("v4 admitted a seven-file part", groups[0])
}
} else if len(groups[0].Missing) != 0 || len(groups[0].SourceKeys) != 7 {
t.Fatal("legacy intake must keep metadata optional", groups[0])
}
}
}
func TestV4CaptureMetadataMustBeNonemptyAndAtItsOwnCaptureRoot(t *testing.T) {
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
for _, location := range []string{"missing", "other-capture", "inside-part", "empty", "complete"} {
t.Run(location, func(t *testing.T) {
objects := captureV4Objects()
meta := &objects[len(objects)-1]
switch location {
case "missing":
objects = objects[:len(objects)-1]
case "other-capture":
meta.Key = "incoming/other/meta_" + layoutCapture + ".json"
case "inside-part":
meta.Key = "incoming/" + layoutCapture + "/part0001/meta_" + layoutCapture + ".json"
case "empty":
meta.Size = 0
}
groups, err := discoverCaptureGroupsForWorkflow(objects, config, captureV4Workflow(true))
if err != nil {
t.Fatal(err)
}
for _, group := range groups {
if group.CaptureID != layoutCapture+"/part0001" {
continue
}
if (len(group.Missing) == 0) != (location == "complete") {
t.Fatalf("%s: %+v", location, group)
}
return
}
t.Fatal("target part not discovered")
})
}
}
func TestV4CapturePreflightRejectsIncompleteOrUnprovableSelections(t *testing.T) {
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
groups, err := discoverCaptureGroups(captureV4Objects(), config)
if err != nil || len(groups) != 1 || len(groups[0].Missing) != 0 {
t.Fatalf("complete fixture: %+v, %v", groups, err)
}
group := groups[0]
base := domain.CaptureRecord{Bucket: config.Bucket, CaptureID: group.CaptureID, Prefix: group.Prefix,
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys, Fingerprint: group.Fingerprint}
for i, key := range base.SourceKeys {
t.Run("missing/"+key, func(t *testing.T) {
record := base
record.SourceKeys = append(append([]string{}, base.SourceKeys[:i]...), base.SourceKeys[i+1:]...)
if err := validateV4CaptureSourceSelection(captureV4Workflow(true), &record); err == nil {
t.Fatal("incomplete selection admitted", key)
}
if err := validateV4CaptureSourceSelection(captureV4Workflow(false), &record); err != nil {
t.Fatal("legacy preflight changed", err)
}
})
}
for _, kind := range []string{"whole-capture", "cross-part", "duplicate-key", "duplicate-role", "wrong-metadata", "unsafe-key", "missing-fingerprint"} {
t.Run(kind, func(t *testing.T) {
record := base
record.SourceKeys = append([]string{}, base.SourceKeys...)
switch kind {
case "whole-capture":
record.Prefix, record.CaptureID = record.SourcePrefix, layoutCapture
case "cross-part":
record.SourceKeys = append(record.SourceKeys, record.SourcePrefix+"/part0002/foreign.mp4")
case "duplicate-key":
record.SourceKeys = append(record.SourceKeys, record.SourceKeys[0])
case "duplicate-role":
record.SourceKeys = append(record.SourceKeys, record.Prefix+"/"+layoutCapture+"_camera_left_part0001.mp4")
case "wrong-metadata":
for i, key := range record.SourceKeys {
if strings.HasSuffix(key, ".json") {
record.SourceKeys[i] = record.SourcePrefix + "/meta_another_capture.json"
}
}
case "unsafe-key":
record.SourceKeys[0] = record.SourcePrefix + "/../foreign.yaml"
case "missing-fingerprint":
record.Fingerprint = ""
}
if err := validateV4CaptureSourceSelection(captureV4Workflow(true), &record); err == nil {
t.Fatal("unprovable v4 selection admitted", kind)
}
if err := validateV4CaptureSourceSelection(captureV4Workflow(false), &record); err != nil {
t.Fatal("legacy preflight changed", err)
}
})
}
}
func TestV4CaptureSourceAndAuditMatchPackageBooleanValues(t *testing.T) {
ctx := context.Background()
ds := store.NewMemoryStore()
runtime := &Runtime{store: ds}
for _, tc := range []struct {
value string
enabled bool
}{
{"true", true}, {"TRUE", true}, {"1", true}, {"yes", true}, {" on ", true},
{"false", false}, {"FALSE", false}, {"0", false}, {"no", false}, {"off", false}, {"", false},
} {
workflow := captureV4Workflow(true)
workflow.ID = "source-audit-alias"
workflow.Nodes[0].Data.Hyperparams[0].Value = tc.value
if err := ds.UpsertWorkflow(ctx, workflow); err != nil {
t.Fatal(err)
}
if err := validateV4CaptureSourceSelection(workflow, &domain.CaptureRecord{}); (err != nil) != tc.enabled {
t.Fatalf("source preflight for package boolean %q: %v", tc.value, err)
}
if audit, err := runtime.classificationDeliveryEnabled(ctx, workflow.ID); err != nil || audit != tc.enabled {
t.Fatalf("audit for package boolean %q: enabled=%v, err=%v", tc.value, audit, err)
}
}
workflow := captureV4Workflow(true)
workflow.ID = "non-package-alias"
workflow.Nodes[0].Data.OperatorID = "unrelated-operator"
if err := ds.UpsertWorkflow(ctx, workflow); err != nil {
t.Fatal(err)
}
if err := validateV4CaptureSourceSelection(workflow, &domain.CaptureRecord{}); err != nil {
t.Fatal("non-package parameter enabled the v4 gate", err)
}
if audit, err := runtime.classificationDeliveryEnabled(ctx, workflow.ID); err != nil || audit {
t.Fatal("non-package parameter enabled v4 audit", audit, err)
}
if audit, err := runtime.classificationDeliveryEnabled(ctx, "missing-workflow"); err != nil || audit {
t.Fatal("missing workflow enabled v4 audit", audit, err)
}
}
func TestV4CaptureLaunchFreezesCompletePartSourceSelection(t *testing.T) {
ctx := context.Background()
ds := store.NewMemoryStore()
workflow, err := ds.CreateWorkflow(ctx, domain.CreateWorkflowInput{
OrganizationID: domain.DefaultOrganizationID, Name: "new v4 capture workflow", Type: domain.WorkflowTypeStreaming,
WorkflowInputs: []domain.WorkflowInputDefinition{{ID: "sourceDirectory", Type: domain.PortTypeFile, Container: domain.PortContainerDirectory, PayloadMode: domain.ArtifactTransferModeRef, Required: true}},
})
if err != nil {
t.Fatal(err)
}
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{
Enabled: true, ConnectionID: "synthetic-source", Bucket: "synthetic-bucket", Prefix: "incoming",
DeliveryPrefix: "deliveries", DeliveryProjectDirectory: "Synthetic__project",
})
workflow.Nodes = captureV4Workflow(true).Nodes
workflow.DefinitionState = domain.WorkflowDefinitionStateReleased
workflow.CaptureAutomation = &config
if err = ds.UpsertWorkflow(ctx, *workflow); err != nil {
t.Fatal(err)
}
objects := append(captureV4Objects(), partObjects("part0002")...)
groups, err := discoverCaptureGroupsForWorkflow(objects, config, *workflow)
if err != nil || len(groups) != 2 {
t.Fatalf("groups: %+v, %v", groups, err)
}
runtime := &Runtime{store: ds}
for _, group := range groups {
if len(group.Missing) != 0 {
t.Fatal("complete source rejected", group)
}
record := domain.CaptureRecord{
ID: captureRecordID(workflow.ID, group.Prefix), OrganizationID: workflow.OrganizationID, WorkflowID: workflow.ID,
ConnectionID: config.ConnectionID, Bucket: config.Bucket, CaptureID: group.CaptureID, Prefix: group.Prefix,
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys, Fingerprint: group.Fingerprint, Status: domain.CaptureRecordReady,
}
if err = validateV4CaptureSourceSelection(*workflow, &record); err != nil {
t.Fatal(err)
}
if err = ds.UpsertCaptureRecord(ctx, record); err != nil {
t.Fatal(err)
}
if err = runtime.launchCaptureRun(ctx, *workflow, config, &record); err != nil {
t.Fatal(err)
}
run, err := ds.GetRun(ctx, record.RunID)
if err != nil || run == nil {
t.Fatalf("run missing: %v", err)
}
ref := run.Inputs["sourceDirectory"].Ref
if ref == nil || ref.Prefix != "incoming/"+layoutCapture || ref.CaptureUnit != group.CaptureID || ref.VersionID != group.Fingerprint || len(ref.ObjectKeys) != 8 || !reflect.DeepEqual(ref.ObjectKeys, group.SourceKeys) {
t.Fatalf("source proof did not survive run creation: %+v", ref)
}
if err = domain.ValidateArtifactPayload(run.Inputs["sourceDirectory"], domain.PortTypeFile); err != nil {
t.Fatal("v5.8 input contract rejected selected part", err)
}
// Later discovery cannot rewrite the input used by annotationOriginalSource.
before := append([]string(nil), ref.ObjectKeys...)
record.SourceKeys[0] = "incoming/unrelated/file"
if err = ds.UpsertCaptureRecord(ctx, record); err != nil {
t.Fatal(err)
}
run, err = ds.GetRun(ctx, record.RunID)
if err != nil || !reflect.DeepEqual(run.Inputs["sourceDirectory"].Ref.ObjectKeys, before) {
t.Fatal("frozen run selection followed later discovery", err)
}
}
}