forked from longtao.wu/droboticflow
243 lines
9.9 KiB
Go
243 lines
9.9 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
func captureV4Workflow(enabled bool) domain.WorkflowRecord {
|
|
value := "false"
|
|
if enabled {
|
|
value = "true"
|
|
}
|
|
return domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{ID: "package", Data: domain.WorkflowNodeData{
|
|
OperatorID: "operator-builtin-package", Hyperparams: []domain.HyperParam{{Key: "DELIVERY_JSON_V4", Value: value}},
|
|
}}}}
|
|
}
|
|
|
|
func captureV4Objects() []captureObject {
|
|
objects := append(calibrationObjects(), partObjects("part0001")...)
|
|
return append(objects, captureObject{Key: "incoming/" + layoutCapture + "/meta_" + layoutCapture + ".json", Size: 20, ETag: "metadata-version"})
|
|
}
|
|
|
|
func TestDiscoverCaptureGroupsForWorkflowV4RequiresCaptureMetadata(t *testing.T) {
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
|
|
objects := append(calibrationObjects(), partObjects("part0001")...)
|
|
for _, enabled := range []bool{false, true} {
|
|
groups, err := discoverCaptureGroupsForWorkflow(objects, config, captureV4Workflow(enabled))
|
|
if err != nil || len(groups) != 1 {
|
|
t.Fatalf("groups: %+v, %v", groups, err)
|
|
}
|
|
if enabled {
|
|
if !strings.Contains(strings.Join(groups[0].Missing, " "), "meta_"+layoutCapture+".json") {
|
|
t.Fatal("v4 admitted a seven-file part", groups[0])
|
|
}
|
|
} else if len(groups[0].Missing) != 0 || len(groups[0].SourceKeys) != 7 {
|
|
t.Fatal("legacy intake must keep metadata optional", groups[0])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestV4CaptureMetadataMustBeNonemptyAndAtItsOwnCaptureRoot(t *testing.T) {
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
|
|
for _, location := range []string{"missing", "other-capture", "inside-part", "empty", "complete"} {
|
|
t.Run(location, func(t *testing.T) {
|
|
objects := captureV4Objects()
|
|
meta := &objects[len(objects)-1]
|
|
switch location {
|
|
case "missing":
|
|
objects = objects[:len(objects)-1]
|
|
case "other-capture":
|
|
meta.Key = "incoming/other/meta_" + layoutCapture + ".json"
|
|
case "inside-part":
|
|
meta.Key = "incoming/" + layoutCapture + "/part0001/meta_" + layoutCapture + ".json"
|
|
case "empty":
|
|
meta.Size = 0
|
|
}
|
|
groups, err := discoverCaptureGroupsForWorkflow(objects, config, captureV4Workflow(true))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, group := range groups {
|
|
if group.CaptureID != layoutCapture+"/part0001" {
|
|
continue
|
|
}
|
|
if (len(group.Missing) == 0) != (location == "complete") {
|
|
t.Fatalf("%s: %+v", location, group)
|
|
}
|
|
return
|
|
}
|
|
t.Fatal("target part not discovered")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestV4CapturePreflightRejectsIncompleteOrUnprovableSelections(t *testing.T) {
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
|
|
groups, err := discoverCaptureGroups(captureV4Objects(), config)
|
|
if err != nil || len(groups) != 1 || len(groups[0].Missing) != 0 {
|
|
t.Fatalf("complete fixture: %+v, %v", groups, err)
|
|
}
|
|
group := groups[0]
|
|
base := domain.CaptureRecord{Bucket: config.Bucket, CaptureID: group.CaptureID, Prefix: group.Prefix,
|
|
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys, Fingerprint: group.Fingerprint}
|
|
for i, key := range base.SourceKeys {
|
|
t.Run("missing/"+key, func(t *testing.T) {
|
|
record := base
|
|
record.SourceKeys = append(append([]string{}, base.SourceKeys[:i]...), base.SourceKeys[i+1:]...)
|
|
if err := validateV4CaptureSourceSelection(captureV4Workflow(true), &record); err == nil {
|
|
t.Fatal("incomplete selection admitted", key)
|
|
}
|
|
if err := validateV4CaptureSourceSelection(captureV4Workflow(false), &record); err != nil {
|
|
t.Fatal("legacy preflight changed", err)
|
|
}
|
|
})
|
|
}
|
|
for _, kind := range []string{"whole-capture", "cross-part", "duplicate-key", "duplicate-role", "wrong-metadata", "unsafe-key", "missing-fingerprint"} {
|
|
t.Run(kind, func(t *testing.T) {
|
|
record := base
|
|
record.SourceKeys = append([]string{}, base.SourceKeys...)
|
|
switch kind {
|
|
case "whole-capture":
|
|
record.Prefix, record.CaptureID = record.SourcePrefix, layoutCapture
|
|
case "cross-part":
|
|
record.SourceKeys = append(record.SourceKeys, record.SourcePrefix+"/part0002/foreign.mp4")
|
|
case "duplicate-key":
|
|
record.SourceKeys = append(record.SourceKeys, record.SourceKeys[0])
|
|
case "duplicate-role":
|
|
record.SourceKeys = append(record.SourceKeys, record.Prefix+"/"+layoutCapture+"_camera_left_part0001.mp4")
|
|
case "wrong-metadata":
|
|
for i, key := range record.SourceKeys {
|
|
if strings.HasSuffix(key, ".json") {
|
|
record.SourceKeys[i] = record.SourcePrefix + "/meta_another_capture.json"
|
|
}
|
|
}
|
|
case "unsafe-key":
|
|
record.SourceKeys[0] = record.SourcePrefix + "/../foreign.yaml"
|
|
case "missing-fingerprint":
|
|
record.Fingerprint = ""
|
|
}
|
|
if err := validateV4CaptureSourceSelection(captureV4Workflow(true), &record); err == nil {
|
|
t.Fatal("unprovable v4 selection admitted", kind)
|
|
}
|
|
if err := validateV4CaptureSourceSelection(captureV4Workflow(false), &record); err != nil {
|
|
t.Fatal("legacy preflight changed", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestV4CaptureSourceAndAuditMatchPackageBooleanValues(t *testing.T) {
|
|
ctx := context.Background()
|
|
ds := store.NewMemoryStore()
|
|
runtime := &Runtime{store: ds}
|
|
for _, tc := range []struct {
|
|
value string
|
|
enabled bool
|
|
}{
|
|
{"true", true}, {"TRUE", true}, {"1", true}, {"yes", true}, {" on ", true},
|
|
{"false", false}, {"FALSE", false}, {"0", false}, {"no", false}, {"off", false}, {"", false},
|
|
} {
|
|
workflow := captureV4Workflow(true)
|
|
workflow.ID = "source-audit-alias"
|
|
workflow.Nodes[0].Data.Hyperparams[0].Value = tc.value
|
|
if err := ds.UpsertWorkflow(ctx, workflow); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := validateV4CaptureSourceSelection(workflow, &domain.CaptureRecord{}); (err != nil) != tc.enabled {
|
|
t.Fatalf("source preflight for package boolean %q: %v", tc.value, err)
|
|
}
|
|
if audit, err := runtime.classificationDeliveryEnabled(ctx, workflow.ID); err != nil || audit != tc.enabled {
|
|
t.Fatalf("audit for package boolean %q: enabled=%v, err=%v", tc.value, audit, err)
|
|
}
|
|
}
|
|
workflow := captureV4Workflow(true)
|
|
workflow.ID = "non-package-alias"
|
|
workflow.Nodes[0].Data.OperatorID = "unrelated-operator"
|
|
if err := ds.UpsertWorkflow(ctx, workflow); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := validateV4CaptureSourceSelection(workflow, &domain.CaptureRecord{}); err != nil {
|
|
t.Fatal("non-package parameter enabled the v4 gate", err)
|
|
}
|
|
if audit, err := runtime.classificationDeliveryEnabled(ctx, workflow.ID); err != nil || audit {
|
|
t.Fatal("non-package parameter enabled v4 audit", audit, err)
|
|
}
|
|
if audit, err := runtime.classificationDeliveryEnabled(ctx, "missing-workflow"); err != nil || audit {
|
|
t.Fatal("missing workflow enabled v4 audit", audit, err)
|
|
}
|
|
}
|
|
|
|
func TestV4CaptureLaunchFreezesCompletePartSourceSelection(t *testing.T) {
|
|
ctx := context.Background()
|
|
ds := store.NewMemoryStore()
|
|
workflow, err := ds.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
OrganizationID: domain.DefaultOrganizationID, Name: "new v4 capture workflow", Type: domain.WorkflowTypeStreaming,
|
|
WorkflowInputs: []domain.WorkflowInputDefinition{{ID: "sourceDirectory", Type: domain.PortTypeFile, Container: domain.PortContainerDirectory, PayloadMode: domain.ArtifactTransferModeRef, Required: true}},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{
|
|
Enabled: true, ConnectionID: "synthetic-source", Bucket: "synthetic-bucket", Prefix: "incoming",
|
|
DeliveryPrefix: "deliveries", DeliveryProjectDirectory: "Synthetic__project",
|
|
})
|
|
workflow.Nodes = captureV4Workflow(true).Nodes
|
|
workflow.DefinitionState = domain.WorkflowDefinitionStateReleased
|
|
workflow.CaptureAutomation = &config
|
|
if err = ds.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
objects := append(captureV4Objects(), partObjects("part0002")...)
|
|
groups, err := discoverCaptureGroupsForWorkflow(objects, config, *workflow)
|
|
if err != nil || len(groups) != 2 {
|
|
t.Fatalf("groups: %+v, %v", groups, err)
|
|
}
|
|
runtime := &Runtime{store: ds}
|
|
for _, group := range groups {
|
|
if len(group.Missing) != 0 {
|
|
t.Fatal("complete source rejected", group)
|
|
}
|
|
record := domain.CaptureRecord{
|
|
ID: captureRecordID(workflow.ID, group.Prefix), OrganizationID: workflow.OrganizationID, WorkflowID: workflow.ID,
|
|
ConnectionID: config.ConnectionID, Bucket: config.Bucket, CaptureID: group.CaptureID, Prefix: group.Prefix,
|
|
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys, Fingerprint: group.Fingerprint, Status: domain.CaptureRecordReady,
|
|
}
|
|
if err = validateV4CaptureSourceSelection(*workflow, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = ds.UpsertCaptureRecord(ctx, record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = runtime.launchCaptureRun(ctx, *workflow, config, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run, err := ds.GetRun(ctx, record.RunID)
|
|
if err != nil || run == nil {
|
|
t.Fatalf("run missing: %v", err)
|
|
}
|
|
ref := run.Inputs["sourceDirectory"].Ref
|
|
if ref == nil || ref.Prefix != "incoming/"+layoutCapture || ref.CaptureUnit != group.CaptureID || ref.VersionID != group.Fingerprint || len(ref.ObjectKeys) != 8 || !reflect.DeepEqual(ref.ObjectKeys, group.SourceKeys) {
|
|
t.Fatalf("source proof did not survive run creation: %+v", ref)
|
|
}
|
|
if err = domain.ValidateArtifactPayload(run.Inputs["sourceDirectory"], domain.PortTypeFile); err != nil {
|
|
t.Fatal("v5.8 input contract rejected selected part", err)
|
|
}
|
|
// Later discovery cannot rewrite the input used by annotationOriginalSource.
|
|
before := append([]string(nil), ref.ObjectKeys...)
|
|
record.SourceKeys[0] = "incoming/unrelated/file"
|
|
if err = ds.UpsertCaptureRecord(ctx, record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run, err = ds.GetRun(ctx, record.RunID)
|
|
if err != nil || !reflect.DeepEqual(run.Inputs["sourceDirectory"].Ref.ObjectKeys, before) {
|
|
t.Fatal("frozen run selection followed later discovery", err)
|
|
}
|
|
}
|
|
}
|