forked from longtao.wu/droboticflow
398 lines
17 KiB
Go
398 lines
17 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"regexp"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
func TestCaptureUserCancellationWinsUntilManualReprocess(t *testing.T) {
|
|
record := &domain.CaptureRecord{
|
|
Status: domain.CaptureRecordCancelled,
|
|
Actions: []domain.CaptureActionRecord{{Action: domain.CaptureActionCancelled}},
|
|
}
|
|
if !captureCancelledByUser(record) {
|
|
t.Fatal("expected user cancellation to win over a still-stopping Run")
|
|
}
|
|
record.Actions = append(record.Actions, domain.CaptureActionRecord{Action: domain.CaptureActionReprocessRequested})
|
|
if captureCancelledByUser(record) {
|
|
t.Fatal("manual reprocess must release the cancellation guard")
|
|
}
|
|
}
|
|
|
|
func completeCaptureObjects(root, captureID string) []captureObject {
|
|
prefix := root + "/" + captureID + "/"
|
|
return []captureObject{
|
|
{Key: prefix + "camera_left_part0001.mp4", ETag: "1", Size: 10},
|
|
{Key: prefix + "camera_right_part0001.mp4", ETag: "2", Size: 10},
|
|
{Key: prefix + "camera_left_part0001_pts.csv", ETag: "3", Size: 2},
|
|
{Key: prefix + "camera_right_part0001_pts.csv", ETag: "4", Size: 2},
|
|
{Key: prefix + "imu.csv", ETag: "5", Size: 2},
|
|
{Key: prefix + "calibration_camera.yaml", ETag: "6", Size: 2},
|
|
{Key: prefix + "calibration_imu.yaml", ETag: "7", Size: 2},
|
|
{Key: prefix + "source-manifest.json", ETag: "8", Size: 2},
|
|
}
|
|
}
|
|
|
|
func TestCaptureIMURecognizesLegacyAndSegmentNames(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
valid bool
|
|
}{
|
|
{"imu.csv", true},
|
|
{"Orbbec_Ego_AZER76400EH_19700101_000046_imu.csv", true},
|
|
{"part0001/FYLX_S03_AZEC8640166_19700101_000304_imu_part0001.csv", true},
|
|
{"part0015/FYLX_S03_AZEC8640166_19700102_171436_imu_part0015.csv", true},
|
|
{"part0015/CAPTURE_IMU_PART0015.CSV", true},
|
|
{"calibration_imu.yaml", false},
|
|
{"capture_imu_part0001.csv.bak", false},
|
|
{"capture_imu_partabc.csv", false},
|
|
{"capture_imu_part0001_pts.csv", false},
|
|
{"notimu.csv", false},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if regexp.MustCompile(defaultOrbbecCaptureRequirements[4].Pattern).MatchString(tc.name) != tc.valid {
|
|
t.Fatalf("unexpected IMU filename match: %s", tc.name)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCaptureSegmentIMUStillRejectsEmptyAndDoesNotFillOtherCapture(t *testing.T) {
|
|
objects := completeCaptureObjects("incoming", "a")
|
|
objects[4].Key = "incoming/a/part0001/a_imu_part0001.csv"
|
|
objects[4].Size = 0
|
|
other := completeCaptureObjects("incoming", "b")
|
|
other[4].Key = "incoming/b/readme.txt"
|
|
objects = append(objects, other...)
|
|
groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"}))
|
|
if err != nil || len(groups) != 2 {
|
|
t.Fatalf("groups=%v err=%v", groups, err)
|
|
}
|
|
if len(groups[0].Missing) != 1 || groups[0].Missing[0] != "目录结构冲突:根目录数据与 part 数据不能混合接入" {
|
|
t.Fatalf("empty IMU accepted: %v", groups[0].Missing)
|
|
}
|
|
if len(groups[1].Missing) != 1 || groups[1].Missing[0] != "IMU" {
|
|
t.Fatalf("cross-Capture IMU matched: %v", groups[1].Missing)
|
|
}
|
|
}
|
|
|
|
func TestCaptureRejectsEmptyLaterPTSSegment(t *testing.T) {
|
|
objects := completeCaptureObjects("incoming", "capture")
|
|
objects = append(objects, captureObject{Key: "incoming/capture/camera_right_part0004_pts.csv", Size: 0})
|
|
groups, err := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"}))
|
|
if err != nil || len(groups) != 1 {
|
|
t.Fatalf("groups=%v err=%v", groups, err)
|
|
}
|
|
found := false
|
|
for _, missing := range groups[0].Missing {
|
|
if missing == "空文件 camera_right_part0004_pts.csv" {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("empty later segment not reported: %v", groups[0].Missing)
|
|
}
|
|
}
|
|
|
|
func TestDiscoverCaptureGroupsWaitsForEveryRequiredType(t *testing.T) {
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming"})
|
|
objects := completeCaptureObjects("incoming", "record-one")
|
|
objects = append(objects, completeCaptureObjects("incoming", "任意业务目录")...)
|
|
objects = objects[:len(objects)-5] // second capture remains partial
|
|
objects = append(objects, captureObject{Key: "incoming/_droboticflow/internal.json", ETag: "system", Size: 42})
|
|
|
|
groups, err := discoverCaptureGroups(objects, config)
|
|
if err != nil {
|
|
t.Fatalf("discover captures: %v", err)
|
|
}
|
|
if len(groups) != 2 {
|
|
t.Fatalf("groups = %d, want 2", len(groups))
|
|
}
|
|
if len(groups[0].Missing) != 0 || groups[0].CaptureID != "record-one" {
|
|
t.Fatalf("first capture should be complete: %#v", groups[0])
|
|
}
|
|
if len(groups[1].Missing) == 0 || groups[1].CaptureID != "任意业务目录" {
|
|
t.Fatalf("second capture should remain incomplete: %#v", groups[1])
|
|
}
|
|
}
|
|
|
|
func TestDiscoverCaptureGroupsSupportsHistoricalRequiredFilesMode(t *testing.T) {
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "test", CompletionMode: "required_files"})
|
|
objects := completeCaptureObjects("test", "ready-record-without-name-contract")
|
|
objects = objects[:len(objects)-1] // historical test directories do not have source-manifest.json
|
|
objects = append(objects, captureObject{Key: "test/ready-record-without-name-contract/_droboticflow/runs/internal.json", ETag: "internal", Size: 42})
|
|
|
|
groups, err := discoverCaptureGroups(objects, config)
|
|
if err != nil {
|
|
t.Fatalf("discover historical capture: %v", err)
|
|
}
|
|
if len(groups) != 1 || len(groups[0].Missing) != 0 {
|
|
t.Fatalf("required-files mode should accept the complete sensor bundle: %#v", groups)
|
|
}
|
|
if len(groups[0].Objects) != len(objects)-1 {
|
|
t.Fatalf("platform-generated _droboticflow objects must not mutate source identity: %#v", groups[0].Objects)
|
|
}
|
|
}
|
|
|
|
func TestCaptureCompletionNeverRequiresManifest(t *testing.T) {
|
|
for _, mode := range []string{"", "manifest_and_files", "required_files"} {
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", CompletionMode: mode})
|
|
if config.CompletionMode != "required_files" {
|
|
t.Fatalf("legacy mode not normalized: %s", mode)
|
|
}
|
|
objects := completeCaptureObjects("incoming", "record")
|
|
objects = objects[:len(objects)-1]
|
|
groups, err := discoverCaptureGroups(objects, config)
|
|
if err != nil || len(groups) != 1 || len(groups[0].Missing) != 0 {
|
|
t.Fatalf("mode=%s groups=%v err=%v", mode, groups, err)
|
|
}
|
|
objects = objects[:len(objects)-1]
|
|
groups, err = discoverCaptureGroups(objects, config)
|
|
if err != nil || len(groups[0].Missing) == 0 {
|
|
t.Fatal("missing IMU calibration must still block intake")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReconcileCaptureFollowsLatestRetryRun(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{OrganizationID: domain.DefaultOrganizationID, Name: "Fanyu"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID, Name: "Fanyu retry workflow",
|
|
Type: domain.WorkflowTypeStreaming,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
record := domain.CaptureRecord{
|
|
ID: "capture-record-retry", OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID,
|
|
WorkflowID: workflow.ID, Prefix: "incoming/arbitrary-directory", CaptureID: "arbitrary-directory",
|
|
Fingerprint: "fingerprint-retry", Status: domain.CaptureRecordRunning, DiscoveredAt: nowISO(), UpdatedAt: nowISO(),
|
|
}
|
|
initial, err := dataStore.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerScheduled, nil, record.ID)
|
|
if err != nil {
|
|
t.Fatalf("create initial run: %v", err)
|
|
}
|
|
initial.Status = domain.RunStatusFailed
|
|
initial.Error = "transient failure"
|
|
initial.EndedAt = nowISO()
|
|
if err := dataStore.UpsertRun(ctx, *initial); err != nil {
|
|
t.Fatalf("store initial run: %v", err)
|
|
}
|
|
record.RunID = initial.ID
|
|
record.CompletedAt = initial.EndedAt
|
|
if err := dataStore.UpsertCaptureRecord(ctx, record); err != nil {
|
|
t.Fatalf("store capture: %v", err)
|
|
}
|
|
retryRun, err := dataStore.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, nil, record.ID)
|
|
if err != nil {
|
|
t.Fatalf("create retry run: %v", err)
|
|
}
|
|
retryRun.CreatedAt = "2099-01-01T00:00:00Z"
|
|
if err := dataStore.UpsertRun(ctx, *retryRun); err != nil {
|
|
t.Fatalf("store retry run: %v", err)
|
|
}
|
|
runtime := &Runtime{store: dataStore}
|
|
if err := runtime.reconcileCaptureRun(ctx, &record); err != nil {
|
|
t.Fatalf("reconcile capture: %v", err)
|
|
}
|
|
if record.RunID != retryRun.ID || record.Status != domain.CaptureRecordQueued || record.CompletedAt != "" {
|
|
t.Fatalf("capture did not follow queued retry: %#v retry=%#v", record, retryRun)
|
|
}
|
|
retryRun.Status = domain.RunStatusCompleted
|
|
retryRun.StartedAt = "2099-01-01T00:00:01Z"
|
|
retryRun.EndedAt = "2099-01-01T00:00:02Z"
|
|
if err := dataStore.UpsertRun(ctx, *retryRun); err != nil {
|
|
t.Fatalf("complete retry run: %v", err)
|
|
}
|
|
if err := runtime.reconcileCaptureRun(ctx, &record); err != nil {
|
|
t.Fatalf("reconcile completed retry: %v", err)
|
|
}
|
|
if record.RunID != retryRun.ID || record.Status != domain.CaptureRecordCompleted {
|
|
t.Fatalf("capture did not follow latest retry: %#v retry=%#v", record, retryRun)
|
|
}
|
|
}
|
|
|
|
func TestRecoverStaleCaptureRequeueRestoresPreviousTerminalStatus(t *testing.T) {
|
|
record := domain.CaptureRecord{
|
|
ID: "capture-stale-requeue", Status: domain.CaptureRecordRequeueing,
|
|
RunID: "run-cancelled", UpdatedAt: "2026-01-01T00:00:00Z",
|
|
Actions: []domain.CaptureActionRecord{{
|
|
ID: "action-1", Action: domain.CaptureActionReprocessRequested,
|
|
FromStatus: domain.CaptureRecordCancelled, ToStatus: domain.CaptureRecordRequeueing,
|
|
ActorUserID: "admin", Reason: "retry", CreatedAt: "2026-01-01T00:00:00Z",
|
|
}},
|
|
}
|
|
if !recoverStaleCaptureRequeue(&record, time.Date(2026, 1, 1, 0, 3, 0, 0, time.UTC)) {
|
|
t.Fatalf("stale requeue was not recovered")
|
|
}
|
|
if record.Status != domain.CaptureRecordCancelled || len(record.Actions) != 2 || record.Actions[1].Action != domain.CaptureActionReprocessFailed {
|
|
t.Fatalf("unexpected stale requeue recovery: %#v", record)
|
|
}
|
|
}
|
|
|
|
func TestLaunchCaptureRunBindsOneCompleteDirectory(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{OrganizationID: domain.DefaultOrganizationID, Name: "Fanyu"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
Name: "Fanyu capture workflow",
|
|
Type: domain.WorkflowTypeStreaming,
|
|
WorkflowInputs: []domain.WorkflowInputDefinition{{
|
|
ID: "sourceDirectory", Name: "sourceDirectory", Type: domain.PortTypeFile,
|
|
Container: domain.PortContainerDirectory, PayloadMode: domain.ArtifactTransferModeRef, Required: true,
|
|
}},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
workflow.DefinitionState = domain.WorkflowDefinitionStateReleased
|
|
workflow.CaptureAutomation = &domain.CaptureAutomationConfig{Enabled: true}
|
|
if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
t.Fatalf("release workflow: %v", err)
|
|
}
|
|
record := domain.CaptureRecord{
|
|
ID: "capture-record-1", OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID,
|
|
WorkflowID: workflow.ID, ConnectionID: "bos-fanyu", Bucket: "drobodata-fanyu",
|
|
Prefix: "incoming/capture-000001", CaptureID: "capture-000001",
|
|
ManifestKey: "incoming/capture-000001/source-manifest.json", Fingerprint: "fingerprint-1",
|
|
Status: domain.CaptureRecordReady, DiscoveredAt: nowISO(), UpdatedAt: nowISO(),
|
|
}
|
|
if err := dataStore.UpsertCaptureRecord(ctx, record); err != nil {
|
|
t.Fatalf("upsert capture: %v", err)
|
|
}
|
|
runtime := &Runtime{store: dataStore}
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{
|
|
Enabled: true, ConnectionID: "bos-fanyu", Bucket: "drobodata-fanyu", Prefix: "incoming", CompletionMode: "required_files",
|
|
DeliveryPrefix: "deliveries", DeliveryProjectDirectory: "Fanyu__project",
|
|
})
|
|
paused := config
|
|
paused.Enabled = false
|
|
workflow.CaptureAutomation = &paused
|
|
if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.launchCaptureRun(ctx, *workflow, config, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, _ := dataStore.GetCaptureRecord(ctx, record.ID)
|
|
if before.RunID != "" {
|
|
t.Fatal("paused intake admitted stale scan")
|
|
}
|
|
changed := config
|
|
changed.Prefix = "different-root"
|
|
workflow.CaptureAutomation = &changed
|
|
if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.launchCaptureRun(ctx, *workflow, config, &record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, _ = dataStore.GetCaptureRecord(ctx, record.ID)
|
|
if before.RunID != "" {
|
|
t.Fatal("changed root admitted stale scan")
|
|
}
|
|
workflow.CaptureAutomation = &config
|
|
if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.launchCaptureRun(ctx, *workflow, config, &record); err != nil {
|
|
t.Fatalf("launch capture run: %v", err)
|
|
}
|
|
stored, err := dataStore.GetCaptureRecord(ctx, record.ID)
|
|
if err != nil || stored == nil || stored.RunID == "" || stored.Status != domain.CaptureRecordQueued {
|
|
t.Fatalf("capture was not queued: %#v err=%v", stored, err)
|
|
}
|
|
run, err := dataStore.GetRun(ctx, stored.RunID)
|
|
if stored.Delivery == nil || stored.Delivery.Prefix != "deliveries/Fanyu__project/capture-000001" {
|
|
t.Fatalf("delivery snapshot missing: %+v", stored.Delivery)
|
|
}
|
|
if err != nil || run == nil {
|
|
t.Fatalf("get run: %#v err=%v", run, err)
|
|
}
|
|
if run.TaskID != record.ID || run.Status != domain.RunStatusQueued {
|
|
t.Fatalf("unexpected run identity: %#v", run)
|
|
}
|
|
if len(run.CaptureDirectories) != 1 || run.CaptureDirectories[0] != record.CaptureID {
|
|
t.Fatalf("unexpected Capture directory: %#v", run.CaptureDirectories)
|
|
}
|
|
input := run.Inputs["sourceDirectory"]
|
|
if input.Ref == nil || input.Ref.Prefix != record.Prefix || input.Ref.ManifestKey != "" || input.Ref.Kind != domain.ArtifactRefKindDirectory {
|
|
t.Fatalf("unexpected directory binding: %#v", input)
|
|
}
|
|
// New part units retain the parent source root plus an explicit file selection.
|
|
part := record
|
|
part.ID = "capture-record-part-1"
|
|
part.RunID = ""
|
|
part.Status = domain.CaptureRecordReady
|
|
part.SourcePrefix = record.Prefix
|
|
part.Prefix = record.Prefix + "/part0001"
|
|
part.CaptureID = record.CaptureID + "/part0001"
|
|
part.SourceKeys = []string{part.Prefix + "/video.mp4", record.Prefix + "/calibration_camera.yaml"}
|
|
if err := dataStore.UpsertCaptureRecord(ctx, part); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
partRun, err := dataStore.GetRun(ctx, part.RunID)
|
|
if err != nil || partRun == nil {
|
|
t.Fatal(err)
|
|
}
|
|
ref := partRun.Inputs["sourceDirectory"].Ref
|
|
if ref.Prefix != record.Prefix || len(ref.ObjectKeys) != 2 || ref.CaptureUnit != part.CaptureID {
|
|
t.Fatalf("part selection lost: %+v", ref)
|
|
}
|
|
if len(partRun.CaptureDirectories) != 1 || partRun.CaptureDirectories[0] != part.CaptureID {
|
|
t.Fatal(partRun.CaptureDirectories)
|
|
}
|
|
firstRunID := part.RunID
|
|
part.Status = domain.CaptureRecordReady
|
|
if err := runtime.launchCaptureRun(ctx, *workflow, config, &part); err != nil || part.RunID != firstRunID {
|
|
t.Fatal("duplicate part launch", err)
|
|
}
|
|
}
|
|
|
|
func TestNormalizedCaptureAutomationDefaultsToThirtySecondStabilityWindow(t *testing.T) {
|
|
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Enabled: true, PollIntervalSeconds: 10})
|
|
if config.StabilityWindowSeconds != 30 {
|
|
t.Fatalf("stability window = %d", config.StabilityWindowSeconds)
|
|
}
|
|
}
|
|
|
|
func TestCaptureRuntimePolicyOverridesReleasedWorkflowConcurrency(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
runtime := &Runtime{store: dataStore}
|
|
workflows := []domain.WorkflowRecord{{
|
|
ID: "workflow-capture-policy", OrganizationID: domain.DefaultOrganizationID,
|
|
CaptureAutomation: &domain.CaptureAutomationConfig{Enabled: true}, MaxConcurrentRuns: 1,
|
|
}}
|
|
runtime.applyWorkflowRuntimePolicies(ctx, workflows)
|
|
if workflows[0].MaxConcurrentRuns != domain.DefaultMaxConcurrentCaptureRuns {
|
|
t.Fatalf("default Capture concurrency=%d", workflows[0].MaxConcurrentRuns)
|
|
}
|
|
policy := domain.WorkflowRuntimePolicyRecord{ID: "policy", OrganizationID: domain.DefaultOrganizationID, WorkflowID: workflows[0].ID, Revision: 1, MaxConcurrentCaptureRuns: 4, AcceptanceSampleProbability: 0.5}
|
|
if updated, err := dataStore.UpdateWorkflowRuntimePolicyVersioned(ctx, policy, 0); err != nil || !updated {
|
|
t.Fatalf("seed policy updated=%v err=%v", updated, err)
|
|
}
|
|
runtime.applyWorkflowRuntimePolicies(ctx, workflows)
|
|
if workflows[0].MaxConcurrentRuns != 4 {
|
|
t.Fatalf("runtime Capture concurrency=%d", workflows[0].MaxConcurrentRuns)
|
|
}
|
|
}
|