package api import ( "archive/tar" "bytes" "context" "encoding/base64" "encoding/json" "math" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "testing" "droboticflow/internal/domain" "droboticflow/internal/store" "go.mongodb.org/mongo-driver/bson" ) func tarDataURL(t *testing.T, files map[string][]byte) string { t.Helper() var buffer bytes.Buffer writer := tar.NewWriter(&buffer) for name, payload := range files { header := &tar.Header{ Name: name, Mode: 0o644, Size: int64(len(payload)), } if err := writer.WriteHeader(header); err != nil { t.Fatalf("write tar header: %v", err) } if _, err := writer.Write(payload); err != nil { t.Fatalf("write tar payload: %v", err) } } if err := writer.Close(); err != nil { t.Fatalf("close tar writer: %v", err) } return "data:application/x-tar;base64," + base64.StdEncoding.EncodeToString(buffer.Bytes()) } func externalizedJSONArtifact(t *testing.T, runID string, nodeID string, name string, payload map[string]any) domain.ExecutionArtifact { t.Helper() root := t.TempDir() payloadPath := filepath.Join(root, name) payloadBytes, err := json.Marshal(payload) if err != nil { t.Fatalf("marshal payload: %v", err) } if err := os.WriteFile(payloadPath, payloadBytes, 0o644); err != nil { t.Fatalf("write payload: %v", err) } manifestPath := filepath.Join(root, strings.TrimSuffix(name, filepath.Ext(name))+"-manifest.json") manifestBytes, err := json.Marshal(map[string]any{ "kind": "dataset", "datasetType": payload["datasetType"], "payloadPath": payloadPath, "summary": map[string]any{"name": name}, }) if err != nil { t.Fatalf("marshal manifest: %v", err) } if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil { t.Fatalf("write manifest: %v", err) } return domain.ExecutionArtifact{ ID: runID + "-" + strings.TrimSuffix(name, filepath.Ext(name)), RunID: runID, NodeID: nodeID, Name: name, Type: domain.PortTypeJSON, Path: payloadPath, Data: map[string]any{ "datasetType": payload["datasetType"], "dataRef": map[string]any{ "kind": "dataset", "datasetType": payload["datasetType"], "manifestPath": manifestPath, "payloadPath": payloadPath, }, }, } } func TestDataPlatformIntegerHelpersRejectLossyValues(t *testing.T) { if got := intValue(float64(4)); got != 4 { t.Fatalf("expected integral float to parse, got %d", got) } if got := intValue(4.9); got != 0 { t.Fatalf("expected lossy float to be ignored, got %d", got) } if got := int64Value(int64(16004000000)); got != 16004000000 { t.Fatalf("expected int64 to be preserved, got %d", got) } if got := int64Value(math.NaN()); got != 0 { t.Fatalf("expected NaN to be ignored, got %d", got) } if got := int64Value(float64(1 << 53)); got != 0 { t.Fatalf("expected unsafe float integer to be ignored, got %d", got) } if got := int64Value(uint64(math.MaxInt64) + 1); got != 0 { t.Fatalf("expected overflowing uint64 to be ignored, got %d", got) } if got := intValue(json.Number("42")); got != 42 { t.Fatalf("expected integer json.Number to parse, got %d", got) } if got := intValue(json.Number("42.5")); got != 0 { t.Fatalf("expected decimal json.Number to be ignored, got %d", got) } } func TestArtifactStructuredDataTreatsLocalPayloadPathAsAuthoritative(t *testing.T) { missingPayloadPath := filepath.Join(t.TempDir(), "missing-payload.json") missingArtifact := domain.ExecutionArtifact{ Name: "source-manifest.json", Type: domain.PortTypeJSON, Data: map[string]any{ "datasetType": "x-platform", "items": []any{map[string]any{"id": "stale-inline"}}, "dataRef": map[string]any{ "payloadPath": missingPayloadPath, }, }, } if got := artifactStructuredData(&missingArtifact); got != nil { t.Fatalf("expected missing external payload to suppress stale inline data, got %#v", got) } invalidPayloadPath := filepath.Join(t.TempDir(), "payload.json") if err := os.WriteFile(invalidPayloadPath, []byte("{"), 0o644); err != nil { t.Fatalf("write invalid payload: %v", err) } invalidArtifact := missingArtifact invalidArtifact.Data = map[string]any{ "datasetType": "x-platform", "items": []any{map[string]any{"id": "stale-inline"}}, "dataRef": map[string]any{ "payloadPath": invalidPayloadPath, }, } if got := artifactStructuredData(&invalidArtifact); got != nil { t.Fatalf("expected invalid external payload to suppress stale inline data, got %#v", got) } } func TestSamplingRulesFromManifestRejectsInvalidRuleShape(t *testing.T) { validRules, ok := samplingRulesFromManifest(map[string]any{ "rules": []any{ map[string]any{ "id": "clarity", "field": "quality.clarity", "operator": "GTE", "value": 0.3, "samplePercent": json.Number("12.5"), "matchedCount": 4, "sampledCount": 2, }, }, }) if !ok || len(validRules) != 1 { t.Fatalf("expected valid sampling rule, got ok=%v rules=%#v", ok, validRules) } if validRules[0].Operator != "gte" || validRules[0].SamplePercent != 12.5 { t.Fatalf("expected normalized operator and sample percent, got %#v", validRules[0]) } cases := map[string]any{ "bool": true, "zero": 0, "over-100": 101, "nan": math.NaN(), "string": "10", "bad-number": json.Number("nan"), "unsupported": nil, } for name, samplePercent := range cases { rule := map[string]any{ "id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.3, "samplePercent": samplePercent, } if name == "unsupported" { rule["operator"] = "regex" rule["samplePercent"] = 10 } if rules, ok := samplingRulesFromManifest(map[string]any{"rules": []any{rule}}); ok { t.Fatalf("expected %s rule to be rejected, got %#v", name, rules) } } duplicate := map[string]any{ "rules": []any{ map[string]any{"id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.3, "samplePercent": 10}, map[string]any{"id": "clarity", "field": "quality.shake", "operator": "lte", "value": 0.1, "samplePercent": 10}, }, } if rules, ok := samplingRulesFromManifest(duplicate); ok { t.Fatalf("expected duplicate rules to be rejected, got %#v", rules) } } func TestSamplingTasksSkipInvalidManifestRuleSamplePercent(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "坏抽检规则项目"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "坏抽检规则工作流", Type: domain.WorkflowTypeBatch, }) if err != nil { t.Fatalf("create workflow: %v", err) } run := domain.RunRecord{ ID: "run-invalid-sampling-rule", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-invalid-sampling-rule-artifact", RunID: "run-invalid-sampling-rule", NodeID: "sample", Name: "audit-samples.json", Type: domain.PortTypeJSON, Data: map[string]any{ "datasetType": "x-platform-audit-samples", "rules": []any{ map[string]any{ "id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.3, "samplePercent": true, "matchedCount": 1, "sampledCount": 1, }, }, "items": []any{ map[string]any{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "sampleRuleId": "clarity", }, }, }, }, }, CreatedAt: "2026-04-10T10:05:00Z", UpdatedAt: "2026-04-10T10:05:00Z", } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } handler := NewHandler(dataStore, nil) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil)) if recorder.Code != http.StatusOK { t.Fatalf("sampling status: %d body=%s", recorder.Code, recorder.Body.String()) } var body struct { Items []domain.SamplingTaskRecord `json:"items"` } if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { t.Fatalf("decode sampling tasks: %v", err) } if len(body.Items) != 0 { t.Fatalf("expected invalid sampling manifest to be skipped, got %#v", body.Items) } } func TestScanConfigBOSSecretsAreReturnedAndPreserved(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} { if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ Name: name, Type: "builtin", Category: "io", Description: name, Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}}, }); err != nil { t.Fatalf("create operator %s: %v", name, err) } } handler := NewHandler(dataStore, nil) createPayload := `{ "name":"BOS 扫描", "description":"扫描真实 BOS 目录", "dataSource":"BOS", "bucket":"datastorage", "prefix":"algo/X-platform-test/", "endpoint":"https://bos.example.test", "accessKeyId":"ak-example-1234", "secretAccessKey":"sk-example-5678", "interval":86400, "enabled":true }` createRecorder := httptest.NewRecorder() handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload))) if createRecorder.Code != http.StatusCreated { t.Fatalf("create scan config status: %d body=%s", createRecorder.Code, createRecorder.Body.String()) } var created domain.WorkflowRecord if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil { t.Fatalf("decode created scan config: %v", err) } if created.ScanConfig == nil { t.Fatalf("expected scan config") } if created.ScanConfig.AccessKeyID != "" || created.ScanConfig.SecretAccessKey != "" { t.Fatalf("scan config secrets should be masked in workflow responses: %+v", created.ScanConfig) } if !created.ScanConfig.HasAccessKeyID || !created.ScanConfig.HasSecretAccessKey { t.Fatalf("expected secret presence flags in API response: %+v", created.ScanConfig) } if created.ScanConfig.AccessKeyPreview == "" || created.ScanConfig.SecretAccessKeyPreview == "" { t.Fatalf("expected secret previews in API response: %+v", created.ScanConfig) } stored, err := dataStore.GetWorkflow(ctx, created.ID) if err != nil { t.Fatalf("get stored workflow: %v", err) } if stored == nil || stored.ScanConfig == nil { t.Fatalf("expected stored scan config") } if stored.ScanConfig.AccessKeyID != "ak-example-1234" || stored.ScanConfig.SecretAccessKey != "sk-example-5678" { t.Fatalf("expected secrets preserved in store: %+v", stored.ScanConfig) } updatePayload := `{ "name":"BOS 扫描更新", "description":"保留原始密钥", "dataSource":"BOS", "bucket":"datastorage", "prefix":"algo/X-platform-test/", "endpoint":"https://bos.example.test", "interval":3600, "enabled":false }` updateRecorder := httptest.NewRecorder() handler.ServeHTTP(updateRecorder, httptest.NewRequest(http.MethodPut, "/api/scan-configs/"+created.ID, strings.NewReader(updatePayload))) if updateRecorder.Code != http.StatusOK { t.Fatalf("update scan config status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) } updated, err := dataStore.GetWorkflow(ctx, created.ID) if err != nil { t.Fatalf("get updated workflow: %v", err) } if updated == nil || updated.ScanConfig == nil { t.Fatalf("expected updated scan config") } if updated.ScanConfig.AccessKeyID != "ak-example-1234" || updated.ScanConfig.SecretAccessKey != "sk-example-5678" { t.Fatalf("expected secrets preserved across updates: %+v", updated.ScanConfig) } } func TestPackagesAndSamplingCanLoadExternalizedManifestPayloads(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "外置成品项目"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "成品工作流", Type: domain.WorkflowTypeBatch, }) if err != nil { t.Fatalf("create workflow: %v", err) } packagesPayload := map[string]any{ "datasetType": "x-platform-packages", "packageCount": 1, "items": []any{ map[string]any{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "fileCount": 1, "storagePath": "/tmp/k8s-job/package", }, }, } samplingPayload := map[string]any{ "datasetType": "x-platform-audit-samples", "rules": []any{ map[string]any{ "id": "confidence-check", "field": "annotation.minConfidence", "operator": "lt", "value": 0.7, "samplePercent": 10, "matchedCount": 1, "sampledCount": 1, }, }, "items": []any{ map[string]any{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "sampleRuleId": "confidence-check", "storagePath": "/tmp/k8s-job/package", }, }, } run := domain.RunRecord{ ID: "run-externalized-package", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ externalizedJSONArtifact(t, "run-externalized-package", "package", "packages-manifest.json", packagesPayload), externalizedJSONArtifact(t, "run-externalized-package", "sample", "audit-samples.json", samplingPayload), }, CreatedAt: "2026-04-10T10:05:00Z", UpdatedAt: "2026-04-10T10:05:00Z", } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } handler := NewHandler(dataStore, nil) packagesRecorder := httptest.NewRecorder() handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) if packagesRecorder.Code != http.StatusOK { t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String()) } var packagesBody struct { Items []domain.PackageRecord `json:"items"` } if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil { t.Fatalf("decode packages: %v", err) } if len(packagesBody.Items) != 1 { t.Fatalf("expected one package from externalized manifest, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String()) } samplingRecorder := httptest.NewRecorder() handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil)) if samplingRecorder.Code != http.StatusOK { t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String()) } var samplingBody struct { Items []domain.SamplingTaskRecord `json:"items"` } if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil { t.Fatalf("decode sampling tasks: %v", err) } if len(samplingBody.Items) != 1 || len(samplingBody.Items[0].Items) != 1 { t.Fatalf("expected one sampling task with one item from externalized manifest, got %#v", samplingBody.Items) } } func TestScanConfigCreateDraftDoesNotEnsureSystemProject(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} { if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{ Name: name, Type: "builtin", Category: "io", Description: name, Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}}, Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}}, }); err != nil { t.Fatalf("create operator %s: %v", name, err) } } handler := NewHandler(dataStore, nil) createPayload := `{ "name":"BOS 草稿扫描", "description":"创建后再配 bucket/prefix", "dataSource":"BOS", "interval":0, "enabled":false }` recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload))) if recorder.Code != http.StatusCreated { t.Fatalf("create draft scan config status: %d body=%s", recorder.Code, recorder.Body.String()) } var workflow domain.WorkflowRecord if err := json.Unmarshal(recorder.Body.Bytes(), &workflow); err != nil { t.Fatalf("decode draft workflow: %v", err) } if workflow.ProjectID != "" { t.Fatalf("expected projectless scan workflow, got projectId=%q", workflow.ProjectID) } if workflow.Kind != domain.WorkflowKindScan { t.Fatalf("expected scan workflow kind, got %s", workflow.Kind) } projects, err := dataStore.ListProjects(ctx) if err != nil { t.Fatalf("list projects: %v", err) } if len(projects) != 0 { t.Fatalf("expected no auto-created system project, got %d", len(projects)) } } func TestScanConfigCreateRejectsInvalidDataSource(t *testing.T) { dataStore := store.NewMemoryStore() handler := NewHandler(dataStore, nil) createPayload := `{ "name":"非法数据源扫描", "description":"invalid data source", "dataSource":"ROW", "interval":0, "enabled":false }` recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload))) if recorder.Code != http.StatusBadRequest { t.Fatalf("expected validation failure, got %d body=%s", recorder.Code, recorder.Body.String()) } if !strings.Contains(recorder.Body.String(), `"field":"dataSource"`) || !strings.Contains(recorder.Body.String(), "dataSource must be BOS for scan workflows") { t.Fatalf("expected dataSource validation issue, got %s", recorder.Body.String()) } } func TestDataPlatformHandlersSupportMongoStyleArtifactData(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{ Name: "数据资产项目", Description: "用于数据资产 API 测试", }) if err != nil { t.Fatalf("create project: %v", err) } scanWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "目录扫描", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, }) if err != nil { t.Fatalf("create scan workflow: %v", err) } pipelineWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "成品流水线", Type: domain.WorkflowTypeBatch, }) if err != nil { t.Fatalf("create pipeline workflow: %v", err) } scanRun := domain.RunRecord{ ID: "run-scan-1", WorkflowID: scanWorkflow.ID, WorkflowName: scanWorkflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-scan-1-source", RunID: "run-scan-1", NodeID: "scan-source", Name: "source-manifest.json", Type: "json", Data: map[string]any{ "dataSource": "BOS", "sourcePath": "bos://datastorage/algo/X-platform-test", "listedCount": 4, "newCount": 4, }, }, { ID: "run-scan-1-index", RunID: "run-scan-1", NodeID: "scan-index", Name: "dataset-index.json", Type: "json", Data: map[string]any{ "archiveSources": bson.M{ "archive-1": bson.M{ "name": "xplatform-dataset.tar", "previewURL": tarDataURL(t, map[string][]byte{ "BJ_001/DJI_001/DJI_0001_D.mp4": []byte("video-preview-bytes"), }), }, }, "collectionCount": 1, "collections": bson.A{ bson.M{ "id": "BJ_001", "path": "/tmp/run-scan-1/index/_materialized_inputs/BJ_001", "relativePath": "BJ_001", "meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"}, "hasIntrinsics": true, "hasVideoMeta": true, "requiredFiles": bson.A{"meta.json", "video_meta.json"}, "missingRequiredFiles": bson.A{}, "videoCount": 1, "fileCount": 4, "totalBytes": 1234, "valid": true, }, }, "items": bson.A{ bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", "filename": "DJI_0001_D.mp4", "codec": "hevc", "resolution": "3840x2160", "frameRate": 29.97, "duration": 41.8, "sizeBytes": 2048, "meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"}, "source": bson.M{ "archiveKey": "archive-1", "archiveMemberPath": "BJ_001/DJI_001/DJI_0001_D.mp4", "mimeType": "video/mp4", "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", }, }, }, }, }, }, CreatedAt: "2026-04-10T10:00:00Z", UpdatedAt: "2026-04-10T10:00:00Z", } if err := dataStore.UpsertRun(ctx, scanRun); err != nil { t.Fatalf("upsert scan run: %v", err) } pipelineRun := domain.RunRecord{ ID: "run-pipeline-1", WorkflowID: pipelineWorkflow.ID, WorkflowName: pipelineWorkflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-pipeline-1-packages", RunID: "run-pipeline-1", NodeID: "package", Name: "packages-manifest.json", Type: "json", Data: map[string]any{ "packageCount": 1, "items": bson.A{ bson.M{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "clipDuration": 300, "fileCount": 4, "pipelineTags": bson.A{"quality_v1", "annotation_v1"}, "storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package", "quality": bson.M{"clarity": 0.92, "shake": 0.08}, "annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}}, }, }, }, }, { ID: "run-pipeline-1-audit", RunID: "run-pipeline-1", NodeID: "sample", Name: "audit-samples.json", Type: "json", Data: map[string]any{ "rules": bson.A{ bson.M{ "id": "confidence-check", "field": "annotation.minConfidence", "operator": "lt", "value": 0.7, "samplePercent": 10, "matchedCount": 1, "sampledCount": 1, }, }, "items": bson.A{ bson.M{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "sampleRuleId": "confidence-check", "storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package", "quality": bson.M{"clarity": 0.92, "shake": 0.08}, "annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}}, }, }, }, }, { ID: "run-pipeline-1-index", RunID: "run-pipeline-1", NodeID: "setup-index", Name: "dataset-index.json", Type: "json", Data: map[string]any{ "archiveSources": bson.M{ "archive-test": bson.M{"name": "synthetic-dataset.tgz", "previewURL": "data:application/x-tar;base64,AAAA"}, }, "collectionCount": 1, "collections": bson.A{ bson.M{ "id": "synthetic-small", "name": "synthetic-small", "path": "/tmp/droboticflow/run-pipeline-1/index/_materialized_inputs/synthetic-small", "relativePath": "synthetic-small", "fileCount": 1, "totalBytes": 21, "valid": true, }, }, }, }, { ID: "run-pipeline-1-video", RunID: "run-pipeline-1", NodeID: "package", Name: "DJI_0001_D_clip001.mp4", Type: "video", Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", Data: map[string]any{ "preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"}, }, }, { ID: "run-pipeline-1-quality", RunID: "run-pipeline-1", NodeID: "package", Name: "DJI_0001_D_clip001_quality.json", Type: "json", Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001_quality.json", Data: map[string]any{"clarity": 0.92}, }, }, CreatedAt: "2026-04-10T10:05:00Z", UpdatedAt: "2026-04-10T10:05:00Z", } if err := dataStore.UpsertRun(ctx, pipelineRun); err != nil { t.Fatalf("upsert pipeline run: %v", err) } handler := NewHandler(dataStore, nil) packagesRecorder := httptest.NewRecorder() handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) if packagesRecorder.Code != http.StatusOK { t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String()) } var packagesBody struct { Items []domain.PackageRecord `json:"items"` } if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil { t.Fatalf("decode packages: %v", err) } if len(packagesBody.Items) != 1 { t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String()) } if packagesBody.Items[0].PackageName != "DJI_0001_D_clip001_package" { t.Fatalf("unexpected package name: %s", packagesBody.Items[0].PackageName) } if packagesBody.Items[0].Storage.Provider != "k8s-job" { t.Fatalf("expected k8s-job package storage, got %+v", packagesBody.Items[0].Storage) } if packagesBody.Items[0].Storage.Prefix != "/tmp/run-pipeline-1/package/packages" { t.Fatalf("unexpected package storage prefix: %+v", packagesBody.Items[0].Storage) } packageDetailRecorder := httptest.NewRecorder() handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package", nil)) if packageDetailRecorder.Code != http.StatusOK { t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String()) } var packageDetail domain.PackageDetail if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil { t.Fatalf("decode package detail: %v", err) } if packageDetail.Package.Storage.Path != "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package" { t.Fatalf("unexpected package storage path: %+v", packageDetail.Package.Storage) } if len(packageDetail.Files) == 0 { t.Fatalf("expected package files") } videoPreview := mapValue(packageDetail.Files[0].Data["preview"]) if !strings.Contains(stringValue(videoPreview["url"]), "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package/content?relativePath=") { t.Fatalf("expected package video preview url to use content api, got %+v", packageDetail.Files[0].Data) } samplingRecorder := httptest.NewRecorder() handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil)) if samplingRecorder.Code != http.StatusOK { t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String()) } var samplingBody struct { Items []domain.SamplingTaskRecord `json:"items"` } if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil { t.Fatalf("decode sampling tasks: %v", err) } if len(samplingBody.Items) != 1 { t.Fatalf("expected one sampling task, got %d body=%s", len(samplingBody.Items), samplingRecorder.Body.String()) } if len(samplingBody.Items[0].Items) != 1 { t.Fatalf("expected one sampling item, got %d", len(samplingBody.Items[0].Items)) } collectionsRecorder := httptest.NewRecorder() handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil)) if collectionsRecorder.Code != http.StatusOK { t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String()) } var collectionsBody struct { Items []domain.SourceCollectionRecord `json:"items"` } if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil { t.Fatalf("decode source collections: %v", err) } if len(collectionsBody.Items) != 1 { t.Fatalf("expected one source collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String()) } if collectionsBody.Items[0].CollectionID != "BJ_001" { t.Fatalf("unexpected collection id: %s", collectionsBody.Items[0].CollectionID) } if collectionsBody.Items[0].CollectionName != "BJ_001" { t.Fatalf("unexpected collection name fallback: %s", collectionsBody.Items[0].CollectionName) } if collectionsBody.Items[0].RunID != "run-scan-1" { t.Fatalf("expected source collections to ignore synthetic pipeline dataset indexes, got run %s", collectionsBody.Items[0].RunID) } if collectionsBody.Items[0].Storage.Provider != "bos" { t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage) } if collectionsBody.Items[0].Storage.Bucket != "datastorage" || collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" { t.Fatalf("unexpected source storage details: %+v", collectionsBody.Items[0].Storage) } detailRecorder := httptest.NewRecorder() handler.ServeHTTP(detailRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001", nil)) if detailRecorder.Code != http.StatusOK { t.Fatalf("source detail status: %d body=%s", detailRecorder.Code, detailRecorder.Body.String()) } var detail domain.SourceCollectionDetail if err := json.Unmarshal(detailRecorder.Body.Bytes(), &detail); err != nil { t.Fatalf("decode source detail: %v", err) } if len(detail.Entries) != 3 { t.Fatalf("expected root entries derived from dataset index, got %d body=%s", len(detail.Entries), detailRecorder.Body.String()) } var videoDirectory *domain.SourceCollectionEntry for index := range detail.Entries { if detail.Entries[index].Kind == "directory" && detail.Entries[index].Path == "DJI_001" { videoDirectory = &detail.Entries[index] break } } if videoDirectory == nil { t.Fatalf("expected DJI_001 directory in derived source entries: %+v", detail.Entries) } if !videoDirectory.DerivedFromIndex { t.Fatalf("expected derived directory to be marked from dataset index: %+v", videoDirectory) } entryRecorder := httptest.NewRecorder() handler.ServeHTTP(entryRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/entries?prefix=DJI_001", nil)) if entryRecorder.Code != http.StatusOK { t.Fatalf("source entries status: %d body=%s", entryRecorder.Code, entryRecorder.Body.String()) } var nestedDetail domain.SourceCollectionDetail if err := json.Unmarshal(entryRecorder.Body.Bytes(), &nestedDetail); err != nil { t.Fatalf("decode nested source entries: %v", err) } if len(nestedDetail.Entries) != 1 { t.Fatalf("expected one nested video entry, got %d body=%s", len(nestedDetail.Entries), entryRecorder.Body.String()) } videoEntry := nestedDetail.Entries[0] if videoEntry.Kind != "video" { t.Fatalf("expected nested entry to be video, got %s", videoEntry.Kind) } if videoEntry.Artifact == nil { t.Fatalf("expected nested video entry artifact") } if !videoEntry.DerivedFromIndex { t.Fatalf("expected nested video entry to be marked from dataset index: %+v", videoEntry) } preview := mapValue(videoEntry.Artifact.Data["preview"]) if !strings.Contains(stringValue(preview["url"]), "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=") { t.Fatalf("expected nested video preview url from archive sources: %+v", videoEntry.Artifact.Data) } contentRecorder := httptest.NewRecorder() contentRequest := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", nil) handler.ServeHTTP(contentRecorder, contentRequest) if contentRecorder.Code != http.StatusOK { t.Fatalf("source content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String()) } if contentRecorder.Header().Get("Content-Type") != "video/mp4" { t.Fatalf("unexpected source content type: %s", contentRecorder.Header().Get("Content-Type")) } if string(contentRecorder.Body.Bytes()) != "video-preview-bytes" { t.Fatalf("unexpected source content payload: %q", contentRecorder.Body.Bytes()) } } func TestSourceCollectionsUseCatalogFieldsForIncrementalScanRuns(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "BOS 增量扫描", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, }) if err != nil { t.Fatalf("create workflow: %v", err) } collection := bson.M{ "id": "BJ_001", "name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", "path": "bos://datastorage/algo/X-platform-test/BJ_001", "relativePath": "BJ_001", "meta": bson.M{"project_id": "BJ_001"}, "hasVideoMeta": true, "videoCount": 1, "fileCount": 4, "totalBytes": int64(16000036740), "valid": true, "requiredFiles": bson.A{"meta.json", "video_meta.json"}, } catalogEntries := bson.A{ bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/meta.json", "sizeBytes": 256, "mimeType": "application/json", "data": bson.M{"project_id": "BJ_001"}, }, bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4", "sizeBytes": int64(16004000000), "mimeType": "video/mp4", "source": bson.M{ "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4", "previewURL": "https://example.com/DJI_20260327132729_0003_D.MP4", }, }, bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.json", "sizeBytes": 4226, "mimeType": "application/json", }, bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D_imu.json", "sizeBytes": int64(36080000), "mimeType": "application/json", }, } run := domain.RunRecord{ ID: "run-incremental-empty", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-incremental-empty-source", RunID: "run-incremental-empty", NodeID: "scan-source", Name: "source-manifest.json", Type: "json", Data: map[string]any{ "dataSource": "BOS", "sourcePath": "bos://datastorage/algo/X-platform-test", "scanMode": "INCREMENTAL", "listedCount": 4, "newCount": 0, "catalogEntries": catalogEntries, }, }, { ID: "run-incremental-empty-index", RunID: "run-incremental-empty", NodeID: "scan-index", Name: "dataset-index.json", Type: "json", Data: map[string]any{ "dataSource": "BOS", "sourcePath": "bos://datastorage/algo/X-platform-test", "scanMode": "INCREMENTAL", "listedCount": 4, "newCount": 0, "collectionCount": 0, "collections": bson.A{}, "items": bson.A{}, "entries": bson.A{}, "catalogCollectionCount": 1, "catalogVideoCount": 1, "catalogCollections": bson.A{collection}, "catalogItems": bson.A{}, "catalogEntries": catalogEntries, }, }, }, CreatedAt: "2026-04-10T12:00:00Z", UpdatedAt: "2026-04-10T12:00:00Z", } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } handler := NewHandler(dataStore, nil) collectionsRecorder := httptest.NewRecorder() handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil)) if collectionsRecorder.Code != http.StatusOK { t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String()) } var collectionsBody struct { Items []domain.SourceCollectionRecord `json:"items"` } if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil { t.Fatalf("decode source collections: %v", err) } if len(collectionsBody.Items) != 1 { t.Fatalf("expected one catalog collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String()) } if collectionsBody.Items[0].FileCount != 4 || collectionsBody.Items[0].TotalBytes != 16000036740 { t.Fatalf("expected catalog file stats, got %+v", collectionsBody.Items[0]) } entriesRecorder := httptest.NewRecorder() handler.ServeHTTP(entriesRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-incremental-empty/BJ_001/entries?prefix=DJI_001", nil)) if entriesRecorder.Code != http.StatusOK { t.Fatalf("entries status: %d body=%s", entriesRecorder.Code, entriesRecorder.Body.String()) } var detail domain.SourceCollectionDetail if err := json.Unmarshal(entriesRecorder.Body.Bytes(), &detail); err != nil { t.Fatalf("decode source detail: %v", err) } if len(detail.Entries) != 3 { t.Fatalf("expected all catalog files under DJI_001, got %d body=%s", len(detail.Entries), entriesRecorder.Body.String()) } var video *domain.SourceCollectionEntry for index := range detail.Entries { if detail.Entries[index].Name == "DJI_20260327132729_0003_D.MP4" { video = &detail.Entries[index] break } } if video == nil || video.SizeBytes != 16004000000 { t.Fatalf("expected real mp4 size from catalog entries, got %+v", video) } if video.DerivedFromIndex { t.Fatalf("expected catalog entry to remain a direct entry, got %+v", video) } if collectionsBody.Items[0].Storage.Provider != "bos" { t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage) } if collectionsBody.Items[0].Storage.Bucket != "datastorage" { t.Fatalf("unexpected source storage bucket: %+v", collectionsBody.Items[0].Storage) } if collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" { t.Fatalf("unexpected source storage prefix: %+v", collectionsBody.Items[0].Storage) } } func TestSourceCollectionsCanLoadExternalizedDatasetIndexPayload(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "BOS 增量扫描", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, }) if err != nil { t.Fatalf("create workflow: %v", err) } indexPayload := map[string]any{ "datasetType": "x-platform-index", "dataSource": "BOS", "sourcePath": "bos://datastorage/algo/X-platform-test", "collectionCount": 0, "collections": []any{}, "items": []any{}, "entries": []any{}, "catalogCollectionCount": 1, "catalogCollections": []any{ map[string]any{ "id": "BJ_001", "name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43", "relativePath": "BJ_001", "videoCount": 1, "valid": true, }, }, "catalogItems": []any{}, "catalogEntries": []any{}, } run := domain.RunRecord{ ID: "run-externalized-index", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ externalizedJSONArtifact(t, "run-externalized-index", "scan-index", "dataset-index.json", indexPayload), }, CreatedAt: "2026-04-10T12:00:00Z", UpdatedAt: "2026-04-10T12:00:00Z", } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } handler := NewHandler(dataStore, nil) collectionsRecorder := httptest.NewRecorder() handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil)) if collectionsRecorder.Code != http.StatusOK { t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String()) } var collectionsBody struct { Items []domain.SourceCollectionRecord `json:"items"` } if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil { t.Fatalf("decode source collections: %v", err) } if len(collectionsBody.Items) != 1 { t.Fatalf("expected externalized payload to produce one collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String()) } } func TestPackageContentCanServeBrowserPreviewVideo(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品预览项目"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "成品预览工作流", Type: domain.WorkflowTypeBatch, }) if err != nil { t.Fatalf("create workflow: %v", err) } videoPath := filepath.Join(t.TempDir(), "DJI_0001_D_clip001.mp4") if err := os.WriteFile(videoPath, []byte("package-video-bytes"), 0o644); err != nil { t.Fatalf("write package video: %v", err) } run := domain.RunRecord{ ID: "run-package-preview", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-package-preview-manifest", RunID: "run-package-preview", NodeID: "package", Name: "packages-manifest.json", Type: "json", Data: map[string]any{ "items": bson.A{ bson.M{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "fileCount": 1, "storagePath": filepath.Dir(videoPath), }, }, }, }, { ID: "run-package-preview-video", RunID: "run-package-preview", NodeID: "package", Name: "DJI_0001_D_clip001.mp4", Type: "video", Path: filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4"), Metadata: &domain.ArtifactMetadata{ MimeType: "video/mp4", }, Data: map[string]any{ "preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"}, }, }, { ID: "run-package-preview-upload-video", RunID: "run-package-preview", NodeID: "upload", Name: "DJI_0001_D_clip001.mp4", Type: "video", Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", Metadata: &domain.ArtifactMetadata{ MimeType: "video/mp4", }, Data: map[string]any{ "source": map[string]any{ "path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", }, }, }, }, } run.Artifacts[1].Path = filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4") if err := os.MkdirAll(filepath.Dir(run.Artifacts[1].Path), 0o755); err != nil { t.Fatalf("mkdir package dir: %v", err) } if err := os.WriteFile(run.Artifacts[1].Path, []byte("package-video-bytes"), 0o644); err != nil { t.Fatalf("write package artifact: %v", err) } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } previewPath := filepath.Join(t.TempDir(), "package-browser-preview.mp4") if err := os.WriteFile(previewPath, []byte("package-browser-preview"), 0o644); err != nil { t.Fatalf("write preview file: %v", err) } originalPreviewResolver := sourceBrowserVideoPreviewPath sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { if request.RedirectURL != run.Artifacts[1].Path { t.Fatalf("unexpected package preview input: %s", request.RedirectURL) } if request.FileName != "DJI_0001_D_clip001.mp4" { t.Fatalf("unexpected file name: %s", request.FileName) } return previewPath, nil } t.Cleanup(func() { sourceBrowserVideoPreviewPath = originalPreviewResolver }) handler := NewHandler(dataStore, nil) packagesRecorder := httptest.NewRecorder() handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) if packagesRecorder.Code != http.StatusOK { t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String()) } var packagesBody struct { Items []domain.PackageRecord `json:"items"` } if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil { t.Fatalf("decode packages: %v", err) } if len(packagesBody.Items) != 1 { t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String()) } if packagesBody.Items[0].Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" { t.Fatalf("expected package storage to prefer BOS upload path, got %+v", packagesBody.Items[0].Storage) } packageDetailRecorder := httptest.NewRecorder() handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-package-preview/DJI_0001_D_clip001_package", nil)) if packageDetailRecorder.Code != http.StatusOK { t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String()) } var packageDetail domain.PackageDetail if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil { t.Fatalf("decode package detail: %v", err) } if packageDetail.Package.Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" { t.Fatalf("expected package detail storage to prefer BOS upload path, got %+v", packageDetail.Package.Storage) } if len(packageDetail.Files) != 1 { t.Fatalf("expected duplicate package files to collapse to one, got %d", len(packageDetail.Files)) } if packageDetail.Files[0].Name != "DJI_0001_D_clip001.mp4" { t.Fatalf("unexpected package file after dedupe: %+v", packageDetail.Files[0]) } recorder := httptest.NewRecorder() request := httptest.NewRequest( http.MethodGet, "/api/packages/run-package-preview/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser", nil, ) handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusOK { t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String()) } if recorder.Header().Get("Content-Type") != "video/mp4" { t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type")) } if string(recorder.Body.Bytes()) != "package-browser-preview" { t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes()) } } func TestPackageContentFallsBackToUploadedBOSVideoWhenLocalPackageFileUnavailable(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品上传回退项目"}) if err != nil { t.Fatalf("create project: %v", err) } connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ Name: "上传 BOS 连接", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test", AccessKeyID: "ak-upload", SecretAccessKey: "sk-upload", }) if err != nil { t.Fatalf("create storage connection: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "成品上传回退工作流", Type: domain.WorkflowTypeBatch, Nodes: []domain.WorkflowNode{ { ID: "upload", Type: "operator", Data: domain.WorkflowNodeData{ Label: "结果上传", ConnectionID: connection.ID, Bucket: "datastorage", Prefix: "factory/output/test-package-preview", Hyperparams: []domain.HyperParam{ {Key: "DATA_SOURCE", Value: "BOS"}, {Key: "SOURCE_PATH", Value: "bos://datastorage/factory/output/test-package-preview"}, }, }, }, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run := domain.RunRecord{ ID: "run-package-upload-fallback", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-package-upload-fallback-manifest", RunID: "run-package-upload-fallback", NodeID: "package", Name: "packages-manifest.json", Type: "json", Data: map[string]any{ "items": bson.A{ bson.M{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "fileCount": 1, "storagePath": "/tmp/droboticflow/missing/package", }, }, }, }, { ID: "run-package-upload-fallback-temp-video", RunID: "run-package-upload-fallback", NodeID: "package", Name: "DJI_0001_D_clip001.mp4", Type: "video", Path: "/tmp/droboticflow/missing/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", Metadata: &domain.ArtifactMetadata{ MimeType: "video/mp4", }, Data: map[string]any{ "preview": map[string]any{ "kind": "video", "url": "/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4", }, }, }, { ID: "run-package-upload-fallback-remote-video", RunID: "run-package-upload-fallback", NodeID: "upload", Name: "DJI_0001_D_clip001.mp4", Type: "video", Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", Metadata: &domain.ArtifactMetadata{ MimeType: "video/mp4", }, Data: map[string]any{ "relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", "source": map[string]any{ "path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", "previewURL": "https://expired.example.com/DJI_0001_D_clip001.mp4", "mimeType": "video/mp4", }, }, }, }, } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } previewPath := filepath.Join(t.TempDir(), "package-upload-browser-preview.mp4") if err := os.WriteFile(previewPath, []byte("package-upload-browser-preview"), 0o644); err != nil { t.Fatalf("write preview file: %v", err) } originalSignedPreviewResolver := sourceCollectionSignedPreviewURL sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { if scanConfig == nil || scanConfig.AccessKeyID != "ak-upload" || scanConfig.SecretAccessKey != "sk-upload" { t.Fatalf("expected upload connection secrets, got %+v", scanConfig) } if sourcePath != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4" { t.Fatalf("unexpected source path: %s", sourcePath) } return "https://signed.example.com/DJI_0001_D_clip001.mp4", nil } t.Cleanup(func() { sourceCollectionSignedPreviewURL = originalSignedPreviewResolver }) originalPreviewResolver := sourceBrowserVideoPreviewPath sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { if request.RedirectURL != "https://signed.example.com/DJI_0001_D_clip001.mp4" { t.Fatalf("unexpected package preview redirect input: %s", request.RedirectURL) } if request.FileName != "DJI_0001_D_clip001.mp4" { t.Fatalf("unexpected file name: %s", request.FileName) } return previewPath, nil } t.Cleanup(func() { sourceBrowserVideoPreviewPath = originalPreviewResolver }) handler := NewHandler(dataStore, nil) recorder := httptest.NewRecorder() request := httptest.NewRequest( http.MethodGet, "/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser", nil, ) handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusOK { t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String()) } if recorder.Header().Get("Content-Type") != "video/mp4" { t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type")) } if string(recorder.Body.Bytes()) != "package-upload-browser-preview" { t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes()) } } func TestPackagesListKeepsLatestPackageVersionPerWorkflow(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "包去重项目"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "包去重工作流", Type: domain.WorkflowTypeBatch, }) if err != nil { t.Fatalf("create workflow: %v", err) } makeRun := func(id string, createdAt string, storagePath string) domain.RunRecord { return domain.RunRecord{ ID: id, WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, CreatedAt: createdAt, Artifacts: []domain.ExecutionArtifact{ { ID: id + "-manifest", RunID: id, NodeID: "package", Name: "packages-manifest.json", Type: "json", Data: map[string]any{ "items": []any{ map[string]any{ "packageName": "DJI_0001_D_clip001_package", "clipId": "DJI_0001_D_clip001", "sourceVideo": "DJI_0001_D.mp4", "fileCount": 1, "storagePath": "/tmp/k8s-job/package", }, }, }, }, { ID: id + "-upload-video", RunID: id, NodeID: "upload", Name: "DJI_0001_D_clip001.mp4", Type: "video", Path: storagePath + "/DJI_0001_D_clip001.mp4", Metadata: &domain.ArtifactMetadata{ MimeType: "video/mp4", }, Data: map[string]any{ "relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", "source": map[string]any{ "path": storagePath + "/DJI_0001_D_clip001.mp4", "mimeType": "video/mp4", "relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4", }, }, }, }, } } if err := dataStore.UpsertRun(ctx, makeRun( "run-package-old", "2026-04-12T10:00:00Z", "bos://datastorage/factory/output/old/packages/DJI_0001_D_clip001_package", )); err != nil { t.Fatalf("upsert old run: %v", err) } if err := dataStore.UpsertRun(ctx, makeRun( "run-package-new", "2026-04-12T11:00:00Z", "bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package", )); err != nil { t.Fatalf("upsert new run: %v", err) } handler := NewHandler(dataStore, nil) recorder := httptest.NewRecorder() handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil)) if recorder.Code != http.StatusOK { t.Fatalf("packages status: %d body=%s", recorder.Code, recorder.Body.String()) } var body struct { Items []domain.PackageRecord `json:"items"` } if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { t.Fatalf("decode packages: %v", err) } if len(body.Items) != 1 { t.Fatalf("expected latest package only, got %d body=%s", len(body.Items), recorder.Body.String()) } if body.Items[0].RunID != "run-package-new" { t.Fatalf("expected latest run package, got %+v", body.Items[0]) } if body.Items[0].Storage.Path != "bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package" { t.Fatalf("expected latest BOS storage path, got %+v", body.Items[0].Storage) } } func TestSourceCollectionContentCanServeBrowserPreviewVideo(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "浏览器预览项目"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "BOS 增量扫描", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, ScanConfig: &domain.ScanConfig{ DataSource: "BOS", Endpoint: "https://bos.example.test", AccessKeyID: "ak", SecretAccessKey: "sk", }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run := domain.RunRecord{ ID: "run-browser-preview", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-browser-preview-index", RunID: "run-browser-preview", NodeID: "scan-index", Name: "dataset-index.json", Type: "json", Data: map[string]any{ "catalogCollections": bson.A{ bson.M{ "id": "BJ_001", "name": "BJ_001", "path": "bos://datastorage/algo/X-platform-test/BJ_001", "relativePath": "BJ_001", "videoCount": 1, "fileCount": 1, "totalBytes": int64(2048), "valid": true, }, }, "catalogEntries": bson.A{ bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", "sizeBytes": int64(2048), "mimeType": "video/mp4", "source": bson.M{ "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", "previewURL": "https://example.com/DJI_0001_D.mp4", }, }, }, }, }, }, } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } previewPath := filepath.Join(t.TempDir(), "preview.mp4") if err := os.WriteFile(previewPath, []byte("browser-preview-video"), 0o644); err != nil { t.Fatalf("write preview file: %v", err) } originalPreviewResolver := sourceBrowserVideoPreviewPath sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { if request.RedirectURL != "https://signed.example.com/DJI_0001_D.mp4" { t.Fatalf("unexpected redirect url: %s", request.RedirectURL) } if request.FileName != "DJI_0001_D.mp4" { t.Fatalf("unexpected file name: %s", request.FileName) } return previewPath, nil } t.Cleanup(func() { sourceBrowserVideoPreviewPath = originalPreviewResolver }) originalSignedPreviewResolver := sourceCollectionSignedPreviewURL sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" { t.Fatalf("unexpected scan config: %+v", scanConfig) } if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" { t.Fatalf("unexpected source path: %s", sourcePath) } return "https://signed.example.com/DJI_0001_D.mp4", nil } t.Cleanup(func() { sourceCollectionSignedPreviewURL = originalSignedPreviewResolver }) handler := NewHandler(dataStore, nil) recorder := httptest.NewRecorder() request := httptest.NewRequest( http.MethodGet, "/api/source-data/collections/run-browser-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4&transcode=browser", nil, ) handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusOK { t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String()) } if recorder.Header().Get("Content-Type") != "video/mp4" { t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type")) } if string(recorder.Body.Bytes()) != "browser-preview-video" { t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes()) } } func TestSourceCollectionContentRefreshesBOSRedirectURL(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 刷新项目"}) if err != nil { t.Fatalf("create project: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "BOS 增量扫描", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, ScanConfig: &domain.ScanConfig{ DataSource: "BOS", Endpoint: "https://bos.example.test", AccessKeyID: "ak", SecretAccessKey: "sk", }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run := domain.RunRecord{ ID: "run-refresh-bos-preview", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-refresh-bos-preview-index", RunID: "run-refresh-bos-preview", NodeID: "scan-index", Name: "dataset-index.json", Type: "json", Data: map[string]any{ "catalogCollections": bson.A{ bson.M{ "id": "BJ_001", "name": "BJ_001", "path": "bos://datastorage/algo/X-platform-test/BJ_001", "relativePath": "BJ_001", "videoCount": 1, "fileCount": 1, "totalBytes": int64(2048), "valid": true, }, }, "catalogEntries": bson.A{ bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", "sizeBytes": int64(2048), "mimeType": "video/mp4", "source": bson.M{ "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", "previewURL": "https://expired.example.com/DJI_0001_D.mp4", }, }, }, }, }, }, } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } originalSignedPreviewResolver := sourceCollectionSignedPreviewURL sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" { t.Fatalf("unexpected scan config: %+v", scanConfig) } if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" { t.Fatalf("unexpected source path: %s", sourcePath) } return "https://signed.example.com/DJI_0001_D.mp4", nil } t.Cleanup(func() { sourceCollectionSignedPreviewURL = originalSignedPreviewResolver }) handler := NewHandler(dataStore, nil) recorder := httptest.NewRecorder() request := httptest.NewRequest( http.MethodGet, "/api/source-data/collections/run-refresh-bos-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", nil, ) handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusTemporaryRedirect { t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String()) } if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" { t.Fatalf("unexpected redirect location: %s", location) } } func TestSourceCollectionContentRefreshesBOSRedirectURLFromStorageConnection(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 连接预览项目"}) if err != nil { t.Fatalf("create project: %v", err) } connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{ Name: "BOS 主连接", Provider: domain.StorageProviderBOS, Endpoint: "https://bos.example.test", AccessKeyID: "ak-from-connection", SecretAccessKey: "sk-from-connection", }) if err != nil { t.Fatalf("create storage connection: %v", err) } workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ ProjectID: project.ID, Name: "BOS 增量扫描", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindScan, ScanConfig: &domain.ScanConfig{ DataSource: "BOS", ConnectionID: connection.ID, }, }) if err != nil { t.Fatalf("create workflow: %v", err) } run := domain.RunRecord{ ID: "run-refresh-bos-preview-connection", WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: project.ID, ProjectName: project.Name, Trigger: domain.RunTriggerManual, Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{ { ID: "run-refresh-bos-preview-connection-index", RunID: "run-refresh-bos-preview-connection", NodeID: "scan-index", Name: "dataset-index.json", Type: "json", Data: map[string]any{ "catalogCollections": bson.A{ bson.M{ "id": "BJ_001", "name": "BJ_001", "path": "bos://datastorage/algo/X-platform-test/BJ_001", "relativePath": "BJ_001", "videoCount": 1, "fileCount": 1, "totalBytes": int64(2048), "valid": true, }, }, "catalogEntries": bson.A{ bson.M{ "collectionId": "BJ_001", "relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4", "sizeBytes": int64(2048), "mimeType": "video/mp4", "source": bson.M{ "path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4", "previewURL": "https://expired.example.com/DJI_0001_D.mp4", }, }, }, }, }, }, } if err := dataStore.UpsertRun(ctx, run); err != nil { t.Fatalf("upsert run: %v", err) } originalSignedPreviewResolver := sourceCollectionSignedPreviewURL sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { if scanConfig == nil || scanConfig.AccessKeyID != "ak-from-connection" || scanConfig.SecretAccessKey != "sk-from-connection" { t.Fatalf("expected resolved connection secrets, got %+v", scanConfig) } if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" { t.Fatalf("unexpected source path: %s", sourcePath) } return "https://signed.example.com/DJI_0001_D.mp4", nil } t.Cleanup(func() { sourceCollectionSignedPreviewURL = originalSignedPreviewResolver }) handler := NewHandler(dataStore, nil) recorder := httptest.NewRecorder() request := httptest.NewRequest( http.MethodGet, "/api/source-data/collections/run-refresh-bos-preview-connection/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", nil, ) handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusTemporaryRedirect { t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String()) } if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" { t.Fatalf("unexpected redirect location: %s", location) } }