droboticflow/internal/api/data_platform_test.go
eust-w 43326d6a5e feat(platform): land workflow, data tooling, and docs updates
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
2026-04-24 12:29:46 +08:00

1737 lines
61 KiB
Go

package api
import (
"archive/tar"
"bytes"
"context"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/bson"
)
func tarDataURL(t *testing.T, files map[string][]byte) string {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
for name, payload := range files {
header := &tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(payload)),
}
if err := writer.WriteHeader(header); err != nil {
t.Fatalf("write tar header: %v", err)
}
if _, err := writer.Write(payload); err != nil {
t.Fatalf("write tar payload: %v", err)
}
}
if err := writer.Close(); err != nil {
t.Fatalf("close tar writer: %v", err)
}
return "data:application/x-tar;base64," + base64.StdEncoding.EncodeToString(buffer.Bytes())
}
func externalizedJSONArtifact(t *testing.T, runID string, nodeID string, name string, payload map[string]any) domain.ExecutionArtifact {
t.Helper()
root := t.TempDir()
payloadPath := filepath.Join(root, name)
payloadBytes, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
if err := os.WriteFile(payloadPath, payloadBytes, 0o644); err != nil {
t.Fatalf("write payload: %v", err)
}
manifestPath := filepath.Join(root, strings.TrimSuffix(name, filepath.Ext(name))+"-manifest.json")
manifestBytes, err := json.Marshal(map[string]any{
"kind": "dataset",
"datasetType": payload["datasetType"],
"payloadPath": payloadPath,
"summary": map[string]any{"name": name},
})
if err != nil {
t.Fatalf("marshal manifest: %v", err)
}
if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
t.Fatalf("write manifest: %v", err)
}
return domain.ExecutionArtifact{
ID: runID + "-" + strings.TrimSuffix(name, filepath.Ext(name)),
RunID: runID,
NodeID: nodeID,
Name: name,
Type: domain.PortTypeJSON,
Path: payloadPath,
Data: map[string]any{
"datasetType": payload["datasetType"],
"dataRef": map[string]any{
"kind": "dataset",
"datasetType": payload["datasetType"],
"manifestPath": manifestPath,
"payloadPath": payloadPath,
},
},
}
}
func TestScanConfigBOSSecretsAreReturnedAndPreserved(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} {
if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
Name: name,
Type: "builtin",
Category: "io",
Description: name,
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}},
}); err != nil {
t.Fatalf("create operator %s: %v", name, err)
}
}
handler := NewHandler(dataStore, nil)
createPayload := `{
"name":"BOS 扫描",
"description":"扫描真实 BOS 目录",
"dataSource":"BOS",
"bucket":"datastorage",
"prefix":"algo/X-platform-test/",
"endpoint":"https://bj.bcebos.com",
"accessKeyId":"ak-example-1234",
"secretAccessKey":"sk-example-5678",
"interval":86400,
"enabled":true
}`
createRecorder := httptest.NewRecorder()
handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
if createRecorder.Code != http.StatusCreated {
t.Fatalf("create scan config status: %d body=%s", createRecorder.Code, createRecorder.Body.String())
}
var created domain.WorkflowRecord
if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil {
t.Fatalf("decode created scan config: %v", err)
}
if created.ScanConfig == nil {
t.Fatalf("expected scan config")
}
if created.ScanConfig.AccessKeyID != "ak-example-1234" || created.ScanConfig.SecretAccessKey != "sk-example-5678" {
t.Fatalf("scan config secrets should be returned in workflow responses: %+v", created.ScanConfig)
}
if !created.ScanConfig.HasAccessKeyID || !created.ScanConfig.HasSecretAccessKey {
t.Fatalf("expected secret presence flags in API response: %+v", created.ScanConfig)
}
if created.ScanConfig.AccessKeyPreview == "" || created.ScanConfig.SecretAccessKeyPreview == "" {
t.Fatalf("expected secret previews in API response: %+v", created.ScanConfig)
}
stored, err := dataStore.GetWorkflow(ctx, created.ID)
if err != nil {
t.Fatalf("get stored workflow: %v", err)
}
if stored == nil || stored.ScanConfig == nil {
t.Fatalf("expected stored scan config")
}
if stored.ScanConfig.AccessKeyID != "ak-example-1234" || stored.ScanConfig.SecretAccessKey != "sk-example-5678" {
t.Fatalf("expected secrets preserved in store: %+v", stored.ScanConfig)
}
updatePayload := `{
"name":"BOS 扫描更新",
"description":"保留原始密钥",
"dataSource":"BOS",
"bucket":"datastorage",
"prefix":"algo/X-platform-test/",
"endpoint":"https://bj.bcebos.com",
"interval":3600,
"enabled":false
}`
updateRecorder := httptest.NewRecorder()
handler.ServeHTTP(updateRecorder, httptest.NewRequest(http.MethodPut, "/api/scan-configs/"+created.ID, strings.NewReader(updatePayload)))
if updateRecorder.Code != http.StatusOK {
t.Fatalf("update scan config status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String())
}
updated, err := dataStore.GetWorkflow(ctx, created.ID)
if err != nil {
t.Fatalf("get updated workflow: %v", err)
}
if updated == nil || updated.ScanConfig == nil {
t.Fatalf("expected updated scan config")
}
if updated.ScanConfig.AccessKeyID != "ak-example-1234" || updated.ScanConfig.SecretAccessKey != "sk-example-5678" {
t.Fatalf("expected secrets preserved across updates: %+v", updated.ScanConfig)
}
}
func TestPackagesAndSamplingCanLoadExternalizedManifestPayloads(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "外置成品项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品工作流",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
packagesPayload := map[string]any{
"datasetType": "x-platform-packages",
"packageCount": 1,
"items": []any{
map[string]any{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": "/tmp/k8s-job/package",
},
},
}
samplingPayload := map[string]any{
"datasetType": "x-platform-audit-samples",
"rules": []any{
map[string]any{
"id": "confidence-check",
"field": "annotation.minConfidence",
"operator": "lt",
"value": 0.7,
"samplePercent": 10,
"matchedCount": 1,
"sampledCount": 1,
},
},
"items": []any{
map[string]any{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"sampleRuleId": "confidence-check",
"storagePath": "/tmp/k8s-job/package",
},
},
}
run := domain.RunRecord{
ID: "run-externalized-package",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
externalizedJSONArtifact(t, "run-externalized-package", "package", "packages-manifest.json", packagesPayload),
externalizedJSONArtifact(t, "run-externalized-package", "sample", "audit-samples.json", samplingPayload),
},
CreatedAt: "2026-04-10T10:05:00Z",
UpdatedAt: "2026-04-10T10:05:00Z",
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
packagesRecorder := httptest.NewRecorder()
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if packagesRecorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
}
var packagesBody struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(packagesBody.Items) != 1 {
t.Fatalf("expected one package from externalized manifest, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
}
samplingRecorder := httptest.NewRecorder()
handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
if samplingRecorder.Code != http.StatusOK {
t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String())
}
var samplingBody struct {
Items []domain.SamplingTaskRecord `json:"items"`
}
if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil {
t.Fatalf("decode sampling tasks: %v", err)
}
if len(samplingBody.Items) != 1 || len(samplingBody.Items[0].Items) != 1 {
t.Fatalf("expected one sampling task with one item from externalized manifest, got %#v", samplingBody.Items)
}
}
func TestScanConfigCreateDraftDoesNotEnsureSystemProject(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} {
if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
Name: name,
Type: "builtin",
Category: "io",
Description: name,
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}},
}); err != nil {
t.Fatalf("create operator %s: %v", name, err)
}
}
handler := NewHandler(dataStore, nil)
createPayload := `{
"name":"BOS 草稿扫描",
"description":"创建后再配 bucket/prefix",
"dataSource":"BOS",
"interval":0,
"enabled":false
}`
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
if recorder.Code != http.StatusCreated {
t.Fatalf("create draft scan config status: %d body=%s", recorder.Code, recorder.Body.String())
}
var workflow domain.WorkflowRecord
if err := json.Unmarshal(recorder.Body.Bytes(), &workflow); err != nil {
t.Fatalf("decode draft workflow: %v", err)
}
if workflow.ProjectID != "" {
t.Fatalf("expected projectless scan workflow, got projectId=%q", workflow.ProjectID)
}
if workflow.Kind != domain.WorkflowKindScan {
t.Fatalf("expected scan workflow kind, got %s", workflow.Kind)
}
projects, err := dataStore.ListProjects(ctx)
if err != nil {
t.Fatalf("list projects: %v", err)
}
if len(projects) != 0 {
t.Fatalf("expected no auto-created system project, got %d", len(projects))
}
}
func TestDataPlatformHandlersSupportMongoStyleArtifactData(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "数据资产项目",
Description: "用于数据资产 API 测试",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
scanWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "目录扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
})
if err != nil {
t.Fatalf("create scan workflow: %v", err)
}
pipelineWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品流水线",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create pipeline workflow: %v", err)
}
scanRun := domain.RunRecord{
ID: "run-scan-1",
WorkflowID: scanWorkflow.ID,
WorkflowName: scanWorkflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-scan-1-source",
RunID: "run-scan-1",
NodeID: "scan-source",
Name: "source-manifest.json",
Type: "json",
Data: map[string]any{
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"listedCount": 4,
"newCount": 4,
},
},
{
ID: "run-scan-1-index",
RunID: "run-scan-1",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"archiveSources": bson.M{
"archive-1": bson.M{
"name": "xplatform-dataset.tar",
"previewURL": tarDataURL(t, map[string][]byte{
"BJ_001/DJI_001/DJI_0001_D.mp4": []byte("video-preview-bytes"),
}),
},
},
"collectionCount": 1,
"collections": bson.A{
bson.M{
"id": "BJ_001",
"path": "/tmp/run-scan-1/index/_materialized_inputs/BJ_001",
"relativePath": "BJ_001",
"meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"},
"hasIntrinsics": true,
"hasVideoMeta": true,
"requiredFiles": bson.A{"meta.json", "video_meta.json"},
"missingRequiredFiles": bson.A{},
"videoCount": 1,
"fileCount": 4,
"totalBytes": 1234,
"valid": true,
},
},
"items": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"filename": "DJI_0001_D.mp4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"duration": 41.8,
"sizeBytes": 2048,
"meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"},
"source": bson.M{
"archiveKey": "archive-1",
"archiveMemberPath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"mimeType": "video/mp4",
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
},
},
},
},
},
},
CreatedAt: "2026-04-10T10:00:00Z",
UpdatedAt: "2026-04-10T10:00:00Z",
}
if err := dataStore.UpsertRun(ctx, scanRun); err != nil {
t.Fatalf("upsert scan run: %v", err)
}
pipelineRun := domain.RunRecord{
ID: "run-pipeline-1",
WorkflowID: pipelineWorkflow.ID,
WorkflowName: pipelineWorkflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-pipeline-1-packages",
RunID: "run-pipeline-1",
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"packageCount": 1,
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"clipDuration": 300,
"fileCount": 4,
"pipelineTags": bson.A{"quality_v1", "annotation_v1"},
"storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package",
"quality": bson.M{"clarity": 0.92, "shake": 0.08},
"annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}},
},
},
},
},
{
ID: "run-pipeline-1-audit",
RunID: "run-pipeline-1",
NodeID: "sample",
Name: "audit-samples.json",
Type: "json",
Data: map[string]any{
"rules": bson.A{
bson.M{
"id": "confidence-check",
"field": "annotation.minConfidence",
"operator": "lt",
"value": 0.7,
"samplePercent": 10,
"matchedCount": 1,
"sampledCount": 1,
},
},
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"sampleRuleId": "confidence-check",
"storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package",
"quality": bson.M{"clarity": 0.92, "shake": 0.08},
"annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}},
},
},
},
},
{
ID: "run-pipeline-1-index",
RunID: "run-pipeline-1",
NodeID: "setup-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"archiveSources": bson.M{
"archive-test": bson.M{"name": "synthetic-dataset.tgz", "previewURL": "data:application/x-tar;base64,AAAA"},
},
"collectionCount": 1,
"collections": bson.A{
bson.M{
"id": "synthetic-small",
"name": "synthetic-small",
"path": "/tmp/droboticflow/run-pipeline-1/index/_materialized_inputs/synthetic-small",
"relativePath": "synthetic-small",
"fileCount": 1,
"totalBytes": 21,
"valid": true,
},
},
},
},
{
ID: "run-pipeline-1-video",
RunID: "run-pipeline-1",
NodeID: "package",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Data: map[string]any{
"preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"},
},
},
{
ID: "run-pipeline-1-quality",
RunID: "run-pipeline-1",
NodeID: "package",
Name: "DJI_0001_D_clip001_quality.json",
Type: "json",
Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001_quality.json",
Data: map[string]any{"clarity": 0.92},
},
},
CreatedAt: "2026-04-10T10:05:00Z",
UpdatedAt: "2026-04-10T10:05:00Z",
}
if err := dataStore.UpsertRun(ctx, pipelineRun); err != nil {
t.Fatalf("upsert pipeline run: %v", err)
}
handler := NewHandler(dataStore, nil)
packagesRecorder := httptest.NewRecorder()
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if packagesRecorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
}
var packagesBody struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(packagesBody.Items) != 1 {
t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
}
if packagesBody.Items[0].PackageName != "DJI_0001_D_clip001_package" {
t.Fatalf("unexpected package name: %s", packagesBody.Items[0].PackageName)
}
if packagesBody.Items[0].Storage.Provider != "k8s-job" {
t.Fatalf("expected k8s-job package storage, got %+v", packagesBody.Items[0].Storage)
}
if packagesBody.Items[0].Storage.Prefix != "/tmp/run-pipeline-1/package/packages" {
t.Fatalf("unexpected package storage prefix: %+v", packagesBody.Items[0].Storage)
}
packageDetailRecorder := httptest.NewRecorder()
handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package", nil))
if packageDetailRecorder.Code != http.StatusOK {
t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String())
}
var packageDetail domain.PackageDetail
if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil {
t.Fatalf("decode package detail: %v", err)
}
if packageDetail.Package.Storage.Path != "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package" {
t.Fatalf("unexpected package storage path: %+v", packageDetail.Package.Storage)
}
if len(packageDetail.Files) == 0 {
t.Fatalf("expected package files")
}
videoPreview := mapValue(packageDetail.Files[0].Data["preview"])
if !strings.Contains(stringValue(videoPreview["url"]), "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package/content?relativePath=") {
t.Fatalf("expected package video preview url to use content api, got %+v", packageDetail.Files[0].Data)
}
samplingRecorder := httptest.NewRecorder()
handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
if samplingRecorder.Code != http.StatusOK {
t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String())
}
var samplingBody struct {
Items []domain.SamplingTaskRecord `json:"items"`
}
if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil {
t.Fatalf("decode sampling tasks: %v", err)
}
if len(samplingBody.Items) != 1 {
t.Fatalf("expected one sampling task, got %d body=%s", len(samplingBody.Items), samplingRecorder.Body.String())
}
if len(samplingBody.Items[0].Items) != 1 {
t.Fatalf("expected one sampling item, got %d", len(samplingBody.Items[0].Items))
}
collectionsRecorder := httptest.NewRecorder()
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
if collectionsRecorder.Code != http.StatusOK {
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
}
var collectionsBody struct {
Items []domain.SourceCollectionRecord `json:"items"`
}
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
t.Fatalf("decode source collections: %v", err)
}
if len(collectionsBody.Items) != 1 {
t.Fatalf("expected one source collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
}
if collectionsBody.Items[0].CollectionID != "BJ_001" {
t.Fatalf("unexpected collection id: %s", collectionsBody.Items[0].CollectionID)
}
if collectionsBody.Items[0].CollectionName != "BJ_001" {
t.Fatalf("unexpected collection name fallback: %s", collectionsBody.Items[0].CollectionName)
}
if collectionsBody.Items[0].RunID != "run-scan-1" {
t.Fatalf("expected source collections to ignore synthetic pipeline dataset indexes, got run %s", collectionsBody.Items[0].RunID)
}
if collectionsBody.Items[0].Storage.Provider != "bos" {
t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage)
}
if collectionsBody.Items[0].Storage.Bucket != "datastorage" || collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" {
t.Fatalf("unexpected source storage details: %+v", collectionsBody.Items[0].Storage)
}
detailRecorder := httptest.NewRecorder()
handler.ServeHTTP(detailRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001", nil))
if detailRecorder.Code != http.StatusOK {
t.Fatalf("source detail status: %d body=%s", detailRecorder.Code, detailRecorder.Body.String())
}
var detail domain.SourceCollectionDetail
if err := json.Unmarshal(detailRecorder.Body.Bytes(), &detail); err != nil {
t.Fatalf("decode source detail: %v", err)
}
if len(detail.Entries) != 3 {
t.Fatalf("expected root entries synthesized from dataset index, got %d body=%s", len(detail.Entries), detailRecorder.Body.String())
}
var videoDirectory *domain.SourceCollectionEntry
for index := range detail.Entries {
if detail.Entries[index].Kind == "directory" && detail.Entries[index].Path == "DJI_001" {
videoDirectory = &detail.Entries[index]
break
}
}
if videoDirectory == nil {
t.Fatalf("expected DJI_001 directory in synthesized source entries: %+v", detail.Entries)
}
entryRecorder := httptest.NewRecorder()
handler.ServeHTTP(entryRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/entries?prefix=DJI_001", nil))
if entryRecorder.Code != http.StatusOK {
t.Fatalf("source entries status: %d body=%s", entryRecorder.Code, entryRecorder.Body.String())
}
var nestedDetail domain.SourceCollectionDetail
if err := json.Unmarshal(entryRecorder.Body.Bytes(), &nestedDetail); err != nil {
t.Fatalf("decode nested source entries: %v", err)
}
if len(nestedDetail.Entries) != 1 {
t.Fatalf("expected one nested video entry, got %d body=%s", len(nestedDetail.Entries), entryRecorder.Body.String())
}
videoEntry := nestedDetail.Entries[0]
if videoEntry.Kind != "video" {
t.Fatalf("expected nested entry to be video, got %s", videoEntry.Kind)
}
if videoEntry.Artifact == nil {
t.Fatalf("expected nested video entry artifact")
}
preview := mapValue(videoEntry.Artifact.Data["preview"])
if !strings.Contains(stringValue(preview["url"]), "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=") {
t.Fatalf("expected nested video preview url from archive sources: %+v", videoEntry.Artifact.Data)
}
contentRecorder := httptest.NewRecorder()
contentRequest := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", nil)
handler.ServeHTTP(contentRecorder, contentRequest)
if contentRecorder.Code != http.StatusOK {
t.Fatalf("source content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String())
}
if contentRecorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected source content type: %s", contentRecorder.Header().Get("Content-Type"))
}
if string(contentRecorder.Body.Bytes()) != "video-preview-bytes" {
t.Fatalf("unexpected source content payload: %q", contentRecorder.Body.Bytes())
}
}
func TestSourceCollectionsUseCatalogFieldsForIncrementalScanRuns(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
collection := bson.M{
"id": "BJ_001",
"name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"meta": bson.M{"project_id": "BJ_001"},
"hasVideoMeta": true,
"videoCount": 1,
"fileCount": 4,
"totalBytes": int64(16000036740),
"valid": true,
"requiredFiles": bson.A{"meta.json", "video_meta.json"},
}
catalogEntries := bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/meta.json",
"sizeBytes": 256,
"mimeType": "application/json",
"data": bson.M{"project_id": "BJ_001"},
},
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4",
"sizeBytes": int64(16004000000),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4",
"previewURL": "https://example.com/DJI_20260327132729_0003_D.MP4",
},
},
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.json",
"sizeBytes": 4226,
"mimeType": "application/json",
},
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D_imu.json",
"sizeBytes": int64(36080000),
"mimeType": "application/json",
},
}
run := domain.RunRecord{
ID: "run-incremental-empty",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-incremental-empty-source",
RunID: "run-incremental-empty",
NodeID: "scan-source",
Name: "source-manifest.json",
Type: "json",
Data: map[string]any{
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"scanMode": "INCREMENTAL",
"listedCount": 4,
"newCount": 0,
"catalogEntries": catalogEntries,
},
},
{
ID: "run-incremental-empty-index",
RunID: "run-incremental-empty",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"scanMode": "INCREMENTAL",
"listedCount": 4,
"newCount": 0,
"collectionCount": 0,
"collections": bson.A{},
"items": bson.A{},
"entries": bson.A{},
"catalogCollectionCount": 1,
"catalogVideoCount": 1,
"catalogCollections": bson.A{collection},
"catalogItems": bson.A{},
"catalogEntries": catalogEntries,
},
},
},
CreatedAt: "2026-04-10T12:00:00Z",
UpdatedAt: "2026-04-10T12:00:00Z",
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
collectionsRecorder := httptest.NewRecorder()
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
if collectionsRecorder.Code != http.StatusOK {
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
}
var collectionsBody struct {
Items []domain.SourceCollectionRecord `json:"items"`
}
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
t.Fatalf("decode source collections: %v", err)
}
if len(collectionsBody.Items) != 1 {
t.Fatalf("expected one catalog collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
}
if collectionsBody.Items[0].FileCount != 4 || collectionsBody.Items[0].TotalBytes != 16000036740 {
t.Fatalf("expected catalog file stats, got %+v", collectionsBody.Items[0])
}
entriesRecorder := httptest.NewRecorder()
handler.ServeHTTP(entriesRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-incremental-empty/BJ_001/entries?prefix=DJI_001", nil))
if entriesRecorder.Code != http.StatusOK {
t.Fatalf("entries status: %d body=%s", entriesRecorder.Code, entriesRecorder.Body.String())
}
var detail domain.SourceCollectionDetail
if err := json.Unmarshal(entriesRecorder.Body.Bytes(), &detail); err != nil {
t.Fatalf("decode source detail: %v", err)
}
if len(detail.Entries) != 3 {
t.Fatalf("expected all catalog files under DJI_001, got %d body=%s", len(detail.Entries), entriesRecorder.Body.String())
}
var video *domain.SourceCollectionEntry
for index := range detail.Entries {
if detail.Entries[index].Name == "DJI_20260327132729_0003_D.MP4" {
video = &detail.Entries[index]
break
}
}
if video == nil || video.SizeBytes != 16004000000 {
t.Fatalf("expected real mp4 size from catalog entries, got %+v", video)
}
if collectionsBody.Items[0].Storage.Provider != "bos" {
t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage)
}
if collectionsBody.Items[0].Storage.Bucket != "datastorage" {
t.Fatalf("unexpected source storage bucket: %+v", collectionsBody.Items[0].Storage)
}
if collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" {
t.Fatalf("unexpected source storage prefix: %+v", collectionsBody.Items[0].Storage)
}
}
func TestSourceCollectionsCanLoadExternalizedDatasetIndexPayload(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
indexPayload := map[string]any{
"datasetType": "x-platform-index",
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"collectionCount": 0,
"collections": []any{},
"items": []any{},
"entries": []any{},
"catalogCollectionCount": 1,
"catalogCollections": []any{
map[string]any{
"id": "BJ_001",
"name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
"relativePath": "BJ_001",
"videoCount": 1,
"valid": true,
},
},
"catalogItems": []any{},
"catalogEntries": []any{},
}
run := domain.RunRecord{
ID: "run-externalized-index",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
externalizedJSONArtifact(t, "run-externalized-index", "scan-index", "dataset-index.json", indexPayload),
},
CreatedAt: "2026-04-10T12:00:00Z",
UpdatedAt: "2026-04-10T12:00:00Z",
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
collectionsRecorder := httptest.NewRecorder()
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
if collectionsRecorder.Code != http.StatusOK {
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
}
var collectionsBody struct {
Items []domain.SourceCollectionRecord `json:"items"`
}
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
t.Fatalf("decode source collections: %v", err)
}
if len(collectionsBody.Items) != 1 {
t.Fatalf("expected externalized payload to produce one collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
}
}
func TestPackageContentCanServeBrowserPreviewVideo(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品预览项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品预览工作流",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
videoPath := filepath.Join(t.TempDir(), "DJI_0001_D_clip001.mp4")
if err := os.WriteFile(videoPath, []byte("package-video-bytes"), 0o644); err != nil {
t.Fatalf("write package video: %v", err)
}
run := domain.RunRecord{
ID: "run-package-preview",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-package-preview-manifest",
RunID: "run-package-preview",
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": filepath.Dir(videoPath),
},
},
},
},
{
ID: "run-package-preview-video",
RunID: "run-package-preview",
NodeID: "package",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4"),
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"},
},
},
{
ID: "run-package-preview-upload-video",
RunID: "run-package-preview",
NodeID: "upload",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"source": map[string]any{
"path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
},
},
},
},
}
run.Artifacts[1].Path = filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4")
if err := os.MkdirAll(filepath.Dir(run.Artifacts[1].Path), 0o755); err != nil {
t.Fatalf("mkdir package dir: %v", err)
}
if err := os.WriteFile(run.Artifacts[1].Path, []byte("package-video-bytes"), 0o644); err != nil {
t.Fatalf("write package artifact: %v", err)
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
previewPath := filepath.Join(t.TempDir(), "package-browser-preview.mp4")
if err := os.WriteFile(previewPath, []byte("package-browser-preview"), 0o644); err != nil {
t.Fatalf("write preview file: %v", err)
}
originalPreviewResolver := sourceBrowserVideoPreviewPath
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
if request.RedirectURL != run.Artifacts[1].Path {
t.Fatalf("unexpected package preview input: %s", request.RedirectURL)
}
if request.FileName != "DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected file name: %s", request.FileName)
}
return previewPath, nil
}
t.Cleanup(func() {
sourceBrowserVideoPreviewPath = originalPreviewResolver
})
handler := NewHandler(dataStore, nil)
packagesRecorder := httptest.NewRecorder()
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if packagesRecorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
}
var packagesBody struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(packagesBody.Items) != 1 {
t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
}
if packagesBody.Items[0].Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" {
t.Fatalf("expected package storage to prefer BOS upload path, got %+v", packagesBody.Items[0].Storage)
}
packageDetailRecorder := httptest.NewRecorder()
handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-package-preview/DJI_0001_D_clip001_package", nil))
if packageDetailRecorder.Code != http.StatusOK {
t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String())
}
var packageDetail domain.PackageDetail
if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil {
t.Fatalf("decode package detail: %v", err)
}
if packageDetail.Package.Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" {
t.Fatalf("expected package detail storage to prefer BOS upload path, got %+v", packageDetail.Package.Storage)
}
if len(packageDetail.Files) != 1 {
t.Fatalf("expected duplicate package files to collapse to one, got %d", len(packageDetail.Files))
}
if packageDetail.Files[0].Name != "DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected package file after dedupe: %+v", packageDetail.Files[0])
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/packages/run-package-preview/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
}
if string(recorder.Body.Bytes()) != "package-browser-preview" {
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
}
}
func TestPackageContentFallsBackToUploadedBOSVideoWhenLocalPackageFileUnavailable(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品上传回退项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "上传 BOS 连接",
Provider: domain.StorageProviderBOS,
Endpoint: "https://bj.bcebos.com",
AccessKeyID: "ak-upload",
SecretAccessKey: "sk-upload",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品上传回退工作流",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
ConnectionID: connection.ID,
Bucket: "datastorage",
Prefix: "factory/output/test-package-preview",
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "BOS"},
{Key: "SOURCE_PATH", Value: "bos://datastorage/factory/output/test-package-preview"},
},
},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-package-upload-fallback",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-package-upload-fallback-manifest",
RunID: "run-package-upload-fallback",
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": "/tmp/droboticflow/missing/package",
},
},
},
},
{
ID: "run-package-upload-fallback-temp-video",
RunID: "run-package-upload-fallback",
NodeID: "package",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "/tmp/droboticflow/missing/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"preview": map[string]any{
"kind": "video",
"url": "/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4",
},
},
},
{
ID: "run-package-upload-fallback-remote-video",
RunID: "run-package-upload-fallback",
NodeID: "upload",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
"source": map[string]any{
"path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
"previewURL": "https://expired.example.com/DJI_0001_D_clip001.mp4",
"mimeType": "video/mp4",
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
previewPath := filepath.Join(t.TempDir(), "package-upload-browser-preview.mp4")
if err := os.WriteFile(previewPath, []byte("package-upload-browser-preview"), 0o644); err != nil {
t.Fatalf("write preview file: %v", err)
}
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak-upload" || scanConfig.SecretAccessKey != "sk-upload" {
t.Fatalf("expected upload connection secrets, got %+v", scanConfig)
}
if sourcePath != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D_clip001.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
originalPreviewResolver := sourceBrowserVideoPreviewPath
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
if request.RedirectURL != "https://signed.example.com/DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected package preview redirect input: %s", request.RedirectURL)
}
if request.FileName != "DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected file name: %s", request.FileName)
}
return previewPath, nil
}
t.Cleanup(func() {
sourceBrowserVideoPreviewPath = originalPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
}
if string(recorder.Body.Bytes()) != "package-upload-browser-preview" {
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
}
}
func TestPackagesListKeepsLatestPackageVersionPerWorkflow(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "包去重项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "包去重工作流",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
makeRun := func(id string, createdAt string, storagePath string) domain.RunRecord {
return domain.RunRecord{
ID: id,
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
CreatedAt: createdAt,
Artifacts: []domain.ExecutionArtifact{
{
ID: id + "-manifest",
RunID: id,
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"items": []any{
map[string]any{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": "/tmp/k8s-job/package",
},
},
},
},
{
ID: id + "-upload-video",
RunID: id,
NodeID: "upload",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: storagePath + "/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
"source": map[string]any{
"path": storagePath + "/DJI_0001_D_clip001.mp4",
"mimeType": "video/mp4",
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
},
},
},
},
}
}
if err := dataStore.UpsertRun(ctx, makeRun(
"run-package-old",
"2026-04-12T10:00:00Z",
"bos://datastorage/factory/output/old/packages/DJI_0001_D_clip001_package",
)); err != nil {
t.Fatalf("upsert old run: %v", err)
}
if err := dataStore.UpsertRun(ctx, makeRun(
"run-package-new",
"2026-04-12T11:00:00Z",
"bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package",
)); err != nil {
t.Fatalf("upsert new run: %v", err)
}
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", recorder.Code, recorder.Body.String())
}
var body struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(body.Items) != 1 {
t.Fatalf("expected latest package only, got %d body=%s", len(body.Items), recorder.Body.String())
}
if body.Items[0].RunID != "run-package-new" {
t.Fatalf("expected latest run package, got %+v", body.Items[0])
}
if body.Items[0].Storage.Path != "bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package" {
t.Fatalf("expected latest BOS storage path, got %+v", body.Items[0].Storage)
}
}
func TestSourceCollectionContentCanServeBrowserPreviewVideo(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "浏览器预览项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
Endpoint: "https://bj.bcebos.com",
AccessKeyID: "ak",
SecretAccessKey: "sk",
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-browser-preview",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-browser-preview-index",
RunID: "run-browser-preview",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"catalogCollections": bson.A{
bson.M{
"id": "BJ_001",
"name": "BJ_001",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"videoCount": 1,
"fileCount": 1,
"totalBytes": int64(2048),
"valid": true,
},
},
"catalogEntries": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"sizeBytes": int64(2048),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"previewURL": "https://example.com/DJI_0001_D.mp4",
},
},
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
previewPath := filepath.Join(t.TempDir(), "preview.mp4")
if err := os.WriteFile(previewPath, []byte("browser-preview-video"), 0o644); err != nil {
t.Fatalf("write preview file: %v", err)
}
originalPreviewResolver := sourceBrowserVideoPreviewPath
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
if request.RedirectURL != "https://signed.example.com/DJI_0001_D.mp4" {
t.Fatalf("unexpected redirect url: %s", request.RedirectURL)
}
if request.FileName != "DJI_0001_D.mp4" {
t.Fatalf("unexpected file name: %s", request.FileName)
}
return previewPath, nil
}
t.Cleanup(func() {
sourceBrowserVideoPreviewPath = originalPreviewResolver
})
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" {
t.Fatalf("unexpected scan config: %+v", scanConfig)
}
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/source-data/collections/run-browser-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4&transcode=browser",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
}
if string(recorder.Body.Bytes()) != "browser-preview-video" {
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
}
}
func TestSourceCollectionContentRefreshesBOSRedirectURL(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 刷新项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
Endpoint: "https://bj.bcebos.com",
AccessKeyID: "ak",
SecretAccessKey: "sk",
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-refresh-bos-preview",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-refresh-bos-preview-index",
RunID: "run-refresh-bos-preview",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"catalogCollections": bson.A{
bson.M{
"id": "BJ_001",
"name": "BJ_001",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"videoCount": 1,
"fileCount": 1,
"totalBytes": int64(2048),
"valid": true,
},
},
"catalogEntries": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"sizeBytes": int64(2048),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"previewURL": "https://expired.example.com/DJI_0001_D.mp4",
},
},
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" {
t.Fatalf("unexpected scan config: %+v", scanConfig)
}
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/source-data/collections/run-refresh-bos-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusTemporaryRedirect {
t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String())
}
if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" {
t.Fatalf("unexpected redirect location: %s", location)
}
}
func TestSourceCollectionContentRefreshesBOSRedirectURLFromStorageConnection(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 连接预览项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS 主连接",
Provider: domain.StorageProviderBOS,
Endpoint: "https://bj.bcebos.com",
AccessKeyID: "ak-from-connection",
SecretAccessKey: "sk-from-connection",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
ConnectionID: connection.ID,
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-refresh-bos-preview-connection",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-refresh-bos-preview-connection-index",
RunID: "run-refresh-bos-preview-connection",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"catalogCollections": bson.A{
bson.M{
"id": "BJ_001",
"name": "BJ_001",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"videoCount": 1,
"fileCount": 1,
"totalBytes": int64(2048),
"valid": true,
},
},
"catalogEntries": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"sizeBytes": int64(2048),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"previewURL": "https://expired.example.com/DJI_0001_D.mp4",
},
},
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak-from-connection" || scanConfig.SecretAccessKey != "sk-from-connection" {
t.Fatalf("expected resolved connection secrets, got %+v", scanConfig)
}
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/source-data/collections/run-refresh-bos-preview-connection/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusTemporaryRedirect {
t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String())
}
if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" {
t.Fatalf("unexpected redirect location: %s", location)
}
}