droboticflow/internal/api/data_platform_test.go
wangbin 57820b16a3 fix(api): refuse deleting a storage connection still referenced by workflows
handleStorageConnectionsDelete performed a hard DeleteOne with only a tenant
check, so a connection still referenced by a workflow node's connectionId could
be deleted — leaving a dangling reference that breaks workflow save/run
validation ("connectionId must reference an existing storage connection") with
no recovery path (the record is hard-deleted and unaudited). Add a reference
guard: before deleting, scan the connection's organization for workflows whose
nodes reference it; if any, return 409 Conflict listing the referencing
workflow/node ids so the caller can remove the reference first. Reuses the
existing ListAllWorkflowsByOrganization store method (no schema/interface change,
no runtime-path impact).

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-09 21:55:13 +08:00

2271 lines
79 KiB
Go

package api
import (
"archive/tar"
"bytes"
"context"
"encoding/base64"
"encoding/json"
"math"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"go.mongodb.org/mongo-driver/bson"
)
func tarDataURL(t *testing.T, files map[string][]byte) string {
t.Helper()
var buffer bytes.Buffer
writer := tar.NewWriter(&buffer)
for name, payload := range files {
header := &tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(payload)),
}
if err := writer.WriteHeader(header); err != nil {
t.Fatalf("write tar header: %v", err)
}
if _, err := writer.Write(payload); err != nil {
t.Fatalf("write tar payload: %v", err)
}
}
if err := writer.Close(); err != nil {
t.Fatalf("close tar writer: %v", err)
}
return "data:application/x-tar;base64," + base64.StdEncoding.EncodeToString(buffer.Bytes())
}
func externalizedJSONArtifact(t *testing.T, runID string, nodeID string, name string, payload map[string]any) domain.ExecutionArtifact {
t.Helper()
root := t.TempDir()
payloadPath := filepath.Join(root, name)
payloadBytes, err := json.Marshal(payload)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
if err := os.WriteFile(payloadPath, payloadBytes, 0o644); err != nil {
t.Fatalf("write payload: %v", err)
}
manifestPath := filepath.Join(root, strings.TrimSuffix(name, filepath.Ext(name))+"-manifest.json")
manifestBytes, err := json.Marshal(map[string]any{
"kind": "dataset",
"datasetType": payload["datasetType"],
"payloadPath": payloadPath,
"summary": map[string]any{"name": name},
})
if err != nil {
t.Fatalf("marshal manifest: %v", err)
}
if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
t.Fatalf("write manifest: %v", err)
}
return domain.ExecutionArtifact{
ID: runID + "-" + strings.TrimSuffix(name, filepath.Ext(name)),
RunID: runID,
NodeID: nodeID,
Name: name,
Type: domain.PortTypeJSON,
Path: payloadPath,
Data: map[string]any{
"datasetType": payload["datasetType"],
"dataRef": map[string]any{
"kind": "dataset",
"datasetType": payload["datasetType"],
"manifestPath": manifestPath,
"payloadPath": payloadPath,
},
},
}
}
func TestDataPlatformIntegerHelpersRejectLossyValues(t *testing.T) {
if got := intValue(float64(4)); got != 4 {
t.Fatalf("expected integral float to parse, got %d", got)
}
if got := intValue(4.9); got != 0 {
t.Fatalf("expected lossy float to be ignored, got %d", got)
}
if got := int64Value(int64(16004000000)); got != 16004000000 {
t.Fatalf("expected int64 to be preserved, got %d", got)
}
if got := int64Value(math.NaN()); got != 0 {
t.Fatalf("expected NaN to be ignored, got %d", got)
}
if got := int64Value(float64(1 << 53)); got != 0 {
t.Fatalf("expected unsafe float integer to be ignored, got %d", got)
}
if got := int64Value(uint64(math.MaxInt64) + 1); got != 0 {
t.Fatalf("expected overflowing uint64 to be ignored, got %d", got)
}
if got := intValue(json.Number("42")); got != 42 {
t.Fatalf("expected integer json.Number to parse, got %d", got)
}
if got := intValue(json.Number("42.5")); got != 0 {
t.Fatalf("expected decimal json.Number to be ignored, got %d", got)
}
}
func TestArtifactStructuredDataTreatsLocalPayloadPathAsAuthoritative(t *testing.T) {
missingPayloadPath := filepath.Join(t.TempDir(), "missing-payload.json")
missingArtifact := domain.ExecutionArtifact{
Name: "source-manifest.json",
Type: domain.PortTypeJSON,
Data: map[string]any{
"datasetType": "x-platform",
"items": []any{map[string]any{"id": "stale-inline"}},
"dataRef": map[string]any{
"payloadPath": missingPayloadPath,
},
},
}
if got := artifactStructuredData(&missingArtifact); got != nil {
t.Fatalf("expected missing external payload to suppress stale inline data, got %#v", got)
}
invalidPayloadPath := filepath.Join(t.TempDir(), "payload.json")
if err := os.WriteFile(invalidPayloadPath, []byte("{"), 0o644); err != nil {
t.Fatalf("write invalid payload: %v", err)
}
invalidArtifact := missingArtifact
invalidArtifact.Data = map[string]any{
"datasetType": "x-platform",
"items": []any{map[string]any{"id": "stale-inline"}},
"dataRef": map[string]any{
"payloadPath": invalidPayloadPath,
},
}
if got := artifactStructuredData(&invalidArtifact); got != nil {
t.Fatalf("expected invalid external payload to suppress stale inline data, got %#v", got)
}
}
func TestSamplingRulesFromManifestRejectsInvalidRuleShape(t *testing.T) {
validRules, ok := samplingRulesFromManifest(map[string]any{
"rules": []any{
map[string]any{
"id": "clarity",
"field": "quality.clarity",
"operator": "GTE",
"value": 0.3,
"samplePercent": json.Number("12.5"),
"matchedCount": 4,
"sampledCount": 2,
},
},
})
if !ok || len(validRules) != 1 {
t.Fatalf("expected valid sampling rule, got ok=%v rules=%#v", ok, validRules)
}
if validRules[0].Operator != "gte" || validRules[0].SamplePercent != 12.5 {
t.Fatalf("expected normalized operator and sample percent, got %#v", validRules[0])
}
cases := map[string]any{
"bool": true,
"zero": 0,
"over-100": 101,
"nan": math.NaN(),
"string": "10",
"bad-number": json.Number("nan"),
"unsupported": nil,
}
for name, samplePercent := range cases {
rule := map[string]any{
"id": "clarity",
"field": "quality.clarity",
"operator": "gte",
"value": 0.3,
"samplePercent": samplePercent,
}
if name == "unsupported" {
rule["operator"] = "regex"
rule["samplePercent"] = 10
}
if rules, ok := samplingRulesFromManifest(map[string]any{"rules": []any{rule}}); ok {
t.Fatalf("expected %s rule to be rejected, got %#v", name, rules)
}
}
duplicate := map[string]any{
"rules": []any{
map[string]any{"id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.3, "samplePercent": 10},
map[string]any{"id": "clarity", "field": "quality.shake", "operator": "lte", "value": 0.1, "samplePercent": 10},
},
}
if rules, ok := samplingRulesFromManifest(duplicate); ok {
t.Fatalf("expected duplicate rules to be rejected, got %#v", rules)
}
}
func TestSamplingTasksSkipInvalidManifestRuleSamplePercent(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "坏抽检规则项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "坏抽检规则工作流",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-invalid-sampling-rule",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-invalid-sampling-rule-artifact",
RunID: "run-invalid-sampling-rule",
NodeID: "sample",
Name: "audit-samples.json",
Type: domain.PortTypeJSON,
Data: map[string]any{
"datasetType": "x-platform-audit-samples",
"rules": []any{
map[string]any{
"id": "clarity",
"field": "quality.clarity",
"operator": "gte",
"value": 0.3,
"samplePercent": true,
"matchedCount": 1,
"sampledCount": 1,
},
},
"items": []any{
map[string]any{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"sampleRuleId": "clarity",
},
},
},
},
},
CreatedAt: "2026-04-10T10:05:00Z",
UpdatedAt: "2026-04-10T10:05:00Z",
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("sampling status: %d body=%s", recorder.Code, recorder.Body.String())
}
var body struct {
Items []domain.SamplingTaskRecord `json:"items"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
t.Fatalf("decode sampling tasks: %v", err)
}
if len(body.Items) != 0 {
t.Fatalf("expected invalid sampling manifest to be skipped, got %#v", body.Items)
}
}
func TestScanConfigBOSSecretsAreReturnedAndPreserved(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} {
if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
Name: name,
Type: "builtin",
Category: "io",
Description: name,
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}},
}); err != nil {
t.Fatalf("create operator %s: %v", name, err)
}
}
handler := NewHandler(dataStore, nil)
createPayload := `{
"name":"BOS 扫描",
"description":"扫描真实 BOS 目录",
"dataSource":"BOS",
"bucket":"datastorage",
"prefix":"algo/X-platform-test/",
"endpoint":"https://bos.example.test",
"accessKeyId":"ak-example-1234",
"secretAccessKey":"sk-example-5678",
"interval":86400,
"enabled":true
}`
createRecorder := httptest.NewRecorder()
handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
if createRecorder.Code != http.StatusCreated {
t.Fatalf("create scan config status: %d body=%s", createRecorder.Code, createRecorder.Body.String())
}
var created domain.WorkflowRecord
if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil {
t.Fatalf("decode created scan config: %v", err)
}
if created.ScanConfig == nil {
t.Fatalf("expected scan config")
}
if created.ScanConfig.AccessKeyID != "" || created.ScanConfig.SecretAccessKey != "" {
t.Fatalf("scan config secrets should be masked in workflow responses: %+v", created.ScanConfig)
}
if !created.ScanConfig.HasAccessKeyID || !created.ScanConfig.HasSecretAccessKey {
t.Fatalf("expected secret presence flags in API response: %+v", created.ScanConfig)
}
if created.ScanConfig.AccessKeyPreview == "" || created.ScanConfig.SecretAccessKeyPreview == "" {
t.Fatalf("expected secret previews in API response: %+v", created.ScanConfig)
}
stored, err := dataStore.GetWorkflow(ctx, created.ID)
if err != nil {
t.Fatalf("get stored workflow: %v", err)
}
if stored == nil || stored.ScanConfig == nil {
t.Fatalf("expected stored scan config")
}
if stored.ScanConfig.AccessKeyID != "ak-example-1234" || stored.ScanConfig.SecretAccessKey != "sk-example-5678" {
t.Fatalf("expected secrets preserved in store: %+v", stored.ScanConfig)
}
updatePayload := `{
"name":"BOS 扫描更新",
"description":"保留原始密钥",
"dataSource":"BOS",
"bucket":"datastorage",
"prefix":"algo/X-platform-test/",
"endpoint":"https://bos.example.test",
"interval":3600,
"enabled":false
}`
updateRecorder := httptest.NewRecorder()
handler.ServeHTTP(updateRecorder, httptest.NewRequest(http.MethodPut, "/api/scan-configs/"+created.ID, strings.NewReader(updatePayload)))
if updateRecorder.Code != http.StatusOK {
t.Fatalf("update scan config status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String())
}
updated, err := dataStore.GetWorkflow(ctx, created.ID)
if err != nil {
t.Fatalf("get updated workflow: %v", err)
}
if updated == nil || updated.ScanConfig == nil {
t.Fatalf("expected updated scan config")
}
if updated.ScanConfig.AccessKeyID != "ak-example-1234" || updated.ScanConfig.SecretAccessKey != "sk-example-5678" {
t.Fatalf("expected secrets preserved across updates: %+v", updated.ScanConfig)
}
}
func TestStorageConnectionDeleteRefusesWhileReferencedAndAllowsAfterClearing(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
createRecorder := httptest.NewRecorder()
handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/storage/connections", strings.NewReader(`{
"name":"BOS Workspace",
"provider":"BOS",
"endpoint":"https://bj.bcebos.com",
"bucket":"testxyz",
"accessKeyId":"ak-example",
"secretAccessKey":"sk-example"
}`)))
if createRecorder.Code != http.StatusCreated {
t.Fatalf("create connection status: %d body=%s", createRecorder.Code, createRecorder.Body.String())
}
var created domain.StorageConnectionRecord
if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil {
t.Fatalf("decode created connection: %v", err)
}
// Workflow in the same org whose node still references the connection.
workflow := domain.WorkflowRecord{
ID: "workflow-ref-test",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "project-ref-test",
Name: "引用存储连接的工作流",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{ID: "op-bos-scan", Data: domain.WorkflowNodeData{Label: "BOS 增量扫描", ConnectionID: created.ID}},
},
}
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
t.Fatalf("upsert referencing workflow: %v", err)
}
refusedRecorder := httptest.NewRecorder()
handler.ServeHTTP(refusedRecorder, httptest.NewRequest(http.MethodDelete, "/api/storage/connections/"+created.ID, nil))
if refusedRecorder.Code != http.StatusConflict {
t.Fatalf("delete referenced connection status: got %d want %d body=%s", refusedRecorder.Code, http.StatusConflict, refusedRecorder.Body.String())
}
var refused map[string]any
if err := json.Unmarshal(refusedRecorder.Body.Bytes(), &refused); err != nil {
t.Fatalf("decode refused body: %v", err)
}
refs, _ := refused["referencedBy"].([]any)
if len(refs) != 1 {
t.Fatalf("expected 1 referencedBy entry, got %v", refs)
}
first, _ := refs[0].(map[string]any)
if first["workflowId"] != "workflow-ref-test" || first["nodeId"] != "op-bos-scan" {
t.Fatalf("unexpected referencedBy entry: %v", first)
}
stillExists, err := dataStore.GetStorageConnection(ctx, created.ID)
if err != nil || stillExists == nil {
t.Fatalf("connection should still exist after refused delete: %v %v", stillExists, err)
}
// Clear the reference → delete must succeed.
workflow.Nodes[0].Data.ConnectionID = ""
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
t.Fatalf("upsert cleared workflow: %v", err)
}
okRecorder := httptest.NewRecorder()
handler.ServeHTTP(okRecorder, httptest.NewRequest(http.MethodDelete, "/api/storage/connections/"+created.ID, nil))
if okRecorder.Code != http.StatusNoContent {
t.Fatalf("delete after clearing status: got %d want %d body=%s", okRecorder.Code, http.StatusNoContent, okRecorder.Body.String())
}
gone, err := dataStore.GetStorageConnection(ctx, created.ID)
if err != nil || gone != nil {
t.Fatalf("connection should be gone after delete: %v %v", gone, err)
}
}
func TestPackagesAndSamplingCanLoadExternalizedManifestPayloads(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "外置成品项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品工作流",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
packagesPayload := map[string]any{
"datasetType": "x-platform-packages",
"packageCount": 1,
"items": []any{
map[string]any{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": "/tmp/k8s-job/package",
},
},
}
samplingPayload := map[string]any{
"datasetType": "x-platform-audit-samples",
"rules": []any{
map[string]any{
"id": "confidence-check",
"field": "annotation.minConfidence",
"operator": "lt",
"value": 0.7,
"samplePercent": 10,
"matchedCount": 1,
"sampledCount": 1,
},
},
"items": []any{
map[string]any{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"sampleRuleId": "confidence-check",
"storagePath": "/tmp/k8s-job/package",
},
},
}
run := domain.RunRecord{
ID: "run-externalized-package",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
externalizedJSONArtifact(t, "run-externalized-package", "package", "packages-manifest.json", packagesPayload),
externalizedJSONArtifact(t, "run-externalized-package", "sample", "audit-samples.json", samplingPayload),
},
CreatedAt: "2026-04-10T10:05:00Z",
UpdatedAt: "2026-04-10T10:05:00Z",
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
packagesRecorder := httptest.NewRecorder()
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if packagesRecorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
}
var packagesBody struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(packagesBody.Items) != 1 {
t.Fatalf("expected one package from externalized manifest, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
}
samplingRecorder := httptest.NewRecorder()
handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
if samplingRecorder.Code != http.StatusOK {
t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String())
}
var samplingBody struct {
Items []domain.SamplingTaskRecord `json:"items"`
}
if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil {
t.Fatalf("decode sampling tasks: %v", err)
}
if len(samplingBody.Items) != 1 || len(samplingBody.Items[0].Items) != 1 {
t.Fatalf("expected one sampling task with one item from externalized manifest, got %#v", samplingBody.Items)
}
}
func TestScanConfigCreateDraftDoesNotEnsureSystemProject(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} {
if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
Name: name,
Type: "builtin",
Category: "io",
Description: name,
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}},
}); err != nil {
t.Fatalf("create operator %s: %v", name, err)
}
}
handler := NewHandler(dataStore, nil)
createPayload := `{
"name":"BOS 草稿扫描",
"description":"创建后再配 bucket/prefix",
"dataSource":"BOS",
"interval":0,
"enabled":false
}`
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
if recorder.Code != http.StatusCreated {
t.Fatalf("create draft scan config status: %d body=%s", recorder.Code, recorder.Body.String())
}
var workflow domain.WorkflowRecord
if err := json.Unmarshal(recorder.Body.Bytes(), &workflow); err != nil {
t.Fatalf("decode draft workflow: %v", err)
}
if workflow.ProjectID != "" {
t.Fatalf("expected projectless scan workflow, got projectId=%q", workflow.ProjectID)
}
if workflow.Kind != domain.WorkflowKindScan {
t.Fatalf("expected scan workflow kind, got %s", workflow.Kind)
}
projects, err := dataStore.ListProjects(ctx)
if err != nil {
t.Fatalf("list projects: %v", err)
}
if len(projects) != 0 {
t.Fatalf("expected no auto-created system project, got %d", len(projects))
}
}
func TestScanConfigCreateRejectsInvalidDataSource(t *testing.T) {
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
createPayload := `{
"name":"非法数据源扫描",
"description":"invalid data source",
"dataSource":"ROW",
"interval":0,
"enabled":false
}`
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("expected validation failure, got %d body=%s", recorder.Code, recorder.Body.String())
}
if !strings.Contains(recorder.Body.String(), `"field":"dataSource"`) ||
!strings.Contains(recorder.Body.String(), "dataSource must be BOS for scan workflows") {
t.Fatalf("expected dataSource validation issue, got %s", recorder.Body.String())
}
}
func TestDataPlatformHandlersSupportMongoStyleArtifactData(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "数据资产项目",
Description: "用于数据资产 API 测试",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
scanWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "目录扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
})
if err != nil {
t.Fatalf("create scan workflow: %v", err)
}
pipelineWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品流水线",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create pipeline workflow: %v", err)
}
scanRun := domain.RunRecord{
ID: "run-scan-1",
WorkflowID: scanWorkflow.ID,
WorkflowName: scanWorkflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-scan-1-source",
RunID: "run-scan-1",
NodeID: "scan-source",
Name: "source-manifest.json",
Type: "json",
Data: map[string]any{
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"listedCount": 4,
"newCount": 4,
},
},
{
ID: "run-scan-1-index",
RunID: "run-scan-1",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"archiveSources": bson.M{
"archive-1": bson.M{
"name": "xplatform-dataset.tar",
"previewURL": tarDataURL(t, map[string][]byte{
"BJ_001/DJI_001/DJI_0001_D.mp4": []byte("video-preview-bytes"),
}),
},
},
"collectionCount": 1,
"collections": bson.A{
bson.M{
"id": "BJ_001",
"path": "/tmp/run-scan-1/index/_materialized_inputs/BJ_001",
"relativePath": "BJ_001",
"meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"},
"hasIntrinsics": true,
"hasVideoMeta": true,
"requiredFiles": bson.A{"meta.json", "video_meta.json"},
"missingRequiredFiles": bson.A{},
"videoCount": 1,
"fileCount": 4,
"totalBytes": 1234,
"valid": true,
},
},
"items": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"filename": "DJI_0001_D.mp4",
"codec": "hevc",
"resolution": "3840x2160",
"frameRate": 29.97,
"duration": 41.8,
"sizeBytes": 2048,
"meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"},
"source": bson.M{
"archiveKey": "archive-1",
"archiveMemberPath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"mimeType": "video/mp4",
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
},
},
},
},
},
},
CreatedAt: "2026-04-10T10:00:00Z",
UpdatedAt: "2026-04-10T10:00:00Z",
}
if err := dataStore.UpsertRun(ctx, scanRun); err != nil {
t.Fatalf("upsert scan run: %v", err)
}
pipelineRun := domain.RunRecord{
ID: "run-pipeline-1",
WorkflowID: pipelineWorkflow.ID,
WorkflowName: pipelineWorkflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-pipeline-1-packages",
RunID: "run-pipeline-1",
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"packageCount": 1,
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"clipDuration": 300,
"fileCount": 4,
"pipelineTags": bson.A{"quality_v1", "annotation_v1"},
"storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package",
"quality": bson.M{"clarity": 0.92, "shake": 0.08},
"annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}},
},
},
},
},
{
ID: "run-pipeline-1-audit",
RunID: "run-pipeline-1",
NodeID: "sample",
Name: "audit-samples.json",
Type: "json",
Data: map[string]any{
"rules": bson.A{
bson.M{
"id": "confidence-check",
"field": "annotation.minConfidence",
"operator": "lt",
"value": 0.7,
"samplePercent": 10,
"matchedCount": 1,
"sampledCount": 1,
},
},
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"sampleRuleId": "confidence-check",
"storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package",
"quality": bson.M{"clarity": 0.92, "shake": 0.08},
"annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}},
},
},
},
},
{
ID: "run-pipeline-1-index",
RunID: "run-pipeline-1",
NodeID: "setup-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"archiveSources": bson.M{
"archive-test": bson.M{"name": "synthetic-dataset.tgz", "previewURL": "data:application/x-tar;base64,AAAA"},
},
"collectionCount": 1,
"collections": bson.A{
bson.M{
"id": "synthetic-small",
"name": "synthetic-small",
"path": "/tmp/droboticflow/run-pipeline-1/index/_materialized_inputs/synthetic-small",
"relativePath": "synthetic-small",
"fileCount": 1,
"totalBytes": 21,
"valid": true,
},
},
},
},
{
ID: "run-pipeline-1-video",
RunID: "run-pipeline-1",
NodeID: "package",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Data: map[string]any{
"preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"},
},
},
{
ID: "run-pipeline-1-quality",
RunID: "run-pipeline-1",
NodeID: "package",
Name: "DJI_0001_D_clip001_quality.json",
Type: "json",
Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001_quality.json",
Data: map[string]any{"clarity": 0.92},
},
},
CreatedAt: "2026-04-10T10:05:00Z",
UpdatedAt: "2026-04-10T10:05:00Z",
}
if err := dataStore.UpsertRun(ctx, pipelineRun); err != nil {
t.Fatalf("upsert pipeline run: %v", err)
}
handler := NewHandler(dataStore, nil)
packagesRecorder := httptest.NewRecorder()
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if packagesRecorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
}
var packagesBody struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(packagesBody.Items) != 1 {
t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
}
if packagesBody.Items[0].PackageName != "DJI_0001_D_clip001_package" {
t.Fatalf("unexpected package name: %s", packagesBody.Items[0].PackageName)
}
if packagesBody.Items[0].Storage.Provider != "k8s-job" {
t.Fatalf("expected k8s-job package storage, got %+v", packagesBody.Items[0].Storage)
}
if packagesBody.Items[0].Storage.Prefix != "/tmp/run-pipeline-1/package/packages" {
t.Fatalf("unexpected package storage prefix: %+v", packagesBody.Items[0].Storage)
}
packageDetailRecorder := httptest.NewRecorder()
handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package", nil))
if packageDetailRecorder.Code != http.StatusOK {
t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String())
}
var packageDetail domain.PackageDetail
if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil {
t.Fatalf("decode package detail: %v", err)
}
if packageDetail.Package.Storage.Path != "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package" {
t.Fatalf("unexpected package storage path: %+v", packageDetail.Package.Storage)
}
if len(packageDetail.Files) == 0 {
t.Fatalf("expected package files")
}
videoPreview := mapValue(packageDetail.Files[0].Data["preview"])
if !strings.Contains(stringValue(videoPreview["url"]), "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package/content?relativePath=") {
t.Fatalf("expected package video preview url to use content api, got %+v", packageDetail.Files[0].Data)
}
samplingRecorder := httptest.NewRecorder()
handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
if samplingRecorder.Code != http.StatusOK {
t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String())
}
var samplingBody struct {
Items []domain.SamplingTaskRecord `json:"items"`
}
if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil {
t.Fatalf("decode sampling tasks: %v", err)
}
if len(samplingBody.Items) != 1 {
t.Fatalf("expected one sampling task, got %d body=%s", len(samplingBody.Items), samplingRecorder.Body.String())
}
if len(samplingBody.Items[0].Items) != 1 {
t.Fatalf("expected one sampling item, got %d", len(samplingBody.Items[0].Items))
}
collectionsRecorder := httptest.NewRecorder()
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
if collectionsRecorder.Code != http.StatusOK {
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
}
var collectionsBody struct {
Items []domain.SourceCollectionRecord `json:"items"`
}
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
t.Fatalf("decode source collections: %v", err)
}
if len(collectionsBody.Items) != 1 {
t.Fatalf("expected one source collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
}
if collectionsBody.Items[0].CollectionID != "BJ_001" {
t.Fatalf("unexpected collection id: %s", collectionsBody.Items[0].CollectionID)
}
if collectionsBody.Items[0].CollectionName != "BJ_001" {
t.Fatalf("unexpected collection name fallback: %s", collectionsBody.Items[0].CollectionName)
}
if collectionsBody.Items[0].RunID != "run-scan-1" {
t.Fatalf("expected source collections to ignore synthetic pipeline dataset indexes, got run %s", collectionsBody.Items[0].RunID)
}
if collectionsBody.Items[0].Storage.Provider != "bos" {
t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage)
}
if collectionsBody.Items[0].Storage.Bucket != "datastorage" || collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" {
t.Fatalf("unexpected source storage details: %+v", collectionsBody.Items[0].Storage)
}
detailRecorder := httptest.NewRecorder()
handler.ServeHTTP(detailRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001", nil))
if detailRecorder.Code != http.StatusOK {
t.Fatalf("source detail status: %d body=%s", detailRecorder.Code, detailRecorder.Body.String())
}
var detail domain.SourceCollectionDetail
if err := json.Unmarshal(detailRecorder.Body.Bytes(), &detail); err != nil {
t.Fatalf("decode source detail: %v", err)
}
if len(detail.Entries) != 3 {
t.Fatalf("expected root entries derived from dataset index, got %d body=%s", len(detail.Entries), detailRecorder.Body.String())
}
var videoDirectory *domain.SourceCollectionEntry
for index := range detail.Entries {
if detail.Entries[index].Kind == "directory" && detail.Entries[index].Path == "DJI_001" {
videoDirectory = &detail.Entries[index]
break
}
}
if videoDirectory == nil {
t.Fatalf("expected DJI_001 directory in derived source entries: %+v", detail.Entries)
}
if !videoDirectory.DerivedFromIndex {
t.Fatalf("expected derived directory to be marked from dataset index: %+v", videoDirectory)
}
entryRecorder := httptest.NewRecorder()
handler.ServeHTTP(entryRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/entries?prefix=DJI_001", nil))
if entryRecorder.Code != http.StatusOK {
t.Fatalf("source entries status: %d body=%s", entryRecorder.Code, entryRecorder.Body.String())
}
var nestedDetail domain.SourceCollectionDetail
if err := json.Unmarshal(entryRecorder.Body.Bytes(), &nestedDetail); err != nil {
t.Fatalf("decode nested source entries: %v", err)
}
if len(nestedDetail.Entries) != 1 {
t.Fatalf("expected one nested video entry, got %d body=%s", len(nestedDetail.Entries), entryRecorder.Body.String())
}
videoEntry := nestedDetail.Entries[0]
if videoEntry.Kind != "video" {
t.Fatalf("expected nested entry to be video, got %s", videoEntry.Kind)
}
if videoEntry.Artifact == nil {
t.Fatalf("expected nested video entry artifact")
}
if !videoEntry.DerivedFromIndex {
t.Fatalf("expected nested video entry to be marked from dataset index: %+v", videoEntry)
}
preview := mapValue(videoEntry.Artifact.Data["preview"])
if !strings.Contains(stringValue(preview["url"]), "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=") {
t.Fatalf("expected nested video preview url from archive sources: %+v", videoEntry.Artifact.Data)
}
contentRecorder := httptest.NewRecorder()
contentRequest := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", nil)
handler.ServeHTTP(contentRecorder, contentRequest)
if contentRecorder.Code != http.StatusOK {
t.Fatalf("source content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String())
}
if contentRecorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected source content type: %s", contentRecorder.Header().Get("Content-Type"))
}
if string(contentRecorder.Body.Bytes()) != "video-preview-bytes" {
t.Fatalf("unexpected source content payload: %q", contentRecorder.Body.Bytes())
}
}
func TestSourceCollectionsUseCatalogFieldsForIncrementalScanRuns(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
collection := bson.M{
"id": "BJ_001",
"name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"meta": bson.M{"project_id": "BJ_001"},
"hasVideoMeta": true,
"videoCount": 1,
"fileCount": 4,
"totalBytes": int64(16000036740),
"valid": true,
"requiredFiles": bson.A{"meta.json", "video_meta.json"},
}
catalogEntries := bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/meta.json",
"sizeBytes": 256,
"mimeType": "application/json",
"data": bson.M{"project_id": "BJ_001"},
},
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4",
"sizeBytes": int64(16004000000),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4",
"previewURL": "https://example.com/DJI_20260327132729_0003_D.MP4",
},
},
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.json",
"sizeBytes": 4226,
"mimeType": "application/json",
},
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D_imu.json",
"sizeBytes": int64(36080000),
"mimeType": "application/json",
},
}
run := domain.RunRecord{
ID: "run-incremental-empty",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-incremental-empty-source",
RunID: "run-incremental-empty",
NodeID: "scan-source",
Name: "source-manifest.json",
Type: "json",
Data: map[string]any{
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"scanMode": "INCREMENTAL",
"listedCount": 4,
"newCount": 0,
"catalogEntries": catalogEntries,
},
},
{
ID: "run-incremental-empty-index",
RunID: "run-incremental-empty",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"scanMode": "INCREMENTAL",
"listedCount": 4,
"newCount": 0,
"collectionCount": 0,
"collections": bson.A{},
"items": bson.A{},
"entries": bson.A{},
"catalogCollectionCount": 1,
"catalogVideoCount": 1,
"catalogCollections": bson.A{collection},
"catalogItems": bson.A{},
"catalogEntries": catalogEntries,
},
},
},
CreatedAt: "2026-04-10T12:00:00Z",
UpdatedAt: "2026-04-10T12:00:00Z",
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
collectionsRecorder := httptest.NewRecorder()
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
if collectionsRecorder.Code != http.StatusOK {
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
}
var collectionsBody struct {
Items []domain.SourceCollectionRecord `json:"items"`
}
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
t.Fatalf("decode source collections: %v", err)
}
if len(collectionsBody.Items) != 1 {
t.Fatalf("expected one catalog collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
}
if collectionsBody.Items[0].FileCount != 4 || collectionsBody.Items[0].TotalBytes != 16000036740 {
t.Fatalf("expected catalog file stats, got %+v", collectionsBody.Items[0])
}
entriesRecorder := httptest.NewRecorder()
handler.ServeHTTP(entriesRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-incremental-empty/BJ_001/entries?prefix=DJI_001", nil))
if entriesRecorder.Code != http.StatusOK {
t.Fatalf("entries status: %d body=%s", entriesRecorder.Code, entriesRecorder.Body.String())
}
var detail domain.SourceCollectionDetail
if err := json.Unmarshal(entriesRecorder.Body.Bytes(), &detail); err != nil {
t.Fatalf("decode source detail: %v", err)
}
if len(detail.Entries) != 3 {
t.Fatalf("expected all catalog files under DJI_001, got %d body=%s", len(detail.Entries), entriesRecorder.Body.String())
}
var video *domain.SourceCollectionEntry
for index := range detail.Entries {
if detail.Entries[index].Name == "DJI_20260327132729_0003_D.MP4" {
video = &detail.Entries[index]
break
}
}
if video == nil || video.SizeBytes != 16004000000 {
t.Fatalf("expected real mp4 size from catalog entries, got %+v", video)
}
if video.DerivedFromIndex {
t.Fatalf("expected catalog entry to remain a direct entry, got %+v", video)
}
if collectionsBody.Items[0].Storage.Provider != "bos" {
t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage)
}
if collectionsBody.Items[0].Storage.Bucket != "datastorage" {
t.Fatalf("unexpected source storage bucket: %+v", collectionsBody.Items[0].Storage)
}
if collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" {
t.Fatalf("unexpected source storage prefix: %+v", collectionsBody.Items[0].Storage)
}
}
func TestSourceCollectionsCanLoadExternalizedDatasetIndexPayload(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
indexPayload := map[string]any{
"datasetType": "x-platform-index",
"dataSource": "BOS",
"sourcePath": "bos://datastorage/algo/X-platform-test",
"collectionCount": 0,
"collections": []any{},
"items": []any{},
"entries": []any{},
"catalogCollectionCount": 1,
"catalogCollections": []any{
map[string]any{
"id": "BJ_001",
"name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
"relativePath": "BJ_001",
"videoCount": 1,
"valid": true,
},
},
"catalogItems": []any{},
"catalogEntries": []any{},
}
run := domain.RunRecord{
ID: "run-externalized-index",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
externalizedJSONArtifact(t, "run-externalized-index", "scan-index", "dataset-index.json", indexPayload),
},
CreatedAt: "2026-04-10T12:00:00Z",
UpdatedAt: "2026-04-10T12:00:00Z",
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
collectionsRecorder := httptest.NewRecorder()
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
if collectionsRecorder.Code != http.StatusOK {
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
}
var collectionsBody struct {
Items []domain.SourceCollectionRecord `json:"items"`
}
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
t.Fatalf("decode source collections: %v", err)
}
if len(collectionsBody.Items) != 1 {
t.Fatalf("expected externalized payload to produce one collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
}
}
func TestPackageContentCanServeBrowserPreviewVideo(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品预览项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品预览工作流",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
videoPath := filepath.Join(t.TempDir(), "DJI_0001_D_clip001.mp4")
if err := os.WriteFile(videoPath, []byte("package-video-bytes"), 0o644); err != nil {
t.Fatalf("write package video: %v", err)
}
run := domain.RunRecord{
ID: "run-package-preview",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-package-preview-manifest",
RunID: "run-package-preview",
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": filepath.Dir(videoPath),
},
},
},
},
{
ID: "run-package-preview-video",
RunID: "run-package-preview",
NodeID: "package",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4"),
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"},
},
},
{
ID: "run-package-preview-upload-video",
RunID: "run-package-preview",
NodeID: "upload",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"source": map[string]any{
"path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
},
},
},
},
}
run.Artifacts[1].Path = filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4")
if err := os.MkdirAll(filepath.Dir(run.Artifacts[1].Path), 0o755); err != nil {
t.Fatalf("mkdir package dir: %v", err)
}
if err := os.WriteFile(run.Artifacts[1].Path, []byte("package-video-bytes"), 0o644); err != nil {
t.Fatalf("write package artifact: %v", err)
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
previewPath := filepath.Join(t.TempDir(), "package-browser-preview.mp4")
if err := os.WriteFile(previewPath, []byte("package-browser-preview"), 0o644); err != nil {
t.Fatalf("write preview file: %v", err)
}
originalPreviewResolver := sourceBrowserVideoPreviewPath
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
if request.RedirectURL != run.Artifacts[1].Path {
t.Fatalf("unexpected package preview input: %s", request.RedirectURL)
}
if request.FileName != "DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected file name: %s", request.FileName)
}
return previewPath, nil
}
t.Cleanup(func() {
sourceBrowserVideoPreviewPath = originalPreviewResolver
})
handler := NewHandler(dataStore, nil)
packagesRecorder := httptest.NewRecorder()
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if packagesRecorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
}
var packagesBody struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(packagesBody.Items) != 1 {
t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
}
if packagesBody.Items[0].Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" {
t.Fatalf("expected package storage to prefer BOS upload path, got %+v", packagesBody.Items[0].Storage)
}
packageDetailRecorder := httptest.NewRecorder()
handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-package-preview/DJI_0001_D_clip001_package", nil))
if packageDetailRecorder.Code != http.StatusOK {
t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String())
}
var packageDetail domain.PackageDetail
if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil {
t.Fatalf("decode package detail: %v", err)
}
if packageDetail.Package.Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" {
t.Fatalf("expected package detail storage to prefer BOS upload path, got %+v", packageDetail.Package.Storage)
}
if len(packageDetail.Files) != 1 {
t.Fatalf("expected duplicate package files to collapse to one, got %d", len(packageDetail.Files))
}
if packageDetail.Files[0].Name != "DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected package file after dedupe: %+v", packageDetail.Files[0])
}
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/packages/run-package-preview/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
}
if string(recorder.Body.Bytes()) != "package-browser-preview" {
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
}
}
func TestPackageContentFallsBackToUploadedBOSVideoWhenLocalPackageFileUnavailable(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品上传回退项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "上传 BOS 连接",
Provider: domain.StorageProviderBOS,
Endpoint: "https://bos.example.test",
AccessKeyID: "ak-upload",
SecretAccessKey: "sk-upload",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "成品上传回退工作流",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "upload",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "结果上传",
ConnectionID: connection.ID,
Bucket: "datastorage",
Prefix: "factory/output/test-package-preview",
Hyperparams: []domain.HyperParam{
{Key: "DATA_SOURCE", Value: "BOS"},
{Key: "SOURCE_PATH", Value: "bos://datastorage/factory/output/test-package-preview"},
},
},
},
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-package-upload-fallback",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-package-upload-fallback-manifest",
RunID: "run-package-upload-fallback",
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"items": bson.A{
bson.M{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": "/tmp/droboticflow/missing/package",
},
},
},
},
{
ID: "run-package-upload-fallback-temp-video",
RunID: "run-package-upload-fallback",
NodeID: "package",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "/tmp/droboticflow/missing/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"preview": map[string]any{
"kind": "video",
"url": "/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4",
},
},
},
{
ID: "run-package-upload-fallback-remote-video",
RunID: "run-package-upload-fallback",
NodeID: "upload",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
"source": map[string]any{
"path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
"previewURL": "https://expired.example.com/DJI_0001_D_clip001.mp4",
"mimeType": "video/mp4",
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
previewPath := filepath.Join(t.TempDir(), "package-upload-browser-preview.mp4")
if err := os.WriteFile(previewPath, []byte("package-upload-browser-preview"), 0o644); err != nil {
t.Fatalf("write preview file: %v", err)
}
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak-upload" || scanConfig.SecretAccessKey != "sk-upload" {
t.Fatalf("expected upload connection secrets, got %+v", scanConfig)
}
if sourcePath != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D_clip001.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
originalPreviewResolver := sourceBrowserVideoPreviewPath
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
if request.RedirectURL != "https://signed.example.com/DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected package preview redirect input: %s", request.RedirectURL)
}
if request.FileName != "DJI_0001_D_clip001.mp4" {
t.Fatalf("unexpected file name: %s", request.FileName)
}
return previewPath, nil
}
t.Cleanup(func() {
sourceBrowserVideoPreviewPath = originalPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
}
if string(recorder.Body.Bytes()) != "package-upload-browser-preview" {
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
}
}
func TestPackagesListIncludesWilorResultArtifacts(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "WiLoR 数据资产项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "WiLoR 3D Hand Pose Verification",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
root := t.TempDir()
outputDir := filepath.Join(root, "wilor_outputs")
if err := os.MkdirAll(outputDir, 0o755); err != nil {
t.Fatalf("mkdir outputs: %v", err)
}
imagePath := filepath.Join(outputDir, "wilor-annotated-image.png")
keypointsPath := filepath.Join(outputDir, "wilor-keypoints.json")
meshPath := filepath.Join(outputDir, "wilor-mesh.obj")
if err := os.WriteFile(imagePath, []byte("png"), 0o644); err != nil {
t.Fatalf("write image: %v", err)
}
if err := os.WriteFile(keypointsPath, []byte(`{"hands":2}`), 0o644); err != nil {
t.Fatalf("write keypoints: %v", err)
}
if err := os.WriteFile(meshPath, []byte("v 0 0 0\nf 1 1 1\n"), 0o644); err != nil {
t.Fatalf("write mesh: %v", err)
}
run := domain.RunRecord{
ID: "run-wilor-assets",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
CreatedAt: "2026-04-12T11:30:00Z",
NodeExecutions: []domain.NodeExecutionRecord{
{
NodeID: "wilor-1",
Status: domain.NodeExecutionStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-wilor-assets-result",
RunID: "run-wilor-assets",
NodeID: "wilor-1",
Name: "wilor-result.json",
Type: domain.PortTypeJSON,
Path: filepath.Join(root, "wilor-result.json"),
Data: map[string]any{
"schemaVersion": wilorHandPoseSchemaVersion,
"operator": "wilor_hand_pose",
"status": "success",
"mediaKind": "image",
"source": map[string]any{
"name": "sample-hands.png",
"ref": "sample-hands.png",
},
"service": map[string]any{
"image": "ccr.example/wilor:gpu",
"imageId": "ccr.example/wilor@sha256:test",
"health": map[string]any{
"runtime": map[string]any{
"device": "cuda",
"gpuName": "NVIDIA A800-SXM4-80GB",
},
},
},
"result": map[string]any{
"handCount": 2,
},
"downloadedOutputs": []any{
map[string]any{"name": "wilor-annotated-image.png", "path": imagePath},
map[string]any{"name": "wilor-keypoints.json", "path": keypointsPath},
map[string]any{"name": "wilor-mesh.obj", "path": meshPath},
},
},
},
{
ID: "run-wilor-assets-image",
RunID: "run-wilor-assets",
NodeID: "wilor-1",
Name: "wilor-annotated-image.png",
Type: domain.PortTypeImage,
Size: "3 B",
Path: imagePath,
Metadata: &domain.ArtifactMetadata{
MimeType: "image/png",
},
},
{
ID: "run-wilor-assets-keypoints",
RunID: "run-wilor-assets",
NodeID: "wilor-1",
Name: "wilor-keypoints.json",
Type: domain.PortTypeJSON,
Size: "11 B",
Path: keypointsPath,
Data: map[string]any{"hands": 2},
},
{
ID: "run-wilor-assets-mesh",
RunID: "run-wilor-assets",
NodeID: "wilor-1",
Name: "wilor-mesh.obj",
Type: domain.PortTypeFile,
Size: "16 B",
Path: meshPath,
Metadata: &domain.ArtifactMetadata{
MimeType: "text/plain; charset=utf-8",
},
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
packagesRecorder := httptest.NewRecorder()
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if packagesRecorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
}
var packagesBody struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(packagesBody.Items) != 1 {
t.Fatalf("expected one WiLoR package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
}
if packagesBody.Items[0].PackageName != "wilor-sample-hands-wilor-1" {
t.Fatalf("unexpected package name: %+v", packagesBody.Items[0])
}
if packagesBody.Items[0].SourceVideo != "sample-hands.png" || packagesBody.Items[0].FileCount != 4 {
t.Fatalf("unexpected WiLoR package summary: %+v", packagesBody.Items[0])
}
detailRecorder := httptest.NewRecorder()
handler.ServeHTTP(detailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-wilor-assets/wilor-sample-hands-wilor-1", nil))
if detailRecorder.Code != http.StatusOK {
t.Fatalf("package detail status: %d body=%s", detailRecorder.Code, detailRecorder.Body.String())
}
var detail domain.PackageDetail
if err := json.Unmarshal(detailRecorder.Body.Bytes(), &detail); err != nil {
t.Fatalf("decode detail: %v", err)
}
if len(detail.Files) != 4 {
t.Fatalf("expected four WiLoR package files, got %d", len(detail.Files))
}
var meshFile *domain.ExecutionArtifact
for index := range detail.Files {
if detail.Files[index].Name == "wilor-mesh.obj" {
meshFile = &detail.Files[index]
break
}
}
if meshFile == nil {
t.Fatalf("expected mesh file in package detail: %+v", detail.Files)
}
if meshFile.Path != "wilor-mesh.obj" {
t.Fatalf("expected mesh relative path, got %+v", meshFile)
}
if !strings.Contains(stringValue(meshFile.Data["contentURL"]), "/api/packages/run-wilor-assets/wilor-sample-hands-wilor-1/content?relativePath=wilor-mesh.obj") {
t.Fatalf("expected mesh contentURL to use package content api, got %+v", meshFile.Data)
}
contentRecorder := httptest.NewRecorder()
handler.ServeHTTP(contentRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-wilor-assets/wilor-sample-hands-wilor-1/content?relativePath=wilor-mesh.obj", nil))
if contentRecorder.Code != http.StatusOK {
t.Fatalf("mesh content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String())
}
if string(contentRecorder.Body.Bytes()) != "v 0 0 0\nf 1 1 1\n" {
t.Fatalf("unexpected mesh content: %q", contentRecorder.Body.Bytes())
}
}
func TestWilorResultArtifactsPrefersUploadedDuplicate(t *testing.T) {
payload := map[string]any{
"schemaVersion": wilorHandPoseSchemaVersion,
"mediaKind": "image",
"source": map[string]any{
"name": "sample-hands.png",
"ref": "https://example.test/sample-hands.png",
},
"result": map[string]any{"handCount": 2},
}
run := domain.RunRecord{
ID: "run-wilor-duplicates",
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-wilor-duplicates-wilor",
RunID: "run-wilor-duplicates",
NodeID: "wilor-1",
Name: "wilor-result.json",
Type: domain.PortTypeJSON,
Data: payload,
},
{
ID: "run-wilor-duplicates-upload",
RunID: "run-wilor-duplicates",
NodeID: "upload-1",
Name: "wilor-result.json",
Type: domain.PortTypeJSON,
Data: payload,
},
},
}
items := wilorResultArtifacts(run)
if len(items) != 1 {
t.Fatalf("expected one deduplicated WiLoR result, got %+v", items)
}
if items[0].NodeID != "upload-1" {
t.Fatalf("expected uploaded duplicate to win, got %+v", items[0])
}
}
func TestPackagesListKeepsLatestPackageVersionPerWorkflow(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "包去重项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "包去重工作流",
Type: domain.WorkflowTypeBatch,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
makeRun := func(id string, createdAt string, storagePath string) domain.RunRecord {
return domain.RunRecord{
ID: id,
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
CreatedAt: createdAt,
Artifacts: []domain.ExecutionArtifact{
{
ID: id + "-manifest",
RunID: id,
NodeID: "package",
Name: "packages-manifest.json",
Type: "json",
Data: map[string]any{
"items": []any{
map[string]any{
"packageName": "DJI_0001_D_clip001_package",
"clipId": "DJI_0001_D_clip001",
"sourceVideo": "DJI_0001_D.mp4",
"fileCount": 1,
"storagePath": "/tmp/k8s-job/package",
},
},
},
},
{
ID: id + "-upload-video",
RunID: id,
NodeID: "upload",
Name: "DJI_0001_D_clip001.mp4",
Type: "video",
Path: storagePath + "/DJI_0001_D_clip001.mp4",
Metadata: &domain.ArtifactMetadata{
MimeType: "video/mp4",
},
Data: map[string]any{
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
"source": map[string]any{
"path": storagePath + "/DJI_0001_D_clip001.mp4",
"mimeType": "video/mp4",
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
},
},
},
},
}
}
if err := dataStore.UpsertRun(ctx, makeRun(
"run-package-old",
"2026-04-12T10:00:00Z",
"bos://datastorage/factory/output/old/packages/DJI_0001_D_clip001_package",
)); err != nil {
t.Fatalf("upsert old run: %v", err)
}
if err := dataStore.UpsertRun(ctx, makeRun(
"run-package-new",
"2026-04-12T11:00:00Z",
"bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package",
)); err != nil {
t.Fatalf("upsert new run: %v", err)
}
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("packages status: %d body=%s", recorder.Code, recorder.Body.String())
}
var body struct {
Items []domain.PackageRecord `json:"items"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
t.Fatalf("decode packages: %v", err)
}
if len(body.Items) != 1 {
t.Fatalf("expected latest package only, got %d body=%s", len(body.Items), recorder.Body.String())
}
if body.Items[0].RunID != "run-package-new" {
t.Fatalf("expected latest run package, got %+v", body.Items[0])
}
if body.Items[0].Storage.Path != "bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package" {
t.Fatalf("expected latest BOS storage path, got %+v", body.Items[0].Storage)
}
}
func TestSourceCollectionContentCanServeBrowserPreviewVideo(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "浏览器预览项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
Endpoint: "https://bos.example.test",
AccessKeyID: "ak",
SecretAccessKey: "sk",
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-browser-preview",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-browser-preview-index",
RunID: "run-browser-preview",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"catalogCollections": bson.A{
bson.M{
"id": "BJ_001",
"name": "BJ_001",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"videoCount": 1,
"fileCount": 1,
"totalBytes": int64(2048),
"valid": true,
},
},
"catalogEntries": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"sizeBytes": int64(2048),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"previewURL": "https://example.com/DJI_0001_D.mp4",
},
},
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
previewPath := filepath.Join(t.TempDir(), "preview.mp4")
if err := os.WriteFile(previewPath, []byte("browser-preview-video"), 0o644); err != nil {
t.Fatalf("write preview file: %v", err)
}
originalPreviewResolver := sourceBrowserVideoPreviewPath
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
if request.RedirectURL != "https://signed.example.com/DJI_0001_D.mp4" {
t.Fatalf("unexpected redirect url: %s", request.RedirectURL)
}
if request.FileName != "DJI_0001_D.mp4" {
t.Fatalf("unexpected file name: %s", request.FileName)
}
return previewPath, nil
}
t.Cleanup(func() {
sourceBrowserVideoPreviewPath = originalPreviewResolver
})
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" {
t.Fatalf("unexpected scan config: %+v", scanConfig)
}
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/source-data/collections/run-browser-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4&transcode=browser",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Content-Type") != "video/mp4" {
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
}
if string(recorder.Body.Bytes()) != "browser-preview-video" {
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
}
}
func TestSourceCollectionContentRefreshesBOSRedirectURL(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 刷新项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
Endpoint: "https://bos.example.test",
AccessKeyID: "ak",
SecretAccessKey: "sk",
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-refresh-bos-preview",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-refresh-bos-preview-index",
RunID: "run-refresh-bos-preview",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"catalogCollections": bson.A{
bson.M{
"id": "BJ_001",
"name": "BJ_001",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"videoCount": 1,
"fileCount": 1,
"totalBytes": int64(2048),
"valid": true,
},
},
"catalogEntries": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"sizeBytes": int64(2048),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"previewURL": "https://expired.example.com/DJI_0001_D.mp4",
},
},
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" {
t.Fatalf("unexpected scan config: %+v", scanConfig)
}
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/source-data/collections/run-refresh-bos-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusTemporaryRedirect {
t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String())
}
if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" {
t.Fatalf("unexpected redirect location: %s", location)
}
}
func TestSourceCollectionContentRefreshesBOSRedirectURLFromStorageConnection(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 连接预览项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS 主连接",
Provider: domain.StorageProviderBOS,
Endpoint: "https://bos.example.test",
AccessKeyID: "ak-from-connection",
SecretAccessKey: "sk-from-connection",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "BOS 增量扫描",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: "BOS",
ConnectionID: connection.ID,
},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run := domain.RunRecord{
ID: "run-refresh-bos-preview-connection",
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
ProjectID: project.ID,
ProjectName: project.Name,
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{
ID: "run-refresh-bos-preview-connection-index",
RunID: "run-refresh-bos-preview-connection",
NodeID: "scan-index",
Name: "dataset-index.json",
Type: "json",
Data: map[string]any{
"catalogCollections": bson.A{
bson.M{
"id": "BJ_001",
"name": "BJ_001",
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
"relativePath": "BJ_001",
"videoCount": 1,
"fileCount": 1,
"totalBytes": int64(2048),
"valid": true,
},
},
"catalogEntries": bson.A{
bson.M{
"collectionId": "BJ_001",
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
"sizeBytes": int64(2048),
"mimeType": "video/mp4",
"source": bson.M{
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
"previewURL": "https://expired.example.com/DJI_0001_D.mp4",
},
},
},
},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil || scanConfig.AccessKeyID != "ak-from-connection" || scanConfig.SecretAccessKey != "sk-from-connection" {
t.Fatalf("expected resolved connection secrets, got %+v", scanConfig)
}
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
t.Fatalf("unexpected source path: %s", sourcePath)
}
return "https://signed.example.com/DJI_0001_D.mp4", nil
}
t.Cleanup(func() {
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
})
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(
http.MethodGet,
"/api/source-data/collections/run-refresh-bos-preview-connection/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4",
nil,
)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusTemporaryRedirect {
t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String())
}
if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" {
t.Fatalf("unexpected redirect location: %s", location)
}
}