2312 lines
81 KiB
Go
2312 lines
81 KiB
Go
package api
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"math"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
)
|
|
|
|
func TestPackageRecordFallsBackToDurationAndAnnotationSourceVideo(t *testing.T) {
|
|
record := packageRecordFromManifest(
|
|
domain.RunRecord{ID: "run-1", WorkflowID: "workflow-1", ProjectID: "project-1"},
|
|
map[string]any{
|
|
"packageName": "package-1",
|
|
"duration": 2.001719,
|
|
"annotation": map[string]any{"sourceVideo": "blurred-source.mp4"},
|
|
},
|
|
nil,
|
|
nil,
|
|
)
|
|
if record.ClipDuration != 2.001719 || record.SourceVideo != "blurred-source.mp4" || stringValue(record.Quality["source_video"]) != "blurred-source.mp4" {
|
|
t.Fatalf("package projection did not use manifest fallbacks: %#v", record)
|
|
}
|
|
}
|
|
|
|
func tarDataURL(t *testing.T, files map[string][]byte) string {
|
|
t.Helper()
|
|
var buffer bytes.Buffer
|
|
writer := tar.NewWriter(&buffer)
|
|
for name, payload := range files {
|
|
header := &tar.Header{
|
|
Name: name,
|
|
Mode: 0o644,
|
|
Size: int64(len(payload)),
|
|
}
|
|
if err := writer.WriteHeader(header); err != nil {
|
|
t.Fatalf("write tar header: %v", err)
|
|
}
|
|
if _, err := writer.Write(payload); err != nil {
|
|
t.Fatalf("write tar payload: %v", err)
|
|
}
|
|
}
|
|
if err := writer.Close(); err != nil {
|
|
t.Fatalf("close tar writer: %v", err)
|
|
}
|
|
return "data:application/x-tar;base64," + base64.StdEncoding.EncodeToString(buffer.Bytes())
|
|
}
|
|
|
|
func externalizedJSONArtifact(t *testing.T, runID string, nodeID string, name string, payload map[string]any) domain.ExecutionArtifact {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
payloadPath := filepath.Join(root, name)
|
|
payloadBytes, err := json.Marshal(payload)
|
|
if err != nil {
|
|
t.Fatalf("marshal payload: %v", err)
|
|
}
|
|
if err := os.WriteFile(payloadPath, payloadBytes, 0o644); err != nil {
|
|
t.Fatalf("write payload: %v", err)
|
|
}
|
|
manifestPath := filepath.Join(root, strings.TrimSuffix(name, filepath.Ext(name))+"-manifest.json")
|
|
manifestBytes, err := json.Marshal(map[string]any{
|
|
"kind": "dataset",
|
|
"datasetType": payload["datasetType"],
|
|
"payloadPath": payloadPath,
|
|
"summary": map[string]any{"name": name},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("marshal manifest: %v", err)
|
|
}
|
|
if err := os.WriteFile(manifestPath, manifestBytes, 0o644); err != nil {
|
|
t.Fatalf("write manifest: %v", err)
|
|
}
|
|
return domain.ExecutionArtifact{
|
|
ID: runID + "-" + strings.TrimSuffix(name, filepath.Ext(name)),
|
|
RunID: runID,
|
|
NodeID: nodeID,
|
|
Name: name,
|
|
Type: domain.PortTypeJSON,
|
|
Path: payloadPath,
|
|
Data: map[string]any{
|
|
"datasetType": payload["datasetType"],
|
|
"dataRef": map[string]any{
|
|
"kind": "dataset",
|
|
"datasetType": payload["datasetType"],
|
|
"manifestPath": manifestPath,
|
|
"payloadPath": payloadPath,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestDataPlatformIntegerHelpersRejectLossyValues(t *testing.T) {
|
|
if got := intValue(float64(4)); got != 4 {
|
|
t.Fatalf("expected integral float to parse, got %d", got)
|
|
}
|
|
if got := intValue(4.9); got != 0 {
|
|
t.Fatalf("expected lossy float to be ignored, got %d", got)
|
|
}
|
|
if got := int64Value(int64(16004000000)); got != 16004000000 {
|
|
t.Fatalf("expected int64 to be preserved, got %d", got)
|
|
}
|
|
if got := int64Value(math.NaN()); got != 0 {
|
|
t.Fatalf("expected NaN to be ignored, got %d", got)
|
|
}
|
|
if got := int64Value(float64(1 << 53)); got != 0 {
|
|
t.Fatalf("expected unsafe float integer to be ignored, got %d", got)
|
|
}
|
|
if got := int64Value(uint64(math.MaxInt64) + 1); got != 0 {
|
|
t.Fatalf("expected overflowing uint64 to be ignored, got %d", got)
|
|
}
|
|
if got := intValue(json.Number("42")); got != 42 {
|
|
t.Fatalf("expected integer json.Number to parse, got %d", got)
|
|
}
|
|
if got := intValue(json.Number("42.5")); got != 0 {
|
|
t.Fatalf("expected decimal json.Number to be ignored, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestArtifactStructuredDataTreatsLocalPayloadPathAsAuthoritative(t *testing.T) {
|
|
missingPayloadPath := filepath.Join(t.TempDir(), "missing-payload.json")
|
|
missingArtifact := domain.ExecutionArtifact{
|
|
Name: "source-manifest.json",
|
|
Type: domain.PortTypeJSON,
|
|
Data: map[string]any{
|
|
"datasetType": "x-platform",
|
|
"items": []any{map[string]any{"id": "stale-inline"}},
|
|
"dataRef": map[string]any{
|
|
"payloadPath": missingPayloadPath,
|
|
},
|
|
},
|
|
}
|
|
if got := artifactStructuredData(&missingArtifact); got != nil {
|
|
t.Fatalf("expected missing external payload to suppress stale inline data, got %#v", got)
|
|
}
|
|
|
|
invalidPayloadPath := filepath.Join(t.TempDir(), "payload.json")
|
|
if err := os.WriteFile(invalidPayloadPath, []byte("{"), 0o644); err != nil {
|
|
t.Fatalf("write invalid payload: %v", err)
|
|
}
|
|
invalidArtifact := missingArtifact
|
|
invalidArtifact.Data = map[string]any{
|
|
"datasetType": "x-platform",
|
|
"items": []any{map[string]any{"id": "stale-inline"}},
|
|
"dataRef": map[string]any{
|
|
"payloadPath": invalidPayloadPath,
|
|
},
|
|
}
|
|
if got := artifactStructuredData(&invalidArtifact); got != nil {
|
|
t.Fatalf("expected invalid external payload to suppress stale inline data, got %#v", got)
|
|
}
|
|
}
|
|
|
|
func TestSamplingRulesFromManifestRejectsInvalidRuleShape(t *testing.T) {
|
|
validRules, ok := samplingRulesFromManifest(map[string]any{
|
|
"rules": []any{
|
|
map[string]any{
|
|
"id": "clarity",
|
|
"field": "quality.clarity",
|
|
"operator": "GTE",
|
|
"value": 0.3,
|
|
"samplePercent": json.Number("12.5"),
|
|
"matchedCount": 4,
|
|
"sampledCount": 2,
|
|
},
|
|
},
|
|
})
|
|
if !ok || len(validRules) != 1 {
|
|
t.Fatalf("expected valid sampling rule, got ok=%v rules=%#v", ok, validRules)
|
|
}
|
|
if validRules[0].Operator != "gte" || validRules[0].SamplePercent != 12.5 {
|
|
t.Fatalf("expected normalized operator and sample percent, got %#v", validRules[0])
|
|
}
|
|
|
|
cases := map[string]any{
|
|
"bool": true,
|
|
"zero": 0,
|
|
"over-100": 101,
|
|
"nan": math.NaN(),
|
|
"string": "10",
|
|
"bad-number": json.Number("nan"),
|
|
"unsupported": nil,
|
|
}
|
|
for name, samplePercent := range cases {
|
|
rule := map[string]any{
|
|
"id": "clarity",
|
|
"field": "quality.clarity",
|
|
"operator": "gte",
|
|
"value": 0.3,
|
|
"samplePercent": samplePercent,
|
|
}
|
|
if name == "unsupported" {
|
|
rule["operator"] = "regex"
|
|
rule["samplePercent"] = 10
|
|
}
|
|
if rules, ok := samplingRulesFromManifest(map[string]any{"rules": []any{rule}}); ok {
|
|
t.Fatalf("expected %s rule to be rejected, got %#v", name, rules)
|
|
}
|
|
}
|
|
|
|
duplicate := map[string]any{
|
|
"rules": []any{
|
|
map[string]any{"id": "clarity", "field": "quality.clarity", "operator": "gte", "value": 0.3, "samplePercent": 10},
|
|
map[string]any{"id": "clarity", "field": "quality.shake", "operator": "lte", "value": 0.1, "samplePercent": 10},
|
|
},
|
|
}
|
|
if rules, ok := samplingRulesFromManifest(duplicate); ok {
|
|
t.Fatalf("expected duplicate rules to be rejected, got %#v", rules)
|
|
}
|
|
}
|
|
|
|
func TestSamplingTasksSkipInvalidManifestRuleSamplePercent(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "坏抽检规则项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "坏抽检规则工作流",
|
|
Type: domain.WorkflowTypeBatch,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-invalid-sampling-rule",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-invalid-sampling-rule-artifact",
|
|
RunID: "run-invalid-sampling-rule",
|
|
NodeID: "sample",
|
|
Name: "audit-samples.json",
|
|
Type: domain.PortTypeJSON,
|
|
Data: map[string]any{
|
|
"datasetType": "x-platform-audit-samples",
|
|
"rules": []any{
|
|
map[string]any{
|
|
"id": "clarity",
|
|
"field": "quality.clarity",
|
|
"operator": "gte",
|
|
"value": 0.3,
|
|
"samplePercent": true,
|
|
"matchedCount": 1,
|
|
"sampledCount": 1,
|
|
},
|
|
},
|
|
"items": []any{
|
|
map[string]any{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"sampleRuleId": "clarity",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
CreatedAt: "2026-04-10T10:05:00Z",
|
|
UpdatedAt: "2026-04-10T10:05:00Z",
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
recorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("sampling status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var body struct {
|
|
Items []domain.SamplingTaskRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
|
|
t.Fatalf("decode sampling tasks: %v", err)
|
|
}
|
|
if len(body.Items) != 0 {
|
|
t.Fatalf("expected invalid sampling manifest to be skipped, got %#v", body.Items)
|
|
}
|
|
}
|
|
|
|
func TestScanConfigBOSSecretsAreReturnedAndPreserved(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} {
|
|
if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
|
|
Name: name,
|
|
Type: "builtin",
|
|
Category: "io",
|
|
Description: name,
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}},
|
|
}); err != nil {
|
|
t.Fatalf("create operator %s: %v", name, err)
|
|
}
|
|
}
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
createPayload := `{
|
|
"name":"BOS 扫描",
|
|
"description":"扫描真实 BOS 目录",
|
|
"dataSource":"BOS",
|
|
"bucket":"datastorage",
|
|
"prefix":"algo/X-platform-test/",
|
|
"endpoint":"https://bos.example.test",
|
|
"accessKeyId":"ak-example-1234",
|
|
"secretAccessKey":"sk-example-5678",
|
|
"interval":86400,
|
|
"enabled":true
|
|
}`
|
|
|
|
createRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create scan config status: %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
var created domain.WorkflowRecord
|
|
if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil {
|
|
t.Fatalf("decode created scan config: %v", err)
|
|
}
|
|
if created.ScanConfig == nil {
|
|
t.Fatalf("expected scan config")
|
|
}
|
|
if created.ScanConfig.AccessKeyID != "" || created.ScanConfig.SecretAccessKey != "" {
|
|
t.Fatalf("scan config secrets should be masked in workflow responses: %+v", created.ScanConfig)
|
|
}
|
|
if !created.ScanConfig.HasAccessKeyID || !created.ScanConfig.HasSecretAccessKey {
|
|
t.Fatalf("expected secret presence flags in API response: %+v", created.ScanConfig)
|
|
}
|
|
if created.ScanConfig.AccessKeyPreview == "" || created.ScanConfig.SecretAccessKeyPreview == "" {
|
|
t.Fatalf("expected secret previews in API response: %+v", created.ScanConfig)
|
|
}
|
|
|
|
stored, err := dataStore.GetWorkflow(ctx, created.ID)
|
|
if err != nil {
|
|
t.Fatalf("get stored workflow: %v", err)
|
|
}
|
|
if stored == nil || stored.ScanConfig == nil {
|
|
t.Fatalf("expected stored scan config")
|
|
}
|
|
if stored.ScanConfig.AccessKeyID != "ak-example-1234" || stored.ScanConfig.SecretAccessKey != "sk-example-5678" {
|
|
t.Fatalf("expected secrets preserved in store: %+v", stored.ScanConfig)
|
|
}
|
|
|
|
updatePayload := `{
|
|
"name":"BOS 扫描更新",
|
|
"description":"保留原始密钥",
|
|
"dataSource":"BOS",
|
|
"bucket":"datastorage",
|
|
"prefix":"algo/X-platform-test/",
|
|
"endpoint":"https://bos.example.test",
|
|
"interval":3600,
|
|
"enabled":false
|
|
}`
|
|
updateRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(updateRecorder, httptest.NewRequest(http.MethodPut, "/api/scan-configs/"+created.ID, strings.NewReader(updatePayload)))
|
|
if updateRecorder.Code != http.StatusOK {
|
|
t.Fatalf("update scan config status: %d body=%s", updateRecorder.Code, updateRecorder.Body.String())
|
|
}
|
|
|
|
updated, err := dataStore.GetWorkflow(ctx, created.ID)
|
|
if err != nil {
|
|
t.Fatalf("get updated workflow: %v", err)
|
|
}
|
|
if updated == nil || updated.ScanConfig == nil {
|
|
t.Fatalf("expected updated scan config")
|
|
}
|
|
if updated.ScanConfig.AccessKeyID != "ak-example-1234" || updated.ScanConfig.SecretAccessKey != "sk-example-5678" {
|
|
t.Fatalf("expected secrets preserved across updates: %+v", updated.ScanConfig)
|
|
}
|
|
}
|
|
|
|
func TestStorageConnectionDeleteRefusesWhileReferencedAndAllowsAfterClearing(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
createRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/storage/connections", strings.NewReader(`{
|
|
"name":"BOS Workspace",
|
|
"provider":"BOS",
|
|
"endpoint":"https://bj.bcebos.com",
|
|
"bucket":"testxyz",
|
|
"accessKeyId":"ak-example",
|
|
"secretAccessKey":"sk-example"
|
|
}`)))
|
|
if createRecorder.Code != http.StatusCreated {
|
|
t.Fatalf("create connection status: %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
|
}
|
|
var created domain.StorageConnectionRecord
|
|
if err := json.Unmarshal(createRecorder.Body.Bytes(), &created); err != nil {
|
|
t.Fatalf("decode created connection: %v", err)
|
|
}
|
|
|
|
// Workflow in the same org whose node still references the connection.
|
|
workflow := domain.WorkflowRecord{
|
|
ID: "workflow-ref-test",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: "project-ref-test",
|
|
Name: "引用存储连接的工作流",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{ID: "op-bos-scan", Data: domain.WorkflowNodeData{Label: "BOS 增量扫描", ConnectionID: created.ID}},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
|
|
t.Fatalf("upsert referencing workflow: %v", err)
|
|
}
|
|
|
|
refusedRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(refusedRecorder, httptest.NewRequest(http.MethodDelete, "/api/storage/connections/"+created.ID, nil))
|
|
if refusedRecorder.Code != http.StatusConflict {
|
|
t.Fatalf("delete referenced connection status: got %d want %d body=%s", refusedRecorder.Code, http.StatusConflict, refusedRecorder.Body.String())
|
|
}
|
|
var refused map[string]any
|
|
if err := json.Unmarshal(refusedRecorder.Body.Bytes(), &refused); err != nil {
|
|
t.Fatalf("decode refused body: %v", err)
|
|
}
|
|
refs, _ := refused["referencedBy"].([]any)
|
|
if len(refs) != 1 {
|
|
t.Fatalf("expected 1 referencedBy entry, got %v", refs)
|
|
}
|
|
first, _ := refs[0].(map[string]any)
|
|
if first["workflowId"] != "workflow-ref-test" || first["nodeId"] != "op-bos-scan" {
|
|
t.Fatalf("unexpected referencedBy entry: %v", first)
|
|
}
|
|
stillExists, err := dataStore.GetStorageConnection(ctx, created.ID)
|
|
if err != nil || stillExists == nil {
|
|
t.Fatalf("connection should still exist after refused delete: %v %v", stillExists, err)
|
|
}
|
|
|
|
// Clear the reference → delete must succeed.
|
|
workflow.Nodes[0].Data.ConnectionID = ""
|
|
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
|
|
t.Fatalf("upsert cleared workflow: %v", err)
|
|
}
|
|
okRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(okRecorder, httptest.NewRequest(http.MethodDelete, "/api/storage/connections/"+created.ID, nil))
|
|
if okRecorder.Code != http.StatusNoContent {
|
|
t.Fatalf("delete after clearing status: got %d want %d body=%s", okRecorder.Code, http.StatusNoContent, okRecorder.Body.String())
|
|
}
|
|
gone, err := dataStore.GetStorageConnection(ctx, created.ID)
|
|
if err != nil || gone != nil {
|
|
t.Fatalf("connection should be gone after delete: %v %v", gone, err)
|
|
}
|
|
}
|
|
|
|
func TestPackagesAndSamplingCanLoadExternalizedManifestPayloads(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "外置成品项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "成品工作流",
|
|
Type: domain.WorkflowTypeBatch,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
packagesPayload := map[string]any{
|
|
"datasetType": "x-platform-packages",
|
|
"packageCount": 1,
|
|
"items": []any{
|
|
map[string]any{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"fileCount": 1,
|
|
"storagePath": "/tmp/k8s-job/package",
|
|
},
|
|
},
|
|
}
|
|
samplingPayload := map[string]any{
|
|
"datasetType": "x-platform-audit-samples",
|
|
"rules": []any{
|
|
map[string]any{
|
|
"id": "confidence-check",
|
|
"field": "annotation.minConfidence",
|
|
"operator": "lt",
|
|
"value": 0.7,
|
|
"samplePercent": 10,
|
|
"matchedCount": 1,
|
|
"sampledCount": 1,
|
|
},
|
|
},
|
|
"items": []any{
|
|
map[string]any{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"sampleRuleId": "confidence-check",
|
|
"storagePath": "/tmp/k8s-job/package",
|
|
},
|
|
},
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-externalized-package",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
externalizedJSONArtifact(t, "run-externalized-package", "package", "packages-manifest.json", packagesPayload),
|
|
externalizedJSONArtifact(t, "run-externalized-package", "sample", "audit-samples.json", samplingPayload),
|
|
},
|
|
CreatedAt: "2026-04-10T10:05:00Z",
|
|
UpdatedAt: "2026-04-10T10:05:00Z",
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
packagesRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
|
|
if packagesRecorder.Code != http.StatusOK {
|
|
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
|
|
}
|
|
var packagesBody struct {
|
|
Items []domain.PackageRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
|
|
t.Fatalf("decode packages: %v", err)
|
|
}
|
|
if len(packagesBody.Items) != 1 {
|
|
t.Fatalf("expected one package from externalized manifest, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
|
|
}
|
|
|
|
samplingRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
|
|
if samplingRecorder.Code != http.StatusOK {
|
|
t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String())
|
|
}
|
|
var samplingBody struct {
|
|
Items []domain.SamplingTaskRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil {
|
|
t.Fatalf("decode sampling tasks: %v", err)
|
|
}
|
|
if len(samplingBody.Items) != 1 || len(samplingBody.Items[0].Items) != 1 {
|
|
t.Fatalf("expected one sampling task with one item from externalized manifest, got %#v", samplingBody.Items)
|
|
}
|
|
}
|
|
|
|
func TestScanConfigCreateDraftDoesNotEnsureSystemProject(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
for _, name := range []string{"BOS 增量扫描", "X 平台数据集索引", "采集目录校验"} {
|
|
if _, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
|
|
Name: name,
|
|
Type: "builtin",
|
|
Category: "io",
|
|
Description: name,
|
|
Inputs: []domain.PortDefinition{{ID: "in-1", Name: "input", Type: domain.PortTypeJSON}},
|
|
Outputs: []domain.PortDefinition{{ID: "out-1", Name: "output", Type: domain.PortTypeJSON}},
|
|
}); err != nil {
|
|
t.Fatalf("create operator %s: %v", name, err)
|
|
}
|
|
}
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
createPayload := `{
|
|
"name":"BOS 草稿扫描",
|
|
"description":"创建后再配 bucket/prefix",
|
|
"dataSource":"BOS",
|
|
"interval":0,
|
|
"enabled":false
|
|
}`
|
|
recorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
|
|
if recorder.Code != http.StatusCreated {
|
|
t.Fatalf("create draft scan config status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
|
|
var workflow domain.WorkflowRecord
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &workflow); err != nil {
|
|
t.Fatalf("decode draft workflow: %v", err)
|
|
}
|
|
if workflow.ProjectID != "" {
|
|
t.Fatalf("expected projectless scan workflow, got projectId=%q", workflow.ProjectID)
|
|
}
|
|
if workflow.Kind != domain.WorkflowKindScan {
|
|
t.Fatalf("expected scan workflow kind, got %s", workflow.Kind)
|
|
}
|
|
|
|
projects, err := dataStore.ListProjects(ctx)
|
|
if err != nil {
|
|
t.Fatalf("list projects: %v", err)
|
|
}
|
|
if len(projects) != 0 {
|
|
t.Fatalf("expected no auto-created system project, got %d", len(projects))
|
|
}
|
|
}
|
|
|
|
func TestScanConfigCreateRejectsInvalidDataSource(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
createPayload := `{
|
|
"name":"非法数据源扫描",
|
|
"description":"invalid data source",
|
|
"dataSource":"ROW",
|
|
"interval":0,
|
|
"enabled":false
|
|
}`
|
|
recorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/scan-configs", strings.NewReader(createPayload)))
|
|
if recorder.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected validation failure, got %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if !strings.Contains(recorder.Body.String(), `"field":"dataSource"`) ||
|
|
!strings.Contains(recorder.Body.String(), "dataSource must be BOS for scan workflows") {
|
|
t.Fatalf("expected dataSource validation issue, got %s", recorder.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestDataPlatformHandlersSupportMongoStyleArtifactData(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "数据资产项目",
|
|
Description: "用于数据资产 API 测试",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
scanWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "目录扫描",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindScan,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create scan workflow: %v", err)
|
|
}
|
|
|
|
pipelineWorkflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "成品流水线",
|
|
Type: domain.WorkflowTypeBatch,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create pipeline workflow: %v", err)
|
|
}
|
|
|
|
scanRun := domain.RunRecord{
|
|
ID: "run-scan-1",
|
|
WorkflowID: scanWorkflow.ID,
|
|
WorkflowName: scanWorkflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-scan-1-source",
|
|
RunID: "run-scan-1",
|
|
NodeID: "scan-source",
|
|
Name: "source-manifest.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test",
|
|
"listedCount": 4,
|
|
"newCount": 4,
|
|
},
|
|
},
|
|
{
|
|
ID: "run-scan-1-index",
|
|
RunID: "run-scan-1",
|
|
NodeID: "scan-index",
|
|
Name: "dataset-index.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"archiveSources": bson.M{
|
|
"archive-1": bson.M{
|
|
"name": "xplatform-dataset.tar",
|
|
"previewURL": tarDataURL(t, map[string][]byte{
|
|
"BJ_001/DJI_001/DJI_0001_D.mp4": []byte("video-preview-bytes"),
|
|
}),
|
|
},
|
|
},
|
|
"collectionCount": 1,
|
|
"collections": bson.A{
|
|
bson.M{
|
|
"id": "BJ_001",
|
|
"path": "/tmp/run-scan-1/index/_materialized_inputs/BJ_001",
|
|
"relativePath": "BJ_001",
|
|
"meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"},
|
|
"hasIntrinsics": true,
|
|
"hasVideoMeta": true,
|
|
"requiredFiles": bson.A{"meta.json", "video_meta.json"},
|
|
"missingRequiredFiles": bson.A{},
|
|
"videoCount": 1,
|
|
"fileCount": 4,
|
|
"totalBytes": 1234,
|
|
"valid": true,
|
|
},
|
|
},
|
|
"items": bson.A{
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"filename": "DJI_0001_D.mp4",
|
|
"codec": "hevc",
|
|
"resolution": "3840x2160",
|
|
"frameRate": 29.97,
|
|
"duration": 41.8,
|
|
"sizeBytes": 2048,
|
|
"meta": bson.M{"type": "家庭", "place": "蓟门里小区", "date": "20260319"},
|
|
"source": bson.M{
|
|
"archiveKey": "archive-1",
|
|
"archiveMemberPath": "BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"mimeType": "video/mp4",
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
CreatedAt: "2026-04-10T10:00:00Z",
|
|
UpdatedAt: "2026-04-10T10:00:00Z",
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, scanRun); err != nil {
|
|
t.Fatalf("upsert scan run: %v", err)
|
|
}
|
|
|
|
pipelineRun := domain.RunRecord{
|
|
ID: "run-pipeline-1",
|
|
WorkflowID: pipelineWorkflow.ID,
|
|
WorkflowName: pipelineWorkflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-pipeline-1-packages",
|
|
RunID: "run-pipeline-1",
|
|
NodeID: "package",
|
|
Name: "packages-manifest.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"packageCount": 1,
|
|
"items": bson.A{
|
|
bson.M{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"clipDuration": 300,
|
|
"fileCount": 4,
|
|
"pipelineTags": bson.A{"quality_v1", "annotation_v1"},
|
|
"storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package",
|
|
"quality": bson.M{"clarity": 0.92, "shake": 0.08},
|
|
"annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-pipeline-1-audit",
|
|
RunID: "run-pipeline-1",
|
|
NodeID: "sample",
|
|
Name: "audit-samples.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"rules": bson.A{
|
|
bson.M{
|
|
"id": "confidence-check",
|
|
"field": "annotation.minConfidence",
|
|
"operator": "lt",
|
|
"value": 0.7,
|
|
"samplePercent": 10,
|
|
"matchedCount": 1,
|
|
"sampledCount": 1,
|
|
},
|
|
},
|
|
"items": bson.A{
|
|
bson.M{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"sampleRuleId": "confidence-check",
|
|
"storagePath": "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package",
|
|
"quality": bson.M{"clarity": 0.92, "shake": 0.08},
|
|
"annotation": bson.M{"minConfidence": 0.65, "objects": bson.A{"人", "桌子"}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-pipeline-1-index",
|
|
RunID: "run-pipeline-1",
|
|
NodeID: "setup-index",
|
|
Name: "dataset-index.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"archiveSources": bson.M{
|
|
"archive-test": bson.M{"name": "synthetic-dataset.tgz", "previewURL": "data:application/x-tar;base64,AAAA"},
|
|
},
|
|
"collectionCount": 1,
|
|
"collections": bson.A{
|
|
bson.M{
|
|
"id": "synthetic-small",
|
|
"name": "synthetic-small",
|
|
"path": "/tmp/droboticflow/run-pipeline-1/index/_materialized_inputs/synthetic-small",
|
|
"relativePath": "synthetic-small",
|
|
"fileCount": 1,
|
|
"totalBytes": 21,
|
|
"valid": true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-pipeline-1-video",
|
|
RunID: "run-pipeline-1",
|
|
NodeID: "package",
|
|
Name: "DJI_0001_D_clip001.mp4",
|
|
Type: "video",
|
|
Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
Data: map[string]any{
|
|
"preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-pipeline-1-quality",
|
|
RunID: "run-pipeline-1",
|
|
NodeID: "package",
|
|
Name: "DJI_0001_D_clip001_quality.json",
|
|
Type: "json",
|
|
Path: "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001_quality.json",
|
|
Data: map[string]any{"clarity": 0.92},
|
|
},
|
|
},
|
|
CreatedAt: "2026-04-10T10:05:00Z",
|
|
UpdatedAt: "2026-04-10T10:05:00Z",
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, pipelineRun); err != nil {
|
|
t.Fatalf("upsert pipeline run: %v", err)
|
|
}
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
packagesRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
|
|
if packagesRecorder.Code != http.StatusOK {
|
|
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
|
|
}
|
|
var packagesBody struct {
|
|
Items []domain.PackageRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
|
|
t.Fatalf("decode packages: %v", err)
|
|
}
|
|
if len(packagesBody.Items) != 1 {
|
|
t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
|
|
}
|
|
if packagesBody.Items[0].PackageName != "DJI_0001_D_clip001_package" {
|
|
t.Fatalf("unexpected package name: %s", packagesBody.Items[0].PackageName)
|
|
}
|
|
if packagesBody.Items[0].Storage.Provider != "k8s-job" {
|
|
t.Fatalf("expected k8s-job package storage, got %+v", packagesBody.Items[0].Storage)
|
|
}
|
|
if packagesBody.Items[0].Storage.Prefix != "/tmp/run-pipeline-1/package/packages" {
|
|
t.Fatalf("unexpected package storage prefix: %+v", packagesBody.Items[0].Storage)
|
|
}
|
|
|
|
packageDetailRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package", nil))
|
|
if packageDetailRecorder.Code != http.StatusOK {
|
|
t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String())
|
|
}
|
|
var packageDetail domain.PackageDetail
|
|
if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil {
|
|
t.Fatalf("decode package detail: %v", err)
|
|
}
|
|
if packageDetail.Package.Storage.Path != "/tmp/run-pipeline-1/package/packages/DJI_0001_D_clip001_package" {
|
|
t.Fatalf("unexpected package storage path: %+v", packageDetail.Package.Storage)
|
|
}
|
|
if len(packageDetail.Files) == 0 {
|
|
t.Fatalf("expected package files")
|
|
}
|
|
videoPreview := mapValue(packageDetail.Files[0].Data["preview"])
|
|
if !strings.Contains(stringValue(videoPreview["url"]), "/api/packages/run-pipeline-1/DJI_0001_D_clip001_package/content?relativePath=") {
|
|
t.Fatalf("expected package video preview url to use content api, got %+v", packageDetail.Files[0].Data)
|
|
}
|
|
|
|
samplingRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(samplingRecorder, httptest.NewRequest(http.MethodGet, "/api/sampling-tasks", nil))
|
|
if samplingRecorder.Code != http.StatusOK {
|
|
t.Fatalf("sampling status: %d body=%s", samplingRecorder.Code, samplingRecorder.Body.String())
|
|
}
|
|
var samplingBody struct {
|
|
Items []domain.SamplingTaskRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(samplingRecorder.Body.Bytes(), &samplingBody); err != nil {
|
|
t.Fatalf("decode sampling tasks: %v", err)
|
|
}
|
|
if len(samplingBody.Items) != 1 {
|
|
t.Fatalf("expected one sampling task, got %d body=%s", len(samplingBody.Items), samplingRecorder.Body.String())
|
|
}
|
|
if len(samplingBody.Items[0].Items) != 1 {
|
|
t.Fatalf("expected one sampling item, got %d", len(samplingBody.Items[0].Items))
|
|
}
|
|
|
|
collectionsRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
|
|
if collectionsRecorder.Code != http.StatusOK {
|
|
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
|
|
}
|
|
var collectionsBody struct {
|
|
Items []domain.SourceCollectionRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
|
|
t.Fatalf("decode source collections: %v", err)
|
|
}
|
|
if len(collectionsBody.Items) != 1 {
|
|
t.Fatalf("expected one source collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
|
|
}
|
|
if collectionsBody.Items[0].CollectionID != "BJ_001" {
|
|
t.Fatalf("unexpected collection id: %s", collectionsBody.Items[0].CollectionID)
|
|
}
|
|
if collectionsBody.Items[0].CollectionName != "BJ_001" {
|
|
t.Fatalf("unexpected collection name fallback: %s", collectionsBody.Items[0].CollectionName)
|
|
}
|
|
if collectionsBody.Items[0].RunID != "run-scan-1" {
|
|
t.Fatalf("expected source collections to ignore synthetic pipeline dataset indexes, got run %s", collectionsBody.Items[0].RunID)
|
|
}
|
|
if collectionsBody.Items[0].Storage.Provider != "bos" {
|
|
t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage)
|
|
}
|
|
if collectionsBody.Items[0].Storage.Bucket != "datastorage" || collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" {
|
|
t.Fatalf("unexpected source storage details: %+v", collectionsBody.Items[0].Storage)
|
|
}
|
|
|
|
detailRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(detailRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001", nil))
|
|
if detailRecorder.Code != http.StatusOK {
|
|
t.Fatalf("source detail status: %d body=%s", detailRecorder.Code, detailRecorder.Body.String())
|
|
}
|
|
var detail domain.SourceCollectionDetail
|
|
if err := json.Unmarshal(detailRecorder.Body.Bytes(), &detail); err != nil {
|
|
t.Fatalf("decode source detail: %v", err)
|
|
}
|
|
if len(detail.Entries) != 3 {
|
|
t.Fatalf("expected root entries derived from dataset index, got %d body=%s", len(detail.Entries), detailRecorder.Body.String())
|
|
}
|
|
var videoDirectory *domain.SourceCollectionEntry
|
|
for index := range detail.Entries {
|
|
if detail.Entries[index].Kind == "directory" && detail.Entries[index].Path == "DJI_001" {
|
|
videoDirectory = &detail.Entries[index]
|
|
break
|
|
}
|
|
}
|
|
if videoDirectory == nil {
|
|
t.Fatalf("expected DJI_001 directory in derived source entries: %+v", detail.Entries)
|
|
}
|
|
if !videoDirectory.DerivedFromIndex {
|
|
t.Fatalf("expected derived directory to be marked from dataset index: %+v", videoDirectory)
|
|
}
|
|
|
|
entryRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(entryRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/entries?prefix=DJI_001", nil))
|
|
if entryRecorder.Code != http.StatusOK {
|
|
t.Fatalf("source entries status: %d body=%s", entryRecorder.Code, entryRecorder.Body.String())
|
|
}
|
|
var nestedDetail domain.SourceCollectionDetail
|
|
if err := json.Unmarshal(entryRecorder.Body.Bytes(), &nestedDetail); err != nil {
|
|
t.Fatalf("decode nested source entries: %v", err)
|
|
}
|
|
if len(nestedDetail.Entries) != 1 {
|
|
t.Fatalf("expected one nested video entry, got %d body=%s", len(nestedDetail.Entries), entryRecorder.Body.String())
|
|
}
|
|
videoEntry := nestedDetail.Entries[0]
|
|
if videoEntry.Kind != "video" {
|
|
t.Fatalf("expected nested entry to be video, got %s", videoEntry.Kind)
|
|
}
|
|
if videoEntry.Artifact == nil {
|
|
t.Fatalf("expected nested video entry artifact")
|
|
}
|
|
if !videoEntry.DerivedFromIndex {
|
|
t.Fatalf("expected nested video entry to be marked from dataset index: %+v", videoEntry)
|
|
}
|
|
preview := mapValue(videoEntry.Artifact.Data["preview"])
|
|
if !strings.Contains(stringValue(preview["url"]), "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=") {
|
|
t.Fatalf("expected nested video preview url from archive sources: %+v", videoEntry.Artifact.Data)
|
|
}
|
|
|
|
contentRecorder := httptest.NewRecorder()
|
|
contentRequest := httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-scan-1/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4", nil)
|
|
handler.ServeHTTP(contentRecorder, contentRequest)
|
|
if contentRecorder.Code != http.StatusOK {
|
|
t.Fatalf("source content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String())
|
|
}
|
|
if contentRecorder.Header().Get("Content-Type") != "video/mp4" {
|
|
t.Fatalf("unexpected source content type: %s", contentRecorder.Header().Get("Content-Type"))
|
|
}
|
|
if string(contentRecorder.Body.Bytes()) != "video-preview-bytes" {
|
|
t.Fatalf("unexpected source content payload: %q", contentRecorder.Body.Bytes())
|
|
}
|
|
}
|
|
|
|
func TestSourceCollectionsUseCatalogFieldsForIncrementalScanRuns(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "BOS 增量扫描",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindScan,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
collection := bson.M{
|
|
"id": "BJ_001",
|
|
"name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
|
|
"relativePath": "BJ_001",
|
|
"meta": bson.M{"project_id": "BJ_001"},
|
|
"hasVideoMeta": true,
|
|
"videoCount": 1,
|
|
"fileCount": 4,
|
|
"totalBytes": int64(16000036740),
|
|
"valid": true,
|
|
"requiredFiles": bson.A{"meta.json", "video_meta.json"},
|
|
}
|
|
catalogEntries := bson.A{
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/meta.json",
|
|
"sizeBytes": 256,
|
|
"mimeType": "application/json",
|
|
"data": bson.M{"project_id": "BJ_001"},
|
|
},
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4",
|
|
"sizeBytes": int64(16004000000),
|
|
"mimeType": "video/mp4",
|
|
"source": bson.M{
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_20260327132729_0003_D.MP4",
|
|
"previewURL": "https://example.com/DJI_20260327132729_0003_D.MP4",
|
|
},
|
|
},
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D.json",
|
|
"sizeBytes": 4226,
|
|
"mimeType": "application/json",
|
|
},
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/DJI_001/DJI_20260327132729_0003_D_imu.json",
|
|
"sizeBytes": int64(36080000),
|
|
"mimeType": "application/json",
|
|
},
|
|
}
|
|
run := domain.RunRecord{
|
|
ID: "run-incremental-empty",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-incremental-empty-source",
|
|
RunID: "run-incremental-empty",
|
|
NodeID: "scan-source",
|
|
Name: "source-manifest.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test",
|
|
"scanMode": "INCREMENTAL",
|
|
"listedCount": 4,
|
|
"newCount": 0,
|
|
"catalogEntries": catalogEntries,
|
|
},
|
|
},
|
|
{
|
|
ID: "run-incremental-empty-index",
|
|
RunID: "run-incremental-empty",
|
|
NodeID: "scan-index",
|
|
Name: "dataset-index.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test",
|
|
"scanMode": "INCREMENTAL",
|
|
"listedCount": 4,
|
|
"newCount": 0,
|
|
"collectionCount": 0,
|
|
"collections": bson.A{},
|
|
"items": bson.A{},
|
|
"entries": bson.A{},
|
|
"catalogCollectionCount": 1,
|
|
"catalogVideoCount": 1,
|
|
"catalogCollections": bson.A{collection},
|
|
"catalogItems": bson.A{},
|
|
"catalogEntries": catalogEntries,
|
|
},
|
|
},
|
|
},
|
|
CreatedAt: "2026-04-10T12:00:00Z",
|
|
UpdatedAt: "2026-04-10T12:00:00Z",
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
collectionsRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
|
|
if collectionsRecorder.Code != http.StatusOK {
|
|
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
|
|
}
|
|
var collectionsBody struct {
|
|
Items []domain.SourceCollectionRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
|
|
t.Fatalf("decode source collections: %v", err)
|
|
}
|
|
if len(collectionsBody.Items) != 1 {
|
|
t.Fatalf("expected one catalog collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
|
|
}
|
|
if collectionsBody.Items[0].FileCount != 4 || collectionsBody.Items[0].TotalBytes != 16000036740 {
|
|
t.Fatalf("expected catalog file stats, got %+v", collectionsBody.Items[0])
|
|
}
|
|
|
|
entriesRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(entriesRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections/run-incremental-empty/BJ_001/entries?prefix=DJI_001", nil))
|
|
if entriesRecorder.Code != http.StatusOK {
|
|
t.Fatalf("entries status: %d body=%s", entriesRecorder.Code, entriesRecorder.Body.String())
|
|
}
|
|
var detail domain.SourceCollectionDetail
|
|
if err := json.Unmarshal(entriesRecorder.Body.Bytes(), &detail); err != nil {
|
|
t.Fatalf("decode source detail: %v", err)
|
|
}
|
|
if len(detail.Entries) != 3 {
|
|
t.Fatalf("expected all catalog files under DJI_001, got %d body=%s", len(detail.Entries), entriesRecorder.Body.String())
|
|
}
|
|
var video *domain.SourceCollectionEntry
|
|
for index := range detail.Entries {
|
|
if detail.Entries[index].Name == "DJI_20260327132729_0003_D.MP4" {
|
|
video = &detail.Entries[index]
|
|
break
|
|
}
|
|
}
|
|
if video == nil || video.SizeBytes != 16004000000 {
|
|
t.Fatalf("expected real mp4 size from catalog entries, got %+v", video)
|
|
}
|
|
if video.DerivedFromIndex {
|
|
t.Fatalf("expected catalog entry to remain a direct entry, got %+v", video)
|
|
}
|
|
if collectionsBody.Items[0].Storage.Provider != "bos" {
|
|
t.Fatalf("expected bos source storage, got %+v", collectionsBody.Items[0].Storage)
|
|
}
|
|
if collectionsBody.Items[0].Storage.Bucket != "datastorage" {
|
|
t.Fatalf("unexpected source storage bucket: %+v", collectionsBody.Items[0].Storage)
|
|
}
|
|
if collectionsBody.Items[0].Storage.Prefix != "algo/X-platform-test" {
|
|
t.Fatalf("unexpected source storage prefix: %+v", collectionsBody.Items[0].Storage)
|
|
}
|
|
}
|
|
|
|
func TestSourceCollectionsCanLoadExternalizedDatasetIndexPayload(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "真实数据资产"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "BOS 增量扫描",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindScan,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
indexPayload := map[string]any{
|
|
"datasetType": "x-platform-index",
|
|
"dataSource": "BOS",
|
|
"sourcePath": "bos://datastorage/algo/X-platform-test",
|
|
"collectionCount": 0,
|
|
"collections": []any{},
|
|
"items": []any{},
|
|
"entries": []any{},
|
|
"catalogCollectionCount": 1,
|
|
"catalogCollections": []any{
|
|
map[string]any{
|
|
"id": "BJ_001",
|
|
"name": "BJ_001_0001_OsmoNano_2026-03-19_14-51-43",
|
|
"relativePath": "BJ_001",
|
|
"videoCount": 1,
|
|
"valid": true,
|
|
},
|
|
},
|
|
"catalogItems": []any{},
|
|
"catalogEntries": []any{},
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-externalized-index",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
externalizedJSONArtifact(t, "run-externalized-index", "scan-index", "dataset-index.json", indexPayload),
|
|
},
|
|
CreatedAt: "2026-04-10T12:00:00Z",
|
|
UpdatedAt: "2026-04-10T12:00:00Z",
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
collectionsRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(collectionsRecorder, httptest.NewRequest(http.MethodGet, "/api/source-data/collections", nil))
|
|
if collectionsRecorder.Code != http.StatusOK {
|
|
t.Fatalf("collections status: %d body=%s", collectionsRecorder.Code, collectionsRecorder.Body.String())
|
|
}
|
|
var collectionsBody struct {
|
|
Items []domain.SourceCollectionRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(collectionsRecorder.Body.Bytes(), &collectionsBody); err != nil {
|
|
t.Fatalf("decode source collections: %v", err)
|
|
}
|
|
if len(collectionsBody.Items) != 1 {
|
|
t.Fatalf("expected externalized payload to produce one collection, got %d body=%s", len(collectionsBody.Items), collectionsRecorder.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestPackageContentCanServeBrowserPreviewVideo(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品预览项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "成品预览工作流",
|
|
Type: domain.WorkflowTypeBatch,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
videoPath := filepath.Join(t.TempDir(), "DJI_0001_D_clip001.mp4")
|
|
if err := os.WriteFile(videoPath, []byte("package-video-bytes"), 0o644); err != nil {
|
|
t.Fatalf("write package video: %v", err)
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-package-preview",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-package-preview-manifest",
|
|
RunID: "run-package-preview",
|
|
NodeID: "package",
|
|
Name: "packages-manifest.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"items": bson.A{
|
|
bson.M{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"fileCount": 1,
|
|
"storagePath": filepath.Dir(videoPath),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-package-preview-video",
|
|
RunID: "run-package-preview",
|
|
NodeID: "package",
|
|
Name: "DJI_0001_D_clip001.mp4",
|
|
Type: "video",
|
|
Path: filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4"),
|
|
Metadata: &domain.ArtifactMetadata{
|
|
MimeType: "video/mp4",
|
|
},
|
|
Data: map[string]any{
|
|
"preview": map[string]any{"kind": "video", "url": "data:video/mp4;base64,AAAA"},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-package-preview-upload-video",
|
|
RunID: "run-package-preview",
|
|
NodeID: "upload",
|
|
Name: "DJI_0001_D_clip001.mp4",
|
|
Type: "video",
|
|
Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
Metadata: &domain.ArtifactMetadata{
|
|
MimeType: "video/mp4",
|
|
},
|
|
Data: map[string]any{
|
|
"source": map[string]any{
|
|
"path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
run.Artifacts[1].Path = filepath.Join(filepath.Dir(videoPath), "packages", "DJI_0001_D_clip001_package", "DJI_0001_D_clip001.mp4")
|
|
if err := os.MkdirAll(filepath.Dir(run.Artifacts[1].Path), 0o755); err != nil {
|
|
t.Fatalf("mkdir package dir: %v", err)
|
|
}
|
|
if err := os.WriteFile(run.Artifacts[1].Path, []byte("package-video-bytes"), 0o644); err != nil {
|
|
t.Fatalf("write package artifact: %v", err)
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
previewPath := filepath.Join(t.TempDir(), "package-browser-preview.mp4")
|
|
if err := os.WriteFile(previewPath, []byte("package-browser-preview"), 0o644); err != nil {
|
|
t.Fatalf("write preview file: %v", err)
|
|
}
|
|
|
|
originalPreviewResolver := sourceBrowserVideoPreviewPath
|
|
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
|
|
if request.RedirectURL != run.Artifacts[1].Path {
|
|
t.Fatalf("unexpected package preview input: %s", request.RedirectURL)
|
|
}
|
|
if request.FileName != "DJI_0001_D_clip001.mp4" {
|
|
t.Fatalf("unexpected file name: %s", request.FileName)
|
|
}
|
|
return previewPath, nil
|
|
}
|
|
t.Cleanup(func() {
|
|
sourceBrowserVideoPreviewPath = originalPreviewResolver
|
|
})
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
packagesRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
|
|
if packagesRecorder.Code != http.StatusOK {
|
|
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
|
|
}
|
|
var packagesBody struct {
|
|
Items []domain.PackageRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
|
|
t.Fatalf("decode packages: %v", err)
|
|
}
|
|
if len(packagesBody.Items) != 1 {
|
|
t.Fatalf("expected one package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
|
|
}
|
|
if packagesBody.Items[0].Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" {
|
|
t.Fatalf("expected package storage to prefer BOS upload path, got %+v", packagesBody.Items[0].Storage)
|
|
}
|
|
|
|
packageDetailRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(packageDetailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-package-preview/DJI_0001_D_clip001_package", nil))
|
|
if packageDetailRecorder.Code != http.StatusOK {
|
|
t.Fatalf("package detail status: %d body=%s", packageDetailRecorder.Code, packageDetailRecorder.Body.String())
|
|
}
|
|
var packageDetail domain.PackageDetail
|
|
if err := json.Unmarshal(packageDetailRecorder.Body.Bytes(), &packageDetail); err != nil {
|
|
t.Fatalf("decode package detail: %v", err)
|
|
}
|
|
if packageDetail.Package.Storage.Path != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package" {
|
|
t.Fatalf("expected package detail storage to prefer BOS upload path, got %+v", packageDetail.Package.Storage)
|
|
}
|
|
if len(packageDetail.Files) != 1 {
|
|
t.Fatalf("expected duplicate package files to collapse to one, got %d", len(packageDetail.Files))
|
|
}
|
|
if packageDetail.Files[0].Name != "DJI_0001_D_clip001.mp4" {
|
|
t.Fatalf("unexpected package file after dedupe: %+v", packageDetail.Files[0])
|
|
}
|
|
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(
|
|
http.MethodGet,
|
|
"/api/packages/run-package-preview/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser",
|
|
nil,
|
|
)
|
|
handler.ServeHTTP(recorder, request)
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if recorder.Header().Get("Content-Type") != "video/mp4" {
|
|
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
|
|
}
|
|
if string(recorder.Body.Bytes()) != "package-browser-preview" {
|
|
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
|
|
}
|
|
}
|
|
|
|
func TestPackageContentFallsBackToUploadedBOSVideoWhenLocalPackageFileUnavailable(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "成品上传回退项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
|
Name: "上传 BOS 连接",
|
|
Provider: domain.StorageProviderBOS,
|
|
Endpoint: "https://bos.example.test",
|
|
AccessKeyID: "ak-upload",
|
|
SecretAccessKey: "sk-upload",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create storage connection: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "成品上传回退工作流",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "upload",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "结果上传",
|
|
ConnectionID: connection.ID,
|
|
Bucket: "datastorage",
|
|
Prefix: "factory/output/test-package-preview",
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "DATA_SOURCE", Value: "BOS"},
|
|
{Key: "SOURCE_PATH", Value: "bos://datastorage/factory/output/test-package-preview"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-package-upload-fallback",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-package-upload-fallback-manifest",
|
|
RunID: "run-package-upload-fallback",
|
|
NodeID: "package",
|
|
Name: "packages-manifest.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"items": bson.A{
|
|
bson.M{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"fileCount": 1,
|
|
"storagePath": "/tmp/droboticflow/missing/package",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-package-upload-fallback-temp-video",
|
|
RunID: "run-package-upload-fallback",
|
|
NodeID: "package",
|
|
Name: "DJI_0001_D_clip001.mp4",
|
|
Type: "video",
|
|
Path: "/tmp/droboticflow/missing/package/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
Metadata: &domain.ArtifactMetadata{
|
|
MimeType: "video/mp4",
|
|
},
|
|
Data: map[string]any{
|
|
"preview": map[string]any{
|
|
"kind": "video",
|
|
"url": "/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-package-upload-fallback-remote-video",
|
|
RunID: "run-package-upload-fallback",
|
|
NodeID: "upload",
|
|
Name: "DJI_0001_D_clip001.mp4",
|
|
Type: "video",
|
|
Path: "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
Metadata: &domain.ArtifactMetadata{
|
|
MimeType: "video/mp4",
|
|
},
|
|
Data: map[string]any{
|
|
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
"source": map[string]any{
|
|
"path": "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
"previewURL": "https://expired.example.com/DJI_0001_D_clip001.mp4",
|
|
"mimeType": "video/mp4",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
previewPath := filepath.Join(t.TempDir(), "package-upload-browser-preview.mp4")
|
|
if err := os.WriteFile(previewPath, []byte("package-upload-browser-preview"), 0o644); err != nil {
|
|
t.Fatalf("write preview file: %v", err)
|
|
}
|
|
|
|
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
|
|
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
|
|
if scanConfig == nil || scanConfig.AccessKeyID != "ak-upload" || scanConfig.SecretAccessKey != "sk-upload" {
|
|
t.Fatalf("expected upload connection secrets, got %+v", scanConfig)
|
|
}
|
|
if sourcePath != "bos://datastorage/factory/output/test-package-preview/packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4" {
|
|
t.Fatalf("unexpected source path: %s", sourcePath)
|
|
}
|
|
return "https://signed.example.com/DJI_0001_D_clip001.mp4", nil
|
|
}
|
|
t.Cleanup(func() {
|
|
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
|
|
})
|
|
|
|
originalPreviewResolver := sourceBrowserVideoPreviewPath
|
|
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
|
|
if request.RedirectURL != "https://signed.example.com/DJI_0001_D_clip001.mp4" {
|
|
t.Fatalf("unexpected package preview redirect input: %s", request.RedirectURL)
|
|
}
|
|
if request.FileName != "DJI_0001_D_clip001.mp4" {
|
|
t.Fatalf("unexpected file name: %s", request.FileName)
|
|
}
|
|
return previewPath, nil
|
|
}
|
|
t.Cleanup(func() {
|
|
sourceBrowserVideoPreviewPath = originalPreviewResolver
|
|
})
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(
|
|
http.MethodGet,
|
|
"/api/packages/run-package-upload-fallback/DJI_0001_D_clip001_package/content?relativePath=DJI_0001_D_clip001.mp4&transcode=browser",
|
|
nil,
|
|
)
|
|
handler.ServeHTTP(recorder, request)
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if recorder.Header().Get("Content-Type") != "video/mp4" {
|
|
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
|
|
}
|
|
if string(recorder.Body.Bytes()) != "package-upload-browser-preview" {
|
|
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
|
|
}
|
|
}
|
|
|
|
func TestPackagesListIncludesWilorResultArtifacts(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "WiLoR 数据资产项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "WiLoR 3D Hand Pose Verification",
|
|
Type: domain.WorkflowTypeBatch,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
root := t.TempDir()
|
|
outputDir := filepath.Join(root, "wilor_outputs")
|
|
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
|
t.Fatalf("mkdir outputs: %v", err)
|
|
}
|
|
imagePath := filepath.Join(outputDir, "wilor-annotated-image.png")
|
|
keypointsPath := filepath.Join(outputDir, "wilor-keypoints.json")
|
|
meshPath := filepath.Join(outputDir, "wilor-mesh.obj")
|
|
if err := os.WriteFile(imagePath, []byte("png"), 0o644); err != nil {
|
|
t.Fatalf("write image: %v", err)
|
|
}
|
|
if err := os.WriteFile(keypointsPath, []byte(`{"hands":2}`), 0o644); err != nil {
|
|
t.Fatalf("write keypoints: %v", err)
|
|
}
|
|
if err := os.WriteFile(meshPath, []byte("v 0 0 0\nf 1 1 1\n"), 0o644); err != nil {
|
|
t.Fatalf("write mesh: %v", err)
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-wilor-assets",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
CreatedAt: "2026-04-12T11:30:00Z",
|
|
NodeExecutions: []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "wilor-1",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-wilor-assets-result",
|
|
RunID: "run-wilor-assets",
|
|
NodeID: "wilor-1",
|
|
Name: "wilor-result.json",
|
|
Type: domain.PortTypeJSON,
|
|
Path: filepath.Join(root, "wilor-result.json"),
|
|
Data: map[string]any{
|
|
"schemaVersion": wilorHandPoseSchemaVersion,
|
|
"operator": "wilor_hand_pose",
|
|
"status": "success",
|
|
"mediaKind": "image",
|
|
"source": map[string]any{
|
|
"name": "sample-hands.png",
|
|
"ref": "sample-hands.png",
|
|
},
|
|
"service": map[string]any{
|
|
"image": "ccr.example/wilor:gpu",
|
|
"imageId": "ccr.example/wilor@sha256:test",
|
|
"health": map[string]any{
|
|
"runtime": map[string]any{
|
|
"device": "cuda",
|
|
"gpuName": "NVIDIA A800-SXM4-80GB",
|
|
},
|
|
},
|
|
},
|
|
"result": map[string]any{
|
|
"handCount": 2,
|
|
},
|
|
"downloadedOutputs": []any{
|
|
map[string]any{"name": "wilor-annotated-image.png", "path": imagePath},
|
|
map[string]any{"name": "wilor-keypoints.json", "path": keypointsPath},
|
|
map[string]any{"name": "wilor-mesh.obj", "path": meshPath},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: "run-wilor-assets-image",
|
|
RunID: "run-wilor-assets",
|
|
NodeID: "wilor-1",
|
|
Name: "wilor-annotated-image.png",
|
|
Type: domain.PortTypeImage,
|
|
Size: "3 B",
|
|
Path: imagePath,
|
|
Metadata: &domain.ArtifactMetadata{
|
|
MimeType: "image/png",
|
|
},
|
|
},
|
|
{
|
|
ID: "run-wilor-assets-keypoints",
|
|
RunID: "run-wilor-assets",
|
|
NodeID: "wilor-1",
|
|
Name: "wilor-keypoints.json",
|
|
Type: domain.PortTypeJSON,
|
|
Size: "11 B",
|
|
Path: keypointsPath,
|
|
Data: map[string]any{"hands": 2},
|
|
},
|
|
{
|
|
ID: "run-wilor-assets-mesh",
|
|
RunID: "run-wilor-assets",
|
|
NodeID: "wilor-1",
|
|
Name: "wilor-mesh.obj",
|
|
Type: domain.PortTypeFile,
|
|
Size: "16 B",
|
|
Path: meshPath,
|
|
Metadata: &domain.ArtifactMetadata{
|
|
MimeType: "text/plain; charset=utf-8",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
packagesRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(packagesRecorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
|
|
if packagesRecorder.Code != http.StatusOK {
|
|
t.Fatalf("packages status: %d body=%s", packagesRecorder.Code, packagesRecorder.Body.String())
|
|
}
|
|
var packagesBody struct {
|
|
Items []domain.PackageRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(packagesRecorder.Body.Bytes(), &packagesBody); err != nil {
|
|
t.Fatalf("decode packages: %v", err)
|
|
}
|
|
if len(packagesBody.Items) != 1 {
|
|
t.Fatalf("expected one WiLoR package, got %d body=%s", len(packagesBody.Items), packagesRecorder.Body.String())
|
|
}
|
|
if packagesBody.Items[0].PackageName != "wilor-sample-hands-wilor-1" {
|
|
t.Fatalf("unexpected package name: %+v", packagesBody.Items[0])
|
|
}
|
|
if packagesBody.Items[0].SourceVideo != "sample-hands.png" || packagesBody.Items[0].FileCount != 4 {
|
|
t.Fatalf("unexpected WiLoR package summary: %+v", packagesBody.Items[0])
|
|
}
|
|
|
|
detailRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(detailRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-wilor-assets/wilor-sample-hands-wilor-1", nil))
|
|
if detailRecorder.Code != http.StatusOK {
|
|
t.Fatalf("package detail status: %d body=%s", detailRecorder.Code, detailRecorder.Body.String())
|
|
}
|
|
var detail domain.PackageDetail
|
|
if err := json.Unmarshal(detailRecorder.Body.Bytes(), &detail); err != nil {
|
|
t.Fatalf("decode detail: %v", err)
|
|
}
|
|
if len(detail.Files) != 4 {
|
|
t.Fatalf("expected four WiLoR package files, got %d", len(detail.Files))
|
|
}
|
|
var meshFile *domain.ExecutionArtifact
|
|
for index := range detail.Files {
|
|
if detail.Files[index].Name == "wilor-mesh.obj" {
|
|
meshFile = &detail.Files[index]
|
|
break
|
|
}
|
|
}
|
|
if meshFile == nil {
|
|
t.Fatalf("expected mesh file in package detail: %+v", detail.Files)
|
|
}
|
|
if meshFile.Path != "wilor-mesh.obj" {
|
|
t.Fatalf("expected mesh relative path, got %+v", meshFile)
|
|
}
|
|
if !strings.Contains(stringValue(meshFile.Data["contentURL"]), "/api/packages/run-wilor-assets/wilor-sample-hands-wilor-1/content?relativePath=wilor-mesh.obj") {
|
|
t.Fatalf("expected mesh contentURL to use package content api, got %+v", meshFile.Data)
|
|
}
|
|
|
|
contentRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(contentRecorder, httptest.NewRequest(http.MethodGet, "/api/packages/run-wilor-assets/wilor-sample-hands-wilor-1/content?relativePath=wilor-mesh.obj", nil))
|
|
if contentRecorder.Code != http.StatusOK {
|
|
t.Fatalf("mesh content status: %d body=%s", contentRecorder.Code, contentRecorder.Body.String())
|
|
}
|
|
if string(contentRecorder.Body.Bytes()) != "v 0 0 0\nf 1 1 1\n" {
|
|
t.Fatalf("unexpected mesh content: %q", contentRecorder.Body.Bytes())
|
|
}
|
|
}
|
|
|
|
func TestWilorResultArtifactsPrefersUploadedDuplicate(t *testing.T) {
|
|
payload := map[string]any{
|
|
"schemaVersion": wilorHandPoseSchemaVersion,
|
|
"mediaKind": "image",
|
|
"source": map[string]any{
|
|
"name": "sample-hands.png",
|
|
"ref": "https://example.test/sample-hands.png",
|
|
},
|
|
"result": map[string]any{"handCount": 2},
|
|
}
|
|
run := domain.RunRecord{
|
|
ID: "run-wilor-duplicates",
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-wilor-duplicates-wilor",
|
|
RunID: "run-wilor-duplicates",
|
|
NodeID: "wilor-1",
|
|
Name: "wilor-result.json",
|
|
Type: domain.PortTypeJSON,
|
|
Data: payload,
|
|
},
|
|
{
|
|
ID: "run-wilor-duplicates-upload",
|
|
RunID: "run-wilor-duplicates",
|
|
NodeID: "upload-1",
|
|
Name: "wilor-result.json",
|
|
Type: domain.PortTypeJSON,
|
|
Data: payload,
|
|
},
|
|
},
|
|
}
|
|
|
|
items := wilorResultArtifacts(run)
|
|
if len(items) != 1 {
|
|
t.Fatalf("expected one deduplicated WiLoR result, got %+v", items)
|
|
}
|
|
if items[0].NodeID != "upload-1" {
|
|
t.Fatalf("expected uploaded duplicate to win, got %+v", items[0])
|
|
}
|
|
}
|
|
|
|
func TestPackagesListKeepsLatestPackageVersionPerWorkflow(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "包去重项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "包去重工作流",
|
|
Type: domain.WorkflowTypeBatch,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
makeRun := func(id string, createdAt string, storagePath string) domain.RunRecord {
|
|
return domain.RunRecord{
|
|
ID: id,
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
CreatedAt: createdAt,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: id + "-manifest",
|
|
RunID: id,
|
|
NodeID: "package",
|
|
Name: "packages-manifest.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"items": []any{
|
|
map[string]any{
|
|
"packageName": "DJI_0001_D_clip001_package",
|
|
"clipId": "DJI_0001_D_clip001",
|
|
"sourceVideo": "DJI_0001_D.mp4",
|
|
"fileCount": 1,
|
|
"storagePath": "/tmp/k8s-job/package",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
ID: id + "-upload-video",
|
|
RunID: id,
|
|
NodeID: "upload",
|
|
Name: "DJI_0001_D_clip001.mp4",
|
|
Type: "video",
|
|
Path: storagePath + "/DJI_0001_D_clip001.mp4",
|
|
Metadata: &domain.ArtifactMetadata{
|
|
MimeType: "video/mp4",
|
|
},
|
|
Data: map[string]any{
|
|
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
"source": map[string]any{
|
|
"path": storagePath + "/DJI_0001_D_clip001.mp4",
|
|
"mimeType": "video/mp4",
|
|
"relativePath": "packages/DJI_0001_D_clip001_package/DJI_0001_D_clip001.mp4",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
if err := dataStore.UpsertRun(ctx, makeRun(
|
|
"run-package-old",
|
|
"2026-04-12T10:00:00Z",
|
|
"bos://datastorage/factory/output/old/packages/DJI_0001_D_clip001_package",
|
|
)); err != nil {
|
|
t.Fatalf("upsert old run: %v", err)
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, makeRun(
|
|
"run-package-new",
|
|
"2026-04-12T11:00:00Z",
|
|
"bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package",
|
|
)); err != nil {
|
|
t.Fatalf("upsert new run: %v", err)
|
|
}
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
recorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/packages", nil))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("packages status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var body struct {
|
|
Items []domain.PackageRecord `json:"items"`
|
|
}
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
|
|
t.Fatalf("decode packages: %v", err)
|
|
}
|
|
if len(body.Items) != 1 {
|
|
t.Fatalf("expected latest package only, got %d body=%s", len(body.Items), recorder.Body.String())
|
|
}
|
|
if body.Items[0].RunID != "run-package-new" {
|
|
t.Fatalf("expected latest run package, got %+v", body.Items[0])
|
|
}
|
|
if body.Items[0].Storage.Path != "bos://datastorage/factory/output/new/packages/DJI_0001_D_clip001_package" {
|
|
t.Fatalf("expected latest BOS storage path, got %+v", body.Items[0].Storage)
|
|
}
|
|
}
|
|
|
|
func TestSourceCollectionContentCanServeBrowserPreviewVideo(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "浏览器预览项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "BOS 增量扫描",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindScan,
|
|
ScanConfig: &domain.ScanConfig{
|
|
DataSource: "BOS",
|
|
Endpoint: "https://bos.example.test",
|
|
AccessKeyID: "ak",
|
|
SecretAccessKey: "sk",
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-browser-preview",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-browser-preview-index",
|
|
RunID: "run-browser-preview",
|
|
NodeID: "scan-index",
|
|
Name: "dataset-index.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"catalogCollections": bson.A{
|
|
bson.M{
|
|
"id": "BJ_001",
|
|
"name": "BJ_001",
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
|
|
"relativePath": "BJ_001",
|
|
"videoCount": 1,
|
|
"fileCount": 1,
|
|
"totalBytes": int64(2048),
|
|
"valid": true,
|
|
},
|
|
},
|
|
"catalogEntries": bson.A{
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": int64(2048),
|
|
"mimeType": "video/mp4",
|
|
"source": bson.M{
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"previewURL": "https://example.com/DJI_0001_D.mp4",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
previewPath := filepath.Join(t.TempDir(), "preview.mp4")
|
|
if err := os.WriteFile(previewPath, []byte("browser-preview-video"), 0o644); err != nil {
|
|
t.Fatalf("write preview file: %v", err)
|
|
}
|
|
|
|
originalPreviewResolver := sourceBrowserVideoPreviewPath
|
|
sourceBrowserVideoPreviewPath = func(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
|
|
if request.RedirectURL != "https://signed.example.com/DJI_0001_D.mp4" {
|
|
t.Fatalf("unexpected redirect url: %s", request.RedirectURL)
|
|
}
|
|
if request.FileName != "DJI_0001_D.mp4" {
|
|
t.Fatalf("unexpected file name: %s", request.FileName)
|
|
}
|
|
return previewPath, nil
|
|
}
|
|
t.Cleanup(func() {
|
|
sourceBrowserVideoPreviewPath = originalPreviewResolver
|
|
})
|
|
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
|
|
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
|
|
if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" {
|
|
t.Fatalf("unexpected scan config: %+v", scanConfig)
|
|
}
|
|
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
|
|
t.Fatalf("unexpected source path: %s", sourcePath)
|
|
}
|
|
return "https://signed.example.com/DJI_0001_D.mp4", nil
|
|
}
|
|
t.Cleanup(func() {
|
|
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
|
|
})
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(
|
|
http.MethodGet,
|
|
"/api/source-data/collections/run-browser-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4&transcode=browser",
|
|
nil,
|
|
)
|
|
handler.ServeHTTP(recorder, request)
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("browser preview status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if recorder.Header().Get("Content-Type") != "video/mp4" {
|
|
t.Fatalf("unexpected browser preview content type: %s", recorder.Header().Get("Content-Type"))
|
|
}
|
|
if string(recorder.Body.Bytes()) != "browser-preview-video" {
|
|
t.Fatalf("unexpected browser preview payload: %q", recorder.Body.Bytes())
|
|
}
|
|
}
|
|
|
|
func TestSourceCollectionContentRefreshesBOSRedirectURL(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 刷新项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "BOS 增量扫描",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindScan,
|
|
ScanConfig: &domain.ScanConfig{
|
|
DataSource: "BOS",
|
|
Endpoint: "https://bos.example.test",
|
|
AccessKeyID: "ak",
|
|
SecretAccessKey: "sk",
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-refresh-bos-preview",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-refresh-bos-preview-index",
|
|
RunID: "run-refresh-bos-preview",
|
|
NodeID: "scan-index",
|
|
Name: "dataset-index.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"catalogCollections": bson.A{
|
|
bson.M{
|
|
"id": "BJ_001",
|
|
"name": "BJ_001",
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
|
|
"relativePath": "BJ_001",
|
|
"videoCount": 1,
|
|
"fileCount": 1,
|
|
"totalBytes": int64(2048),
|
|
"valid": true,
|
|
},
|
|
},
|
|
"catalogEntries": bson.A{
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": int64(2048),
|
|
"mimeType": "video/mp4",
|
|
"source": bson.M{
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"previewURL": "https://expired.example.com/DJI_0001_D.mp4",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
|
|
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
|
|
if scanConfig == nil || scanConfig.AccessKeyID != "ak" || scanConfig.SecretAccessKey != "sk" {
|
|
t.Fatalf("unexpected scan config: %+v", scanConfig)
|
|
}
|
|
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
|
|
t.Fatalf("unexpected source path: %s", sourcePath)
|
|
}
|
|
return "https://signed.example.com/DJI_0001_D.mp4", nil
|
|
}
|
|
t.Cleanup(func() {
|
|
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
|
|
})
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(
|
|
http.MethodGet,
|
|
"/api/source-data/collections/run-refresh-bos-preview/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4",
|
|
nil,
|
|
)
|
|
handler.ServeHTTP(recorder, request)
|
|
if recorder.Code != http.StatusTemporaryRedirect {
|
|
t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" {
|
|
t.Fatalf("unexpected redirect location: %s", location)
|
|
}
|
|
}
|
|
|
|
func TestSourceCollectionContentRefreshesBOSRedirectURLFromStorageConnection(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "BOS 连接预览项目"})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
|
Name: "BOS 主连接",
|
|
Provider: domain.StorageProviderBOS,
|
|
Endpoint: "https://bos.example.test",
|
|
AccessKeyID: "ak-from-connection",
|
|
SecretAccessKey: "sk-from-connection",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create storage connection: %v", err)
|
|
}
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "BOS 增量扫描",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindScan,
|
|
ScanConfig: &domain.ScanConfig{
|
|
DataSource: "BOS",
|
|
ConnectionID: connection.ID,
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
run := domain.RunRecord{
|
|
ID: "run-refresh-bos-preview-connection",
|
|
WorkflowID: workflow.ID,
|
|
WorkflowName: workflow.Name,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{
|
|
ID: "run-refresh-bos-preview-connection-index",
|
|
RunID: "run-refresh-bos-preview-connection",
|
|
NodeID: "scan-index",
|
|
Name: "dataset-index.json",
|
|
Type: "json",
|
|
Data: map[string]any{
|
|
"catalogCollections": bson.A{
|
|
bson.M{
|
|
"id": "BJ_001",
|
|
"name": "BJ_001",
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001",
|
|
"relativePath": "BJ_001",
|
|
"videoCount": 1,
|
|
"fileCount": 1,
|
|
"totalBytes": int64(2048),
|
|
"valid": true,
|
|
},
|
|
},
|
|
"catalogEntries": bson.A{
|
|
bson.M{
|
|
"collectionId": "BJ_001",
|
|
"relativePath": "BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"sizeBytes": int64(2048),
|
|
"mimeType": "video/mp4",
|
|
"source": bson.M{
|
|
"path": "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4",
|
|
"previewURL": "https://expired.example.com/DJI_0001_D.mp4",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
|
|
originalSignedPreviewResolver := sourceCollectionSignedPreviewURL
|
|
sourceCollectionSignedPreviewURL = func(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
|
|
if scanConfig == nil || scanConfig.AccessKeyID != "ak-from-connection" || scanConfig.SecretAccessKey != "sk-from-connection" {
|
|
t.Fatalf("expected resolved connection secrets, got %+v", scanConfig)
|
|
}
|
|
if sourcePath != "bos://datastorage/algo/X-platform-test/BJ_001/DJI_001/DJI_0001_D.mp4" {
|
|
t.Fatalf("unexpected source path: %s", sourcePath)
|
|
}
|
|
return "https://signed.example.com/DJI_0001_D.mp4", nil
|
|
}
|
|
t.Cleanup(func() {
|
|
sourceCollectionSignedPreviewURL = originalSignedPreviewResolver
|
|
})
|
|
|
|
handler := NewHandler(dataStore, nil)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(
|
|
http.MethodGet,
|
|
"/api/source-data/collections/run-refresh-bos-preview-connection/BJ_001/content?relativePath=DJI_001/DJI_0001_D.mp4",
|
|
nil,
|
|
)
|
|
handler.ServeHTTP(recorder, request)
|
|
if recorder.Code != http.StatusTemporaryRedirect {
|
|
t.Fatalf("redirect status: %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
if location := recorder.Header().Get("Location"); location != "https://signed.example.com/DJI_0001_D.mp4" {
|
|
t.Fatalf("unexpected redirect location: %s", location)
|
|
}
|
|
}
|
|
|
|
func TestSourceBrowserPreviewCacheKeyIsStableAcrossPresignedURLs(t *testing.T) {
|
|
// The presigned BOS URL is re-signed on every request (embedded
|
|
// authorization timestamp). A cache key derived from it never matched
|
|
// across requests, so the expensive browser transcode ran cold every
|
|
// time and long videos never finished within the write deadline.
|
|
base := sourceBrowserPreviewRequest{
|
|
FileName: "camera_left_part0001_silent_480p_wm.mp4",
|
|
ObjectBucket: "drobodata-fanyu",
|
|
ObjectKey: "FYLX_B/B13/part0001/camera_left.mp4",
|
|
}
|
|
first := base
|
|
first.RedirectURL = "https://bucket.bcebos.com/key?authorization=bce-auth-v1/ALTAK/2026-10-09T09-00-00Z/3600"
|
|
second := base
|
|
second.RedirectURL = "https://bucket.bcebos.com/key?authorization=bce-auth-v1/ALTAK/2026-10-10T09-00-00Z/3600"
|
|
if sourceBrowserPreviewCacheKey(first) != sourceBrowserPreviewCacheKey(second) {
|
|
t.Fatal("preview cache key changed with a re-signed URL; the transcode cache can never hit")
|
|
}
|
|
// A different object must not share the cache entry.
|
|
other := base
|
|
other.ObjectKey = "FYLX_B/B14/part0001/camera_left.mp4"
|
|
if sourceBrowserPreviewCacheKey(base) == sourceBrowserPreviewCacheKey(other) {
|
|
t.Fatal("preview cache key collided across different objects")
|
|
}
|
|
}
|