forked from longtao.wu/droboticflow
493 lines
22 KiB
Go
493 lines
22 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"droboticflow/internal/deliveryindex"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/sourceidentity"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
type indexTestBrowser struct {
|
|
*downloadTestBrowser
|
|
puts []string
|
|
fail string
|
|
ambiguous bool
|
|
}
|
|
|
|
func (b *indexTestBrowser) GetObjectSummary(_ context.Context, bucket, key string) (domain.ObjectStorageObjectSummary, error) {
|
|
body, ok := b.objects[key]
|
|
if !ok {
|
|
return domain.ObjectStorageObjectSummary{}, os.ErrNotExist
|
|
}
|
|
return domain.ObjectStorageObjectSummary{Bucket: bucket, Key: key, Kind: "file", SizeBytes: int64(len(body)), ETag: deliveryindex.Hash(body)}, nil
|
|
}
|
|
|
|
func (b *indexTestBrowser) PutObjectConditional(_ context.Context, _ string, key string, body []byte, _ string, match, none string, _ map[string]string) (domain.ObjectStorageObjectSummary, error) {
|
|
if match != "" || none != "*" {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("unconditional write")
|
|
}
|
|
b.puts = append(b.puts, key)
|
|
if strings.HasSuffix(key, b.fail) && b.fail != "" {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("injected interruption")
|
|
}
|
|
if _, ok := b.objects[key]; ok {
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("precondition failed")
|
|
}
|
|
b.objects[key] = append([]byte(nil), body...)
|
|
if b.ambiguous {
|
|
b.ambiguous = false
|
|
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("response lost after commit")
|
|
}
|
|
return domain.ObjectStorageObjectSummary{}, nil
|
|
}
|
|
func (b *indexTestBrowser) ListObjectsPage(_ context.Context, _ string, prefix, marker string, limit int) (objectStoragePage, error) {
|
|
if p, ok := b.pages[prefix+"|"+marker]; ok {
|
|
return p, nil
|
|
}
|
|
keys := []string{}
|
|
seen := map[string]bool{}
|
|
for key := range b.objects {
|
|
if strings.HasPrefix(key, prefix) {
|
|
rest := strings.TrimPrefix(key, prefix)
|
|
first, _, nested := strings.Cut(rest, "/")
|
|
item := prefix + first
|
|
if nested {
|
|
item += "/"
|
|
}
|
|
if !seen[item] && item > marker {
|
|
seen[item] = true
|
|
keys = append(keys, item)
|
|
}
|
|
}
|
|
}
|
|
sort.Strings(keys)
|
|
page := objectStoragePage{}
|
|
if limit <= 0 {
|
|
limit = 1000
|
|
}
|
|
if len(keys) > limit {
|
|
page.HasMore = true
|
|
keys = keys[:limit]
|
|
page.NextMarker = keys[len(keys)-1]
|
|
}
|
|
for _, key := range keys {
|
|
kind := "file"
|
|
if strings.HasSuffix(key, "/") {
|
|
kind = "directory"
|
|
}
|
|
page.Items = append(page.Items, domain.ObjectStorageObjectSummary{Key: key, Kind: kind})
|
|
}
|
|
return page, nil
|
|
}
|
|
func indexCall(h http.Handler, p *authenticatedPrincipal, method, target string, body any) *httptest.ResponseRecorder {
|
|
raw, _ := json.Marshal(body)
|
|
r := httptest.NewRequest(method, target, bytes.NewReader(raw))
|
|
r.Header.Set("Content-Type", "application/json")
|
|
r.AddCookie(&http.Cookie{Name: csrfCookieName, Value: "index-test-csrf"})
|
|
r.Header.Set(csrfHeaderName, "index-test-csrf")
|
|
r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, p))
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, r)
|
|
return w
|
|
}
|
|
func seedIndexFixture(t *testing.T, ds store.Store) (*indexTestBrowser, *authenticatedPrincipal, string) {
|
|
t.Helper()
|
|
root := "deliveries/项目A__project-A"
|
|
org := "index-test-org"
|
|
ctx := t.Context()
|
|
b := &indexTestBrowser{downloadTestBrowser: &downloadTestBrowser{objects: map[string][]byte{}, pages: map[string]objectStoragePage{}}}
|
|
must := func(e error) {
|
|
t.Helper()
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
must(ds.UpsertProject(ctx, domain.ProjectRecord{ID: "project-A", OrganizationID: org, Name: "项目A"}))
|
|
must(ds.UpsertStorageConnection(ctx, domain.StorageConnectionRecord{ID: "index-storage", OrganizationID: org, Provider: domain.StorageProviderBOS, Bucket: "test-bucket", Endpoint: "https://bj.bcebos.com"}))
|
|
dir := os.Getenv("DELIVERY_V3_FIXTURE_DIR")
|
|
v4 := os.Getenv("DELIVERY_CLASSIFICATION_TEST") == "1"
|
|
if v4 {
|
|
dir = os.Getenv("DELIVERY_V4_FIXTURE_DIR")
|
|
}
|
|
for _, part := range []string{"part0001", "part0002"} {
|
|
prefix := root + "/capture-A/" + part
|
|
sourceRun := "source-run-" + part
|
|
runID := "package-run-" + part
|
|
artifactID := "directory-" + part
|
|
body := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.part-index/v3","capture":{"captureId":"capture-A","prefix":"incoming/capture-A"},"partId":%q,"revision":1,"sourceMetadata":{"path":"meta_capture-A.json"},"lineage":{"sourceRunId":%q},"acceptance":{"finalAcceptance":{"confirmed":true,"acceptedAt":"2026-10-20T01:00:00Z"}},"slices":[{"sliceId":"slice-1","taskSplit":{"startSecond":0,"endSecond":1,"semantics":"half-open"},"classification":{"deliveryDisposition":"keep","descriptionStatus":"empty_confirmed"},"segments":[{"description":"","isInvalid":false,"contentKind":"gap_placeholder"}]}],"discardedSlices":[],"deliveryClassification":{"keptSliceCount":1,"discardedSliceCount":0,"keptDurationSeconds":1,"discardedDurationSeconds":0}}`, part, sourceRun))
|
|
history := []byte(fmt.Sprintf(`{"schemaVersion":"droboticflow.delivery.review-history/v1","captureId":"capture-A","partId":%q,"revision":1,"qualityReviews":[]}`, part))
|
|
if dir != "" {
|
|
var e error
|
|
body, e = os.ReadFile(filepath.Join(dir, part, "part.json"))
|
|
must(e)
|
|
historyPath := filepath.Join(dir, part, "review-history.json")
|
|
if v4 {
|
|
historyPath = filepath.Join(dir, "internal", part, "review-history.json")
|
|
}
|
|
history, e = os.ReadFile(historyPath)
|
|
must(e)
|
|
}
|
|
if v4 && dir == "" {
|
|
var doc map[string]any
|
|
must(json.Unmarshal(body, &doc))
|
|
acceptance := doc["acceptance"]
|
|
delete(doc, "acceptance")
|
|
delete(doc, "lineage")
|
|
doc["schemaVersion"] = deliveryindex.ClassificationPartSchema
|
|
enrichConsumerFixture(doc)
|
|
body, _ = json.Marshal(doc)
|
|
history, _ = json.Marshal(map[string]any{"schemaVersion": "droboticflow.delivery.review-history/v2", "captureId": "capture-A", "partId": part, "revision": 1, "partSha256": deliveryindex.Hash(body), "acceptance": acceptance, "lineage": map[string]any{"sourceRunId": sourceRun}, "qualityReviews": []any{map[string]any{"comment": "PRIVATE_REWORK"}}, "trace": []any{}})
|
|
}
|
|
b.objects[prefix+"/.delivery-owner.json"] = []byte(`{"owner":"synthetic-source"}`)
|
|
b.objects[prefix+"/part.json"] = body
|
|
b.objects[prefix+"/review-history.json"] = history
|
|
entries := []deliveryindex.Entry{{Path: "part.json", Key: prefix + "/part.json", SizeBytes: int64(len(body)), SHA256: deliveryindex.Hash(body)}, {Path: "review-history.json", Key: prefix + "/review-history.json", SizeBytes: int64(len(history)), SHA256: deliveryindex.Hash(history)}}
|
|
b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 2, "totalBytes": len(body) + len(history), "entries": entries})
|
|
if v4 {
|
|
delete(b.objects, prefix+"/review-history.json")
|
|
entries = entries[:1]
|
|
b.objects[prefix+"/manifest.json"], _ = json.Marshal(map[string]any{"schemaVersion": 1, "kind": "directory", "fileCount": 1, "totalBytes": len(body), "entries": entries})
|
|
}
|
|
b.objects["incoming/capture-A/"+part+"/left.mp4"] = []byte("synthetic")
|
|
if v4 {
|
|
var doc struct {
|
|
Original struct {
|
|
Files map[string]struct {
|
|
Path string `json:"path"`
|
|
} `json:"files"`
|
|
} `json:"originalSource"`
|
|
}
|
|
must(json.Unmarshal(body, &doc))
|
|
for _, file := range doc.Original.Files {
|
|
b.objects["incoming/capture-A/"+file.Path] = []byte("synthetic original")
|
|
}
|
|
}
|
|
run := domain.RunRecord{ID: runID, OrganizationID: org, ProjectID: "project-A", Status: domain.RunStatusCompleted, Artifacts: []domain.ExecutionArtifact{{ID: artifactID, RunID: runID, Name: "delivery.directory.index", Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{Kind: domain.ArtifactRefKindDirectory, Provider: domain.StorageProviderBOS, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix, ManifestKey: prefix + "/manifest.json"}}}}}
|
|
if v4 {
|
|
var audit map[string]any
|
|
must(json.Unmarshal(history, &audit))
|
|
run.Artifacts = append(run.Artifacts, domain.ExecutionArtifact{ID: "audit-" + part, RunID: runID, PortID: "deliveryAudit", Type: domain.PortTypeJSON, Data: audit})
|
|
}
|
|
run.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "package", Artifacts: run.Artifacts}}
|
|
must(ds.UpsertRun(ctx, run))
|
|
must(ds.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{ID: "package-" + part, OrganizationID: org, ProjectID: "project-A", ParentRunID: sourceRun, ChildRunID: runID, Status: domain.DeliveryPackageAccepted, Mode: domain.RunModeProduction, FinalExportArtifactIDs: []string{artifactID}}))
|
|
must(ds.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "record-" + part, OrganizationID: org, ProjectID: "project-A", RunID: sourceRun, CaptureID: "capture-A/" + part, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A/" + part, SourcePrefix: "incoming/capture-A", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: prefix}}))
|
|
}
|
|
b.objects["incoming/capture-A/meta_capture-A.json"] = []byte(`{"synthetic":true}`)
|
|
if v4 {
|
|
for _, part := range []string{"part0001", "part0002"} {
|
|
var doc struct {
|
|
Original struct {
|
|
Files map[string]struct {
|
|
Path string `json:"path"`
|
|
} `json:"files"`
|
|
} `json:"originalSource"`
|
|
}
|
|
must(json.Unmarshal(b.objects[root+"/capture-A/"+part+"/part.json"], &doc))
|
|
keys := []string{}
|
|
objects := []sourceidentity.Object{}
|
|
for _, file := range doc.Original.Files {
|
|
key := "incoming/capture-A/" + file.Path
|
|
keys = append(keys, key)
|
|
objects = append(objects, sourceidentity.Object{Key: key, ETag: deliveryindex.Hash(b.objects[key]), Size: int64(len(b.objects[key]))})
|
|
}
|
|
must(ds.UpsertRun(ctx, domain.RunRecord{ID: "source-run-" + part, OrganizationID: org, ProjectID: "project-A", Inputs: map[string]domain.ArtifactPayload{"source": {Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, Kind: domain.ArtifactRefKindDirectory, ConnectionID: "index-storage", Bucket: "test-bucket", Prefix: "incoming/capture-A", CaptureUnit: "capture-A/" + part, ObjectKeys: keys, VersionID: sourceidentity.Fingerprint(objects)}}}}))
|
|
}
|
|
}
|
|
old := storageConnectionBrowserResolver
|
|
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
|
if c.OrganizationID != org {
|
|
return nil, nil, fmt.Errorf("foreign storage")
|
|
}
|
|
return b, nil, nil
|
|
}
|
|
t.Cleanup(func() { storageConnectionBrowserResolver = old })
|
|
p := &authenticatedPrincipal{User: domain.UserRecord{ID: "index-admin", OrganizationID: org, Role: domain.UserRoleAdmin}, Organization: domain.OrganizationRecord{ID: org}}
|
|
return b, p, root
|
|
}
|
|
func exerciseIndexBatch(t *testing.T, ds store.Store) {
|
|
t.Helper()
|
|
b, p, root := seedIndexFixture(t, ds)
|
|
h := NewHandler(ds, nil)
|
|
selection := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001&packageId=package-part0002"
|
|
preview := indexCall(h, p, "GET", selection, nil)
|
|
if preview.Code != 200 {
|
|
t.Fatal(preview.Code, preview.Body.String())
|
|
}
|
|
if len(b.puts) > 0 {
|
|
t.Fatal("preview wrote storage")
|
|
}
|
|
var plan struct {
|
|
ID string `json:"snapshotId"`
|
|
Root string `json:"rootKey"`
|
|
}
|
|
json.Unmarshal(preview.Body.Bytes(), &plan)
|
|
post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001", "package-part0002"}, SnapshotID: plan.ID}
|
|
stale := post
|
|
stale.SnapshotID = "old"
|
|
if w := indexCall(h, p, "POST", "/api/delivery-index-batches", stale); w.Code != 409 || len(b.puts) > 0 {
|
|
t.Fatal("stale preview wrote")
|
|
}
|
|
b.fail = "part0002.json"
|
|
w := indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
|
if w.Code != 409 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if _, ok := b.objects[plan.Root+"/delivery_index.json"]; ok {
|
|
t.Fatal("partial batch marked complete")
|
|
}
|
|
b.fail = ""
|
|
b.ambiguous = true
|
|
h = NewHandler(ds, nil) // API restart; all progress lives in immutable storage.
|
|
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
|
if w.Code != 200 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if b.puts[len(b.puts)-1] != plan.Root+"/delivery_index.json" {
|
|
t.Fatal("total index must be last")
|
|
}
|
|
before := append([]byte(nil), b.objects[plan.Root+"/delivery_index.json"]...)
|
|
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
|
if w.Code != 200 || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) {
|
|
t.Fatal("idempotent retry")
|
|
}
|
|
target := deliveryDownloadTarget{"index-storage", "test-bucket", plan.Root, pathBase(plan.Root)}
|
|
files, e := readDownloadManifest(t.Context(), b, target)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
for _, f := range files {
|
|
if deliveryindex.Hash(b.objects[f.key]) != f.SHA256 {
|
|
t.Fatal("download hash", f.RelativePath)
|
|
}
|
|
}
|
|
// Exercise the existing public download route, including refreshed signatures.
|
|
download := downloadRequest(h, p, "/api/delivery-downloads/manifest?directory=bos://test-bucket/"+plan.Root+"&connectionId=index-storage")
|
|
if download.Code != 200 || !strings.Contains(download.Body.String(), "delivery_index.json") {
|
|
t.Fatal(download.Code, download.Body.String())
|
|
}
|
|
if os.Getenv("DELIVERY_CLASSIFICATION_TEST") == "1" {
|
|
for _, file := range files {
|
|
if strings.Contains(file.RelativePath, "history") || bytes.Contains(b.objects[file.key], []byte("PRIVATE_REWORK")) {
|
|
t.Fatal("private audit delivered")
|
|
}
|
|
}
|
|
if strings.Contains(download.Body.String(), "history") {
|
|
t.Fatal("download exposes private audit")
|
|
}
|
|
}
|
|
// A later partial snapshot does not alter the old full snapshot.
|
|
first := indexCall(h, p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
|
|
var partial struct {
|
|
ID string `json:"snapshotId"`
|
|
}
|
|
json.Unmarshal(first.Body.Bytes(), &partial)
|
|
w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID})
|
|
if w.Code != 200 || partial.ID == plan.ID || !bytes.Equal(before, b.objects[plan.Root+"/delivery_index.json"]) {
|
|
t.Fatal("immutable snapshots", w.Code, w.Body.String())
|
|
}
|
|
// Occupied, unowned batch directories can never be adopted.
|
|
conflictRoot := root + "/index-batches/" + partial.ID
|
|
delete(b.objects, conflictRoot+"/.delivery-owner.json")
|
|
if w = indexCall(h, p, "POST", "/api/delivery-index-batches", indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: partial.ID}); w.Code != 409 {
|
|
t.Fatal("adopted unowned directory")
|
|
}
|
|
}
|
|
func pathBase(s string) string { p := strings.Split(s, "/"); return p[len(p)-1] }
|
|
func TestIndexBatchHTTPPublicationRecoveryAndDownload(t *testing.T) {
|
|
exerciseIndexBatch(t, store.NewMemoryStore())
|
|
}
|
|
func TestIndexBatchRejectsScopeAndTampering(t *testing.T) {
|
|
ds := store.NewMemoryStore()
|
|
b, p, _ := seedIndexFixture(t, ds)
|
|
h := NewHandler(ds, nil)
|
|
target := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001"
|
|
other := *p
|
|
other.User.Role = domain.UserRoleQualityReviewer
|
|
if w := indexCall(h, &other, "GET", target, nil); w.Code != 403 {
|
|
t.Fatal("non-admin")
|
|
}
|
|
other = *p
|
|
other.Organization.ID = "other"
|
|
other.User.OrganizationID = "other"
|
|
if w := indexCall(h, &other, "GET", target, nil); w.Code == 200 {
|
|
t.Fatal("foreign org")
|
|
}
|
|
b.objects["deliveries/项目A__project-A/capture-A/part0001/part.json"] = []byte(`{"tampered":true}`)
|
|
if w := indexCall(h, p, "GET", target, nil); w.Code != 409 || len(b.puts) > 0 {
|
|
t.Fatal("tampered source accepted")
|
|
}
|
|
}
|
|
|
|
func TestIndexBatchDelegationKeepsSourceAndDeliveryScopesSeparate(t *testing.T) {
|
|
ds := store.NewMemoryStore()
|
|
b, p, root := seedIndexFixture(t, ds)
|
|
c, _ := ds.GetStorageConnection(t.Context(), "index-storage")
|
|
c.OrganizationID = "source-owner"
|
|
if e := ds.UpsertStorageConnection(t.Context(), *c); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
storageConnectionBrowserResolver = func(_ context.Context, c *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
|
if c.OrganizationID != "source-owner" {
|
|
return nil, nil, fmt.Errorf("wrong owner")
|
|
}
|
|
return b, nil, nil
|
|
}
|
|
mapping := func(writes []string, deliveries []string) string {
|
|
raw, _ := json.Marshal(map[string]any{p.Organization.ID: map[string]any{"connectionId": "index-storage", "bucket": "test-bucket", "readConnectionOrganizationId": "source-owner", "writeConnectionOrganizationId": "source-owner", "readPrefixes": []string{"incoming"}, "writePrefixes": writes, "deliveryPrefixes": deliveries}})
|
|
return string(raw)
|
|
}
|
|
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001", "system/runs/source-run-part0002"}, []string{root}))
|
|
h := NewHandler(ds, nil)
|
|
query := "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001"
|
|
w := indexCall(h, p, "GET", query, nil)
|
|
if w.Code != 200 {
|
|
t.Fatal("explicit source read/final-output write grant", w.Code, w.Body.String())
|
|
}
|
|
var preview struct {
|
|
ID string `json:"snapshotId"`
|
|
}
|
|
json.Unmarshal(w.Body.Bytes(), &preview)
|
|
post := indexBatchRequest{ProjectID: "project-A", PackageIDs: []string{"package-part0001"}, SnapshotID: preview.ID}
|
|
w = indexCall(h, p, "POST", "/api/delivery-index-batches", post)
|
|
if w.Code != 200 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
for _, key := range b.puts {
|
|
if !strings.HasPrefix(key, root+"/index-batches/") {
|
|
t.Fatal("write escaped final-output scope", key)
|
|
}
|
|
}
|
|
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/some-other-run"}, []string{root}))
|
|
before := len(b.puts)
|
|
if w = indexCall(h, p, "POST", "/api/delivery-index-batches", post); w.Code != 409 || len(b.puts) != before {
|
|
t.Fatal("unapproved source run wrote")
|
|
}
|
|
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", mapping([]string{"system/runs/source-run-part0001"}, []string{root + "/capture-A"}))
|
|
if w = indexCall(h, p, "GET", query, nil); w.Code != 409 {
|
|
t.Fatal("sibling batch prefix escaped delivery grant", w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestClassificationBatchHTTPPublicationRecoveryAndDownload(t *testing.T) {
|
|
t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1")
|
|
exerciseIndexBatch(t, store.NewMemoryStore())
|
|
}
|
|
|
|
func TestClassificationBatchRejectsMissingOrChangedOriginalSource(t *testing.T) {
|
|
t.Setenv("DELIVERY_CLASSIFICATION_TEST", "1")
|
|
t.Setenv("DELIVERY_V4_FIXTURE_DIR", "")
|
|
for _, failure := range []string{"bucket", "missing-file"} {
|
|
t.Run(failure, func(t *testing.T) {
|
|
ds := store.NewMemoryStore()
|
|
b, p, _ := seedIndexFixture(t, ds)
|
|
if failure == "bucket" {
|
|
record, err := ds.GetCaptureRecordByRunID(t.Context(), "source-run-part0001")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record.Bucket = "changed-bucket"
|
|
if err = ds.UpsertCaptureRecord(t.Context(), *record); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else {
|
|
delete(b.objects, "incoming/capture-A/part0001/capture-A_camera_left_part0001.mp4")
|
|
}
|
|
w := indexCall(NewHandler(ds, nil), p, "GET", "/api/delivery-index-batches?projectId=project-A&packageId=package-part0001", nil)
|
|
if w.Code != 409 || len(b.puts) != 0 || !strings.Contains(w.Body.String(), "原始") {
|
|
t.Fatalf("incomplete source accepted: %d %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func enrichConsumerFixture(p map[string]any) {
|
|
part := p["partId"].(string)
|
|
p["consumerContract"] = "droboticflow.classification-consumer/v1"
|
|
p["timeBases"] = map[string]string{"taskSplitSeconds": "original_left_video", "segmentSeconds": "slice_left_video", "segmentPartSeconds": "original_left_video", "timestampUs": "original_hardware_pts", "interval": "half-open", "frameRanges": "zero-based-half-open"}
|
|
p["pathBases"] = map[string]string{"sources": "original_capture", "sourceMetadata": "original_capture", "calibration": "original_capture"}
|
|
files := map[string]any{}
|
|
sources := map[string]any{}
|
|
for _, role := range []string{"leftVideo", "rightVideo", "leftPts", "rightPts", "imu", "cameraCalibration", "imuCalibration", "metadata"} {
|
|
name := part + "/" + role
|
|
switch role {
|
|
case "leftVideo":
|
|
name = part + "/capture-A_camera_left_" + part + ".mp4"
|
|
case "rightVideo":
|
|
name = part + "/capture-A_camera_right_" + part + ".mp4"
|
|
case "leftPts":
|
|
name = part + "/capture-A_camera_left_" + part + "_pts.csv"
|
|
case "rightPts":
|
|
name = part + "/capture-A_camera_right_" + part + "_pts.csv"
|
|
case "imu":
|
|
name = part + "/capture-A_imu_" + part + ".csv"
|
|
}
|
|
if role == "cameraCalibration" || role == "imuCalibration" {
|
|
name = "capture-A_calibration_camera.yaml"
|
|
if role == "imuCalibration" {
|
|
name = "capture-A_calibration_imu.yaml"
|
|
}
|
|
}
|
|
if role == "metadata" {
|
|
name = "meta_capture-A.json"
|
|
}
|
|
files[role] = map[string]any{"path": name}
|
|
if role == "leftVideo" || role == "rightVideo" || role == "leftPts" || role == "rightPts" || role == "imu" {
|
|
sources[role] = files[role]
|
|
}
|
|
}
|
|
p["sources"] = sources
|
|
p["sourceMetadata"] = files["metadata"]
|
|
p["originalSource"] = map[string]any{"provider": "BOS", "bucket": "test-bucket", "prefix": "incoming/capture-A", "partPrefix": "incoming/capture-A/" + part, "files": files}
|
|
for _, kind := range []string{"slices", "discardedSlices"} {
|
|
for _, raw := range p[kind].([]any) {
|
|
item := raw.(map[string]any)
|
|
split := item["taskSplit"].(map[string]any)
|
|
number := func(v any) float64 {
|
|
switch n := v.(type) {
|
|
case float64:
|
|
return n
|
|
case int:
|
|
return float64(n)
|
|
default:
|
|
panic("invalid fixture time")
|
|
}
|
|
}
|
|
start, end := number(split["startSecond"]), number(split["endSecond"])
|
|
split["startTimestampUs"] = int64(1e6 + start*1e6)
|
|
split["endTimestampUs"] = int64(1e6 + end*1e6)
|
|
if kind == "discardedSlices" {
|
|
continue
|
|
}
|
|
class := item["classification"].(map[string]any)
|
|
class["dictionary"] = map[string]any{"id": "base", "kind": "slice", "version": 1, "name": "操作", "tags": []string{"操作"}}
|
|
class["tags"] = []string{"操作"}
|
|
actions := item["segments"].([]any)
|
|
for i, raw := range actions {
|
|
a := raw.(map[string]any)
|
|
low, high := float64(i)*(end-start)/float64(len(actions)), float64(i+1)*(end-start)/float64(len(actions))
|
|
a["startSecond"] = low
|
|
a["endSecond"] = high
|
|
a["partStartSecond"] = start + low
|
|
a["partEndSecond"] = start + high
|
|
a["startTimestampUs"] = int64(1e6 + (start+low)*1e6)
|
|
a["endTimestampUs"] = int64(1e6 + (start+high)*1e6)
|
|
a["tags"] = []string{"操作"}
|
|
a["tagSource"] = "slice"
|
|
}
|
|
}
|
|
}
|
|
}
|