Contract step of the account-phone-login expand-migrate-contract rollout. Phone is now the sole account identifier end to end: - domain.UserRecord/CreateUserInput: delete Email, drop Phone's omitempty (required going forward). - store: delete email_unique Mongo index; phone_unique is now non-sparse (unique, not sparse) since every account creation path requires a phone. Delete FindUserByEmail, ensureUserEmailAvailableLocked, and ErrDuplicateUserEmail (confirmed dead via grep - all real call sites were already migrated to the *Phone equivalents by tickets 01-03). Mongo duplicate-key fallback and the account-mutation-guard's stale-write snapshot/filter now key off Phone instead of Email. - api: drop the now-impossible ErrDuplicateUserEmail branch in renderUserMutationError. The account-hierarchy audit endpoint and the supplier-deletion receipt struct both switch their identifying field from Email to Phone, since both are populated fresh from current account state (not frozen historical snapshots) - continuing to read a "email" key would silently go blank for every account created after this change. - frontend: src/types.ts User.phone is now required (email removed); AccountHierarchyIssue.email -> .phone in services/api/users.ts and pages/Users.tsx to match the backend audit endpoint's renamed field. - Rewrote the wide blast radius of Go test fixtures that constructed users by email across internal/api/*_test.go and internal/store/*_test.go, including the two dedicated memory-store uniqueness tests (merged into phone-only equivalents) and several opt-in "live" acceptance-fixture tests gated behind env vars for real Fanyu deployments. Repo-wide grep of Go and TS/TSX for Email/email/邮箱 turned up no remaining account-identifier references outside of a few harmless test-only cosmetic labels (e.g. "Email Tenant" as an org display name in a seed helper) and assertions that explicitly check "邮箱" text is absent from API responses. go build/vet clean. go test ./internal/api/... ./internal/store/... ./internal/domain/... : 75 failures, identical test names to the pre-ticket-05 baseline (diffed against a throwaway worktree at the prior tip). npm run lint:web (tsc --noEmit) clean. npm run test:frontend: 599/586/13, matching the stated baseline exactly. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
286 lines
12 KiB
Go
286 lines
12 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"encoding/json"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// Opt-in seeding only. Original production task/files remain read-only; every
|
|
// write goes to a uniquely named sandbox database and private test Run prefixes.
|
|
func TestSeedAnnotationPerformanceSandbox(t *testing.T) {
|
|
if os.Getenv("FANYU_PERF_SEED") != "1" {
|
|
t.Skip("explicit isolated performance fixture")
|
|
}
|
|
nonce := os.Getenv("FANYU_PERF_NONCE")
|
|
if !regexp.MustCompile(`^2026[0-9a-z]{8,24}$`).MatchString(nonce) {
|
|
t.Fatal("invalid nonce")
|
|
}
|
|
name := "annotation_perf_" + nonce
|
|
if name == os.Getenv("MONGO_DB") {
|
|
t.Fatal("sandbox must differ from source")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cancel()
|
|
client, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")))
|
|
if e != nil {
|
|
t.Fatal("mongo connect")
|
|
}
|
|
defer client.Disconnect(ctx)
|
|
source := store.NewMongoStore(client.Database(os.Getenv("MONGO_DB")))
|
|
target := store.NewMongoStore(client.Database(name))
|
|
mode := os.Getenv("FANYU_PERF_MODE")
|
|
if mode == "audit" {
|
|
db := client.Database(name)
|
|
indexes, e := db.Collection("annotationTimelineIndexes").CountDocuments(ctx, bson.M{})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
cur, e := db.Collection("humanItems").Find(ctx, bson.M{"projectId": "project-perf-" + nonce}, options.Find().SetProjection(bson.M{"_id": 0, "id": 1, "legacyTaskId": 1, "currentStageId": 1, "processingJob.status": 1, "processingJob.stage": 1, "processingJob.attempt": 1, "processingJob.error": 1}))
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
var rows []bson.M
|
|
if e = cur.All(ctx, &rows); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
out, _ := json.Marshal(map[string]any{"indexes": indexes, "items": rows})
|
|
fmt.Println("PERFORMANCE_AUDIT " + string(out))
|
|
return
|
|
}
|
|
if mode == "verify" {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
|
defer cancel()
|
|
scratch, e := os.MkdirTemp("", "annotation-perf-verify-")
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
defer os.RemoveAll(scratch)
|
|
reader := &server{store: target, workspaceMountPath: "/workspace"}
|
|
tasks, e := target.ListAnnotationTasks(ctx, "org-29cd7a684faba19f630f8ad5890e18bb", "project-perf-"+nonce)
|
|
if e != nil || len(tasks) == 0 {
|
|
t.Fatal("no performance tasks")
|
|
}
|
|
sort.Slice(tasks, func(i, j int) bool { return tasks[i].ID < tasks[j].ID })
|
|
task := tasks[0]
|
|
run, e := target.GetRun(ctx, task.RunID)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
var chosen *domain.ExecutionArtifact
|
|
for i := range run.Artifacts {
|
|
a := &run.Artifacts[i]
|
|
if boolValue(a.Data["metadataOnly"]) {
|
|
chosen = a
|
|
}
|
|
}
|
|
if chosen == nil || !strings.HasPrefix(chosen.Path, "/workspace/system/runs/run-perf-"+nonce+"-") {
|
|
t.Fatal("missing scoped metadata-only generation")
|
|
}
|
|
manifest, e := readVerifiedAnnotationJSON(filepath.Join(chosen.Path, "revision-manifest.json"), chosen.Metadata.SHA256)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
frames, files := 0, 0
|
|
for _, raw := range listValue(manifest["items"]) {
|
|
entry := mapValue(raw)
|
|
directory := filepath.Join(chosen.Path, stringValue(entry["directory"]))
|
|
for kind, rawFile := range mapValue(entry["files"]) {
|
|
f := mapValue(rawFile)
|
|
path := filepath.Join(directory, stringValue(f["name"]))
|
|
if !strings.HasPrefix(path, chosen.Path+"/") {
|
|
t.Fatal("unsafe member path")
|
|
}
|
|
local, e := reader.localAnnotationSource(ctx, task.OrganizationID, path, filepath.Join(scratch, fmt.Sprintf("member-%d-%s", files, filepath.Base(path))))
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
sha, size, e := sha256File(local)
|
|
if e != nil || sha != stringValue(f["sha256"]) || size != int64Value(f["bytes"]) {
|
|
t.Fatal("file digest mismatch", kind, e)
|
|
}
|
|
files++
|
|
if kind == "leftVideo" || kind == "rightVideo" {
|
|
n, e := ffprobeFrameCount(ctx, local)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
ptsKey := "leftPts"
|
|
if kind == "rightVideo" {
|
|
ptsKey = "rightPts"
|
|
}
|
|
if n != intValue(mapValue(mapValue(entry["files"])[ptsKey])["rowCount"]) {
|
|
t.Fatal("video/PTS mismatch")
|
|
}
|
|
frames += n
|
|
}
|
|
}
|
|
}
|
|
out, _ := json.Marshal(map[string]any{"taskId": task.ID, "files": files, "stereoFrames": frames, "metadataOnly": true, "sourceDuration": task.SourceDurationSeconds})
|
|
fmt.Println("PERFORMANCE_VERIFIED " + string(out))
|
|
return
|
|
}
|
|
if mode == "disable-users" {
|
|
users, e := target.ListUsers(ctx)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
for _, u := range users {
|
|
if !regexp.MustCompile(`^perf-` + nonce + `-`).MatchString(u.ID) {
|
|
continue
|
|
}
|
|
u.Status = domain.UserStatusDisabled
|
|
if e = target.UpsertUser(ctx, u); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
fmt.Println("disabled only isolated performance accounts")
|
|
_, e = client.Database(name).Collection("storageConnections").UpdateOne(ctx, bson.M{"id": "storage-d0a49fe4b9881c324f0640ab62359603", "organizationId": "org-29cd7a684faba19f630f8ad5890e18bb"}, bson.M{"$unset": bson.M{"accessKeyId": "", "secretAccessKey": ""}})
|
|
if e != nil {
|
|
t.Fatal("remove copied sandbox credentials")
|
|
}
|
|
return
|
|
}
|
|
collections, e := client.Database(name).ListCollectionNames(ctx, bson.M{})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if len(collections) > 0 {
|
|
t.Fatal("sandbox already exists; use a fresh nonce")
|
|
}
|
|
password := os.Getenv("FANYU_PERF_PASSWORD")
|
|
if len(password) < 24 {
|
|
t.Fatal("ephemeral password required")
|
|
}
|
|
count := 5
|
|
if v, e := strconv.Atoi(os.Getenv("FANYU_PERF_USERS")); e == nil {
|
|
count = v
|
|
}
|
|
if count < 2 || count > 10 {
|
|
t.Fatal("bounded concurrency 2..10 required")
|
|
}
|
|
original, e := source.GetAnnotationTask(ctx, "annotation-task-7d738b9fc2bd9e9e778308029a7d6a50")
|
|
if e != nil || original == nil || original.OrganizationID != "org-29cd7a684faba19f630f8ad5890e18bb" || original.SourceDurationSeconds < 800 {
|
|
t.Fatal("authorized long source task unavailable")
|
|
}
|
|
run, e := source.GetRun(ctx, original.RunID)
|
|
if e != nil || run == nil {
|
|
t.Fatal("source run unavailable")
|
|
}
|
|
var input domain.NodeExecutionRecord
|
|
for _, n := range run.NodeExecutions {
|
|
if n.NodeID == original.NodeID {
|
|
input = n
|
|
}
|
|
}
|
|
org, e := source.GetOrganization(ctx, original.OrganizationID)
|
|
if e != nil || org == nil {
|
|
t.Fatal("source organization")
|
|
}
|
|
connection, e := source.GetStorageConnection(ctx, "storage-d0a49fe4b9881c324f0640ab62359603")
|
|
if e != nil || connection == nil || connection.OrganizationID != org.ID {
|
|
t.Fatal("workspace connection scope")
|
|
}
|
|
if e = target.UpsertOrganization(ctx, *org); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e = target.UpsertStorageConnection(ctx, *connection); e != nil {
|
|
t.Fatal("copy scoped storage configuration")
|
|
}
|
|
if e = target.EnsureIndexes(ctx); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
projectID := "project-perf-" + nonce
|
|
workflowID := "workflow-perf-" + nonce
|
|
supplierID := "supplier-perf-" + nonce
|
|
must := func(e error) {
|
|
t.Helper()
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
must(target.UpsertProject(ctx, domain.ProjectRecord{ID: projectID, OrganizationID: org.ID, Name: "长视频并发隔离验收", Description: "Isolated performance fixture " + nonce}))
|
|
must(target.UpsertSupplier(ctx, domain.SupplierRecord{ID: supplierID, OrganizationID: org.ID, Name: "性能测试专用供应商", Status: "active"}))
|
|
must(target.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: workflowID, OrganizationID: org.ID, ProjectID: projectID, Name: "隔离人工流程"}))
|
|
templateID := "template-perf-" + nonce
|
|
must(target.UpsertAnnotationTemplate(ctx, domain.AnnotationTemplateRecord{ID: templateID, OrganizationID: org.ID, ProjectID: projectID, Version: 6, Status: domain.AnnotationTemplatePublished, Labels: domain.ConvenienceStore22SemanticLabels()}))
|
|
labelID := "slice-perf-" + nonce
|
|
must(target.UpsertAnnotationLabel(ctx, domain.AnnotationLabelRecord{ID: labelID, OrganizationID: org.ID, Kind: "slice", Code: labelID, Name: "压力切片", Status: domain.AnnotationLabelActive}))
|
|
must(target.UpsertProjectSliceLabels(ctx, domain.ProjectAnnotationLabelsRecord{ID: "labels-" + nonce, OrganizationID: org.ID, ProjectID: projectID, SliceLabelIDs: []string{labelID}}))
|
|
users, tasks := []map[string]string{}, []string{}
|
|
for i := 0; i < count; i++ {
|
|
suffix := fmt.Sprintf("%02d", i+1)
|
|
uid := "perf-" + nonce + "-ann-" + suffix
|
|
phone := testPhoneFromSeed(uid)
|
|
hash, e := hashPassword(password)
|
|
must(e)
|
|
must(target.UpsertUser(ctx, domain.UserRecord{ID: uid, OrganizationID: org.ID, Name: "压力标注员" + suffix, Phone: phone, PasswordHash: hash, Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive, SupplierID: supplierID}))
|
|
must(target.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "member-" + uid, OrganizationID: org.ID, ProjectID: projectID, UserID: uid, SupplierID: supplierID, Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}))
|
|
runID := "run-perf-" + nonce + "-" + suffix
|
|
taskID := "task-perf-" + nonce + "-" + suffix
|
|
batchID := "batch-perf-" + nonce + "-" + suffix
|
|
itemID := "item-perf-" + nonce + "-" + suffix
|
|
bindings := []domain.NodeInputBinding{}
|
|
artifacts := []domain.ExecutionArtifact{}
|
|
leftID, rightID := "", ""
|
|
for _, port := range []string{"fullLeftVideos", "fullRightVideos", "leftTimestamps", "rightTimestamps", "imu"} {
|
|
a := artifactsForTarget(input, port)
|
|
if len(a) != 1 {
|
|
t.Fatalf("expected one source per port %s, got %d", port, len(a))
|
|
}
|
|
v := a[0]
|
|
v.ID = runID + "-" + port
|
|
v.RunID = runID
|
|
v.NodeID = "source"
|
|
bindings = append(bindings, domain.NodeInputBinding{TargetPortID: port, SourceNodeID: "source", SourcePortID: port, Artifacts: []domain.ExecutionArtifact{v}})
|
|
artifacts = append(artifacts, v)
|
|
if port == "fullLeftVideos" {
|
|
leftID = v.ID
|
|
}
|
|
if port == "fullRightVideos" {
|
|
rightID = v.ID
|
|
}
|
|
}
|
|
must(target.UpsertRun(ctx, domain.RunRecord{ID: runID, OrganizationID: org.ID, ProjectID: projectID, WorkflowID: workflowID, ProcessingProfile: domain.ProcessingProfileRawHuman, Status: domain.RunStatusWaitingForHuman, Mode: domain.RunModeProduction, Artifacts: artifacts, NodeExecutions: []domain.NodeExecutionRecord{{NodeID: "source", Status: domain.NodeExecutionStatusCompleted, Artifacts: artifacts}, {NodeID: "human", Status: domain.NodeExecutionStatusWaitingForHuman, InputBindings: bindings, InputArtifacts: artifacts}}}))
|
|
must(target.UpsertHumanBatch(ctx, domain.HumanBatchRecord{ID: batchID, OrganizationID: org.ID, PluginID: "annotation-qa", ProjectID: projectID, SupplierID: supplierID, WorkflowID: workflowID, RunID: runID, GateNodeID: "human", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages(), RequirePrePackageAcceptance: true, ReleasePolicy: domain.HumanReleaseAllAccepted, ItemCount: 1}))
|
|
must(target.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: itemID, OrganizationID: org.ID, PluginID: "annotation-qa", ProjectID: projectID, SupplierID: supplierID, BatchID: batchID, WorkflowID: workflowID, RunID: runID, GateNodeID: "human", LegacyTaskID: taskID, ProcessingProfile: domain.ProcessingProfileRawHuman, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 1, Subject: domain.HumanSubjectRef{ItemKey: "pressure-" + suffix, Payload: map[string]any{"captureDirectory": "真实长视频压力-" + suffix, "sliceWorkflow": map[string]any{"status": "editing", "phase": "slice"}}}}))
|
|
copy := *original
|
|
copy.ID = taskID
|
|
copy.ProjectID = projectID
|
|
copy.WorkflowID = workflowID
|
|
copy.RunID = runID
|
|
copy.NodeID = "human"
|
|
copy.ArtifactID = leftID
|
|
copy.RightArtifactID = rightID
|
|
copy.SourceVideo = "/api/runs/" + runID + "/artifacts/" + leftID + "/content"
|
|
copy.RightSourceVideo = "/api/runs/" + runID + "/artifacts/" + rightID + "/content"
|
|
copy.TemplateID = templateID
|
|
copy.TemplateVersion = 6
|
|
copy.SupplierID = supplierID
|
|
copy.AssignedUserID = ""
|
|
copy.Status = domain.AnnotationTaskStatusPendingAnnotation
|
|
copy.CurrentRevision = 0
|
|
copy.ProcessingProfile = domain.ProcessingProfileRawHuman
|
|
must(target.UpsertAnnotationTask(ctx, copy))
|
|
users = append(users, map[string]string{"id": uid, "phone": phone})
|
|
tasks = append(tasks, taskID)
|
|
}
|
|
_, e = client.Database(name).Collection("performanceScope").InsertOne(ctx, bson.M{"id": nonce, "sourceTaskId": original.ID, "projectId": projectID})
|
|
must(e)
|
|
out, _ := json.Marshal(map[string]any{"database": name, "nonce": nonce, "projectId": projectID, "users": users, "tasks": tasks, "sourceTaskId": original.ID, "sourceDurationSeconds": original.SourceDurationSeconds})
|
|
fmt.Println("PERFORMANCE_FIXTURE " + string(out))
|
|
}
|