diff --git a/docs/supplier-cascade-deletion.md b/docs/supplier-cascade-deletion.md index 56f08713..6a9a0f11 100644 --- a/docs/supplier-cascade-deletion.md +++ b/docs/supplier-cascade-deletion.md @@ -15,3 +15,11 @@ Execution uses a renewable lease and idempotent steps. Closing the dialog does n `SUPPLIER_CASCADE_MONGO_TEST=1 go test -race ./internal/store ./internal/api -run TestSupplierCascade -count=1` runs the memory and standalone MongoDB contracts. The Mongo fixture uses the configured local isolated Mongo server. Frontend checks are `npm run lint:web` and `npm run build:web`. `TestSupplierCascadeLiveFixture` is opt-in and requires an explicitly isolated nonce. It seeds two suppliers in one dedicated project, mixed evidence, exclusive and shared derived directories, and protected raw input. A dedicated test Pod finalizer can hold runtime termination to exercise failure and retry; remove only that test finalizer before retrying. Mounted object storage may not support POSIX chmod, so permission injection must not be assumed to work there. Verify mode checks supplier A cleanup, supplier B preservation, shared references and late-write fencing. Cleanup mode removes only that isolated fixture and retains the deletion audit. Never use this fixture to delete existing customer suppliers. + +## Preview performance and cancellation + +The planner indexes children by batch/run once instead of comparing every run with every task for each ownership pass. Mongo reads up to four collections concurrently, projects ownership metadata, and extracts run references from raw BSON without materializing inline artifact payloads. Inline payload token matching is conservative: ambiguous binary matches preserve extra data rather than widening deletion. Cross-organization and transitive artifact references remain protected. + +A preview uses one request-local run inventory, with additional reads for other organizations. No run inventory or deletion plan is cached across requests, confirmations or retries. Backend planning has a 20-second deadline and at most two concurrent previews per process. The dialog aborts reads when closed/switched, times out after 25 seconds, and exposes retry. Timeout responses use HTTP 408 so they do not enter automatic gateway-error retries. + +`TestSupplierCascadeProductionScaleAndCancellation` covers 10,001 runs, 5,001 items and 5,000 batches, including shared records and cancelled requests. `TestSupplierCascadeLivePreviewPerformance` is an opt-in read-only probe; it never begins deletion. During 2026-09-21 verification, standalone Mongo/race regressions passed, the synthetic planning benchmark took approximately 17 ms, and the complete production-data preview took approximately 8.2 seconds for three accounts / 88 records / 14 targets. These are different measurements, not an HTTP latency guarantee. Local browser validation confirmed request abort on close, timeout feedback, retry recovery and no console errors. diff --git a/internal/api/supplier_deletion.go b/internal/api/supplier_deletion.go index ddd212a8..5d184dbe 100644 --- a/internal/api/supplier_deletion.go +++ b/internal/api/supplier_deletion.go @@ -5,9 +5,11 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "os" + "regexp" "sort" "strings" "sync" @@ -38,7 +40,22 @@ func (s *server) supplierDeletionAdmin(w http.ResponseWriter, r *http.Request) b } return true } + +// Bound concurrent scans and release all work when the requesting browser leaves. +var supplierPreviewSlots = make(chan struct{}, 2) +var supplierRunReferencePattern = regexp.MustCompile(`run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}`) + func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) { + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + ctx = store.WithSupplierDeletionReadScope(ctx) + select { + case supplierPreviewSlots <- struct{}{}: + defer func() { <-supplierPreviewSlots }() + case <-ctx.Done(): + return domain.SupplierDeletionPlan{}, ctx.Err() + } + p, e := s.store.PreviewSupplierDeletion(ctx, org, id) if e != nil { return p, e @@ -53,7 +70,13 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin p.ProjectNames = append(p.ProjectNames, project.Name) } } - runs, e := s.store.ListRunArtifacts(ctx, "") + if len(p.RunIDs) == 0 { + p.RunIDs = []string{} + p.Counts["fileTargets"], p.Counts["runningTasks"] = 0, 0 + store.FinalizeSupplierDeletionPlan(&p) + return p, ctx.Err() + } + runs, e := s.store.ListSupplierDeletionRunReferences(ctx) if e != nil { return p, e } @@ -70,22 +93,36 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin owns[id] = true } byID := map[string]domain.RunRecord{} + runJSON := map[string]string{} + runsByWorkflow := map[string][]domain.RunRecord{} for _, r := range runs { - byID[r.ID] = r - } - // Foreign references are checked across projects as well as across suppliers - // within the same project. Preserve the producer record and its files. - for _, r := range runs { - if owns[r.ID] { - continue + if err := ctx.Err(); err != nil { + return p, err } + byID[r.ID] = r raw, _ := json.Marshal(r) - for id := range owns { - if strings.Contains(string(raw), "/"+id+"/") || strings.Contains(string(raw), `"runId":"`+id+`"`) { + runJSON[r.ID] = string(raw) + runsByWorkflow[r.WorkflowID] = append(runsByWorkflow[r.WorkflowID], r) + } + // Index foreign references once, including across projects/organizations. + // A newly shared producer is queued so transitive sharing is preserved. + queue := []string{} + for _, r := range runs { + if !owns[r.ID] { + queue = append(queue, r.ID) + } + } + for head := 0; head < len(queue); head++ { + if err := ctx.Err(); err != nil { + return p, err + } + for _, id := range supplierRunReferencePattern.FindAllString(runJSON[queue[head]], -1) { + if owns[id] { delete(owns, id) p.SharedRunIDs = append(p.SharedRunIDs, id) p.SharedRecords["runs"] = append(p.SharedRecords["runs"], id) p.Preserved = append(p.Preserved, "其他运行引用的产物及运行记录:"+id) + queue = append(queue, id) } } } @@ -109,9 +146,12 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin retainedPaths := []retainedPath{} uncertainStores := map[string]bool{} for _, w := range workflows { + if err := ctx.Err(); err != nil { + return p, err + } retained := []domain.RunRecord{} - for _, r := range runs { - if r.WorkflowID == w.ID && !owns[r.ID] { + for _, r := range runsByWorkflow[w.ID] { + if !owns[r.ID] { retained = append(retained, r) } } @@ -169,7 +209,7 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin if owns[r.ID] { continue } - raw, _ := json.Marshal(r) + raw := runJSON[r.ID] if strings.Contains(string(raw), "bos://"+bucket+"/"+strings.Trim(prefix, "/")) { p.Preserved = append(p.Preserved, "其他运行引用的文件:"+prefix) return @@ -195,7 +235,7 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin for _, w := range workflows { ownedRuns := []domain.RunRecord{} foreignRun := false - for _, r := range runs { + for _, r := range runsByWorkflow[w.ID] { if r.WorkflowID == w.ID { if owns[r.ID] { ownedRuns = append(ownedRuns, r) @@ -253,7 +293,7 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin } } store.FinalizeSupplierDeletionPlan(&p) - return p, nil + return p, ctx.Err() } func (s *server) handleSupplierDeletionPreview(w http.ResponseWriter, r *http.Request) { if !s.supplierDeletionAdmin(w, r) { @@ -261,6 +301,13 @@ func (s *server) handleSupplierDeletionPreview(w http.ResponseWriter, r *http.Re } id := chi.URLParam(r, "supplierId") p, e := s.buildSupplierDeletionPreview(r.Context(), s.requestOrganizationID(r), id) + if errors.Is(e, context.DeadlineExceeded) { + renderJSON(w, http.StatusRequestTimeout, map[string]string{"error": "删除范围核对超时,请稍后重试", "code": "supplier_preview_timeout"}) + return + } + if errors.Is(e, context.Canceled) { + return + } if e != nil { renderSupplierMutationError(w, e) return diff --git a/internal/api/supplier_deletion_preview_live_test.go b/internal/api/supplier_deletion_preview_live_test.go new file mode 100644 index 00000000..b8332682 --- /dev/null +++ b/internal/api/supplier_deletion_preview_live_test.go @@ -0,0 +1,48 @@ +package api + +import ( + "context" + "droboticflow/internal/store" + "fmt" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "os" + "testing" + "time" +) + +// Opt-in read-only performance probe. It never starts or resumes a deletion. +func TestSupplierCascadeLivePreviewPerformance(t *testing.T) { + if os.Getenv("SUPPLIER_PREVIEW_READONLY") != "1" { + t.Skip("explicit read-only preview probe") + } + id := os.Getenv("SUPPLIER_PREVIEW_ID") + if id == "" { + t.Fatal("supplier required") + } + ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI"))) + if err != nil { + t.Fatal("database connection failed") + } + defer client.Disconnect(context.Background()) + ds := store.NewMongoStore(client.Database(os.Getenv("MONGO_DB"))) + supplier, err := ds.GetSupplier(ctx, id) + if err != nil || supplier == nil { + t.Fatal("supplier unavailable") + } + start := time.Now() + p, err := ds.PreviewSupplierDeletion(ctx, supplier.OrganizationID, id) + if err != nil { + t.Fatal(err) + } + fmt.Printf("ownership_preview_ms=%d records=%d owned_runs=%d shared_runs=%d\n", time.Since(start).Milliseconds(), p.Counts["totalRecords"], len(p.RunIDs), len(p.SharedRunIDs)) + server := &server{store: ds, workspaceMountPath: os.Getenv("WORKSPACE_MOUNT_PATH")} + start = time.Now() + p, err = server.buildSupplierDeletionPreview(ctx, supplier.OrganizationID, id) + if err != nil { + t.Fatal(err) + } + fmt.Printf("complete_preview_ms=%d accounts=%d records=%d targets=%d blockers=%d\n", time.Since(start).Milliseconds(), len(p.Accounts), p.Counts["totalRecords"], len(p.Targets), len(p.Blockers)) +} diff --git a/internal/api/supplier_deletion_test.go b/internal/api/supplier_deletion_test.go index 80ef8b2b..b51812c5 100644 --- a/internal/api/supplier_deletion_test.go +++ b/internal/api/supplier_deletion_test.go @@ -189,3 +189,39 @@ func TestSupplierCascadePlatformWorkspaceScope(t *testing.T) { }) } } + +func TestSupplierCascadePreviewPreservesCrossOrganizationArtifactChain(t *testing.T) { + ds := store.NewMemoryStore() + ctx := t.Context() + if err := ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: "A", OrganizationID: "org-a", Name: "A", Status: "active"}); err != nil { + t.Fatal(err) + } + for _, id := range []string{"run-first", "run-second", "run-adjacent"} { + r := domain.RunRecord{ID: id, OrganizationID: "org-a", ProjectID: "project-a", Status: domain.RunStatusCompleted} + if id == "run-second" { + r.Artifacts = []domain.ExecutionArtifact{{Path: "/workspace/run-first/file"}} + } + if err := ds.UpsertRun(ctx, r); err != nil { + t.Fatal(err) + } + if err := ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item-" + id, OrganizationID: "org-a", ProjectID: "project-a", SupplierID: "A", RunID: id}); err != nil { + t.Fatal(err) + } + } + if err := ds.UpsertRun(ctx, domain.RunRecord{ID: "run-foreign", OrganizationID: "org-b", Artifacts: []domain.ExecutionArtifact{{Path: "/workspace/run-second/run-adjacent/file"}}}); err != nil { + t.Fatal(err) + } + s := &server{store: ds} + plan, err := s.buildSupplierDeletionPreview(ctx, "org-a", "A") + if err != nil { + t.Fatal(err) + } + if len(plan.RunIDs) != 0 || len(plan.SharedRunIDs) != 3 || len(plan.Targets) != 0 { + t.Fatalf("foreign/transitive artifact references lost: %+v", plan) + } + cancelled, cancel := context.WithCancel(ctx) + cancel() + if _, err = s.buildSupplierDeletionPreview(cancelled, "org-a", "A"); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation not propagated: %v", err) + } +} diff --git a/internal/store/store.go b/internal/store/store.go index f3094f75..4126c81f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -159,6 +159,7 @@ type Store interface { ListProjectAnnotationMemberships(context.Context, string, string) ([]domain.ProjectAnnotationMembershipRecord, error) UpsertProjectAnnotationMembership(context.Context, domain.ProjectAnnotationMembershipRecord) error DeleteProjectAnnotationMembership(context.Context, string) (bool, error) + ListSupplierDeletionRunReferences(context.Context) ([]domain.RunRecord, error) PreviewSupplierDeletion(context.Context, string, string) (domain.SupplierDeletionPlan, error) GetSupplierDeletion(context.Context, string) (*domain.SupplierDeletionTask, error) ListSupplierDeletions(context.Context, string) ([]domain.SupplierDeletionTask, error) diff --git a/internal/store/supplier_deletion.go b/internal/store/supplier_deletion.go index 8b69d3c0..26e10d0c 100644 --- a/internal/store/supplier_deletion.go +++ b/internal/store/supplier_deletion.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "golang.org/x/sync/errgroup" "sort" + "sync" "time" "droboticflow/internal/domain" @@ -69,25 +71,79 @@ func (s *MemoryStore) supplierSnapshotLocked(org string) supplierSnapshot { "datasetVersions": supplierMapDocs(s.datasetVersions, org), } } + +// Only relationship metadata is needed for ownership. Run references remain +// structurally complete so references buried in legacy payloads stay protected. +func supplierDeletionProjection(col string) bson.M { + if col == "runs" { + return bson.M{"logs": 0, "resourceSeries": 0, "nodeExecutions.logs": 0, "nodeExecutions.resourceSeries": 0} + } + p := bson.M{"_id": 0} + for _, field := range []string{"id", "organizationId", "supplierId", "vendorId", "projectId", "workflowId", "taskId", "itemId", "batchId", "runId", "childRunId", "parentRunId", "legacyTaskId", "userId"} { + p[field] = 1 + } + switch col { + case "users": + for _, field := range []string{"name", "email", "role", "isSuperAdmin"} { + p[field] = 1 + } + case "workflows": + p["nodes.data.hyperparams"] = 1 + case "humanItems": + p["subject.payload.items.sourceHumanItemId"] = 1 + p["subject.payload.items.taskId"] = 1 + case "humanReleasePartitions", "deliveryPackages": + p["itemIds"] = 1 + } + return p +} func (s *MongoStore) loadSupplierSnapshot(ctx context.Context, org string) (supplierSnapshot, error) { data := supplierSnapshot{} - for _, col := range supplierDeletionCollections { - cursor, e := s.db.Collection(col).Find(ctx, orgCompatibilityFilter(org), options.Find().SetProjection(bson.M{"passwordHash": 0, "tokenHash": 0, "body": 0, "logs": 0, "resourceSeries": 0, "nodeExecutions.logs": 0, "nodeExecutions.resourceSeries": 0, "processingJob.input": 0, "subject.payload.annotationWorkspace": 0, "subject.payload.annotationDraft": 0}).SetLimit(50001)) - if e != nil { - return nil, e - } - rows, e := decodeMany[bson.M](ctx, cursor) - if e != nil { - return nil, e - } - if len(rows) > 50000 { - return nil, fmt.Errorf("%s 超过安全预览上限", col) - } - data[col] = rows + var mu sync.Mutex + group, ctx := errgroup.WithContext(ctx) + group.SetLimit(4) + for _, collection := range supplierDeletionCollections { + col := collection + group.Go(func() error { + var rows []bson.M + if col == "runs" { + runs, err := s.supplierRunReferences(ctx, orgCompatibilityFilter(org)) + if err != nil { + return err + } + if scope, ok := ctx.Value(supplierReadScopeKey{}).(*supplierReadScope); ok { + scope.organization = org + scope.runs = runs + scope.loaded = true + } + for _, r := range runs { + rows = append(rows, supplierDoc(r)) + } + } else { + cursor, err := s.db.Collection(col).Find(ctx, orgCompatibilityFilter(org), options.Find().SetProjection(supplierDeletionProjection(col)).SetBatchSize(5000).SetLimit(50001)) + if err != nil { + return err + } + rows, err = decodeMany[bson.M](ctx, cursor) + if err != nil { + return err + } + } + if len(rows) > 50000 { + return fmt.Errorf("%s 超过安全预览上限", col) + } + mu.Lock() + data[col] = rows + mu.Unlock() + return nil + }) + } + if err := group.Wait(); err != nil { + return nil, err } return data, nil } -func (s *MemoryStore) PreviewSupplierDeletion(_ context.Context, org, id string) (domain.SupplierDeletionPlan, error) { +func (s *MemoryStore) PreviewSupplierDeletion(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) { s.mu.Lock() defer s.mu.Unlock() supplier, ok := s.suppliers[id] @@ -97,7 +153,7 @@ func (s *MemoryStore) PreviewSupplierDeletion(_ context.Context, org, id string) if supplier.IsDefault || id == domain.DefaultSupplierID(org) { return domain.SupplierDeletionPlan{}, ErrDefaultSupplierProtected } - return buildSupplierDeletionPlan(supplier, s.supplierSnapshotLocked(org)), nil + return buildSupplierDeletionPlan(ctx, supplier, s.supplierSnapshotLocked(org)) } func (s *MongoStore) PreviewSupplierDeletion(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) { supplier, e := s.GetSupplier(ctx, id) @@ -114,7 +170,7 @@ func (s *MongoStore) PreviewSupplierDeletion(ctx context.Context, org, id string if e != nil { return domain.SupplierDeletionPlan{}, e } - return buildSupplierDeletionPlan(*supplier, data), nil + return buildSupplierDeletionPlan(ctx, *supplier, data) } func (s *MemoryStore) GetSupplierDeletion(_ context.Context, id string) (*domain.SupplierDeletionTask, error) { s.mu.Lock() @@ -154,7 +210,7 @@ func (s *MongoStore) ListSupplierDeletions(ctx context.Context, org string) ([]d } return decodeMany[domain.SupplierDeletionTask](ctx, cur) } -func (s *MemoryStore) BeginSupplierDeletion(_ context.Context, t domain.SupplierDeletionTask) (*domain.SupplierDeletionTask, error) { +func (s *MemoryStore) BeginSupplierDeletion(ctx context.Context, t domain.SupplierDeletionTask) (*domain.SupplierDeletionTask, error) { s.mu.Lock() defer s.mu.Unlock() if old, ok := s.supplierDeletions[t.ID]; ok { @@ -167,7 +223,10 @@ func (s *MemoryStore) BeginSupplierDeletion(_ context.Context, t domain.Supplier if supplier.IsDefault || supplier.ID == domain.DefaultSupplierID(t.OrganizationID) { return nil, ErrDefaultSupplierProtected } - fresh := buildSupplierDeletionPlan(supplier, s.supplierSnapshotLocked(t.OrganizationID)) + fresh, err := buildSupplierDeletionPlan(ctx, supplier, s.supplierSnapshotLocked(t.OrganizationID)) + if err != nil { + return nil, err + } if fresh.Digest != t.Plan.ScopeDigest { return nil, ErrSupplierDeletionScopeChanged } diff --git a/internal/store/supplier_deletion_performance_test.go b/internal/store/supplier_deletion_performance_test.go new file mode 100644 index 00000000..a5c573be --- /dev/null +++ b/internal/store/supplier_deletion_performance_test.go @@ -0,0 +1,80 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "errors" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "testing" + "time" +) + +func productionScaleSupplierSnapshot() supplierSnapshot { + data := supplierSnapshot{} + for i := 0; i < 10000; i++ { + id := fmt.Sprintf("run-%d", i) + data["runs"] = append(data["runs"], bson.M{"id": id, "projectId": "shared-project"}) + } + for i := 0; i < 5000; i++ { + sup := "other" + if i < 10 { + sup = "target" + } + run, batch, task, item := fmt.Sprintf("run-%d", i), fmt.Sprintf("batch-%d", i), fmt.Sprintf("task-%d", i), fmt.Sprintf("item-%d", i) + data["annotationTasks"] = append(data["annotationTasks"], bson.M{"id": task, "supplierId": sup, "projectId": "shared-project", "runId": run}) + data["humanItems"] = append(data["humanItems"], bson.M{"id": item, "supplierId": sup, "projectId": "shared-project", "runId": run, "batchId": batch, "legacyTaskId": task}) + data["humanBatches"] = append(data["humanBatches"], bson.M{"id": batch, "projectId": "shared-project", "runId": run}) + } + data["humanItems"] = append(data["humanItems"], bson.M{"id": "other-shared-item", "supplierId": "other", "batchId": "batch-0", "runId": "run-0", "projectId": "shared-project"}) + data["runs"] = append(data["runs"], bson.M{"id": "run-borrower", "inputs": bson.M{"nested": bson.M{"sourceRunId": "run-1"}}}) + return data +} +func TestSupplierCascadeProductionScaleAndCancellation(t *testing.T) { + data := productionScaleSupplierSnapshot() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + start := time.Now() + plan, err := buildSupplierDeletionPlan(ctx, domain.SupplierRecord{ID: "target", OrganizationID: "org", Name: "target"}, data) + if err != nil { + t.Fatalf("production-scale preview failed: %v", err) + } + t.Logf("10001 runs / 5001 items / 5000 batches planned in %s", time.Since(start)) + if len(plan.Records["annotationTasks"]) != 10 || len(plan.RunIDs) != 8 || len(plan.SharedRunIDs) != 2 || len(plan.SharedRecords["humanBatches"]) != 1 { + t.Fatalf("wrong ownership scope: %+v", plan.Counts) + } + cancelled, stop := context.WithCancel(t.Context()) + stop() + if _, err := buildSupplierDeletionPlan(cancelled, domain.SupplierRecord{ID: "target"}, data); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled planner: %v", err) + } +} +func BenchmarkSupplierCascadeProductionScale(b *testing.B) { + data := productionScaleSupplierSnapshot() + s := domain.SupplierRecord{ID: "target", OrganizationID: "org", Name: "target"} + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := buildSupplierDeletionPlan(context.Background(), s, data); err != nil { + b.Fatal(err) + } + } +} + +func TestSupplierCascadeCompactRunRetainsNestedReferences(t *testing.T) { + raw, err := bson.Marshal(bson.M{"id": "run-owner", "organizationId": "org", "workflowId": "workflow", "projectId": "project", "status": "completed", "logs": []bson.M{{"message": "run-log-only"}}, "nodeExecutions": []bson.M{{"nodeId": "node-a", "artifacts": []bson.M{{"payload": bson.M{"nested": []any{bson.M{"sourceRunId": "run-producer", "uri": "bos://bucket/shared/run-producer/file"}, []byte{1, 2, 3}}}}}}}}) + if err != nil { + t.Fatal(err) + } + r, err := compactSupplierRun(raw) + if err != nil { + t.Fatal(err) + } + refs := r.Artifacts[0].Data["runIds"].([]string) + found := map[string]bool{} + for _, ref := range refs { + found[ref] = true + } + if !found["run-producer"] || !found["bos://bucket/shared/run-producer/file"] || found["run-log-only"] || r.ID != "run-owner" || len(r.NodeExecutions) != 1 || r.NodeExecutions[0].NodeID != "node-a" { + t.Fatalf("references not preserved: %+v", r) + } +} diff --git a/internal/store/supplier_deletion_plan.go b/internal/store/supplier_deletion_plan.go index ef180164..1a31b354 100644 --- a/internal/store/supplier_deletion_plan.go +++ b/internal/store/supplier_deletion_plan.go @@ -1,6 +1,7 @@ package store import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -101,12 +102,15 @@ func payloadItems(d bson.M) []any { return deletionArray(deletionDoc(deletionDoc(d["subject"])["payload"])["items"]) } -func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnapshot) domain.SupplierDeletionPlan { +func buildSupplierDeletionPlan(ctx context.Context, supplier domain.SupplierRecord, data supplierSnapshot) (domain.SupplierDeletionPlan, error) { p := domain.SupplierDeletionPlan{SupplierID: supplier.ID, SupplierName: supplier.Name, OrganizationID: supplier.OrganizationID, Records: map[string][]string{}, SharedRecords: map[string][]string{}, Counts: map[string]int{}, Targets: []domain.ProjectDeletionTarget{}, Preserved: []string{"共享项目、其他供应商账号和任务、原始采集文件、共享文件、公共模板与基础设施、删除审计"}, Blockers: []string{}} owned := map[string]map[string]bool{} shared := map[string]map[string]bool{} index := map[string]map[string]bson.M{} for _, col := range supplierDeletionCollections { + if err := ctx.Err(); err != nil { + return p, err + } owned[col] = map[string]bool{} shared[col] = map[string]bool{} index[col] = map[string]bson.M{} @@ -121,6 +125,27 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap } } } + // Index immutable relationships once. Ownership changes between passes, + // but each child is visited only through its actual batch/run parents. + type childRef struct{ collection, id string } + batchChildren := map[string][]string{} + runChildren := map[string][]childRef{} + for id, d := range index["humanItems"] { + batch := deletionString(d, "batchId") + batchChildren[batch] = append(batchChildren[batch], id) + } + for _, col := range []string{"annotationTasks", "humanItems", "humanBatches", "deliveryPackages"} { + for id, d := range index[col] { + seen := map[string]bool{} + for _, field := range []string{"runId", "childRunId", "parentRunId"} { + run := deletionString(d, field) + if run != "" && !seen[run] { + runChildren[run] = append(runChildren[run], childRef{col, id}) + seen[run] = true + } + } + } + } for id := range owned["users"] { u := index["users"][id] p.Accounts = append(p.Accounts, domain.SupplierDeletionAccount{ID: id, Name: deletionString(u, "name"), Email: deletionString(u, "email"), Role: deletionString(u, "role")}) @@ -174,6 +199,9 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap // Resolve ownership from leaves upwards; unknown or foreign siblings make a // batch/run shared. Fixed-point passes also cover aggregate acceptance items. for pass := 0; pass < 8; pass++ { + if err := ctx.Err(); err != nil { + return p, err + } for id, d := range index["annotationTasks"] { if docSupplier(d) == "" && wfOwned[deletionString(d, "workflowId")] { owned["annotationTasks"][id] = true @@ -205,13 +233,11 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap } for id := range index["humanBatches"] { mine, other := 0, 0 - for cid, child := range index["humanItems"] { - if deletionString(child, "batchId") == id { - if owned["humanItems"][cid] { - mine++ - } else { - other++ - } + for _, cid := range batchChildren[id] { + if owned["humanItems"][cid] { + mine++ + } else { + other++ } } if other > 0 && (mine > 0 || owned["humanBatches"][id]) { @@ -245,15 +271,11 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap } for id, d := range index["runs"] { mine, other := 0, 0 - for _, col := range []string{"annotationTasks", "humanItems", "humanBatches", "deliveryPackages"} { - for cid, child := range index[col] { - if deletionString(child, "runId") == id || deletionString(child, "childRunId") == id || deletionString(child, "parentRunId") == id { - if owned[col][cid] { - mine++ - } else { - other++ - } - } + for _, child := range runChildren[id] { + if owned[child.collection][child.id] { + mine++ + } else { + other++ } } if mine > 0 && other == 0 { @@ -269,18 +291,21 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap } // A retained run borrowing an artifact makes that producer's directory shared, // even when both runs are in the same project. Preserve the producer record. - for changed := true; changed; { - changed = false - for id, d := range index["runs"] { - if owned["runs"][id] { - continue - } - for _, ref := range nestedReferenceIDs(d, map[string]bool{"runId": true, "parentRunId": true, "sourceRunId": true}) { - if owned["runs"][ref] { - delete(owned["runs"], ref) - shared["runs"][ref] = true - changed = true - } + queue := []string{} + for id := range index["runs"] { + if !owned["runs"][id] { + queue = append(queue, id) + } + } + for head := 0; head < len(queue); head++ { + if err := ctx.Err(); err != nil { + return p, err + } + for _, ref := range nestedReferenceIDs(index["runs"][queue[head]], map[string]bool{"runId": true, "parentRunId": true, "sourceRunId": true, "runIds": true}) { + if owned["runs"][ref] { + delete(owned["runs"], ref) + shared["runs"][ref] = true + queue = append(queue, ref) } } } @@ -348,7 +373,7 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap } } FinalizeSupplierDeletionPlan(&p) - return p + return p, ctx.Err() } func FinalizeSupplierDeletionPlan(p *domain.SupplierDeletionPlan) { sort.Slice(p.Accounts, func(i, j int) bool { return p.Accounts[i].ID < p.Accounts[j].ID }) diff --git a/internal/store/supplier_deletion_run_refs.go b/internal/store/supplier_deletion_run_refs.go new file mode 100644 index 00000000..20a37adc --- /dev/null +++ b/internal/store/supplier_deletion_run_refs.go @@ -0,0 +1,158 @@ +package store + +import ( + "bytes" + "context" + "droboticflow/internal/domain" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/bsontype" + "go.mongodb.org/mongo-driver/mongo/options" + "regexp" + "strings" +) + +var supplierRunIDs = regexp.MustCompile(`run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}`) +var supplierBOSURLs = regexp.MustCompile(`bos://[^\x00\s"'<>\\]+`) + +// Walk BSON without materializing large artifact payloads as interface maps. +// Retain reference tokens from every non-log field, including legacy nested data. +func compactSupplierRun(raw bson.Raw) (domain.RunRecord, error) { + str := func(key string) string { s, _ := raw.Lookup(key).StringValueOK(); return s } + r := domain.RunRecord{ID: str("id"), OrganizationID: str("organizationId"), ProjectID: str("projectId"), WorkflowID: str("workflowId"), Status: domain.RunStatus(str("status"))} + refs := map[string]bool{} + nodes := map[string]bool{} + // Inline artifact data can contain millions of numeric cells. Scan its raw + // bytes for reference tokens instead of allocating each cell. False positives + // in binary values only preserve extra data; references are never omitted. + scanPayload := func(raw []byte) { + if bytes.Contains(raw, []byte("run-")) { + for _, id := range supplierRunIDs.FindAll(raw, -1) { + refs[string(id)] = true + } + } + if bytes.Contains(raw, []byte("bos://")) { + for _, uri := range supplierBOSURLs.FindAll(raw, -1) { + refs[string(uri)] = true + } + } + } + var walk func(bson.Raw) error + walk = func(doc bson.Raw) error { + elements, err := doc.Elements() + if err != nil { + return err + } + for _, el := range elements { + if el.Key() == "logs" || el.Key() == "resourceSeries" { + continue + } + v := el.Value() + if el.Key() == "data" || el.Key() == "payload" { + scanPayload(v.Value) + continue + } + switch v.Type { + case bsontype.String: + s := v.StringValue() + if el.Key() == "nodeId" { + nodes[s] = true + } + if strings.Contains(s, "run-") { + for _, id := range supplierRunIDs.FindAllString(s, -1) { + refs[id] = true + } + } + if strings.Contains(s, "bos://") { + for _, url := range supplierBOSURLs.FindAllString(s, -1) { + refs[url] = true + } + } + case bsontype.EmbeddedDocument: + if err := walk(v.Document()); err != nil { + return err + } + case bsontype.Array: + if err := walk(bson.Raw(v.Array())); err != nil { + return err + } + } + } + return nil + } + if err := walk(raw); err != nil { + return r, err + } + r.Artifacts = []domain.ExecutionArtifact{{Data: map[string]any{"runIds": sortedKeys(refs)}}} + for _, id := range sortedKeys(nodes) { + r.NodeExecutions = append(r.NodeExecutions, domain.NodeExecutionRecord{NodeID: id}) + } + return r, nil +} +func (s *MongoStore) supplierRunReferences(ctx context.Context, filter bson.M) ([]domain.RunRecord, error) { + cur, err := s.runs.Find(ctx, filter, options.Find().SetProjection(bson.M{"logs": 0, "resourceSeries": 0, "nodeExecutions.logs": 0, "nodeExecutions.resourceSeries": 0}).SetBatchSize(1000)) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + runs := []domain.RunRecord{} + for cur.Next(ctx) { + if err := ctx.Err(); err != nil { + return nil, err + } + r, err := compactSupplierRun(cur.Current) + if err != nil { + return nil, err + } + runs = append(runs, r) + if len(runs) > 100000 { + return nil, fmt.Errorf("运行引用超过安全核对上限") + } + } + return runs, cur.Err() +} + +type supplierReadScopeKey struct{} +type supplierReadScope struct { + organization string + runs []domain.RunRecord + loaded bool +} + +// Request-scoped reuse only: admission and retries always create a fresh scope. +// Never reuse this inventory across requests or across deletion attempts. +func WithSupplierDeletionReadScope(ctx context.Context) context.Context { + return context.WithValue(ctx, supplierReadScopeKey{}, &supplierReadScope{}) +} +func (s *MongoStore) ListSupplierDeletionRunReferences(ctx context.Context) ([]domain.RunRecord, error) { + if scope, ok := ctx.Value(supplierReadScopeKey{}).(*supplierReadScope); ok && scope.loaded { + foreign, err := s.supplierRunReferences(ctx, bson.M{"$nor": bson.A{orgCompatibilityFilter(scope.organization)}}) + if err != nil { + return nil, err + } + return append(foreign, scope.runs...), nil + } + return s.supplierRunReferences(ctx, bson.M{}) +} +func (s *MemoryStore) ListSupplierDeletionRunReferences(ctx context.Context) ([]domain.RunRecord, error) { + runs, err := s.ListRunArtifacts(ctx, "") + if err != nil { + return nil, err + } + out := make([]domain.RunRecord, 0, len(runs)) + for _, r := range runs { + if err := ctx.Err(); err != nil { + return nil, err + } + raw, err := bson.Marshal(r) + if err != nil { + return nil, err + } + next, err := compactSupplierRun(raw) + if err != nil { + return nil, err + } + out = append(out, next) + } + return out, nil +} diff --git a/src/components/SupplierDeletionDialog.tsx b/src/components/SupplierDeletionDialog.tsx index 42405daa..656e5dd3 100644 --- a/src/components/SupplierDeletionDialog.tsx +++ b/src/components/SupplierDeletionDialog.tsx @@ -35,8 +35,17 @@ export function SupplierDeletionDialog({ const [error, setError] = useState(""); const [loading, setLoading] = useState(true); const [saving, setSaving] = useState(false); + const [reload, setReload] = useState(0); useEffect(() => { let live = true; + const controller = new AbortController(); + const timeout = setTimeout(() => { + controller.abort(); + if (live) { + setError("删除范围核对超时,请稍后重试"); + setLoading(false); + } + }, 25000); setLoading(true); setError(""); setName(""); @@ -46,22 +55,25 @@ export function SupplierDeletionDialog({ void (async () => { try { let existing: SupplierDeletionTask | null = null; - if (existingDeletionId) existing = await supplierDeletionsApi.get(existingDeletionId); - const next = existing?.plan || (await supplierDeletionsApi.preview(supplierId)); + if (existingDeletionId) existing = await supplierDeletionsApi.get(existingDeletionId, controller.signal); + const next = existing?.plan || (await supplierDeletionsApi.preview(supplierId, controller.signal)); if (live) { setTask(existing); setPlan(next); } } catch (e) { - if (live) setError(getActionErrorMessage(e, "加载删除范围失败")); + if (live && !controller.signal.aborted) setError(getActionErrorMessage(e, "加载删除范围失败")); } finally { + clearTimeout(timeout); if (live) setLoading(false); } })(); return () => { live = false; + clearTimeout(timeout); + controller.abort(); }; - }, [supplierId, existingDeletionId]); + }, [supplierId, existingDeletionId, reload]); const active = task && !task.retryable && task.status !== "completed" && task.status !== "failed"; useEffect(() => { if (!active || !task) return; @@ -145,6 +157,9 @@ export function SupplierDeletionDialog({ {error && (
{error} + {!loading && !plan && ( + + )}
)} {plan && ( diff --git a/src/services/api/suppliers.ts b/src/services/api/suppliers.ts index 13e7bda0..49c7b875 100644 --- a/src/services/api/suppliers.ts +++ b/src/services/api/suppliers.ts @@ -39,10 +39,10 @@ export type SupplierDeletionTask = { attempt: number; }; export const supplierDeletionsApi = { - preview: (id: string) => - apiRequest(`/api/suppliers/${encodeURIComponent(id)}/deletion-preview`), + preview: (id: string, signal?: AbortSignal) => + apiRequest(`/api/suppliers/${encodeURIComponent(id)}/deletion-preview`, { signal }), list: async () => (await apiRequest<{ items: SupplierDeletionTask[] }>("/api/supplier-deletions")).items, - get: (id: string) => apiRequest(`/api/supplier-deletions/${encodeURIComponent(id)}`), + get: (id: string, signal?: AbortSignal) => apiRequest(`/api/supplier-deletions/${encodeURIComponent(id)}`, { signal }), start: (id: string, payload: { confirmationName: string; digest: string; stopRunning: boolean; reason: string }) => apiRequest(`/api/suppliers/${encodeURIComponent(id)}/deletions`, { method: "POST" }, payload), };