⚡ Bound supplier deletion previews and index ownership relationships

This commit is contained in:
eust-w 2026-09-21 16:21:16 +08:00
parent 5096876134
commit f7ed44b8f8
11 changed files with 547 additions and 70 deletions

View File

@ -15,3 +15,11 @@ Execution uses a renewable lease and idempotent steps. Closing the dialog does n
`SUPPLIER_CASCADE_MONGO_TEST=1 go test -race ./internal/store ./internal/api -run TestSupplierCascade -count=1` runs the memory and standalone MongoDB contracts. The Mongo fixture uses the configured local isolated Mongo server. Frontend checks are `npm run lint:web` and `npm run build:web`.
`TestSupplierCascadeLiveFixture` is opt-in and requires an explicitly isolated nonce. It seeds two suppliers in one dedicated project, mixed evidence, exclusive and shared derived directories, and protected raw input. A dedicated test Pod finalizer can hold runtime termination to exercise failure and retry; remove only that test finalizer before retrying. Mounted object storage may not support POSIX chmod, so permission injection must not be assumed to work there. Verify mode checks supplier A cleanup, supplier B preservation, shared references and late-write fencing. Cleanup mode removes only that isolated fixture and retains the deletion audit. Never use this fixture to delete existing customer suppliers.
## Preview performance and cancellation
The planner indexes children by batch/run once instead of comparing every run with every task for each ownership pass. Mongo reads up to four collections concurrently, projects ownership metadata, and extracts run references from raw BSON without materializing inline artifact payloads. Inline payload token matching is conservative: ambiguous binary matches preserve extra data rather than widening deletion. Cross-organization and transitive artifact references remain protected.
A preview uses one request-local run inventory, with additional reads for other organizations. No run inventory or deletion plan is cached across requests, confirmations or retries. Backend planning has a 20-second deadline and at most two concurrent previews per process. The dialog aborts reads when closed/switched, times out after 25 seconds, and exposes retry. Timeout responses use HTTP 408 so they do not enter automatic gateway-error retries.
`TestSupplierCascadeProductionScaleAndCancellation` covers 10,001 runs, 5,001 items and 5,000 batches, including shared records and cancelled requests. `TestSupplierCascadeLivePreviewPerformance` is an opt-in read-only probe; it never begins deletion. During 2026-09-21 verification, standalone Mongo/race regressions passed, the synthetic planning benchmark took approximately 17 ms, and the complete production-data preview took approximately 8.2 seconds for three accounts / 88 records / 14 targets. These are different measurements, not an HTTP latency guarantee. Local browser validation confirmed request abort on close, timeout feedback, retry recovery and no console errors.

View File

@ -5,9 +5,11 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"regexp"
"sort"
"strings"
"sync"
@ -38,7 +40,22 @@ func (s *server) supplierDeletionAdmin(w http.ResponseWriter, r *http.Request) b
}
return true
}
// Bound concurrent scans and release all work when the requesting browser leaves.
var supplierPreviewSlots = make(chan struct{}, 2)
var supplierRunReferencePattern = regexp.MustCompile(`run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}`)
func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) {
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
ctx = store.WithSupplierDeletionReadScope(ctx)
select {
case supplierPreviewSlots <- struct{}{}:
defer func() { <-supplierPreviewSlots }()
case <-ctx.Done():
return domain.SupplierDeletionPlan{}, ctx.Err()
}
p, e := s.store.PreviewSupplierDeletion(ctx, org, id)
if e != nil {
return p, e
@ -53,7 +70,13 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin
p.ProjectNames = append(p.ProjectNames, project.Name)
}
}
runs, e := s.store.ListRunArtifacts(ctx, "")
if len(p.RunIDs) == 0 {
p.RunIDs = []string{}
p.Counts["fileTargets"], p.Counts["runningTasks"] = 0, 0
store.FinalizeSupplierDeletionPlan(&p)
return p, ctx.Err()
}
runs, e := s.store.ListSupplierDeletionRunReferences(ctx)
if e != nil {
return p, e
}
@ -70,22 +93,36 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin
owns[id] = true
}
byID := map[string]domain.RunRecord{}
runJSON := map[string]string{}
runsByWorkflow := map[string][]domain.RunRecord{}
for _, r := range runs {
byID[r.ID] = r
}
// Foreign references are checked across projects as well as across suppliers
// within the same project. Preserve the producer record and its files.
for _, r := range runs {
if owns[r.ID] {
continue
if err := ctx.Err(); err != nil {
return p, err
}
byID[r.ID] = r
raw, _ := json.Marshal(r)
for id := range owns {
if strings.Contains(string(raw), "/"+id+"/") || strings.Contains(string(raw), `"runId":"`+id+`"`) {
runJSON[r.ID] = string(raw)
runsByWorkflow[r.WorkflowID] = append(runsByWorkflow[r.WorkflowID], r)
}
// Index foreign references once, including across projects/organizations.
// A newly shared producer is queued so transitive sharing is preserved.
queue := []string{}
for _, r := range runs {
if !owns[r.ID] {
queue = append(queue, r.ID)
}
}
for head := 0; head < len(queue); head++ {
if err := ctx.Err(); err != nil {
return p, err
}
for _, id := range supplierRunReferencePattern.FindAllString(runJSON[queue[head]], -1) {
if owns[id] {
delete(owns, id)
p.SharedRunIDs = append(p.SharedRunIDs, id)
p.SharedRecords["runs"] = append(p.SharedRecords["runs"], id)
p.Preserved = append(p.Preserved, "其他运行引用的产物及运行记录:"+id)
queue = append(queue, id)
}
}
}
@ -109,9 +146,12 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin
retainedPaths := []retainedPath{}
uncertainStores := map[string]bool{}
for _, w := range workflows {
if err := ctx.Err(); err != nil {
return p, err
}
retained := []domain.RunRecord{}
for _, r := range runs {
if r.WorkflowID == w.ID && !owns[r.ID] {
for _, r := range runsByWorkflow[w.ID] {
if !owns[r.ID] {
retained = append(retained, r)
}
}
@ -169,7 +209,7 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin
if owns[r.ID] {
continue
}
raw, _ := json.Marshal(r)
raw := runJSON[r.ID]
if strings.Contains(string(raw), "bos://"+bucket+"/"+strings.Trim(prefix, "/")) {
p.Preserved = append(p.Preserved, "其他运行引用的文件:"+prefix)
return
@ -195,7 +235,7 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin
for _, w := range workflows {
ownedRuns := []domain.RunRecord{}
foreignRun := false
for _, r := range runs {
for _, r := range runsByWorkflow[w.ID] {
if r.WorkflowID == w.ID {
if owns[r.ID] {
ownedRuns = append(ownedRuns, r)
@ -253,7 +293,7 @@ func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id strin
}
}
store.FinalizeSupplierDeletionPlan(&p)
return p, nil
return p, ctx.Err()
}
func (s *server) handleSupplierDeletionPreview(w http.ResponseWriter, r *http.Request) {
if !s.supplierDeletionAdmin(w, r) {
@ -261,6 +301,13 @@ func (s *server) handleSupplierDeletionPreview(w http.ResponseWriter, r *http.Re
}
id := chi.URLParam(r, "supplierId")
p, e := s.buildSupplierDeletionPreview(r.Context(), s.requestOrganizationID(r), id)
if errors.Is(e, context.DeadlineExceeded) {
renderJSON(w, http.StatusRequestTimeout, map[string]string{"error": "删除范围核对超时,请稍后重试", "code": "supplier_preview_timeout"})
return
}
if errors.Is(e, context.Canceled) {
return
}
if e != nil {
renderSupplierMutationError(w, e)
return

View File

@ -0,0 +1,48 @@
package api
import (
"context"
"droboticflow/internal/store"
"fmt"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"os"
"testing"
"time"
)
// Opt-in read-only performance probe. It never starts or resumes a deletion.
func TestSupplierCascadeLivePreviewPerformance(t *testing.T) {
if os.Getenv("SUPPLIER_PREVIEW_READONLY") != "1" {
t.Skip("explicit read-only preview probe")
}
id := os.Getenv("SUPPLIER_PREVIEW_ID")
if id == "" {
t.Fatal("supplier required")
}
ctx, cancel := context.WithTimeout(t.Context(), 45*time.Second)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")))
if err != nil {
t.Fatal("database connection failed")
}
defer client.Disconnect(context.Background())
ds := store.NewMongoStore(client.Database(os.Getenv("MONGO_DB")))
supplier, err := ds.GetSupplier(ctx, id)
if err != nil || supplier == nil {
t.Fatal("supplier unavailable")
}
start := time.Now()
p, err := ds.PreviewSupplierDeletion(ctx, supplier.OrganizationID, id)
if err != nil {
t.Fatal(err)
}
fmt.Printf("ownership_preview_ms=%d records=%d owned_runs=%d shared_runs=%d\n", time.Since(start).Milliseconds(), p.Counts["totalRecords"], len(p.RunIDs), len(p.SharedRunIDs))
server := &server{store: ds, workspaceMountPath: os.Getenv("WORKSPACE_MOUNT_PATH")}
start = time.Now()
p, err = server.buildSupplierDeletionPreview(ctx, supplier.OrganizationID, id)
if err != nil {
t.Fatal(err)
}
fmt.Printf("complete_preview_ms=%d accounts=%d records=%d targets=%d blockers=%d\n", time.Since(start).Milliseconds(), len(p.Accounts), p.Counts["totalRecords"], len(p.Targets), len(p.Blockers))
}

View File

@ -189,3 +189,39 @@ func TestSupplierCascadePlatformWorkspaceScope(t *testing.T) {
})
}
}
func TestSupplierCascadePreviewPreservesCrossOrganizationArtifactChain(t *testing.T) {
ds := store.NewMemoryStore()
ctx := t.Context()
if err := ds.UpsertSupplier(ctx, domain.SupplierRecord{ID: "A", OrganizationID: "org-a", Name: "A", Status: "active"}); err != nil {
t.Fatal(err)
}
for _, id := range []string{"run-first", "run-second", "run-adjacent"} {
r := domain.RunRecord{ID: id, OrganizationID: "org-a", ProjectID: "project-a", Status: domain.RunStatusCompleted}
if id == "run-second" {
r.Artifacts = []domain.ExecutionArtifact{{Path: "/workspace/run-first/file"}}
}
if err := ds.UpsertRun(ctx, r); err != nil {
t.Fatal(err)
}
if err := ds.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: "item-" + id, OrganizationID: "org-a", ProjectID: "project-a", SupplierID: "A", RunID: id}); err != nil {
t.Fatal(err)
}
}
if err := ds.UpsertRun(ctx, domain.RunRecord{ID: "run-foreign", OrganizationID: "org-b", Artifacts: []domain.ExecutionArtifact{{Path: "/workspace/run-second/run-adjacent/file"}}}); err != nil {
t.Fatal(err)
}
s := &server{store: ds}
plan, err := s.buildSupplierDeletionPreview(ctx, "org-a", "A")
if err != nil {
t.Fatal(err)
}
if len(plan.RunIDs) != 0 || len(plan.SharedRunIDs) != 3 || len(plan.Targets) != 0 {
t.Fatalf("foreign/transitive artifact references lost: %+v", plan)
}
cancelled, cancel := context.WithCancel(ctx)
cancel()
if _, err = s.buildSupplierDeletionPreview(cancelled, "org-a", "A"); !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation not propagated: %v", err)
}
}

View File

@ -159,6 +159,7 @@ type Store interface {
ListProjectAnnotationMemberships(context.Context, string, string) ([]domain.ProjectAnnotationMembershipRecord, error)
UpsertProjectAnnotationMembership(context.Context, domain.ProjectAnnotationMembershipRecord) error
DeleteProjectAnnotationMembership(context.Context, string) (bool, error)
ListSupplierDeletionRunReferences(context.Context) ([]domain.RunRecord, error)
PreviewSupplierDeletion(context.Context, string, string) (domain.SupplierDeletionPlan, error)
GetSupplierDeletion(context.Context, string) (*domain.SupplierDeletionTask, error)
ListSupplierDeletions(context.Context, string) ([]domain.SupplierDeletionTask, error)

View File

@ -4,7 +4,9 @@ import (
"context"
"errors"
"fmt"
"golang.org/x/sync/errgroup"
"sort"
"sync"
"time"
"droboticflow/internal/domain"
@ -69,25 +71,79 @@ func (s *MemoryStore) supplierSnapshotLocked(org string) supplierSnapshot {
"datasetVersions": supplierMapDocs(s.datasetVersions, org),
}
}
// Only relationship metadata is needed for ownership. Run references remain
// structurally complete so references buried in legacy payloads stay protected.
func supplierDeletionProjection(col string) bson.M {
if col == "runs" {
return bson.M{"logs": 0, "resourceSeries": 0, "nodeExecutions.logs": 0, "nodeExecutions.resourceSeries": 0}
}
p := bson.M{"_id": 0}
for _, field := range []string{"id", "organizationId", "supplierId", "vendorId", "projectId", "workflowId", "taskId", "itemId", "batchId", "runId", "childRunId", "parentRunId", "legacyTaskId", "userId"} {
p[field] = 1
}
switch col {
case "users":
for _, field := range []string{"name", "email", "role", "isSuperAdmin"} {
p[field] = 1
}
case "workflows":
p["nodes.data.hyperparams"] = 1
case "humanItems":
p["subject.payload.items.sourceHumanItemId"] = 1
p["subject.payload.items.taskId"] = 1
case "humanReleasePartitions", "deliveryPackages":
p["itemIds"] = 1
}
return p
}
func (s *MongoStore) loadSupplierSnapshot(ctx context.Context, org string) (supplierSnapshot, error) {
data := supplierSnapshot{}
for _, col := range supplierDeletionCollections {
cursor, e := s.db.Collection(col).Find(ctx, orgCompatibilityFilter(org), options.Find().SetProjection(bson.M{"passwordHash": 0, "tokenHash": 0, "body": 0, "logs": 0, "resourceSeries": 0, "nodeExecutions.logs": 0, "nodeExecutions.resourceSeries": 0, "processingJob.input": 0, "subject.payload.annotationWorkspace": 0, "subject.payload.annotationDraft": 0}).SetLimit(50001))
if e != nil {
return nil, e
}
rows, e := decodeMany[bson.M](ctx, cursor)
if e != nil {
return nil, e
}
if len(rows) > 50000 {
return nil, fmt.Errorf("%s 超过安全预览上限", col)
}
data[col] = rows
var mu sync.Mutex
group, ctx := errgroup.WithContext(ctx)
group.SetLimit(4)
for _, collection := range supplierDeletionCollections {
col := collection
group.Go(func() error {
var rows []bson.M
if col == "runs" {
runs, err := s.supplierRunReferences(ctx, orgCompatibilityFilter(org))
if err != nil {
return err
}
if scope, ok := ctx.Value(supplierReadScopeKey{}).(*supplierReadScope); ok {
scope.organization = org
scope.runs = runs
scope.loaded = true
}
for _, r := range runs {
rows = append(rows, supplierDoc(r))
}
} else {
cursor, err := s.db.Collection(col).Find(ctx, orgCompatibilityFilter(org), options.Find().SetProjection(supplierDeletionProjection(col)).SetBatchSize(5000).SetLimit(50001))
if err != nil {
return err
}
rows, err = decodeMany[bson.M](ctx, cursor)
if err != nil {
return err
}
}
if len(rows) > 50000 {
return fmt.Errorf("%s 超过安全预览上限", col)
}
mu.Lock()
data[col] = rows
mu.Unlock()
return nil
})
}
if err := group.Wait(); err != nil {
return nil, err
}
return data, nil
}
func (s *MemoryStore) PreviewSupplierDeletion(_ context.Context, org, id string) (domain.SupplierDeletionPlan, error) {
func (s *MemoryStore) PreviewSupplierDeletion(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) {
s.mu.Lock()
defer s.mu.Unlock()
supplier, ok := s.suppliers[id]
@ -97,7 +153,7 @@ func (s *MemoryStore) PreviewSupplierDeletion(_ context.Context, org, id string)
if supplier.IsDefault || id == domain.DefaultSupplierID(org) {
return domain.SupplierDeletionPlan{}, ErrDefaultSupplierProtected
}
return buildSupplierDeletionPlan(supplier, s.supplierSnapshotLocked(org)), nil
return buildSupplierDeletionPlan(ctx, supplier, s.supplierSnapshotLocked(org))
}
func (s *MongoStore) PreviewSupplierDeletion(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) {
supplier, e := s.GetSupplier(ctx, id)
@ -114,7 +170,7 @@ func (s *MongoStore) PreviewSupplierDeletion(ctx context.Context, org, id string
if e != nil {
return domain.SupplierDeletionPlan{}, e
}
return buildSupplierDeletionPlan(*supplier, data), nil
return buildSupplierDeletionPlan(ctx, *supplier, data)
}
func (s *MemoryStore) GetSupplierDeletion(_ context.Context, id string) (*domain.SupplierDeletionTask, error) {
s.mu.Lock()
@ -154,7 +210,7 @@ func (s *MongoStore) ListSupplierDeletions(ctx context.Context, org string) ([]d
}
return decodeMany[domain.SupplierDeletionTask](ctx, cur)
}
func (s *MemoryStore) BeginSupplierDeletion(_ context.Context, t domain.SupplierDeletionTask) (*domain.SupplierDeletionTask, error) {
func (s *MemoryStore) BeginSupplierDeletion(ctx context.Context, t domain.SupplierDeletionTask) (*domain.SupplierDeletionTask, error) {
s.mu.Lock()
defer s.mu.Unlock()
if old, ok := s.supplierDeletions[t.ID]; ok {
@ -167,7 +223,10 @@ func (s *MemoryStore) BeginSupplierDeletion(_ context.Context, t domain.Supplier
if supplier.IsDefault || supplier.ID == domain.DefaultSupplierID(t.OrganizationID) {
return nil, ErrDefaultSupplierProtected
}
fresh := buildSupplierDeletionPlan(supplier, s.supplierSnapshotLocked(t.OrganizationID))
fresh, err := buildSupplierDeletionPlan(ctx, supplier, s.supplierSnapshotLocked(t.OrganizationID))
if err != nil {
return nil, err
}
if fresh.Digest != t.Plan.ScopeDigest {
return nil, ErrSupplierDeletionScopeChanged
}

View File

@ -0,0 +1,80 @@
package store
import (
"context"
"droboticflow/internal/domain"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"testing"
"time"
)
func productionScaleSupplierSnapshot() supplierSnapshot {
data := supplierSnapshot{}
for i := 0; i < 10000; i++ {
id := fmt.Sprintf("run-%d", i)
data["runs"] = append(data["runs"], bson.M{"id": id, "projectId": "shared-project"})
}
for i := 0; i < 5000; i++ {
sup := "other"
if i < 10 {
sup = "target"
}
run, batch, task, item := fmt.Sprintf("run-%d", i), fmt.Sprintf("batch-%d", i), fmt.Sprintf("task-%d", i), fmt.Sprintf("item-%d", i)
data["annotationTasks"] = append(data["annotationTasks"], bson.M{"id": task, "supplierId": sup, "projectId": "shared-project", "runId": run})
data["humanItems"] = append(data["humanItems"], bson.M{"id": item, "supplierId": sup, "projectId": "shared-project", "runId": run, "batchId": batch, "legacyTaskId": task})
data["humanBatches"] = append(data["humanBatches"], bson.M{"id": batch, "projectId": "shared-project", "runId": run})
}
data["humanItems"] = append(data["humanItems"], bson.M{"id": "other-shared-item", "supplierId": "other", "batchId": "batch-0", "runId": "run-0", "projectId": "shared-project"})
data["runs"] = append(data["runs"], bson.M{"id": "run-borrower", "inputs": bson.M{"nested": bson.M{"sourceRunId": "run-1"}}})
return data
}
func TestSupplierCascadeProductionScaleAndCancellation(t *testing.T) {
data := productionScaleSupplierSnapshot()
ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
defer cancel()
start := time.Now()
plan, err := buildSupplierDeletionPlan(ctx, domain.SupplierRecord{ID: "target", OrganizationID: "org", Name: "target"}, data)
if err != nil {
t.Fatalf("production-scale preview failed: %v", err)
}
t.Logf("10001 runs / 5001 items / 5000 batches planned in %s", time.Since(start))
if len(plan.Records["annotationTasks"]) != 10 || len(plan.RunIDs) != 8 || len(plan.SharedRunIDs) != 2 || len(plan.SharedRecords["humanBatches"]) != 1 {
t.Fatalf("wrong ownership scope: %+v", plan.Counts)
}
cancelled, stop := context.WithCancel(t.Context())
stop()
if _, err := buildSupplierDeletionPlan(cancelled, domain.SupplierRecord{ID: "target"}, data); !errors.Is(err, context.Canceled) {
t.Fatalf("cancelled planner: %v", err)
}
}
func BenchmarkSupplierCascadeProductionScale(b *testing.B) {
data := productionScaleSupplierSnapshot()
s := domain.SupplierRecord{ID: "target", OrganizationID: "org", Name: "target"}
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := buildSupplierDeletionPlan(context.Background(), s, data); err != nil {
b.Fatal(err)
}
}
}
func TestSupplierCascadeCompactRunRetainsNestedReferences(t *testing.T) {
raw, err := bson.Marshal(bson.M{"id": "run-owner", "organizationId": "org", "workflowId": "workflow", "projectId": "project", "status": "completed", "logs": []bson.M{{"message": "run-log-only"}}, "nodeExecutions": []bson.M{{"nodeId": "node-a", "artifacts": []bson.M{{"payload": bson.M{"nested": []any{bson.M{"sourceRunId": "run-producer", "uri": "bos://bucket/shared/run-producer/file"}, []byte{1, 2, 3}}}}}}}})
if err != nil {
t.Fatal(err)
}
r, err := compactSupplierRun(raw)
if err != nil {
t.Fatal(err)
}
refs := r.Artifacts[0].Data["runIds"].([]string)
found := map[string]bool{}
for _, ref := range refs {
found[ref] = true
}
if !found["run-producer"] || !found["bos://bucket/shared/run-producer/file"] || found["run-log-only"] || r.ID != "run-owner" || len(r.NodeExecutions) != 1 || r.NodeExecutions[0].NodeID != "node-a" {
t.Fatalf("references not preserved: %+v", r)
}
}

View File

@ -1,6 +1,7 @@
package store
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
@ -101,12 +102,15 @@ func payloadItems(d bson.M) []any {
return deletionArray(deletionDoc(deletionDoc(d["subject"])["payload"])["items"])
}
func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnapshot) domain.SupplierDeletionPlan {
func buildSupplierDeletionPlan(ctx context.Context, supplier domain.SupplierRecord, data supplierSnapshot) (domain.SupplierDeletionPlan, error) {
p := domain.SupplierDeletionPlan{SupplierID: supplier.ID, SupplierName: supplier.Name, OrganizationID: supplier.OrganizationID, Records: map[string][]string{}, SharedRecords: map[string][]string{}, Counts: map[string]int{}, Targets: []domain.ProjectDeletionTarget{}, Preserved: []string{"共享项目、其他供应商账号和任务、原始采集文件、共享文件、公共模板与基础设施、删除审计"}, Blockers: []string{}}
owned := map[string]map[string]bool{}
shared := map[string]map[string]bool{}
index := map[string]map[string]bson.M{}
for _, col := range supplierDeletionCollections {
if err := ctx.Err(); err != nil {
return p, err
}
owned[col] = map[string]bool{}
shared[col] = map[string]bool{}
index[col] = map[string]bson.M{}
@ -121,6 +125,27 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap
}
}
}
// Index immutable relationships once. Ownership changes between passes,
// but each child is visited only through its actual batch/run parents.
type childRef struct{ collection, id string }
batchChildren := map[string][]string{}
runChildren := map[string][]childRef{}
for id, d := range index["humanItems"] {
batch := deletionString(d, "batchId")
batchChildren[batch] = append(batchChildren[batch], id)
}
for _, col := range []string{"annotationTasks", "humanItems", "humanBatches", "deliveryPackages"} {
for id, d := range index[col] {
seen := map[string]bool{}
for _, field := range []string{"runId", "childRunId", "parentRunId"} {
run := deletionString(d, field)
if run != "" && !seen[run] {
runChildren[run] = append(runChildren[run], childRef{col, id})
seen[run] = true
}
}
}
}
for id := range owned["users"] {
u := index["users"][id]
p.Accounts = append(p.Accounts, domain.SupplierDeletionAccount{ID: id, Name: deletionString(u, "name"), Email: deletionString(u, "email"), Role: deletionString(u, "role")})
@ -174,6 +199,9 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap
// Resolve ownership from leaves upwards; unknown or foreign siblings make a
// batch/run shared. Fixed-point passes also cover aggregate acceptance items.
for pass := 0; pass < 8; pass++ {
if err := ctx.Err(); err != nil {
return p, err
}
for id, d := range index["annotationTasks"] {
if docSupplier(d) == "" && wfOwned[deletionString(d, "workflowId")] {
owned["annotationTasks"][id] = true
@ -205,13 +233,11 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap
}
for id := range index["humanBatches"] {
mine, other := 0, 0
for cid, child := range index["humanItems"] {
if deletionString(child, "batchId") == id {
if owned["humanItems"][cid] {
mine++
} else {
other++
}
for _, cid := range batchChildren[id] {
if owned["humanItems"][cid] {
mine++
} else {
other++
}
}
if other > 0 && (mine > 0 || owned["humanBatches"][id]) {
@ -245,15 +271,11 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap
}
for id, d := range index["runs"] {
mine, other := 0, 0
for _, col := range []string{"annotationTasks", "humanItems", "humanBatches", "deliveryPackages"} {
for cid, child := range index[col] {
if deletionString(child, "runId") == id || deletionString(child, "childRunId") == id || deletionString(child, "parentRunId") == id {
if owned[col][cid] {
mine++
} else {
other++
}
}
for _, child := range runChildren[id] {
if owned[child.collection][child.id] {
mine++
} else {
other++
}
}
if mine > 0 && other == 0 {
@ -269,18 +291,21 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap
}
// A retained run borrowing an artifact makes that producer's directory shared,
// even when both runs are in the same project. Preserve the producer record.
for changed := true; changed; {
changed = false
for id, d := range index["runs"] {
if owned["runs"][id] {
continue
}
for _, ref := range nestedReferenceIDs(d, map[string]bool{"runId": true, "parentRunId": true, "sourceRunId": true}) {
if owned["runs"][ref] {
delete(owned["runs"], ref)
shared["runs"][ref] = true
changed = true
}
queue := []string{}
for id := range index["runs"] {
if !owned["runs"][id] {
queue = append(queue, id)
}
}
for head := 0; head < len(queue); head++ {
if err := ctx.Err(); err != nil {
return p, err
}
for _, ref := range nestedReferenceIDs(index["runs"][queue[head]], map[string]bool{"runId": true, "parentRunId": true, "sourceRunId": true, "runIds": true}) {
if owned["runs"][ref] {
delete(owned["runs"], ref)
shared["runs"][ref] = true
queue = append(queue, ref)
}
}
}
@ -348,7 +373,7 @@ func buildSupplierDeletionPlan(supplier domain.SupplierRecord, data supplierSnap
}
}
FinalizeSupplierDeletionPlan(&p)
return p
return p, ctx.Err()
}
func FinalizeSupplierDeletionPlan(p *domain.SupplierDeletionPlan) {
sort.Slice(p.Accounts, func(i, j int) bool { return p.Accounts[i].ID < p.Accounts[j].ID })

View File

@ -0,0 +1,158 @@
package store
import (
"bytes"
"context"
"droboticflow/internal/domain"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/bsontype"
"go.mongodb.org/mongo-driver/mongo/options"
"regexp"
"strings"
)
var supplierRunIDs = regexp.MustCompile(`run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}`)
var supplierBOSURLs = regexp.MustCompile(`bos://[^\x00\s"'<>\\]+`)
// Walk BSON without materializing large artifact payloads as interface maps.
// Retain reference tokens from every non-log field, including legacy nested data.
func compactSupplierRun(raw bson.Raw) (domain.RunRecord, error) {
str := func(key string) string { s, _ := raw.Lookup(key).StringValueOK(); return s }
r := domain.RunRecord{ID: str("id"), OrganizationID: str("organizationId"), ProjectID: str("projectId"), WorkflowID: str("workflowId"), Status: domain.RunStatus(str("status"))}
refs := map[string]bool{}
nodes := map[string]bool{}
// Inline artifact data can contain millions of numeric cells. Scan its raw
// bytes for reference tokens instead of allocating each cell. False positives
// in binary values only preserve extra data; references are never omitted.
scanPayload := func(raw []byte) {
if bytes.Contains(raw, []byte("run-")) {
for _, id := range supplierRunIDs.FindAll(raw, -1) {
refs[string(id)] = true
}
}
if bytes.Contains(raw, []byte("bos://")) {
for _, uri := range supplierBOSURLs.FindAll(raw, -1) {
refs[string(uri)] = true
}
}
}
var walk func(bson.Raw) error
walk = func(doc bson.Raw) error {
elements, err := doc.Elements()
if err != nil {
return err
}
for _, el := range elements {
if el.Key() == "logs" || el.Key() == "resourceSeries" {
continue
}
v := el.Value()
if el.Key() == "data" || el.Key() == "payload" {
scanPayload(v.Value)
continue
}
switch v.Type {
case bsontype.String:
s := v.StringValue()
if el.Key() == "nodeId" {
nodes[s] = true
}
if strings.Contains(s, "run-") {
for _, id := range supplierRunIDs.FindAllString(s, -1) {
refs[id] = true
}
}
if strings.Contains(s, "bos://") {
for _, url := range supplierBOSURLs.FindAllString(s, -1) {
refs[url] = true
}
}
case bsontype.EmbeddedDocument:
if err := walk(v.Document()); err != nil {
return err
}
case bsontype.Array:
if err := walk(bson.Raw(v.Array())); err != nil {
return err
}
}
}
return nil
}
if err := walk(raw); err != nil {
return r, err
}
r.Artifacts = []domain.ExecutionArtifact{{Data: map[string]any{"runIds": sortedKeys(refs)}}}
for _, id := range sortedKeys(nodes) {
r.NodeExecutions = append(r.NodeExecutions, domain.NodeExecutionRecord{NodeID: id})
}
return r, nil
}
func (s *MongoStore) supplierRunReferences(ctx context.Context, filter bson.M) ([]domain.RunRecord, error) {
cur, err := s.runs.Find(ctx, filter, options.Find().SetProjection(bson.M{"logs": 0, "resourceSeries": 0, "nodeExecutions.logs": 0, "nodeExecutions.resourceSeries": 0}).SetBatchSize(1000))
if err != nil {
return nil, err
}
defer cur.Close(ctx)
runs := []domain.RunRecord{}
for cur.Next(ctx) {
if err := ctx.Err(); err != nil {
return nil, err
}
r, err := compactSupplierRun(cur.Current)
if err != nil {
return nil, err
}
runs = append(runs, r)
if len(runs) > 100000 {
return nil, fmt.Errorf("运行引用超过安全核对上限")
}
}
return runs, cur.Err()
}
type supplierReadScopeKey struct{}
type supplierReadScope struct {
organization string
runs []domain.RunRecord
loaded bool
}
// Request-scoped reuse only: admission and retries always create a fresh scope.
// Never reuse this inventory across requests or across deletion attempts.
func WithSupplierDeletionReadScope(ctx context.Context) context.Context {
return context.WithValue(ctx, supplierReadScopeKey{}, &supplierReadScope{})
}
func (s *MongoStore) ListSupplierDeletionRunReferences(ctx context.Context) ([]domain.RunRecord, error) {
if scope, ok := ctx.Value(supplierReadScopeKey{}).(*supplierReadScope); ok && scope.loaded {
foreign, err := s.supplierRunReferences(ctx, bson.M{"$nor": bson.A{orgCompatibilityFilter(scope.organization)}})
if err != nil {
return nil, err
}
return append(foreign, scope.runs...), nil
}
return s.supplierRunReferences(ctx, bson.M{})
}
func (s *MemoryStore) ListSupplierDeletionRunReferences(ctx context.Context) ([]domain.RunRecord, error) {
runs, err := s.ListRunArtifacts(ctx, "")
if err != nil {
return nil, err
}
out := make([]domain.RunRecord, 0, len(runs))
for _, r := range runs {
if err := ctx.Err(); err != nil {
return nil, err
}
raw, err := bson.Marshal(r)
if err != nil {
return nil, err
}
next, err := compactSupplierRun(raw)
if err != nil {
return nil, err
}
out = append(out, next)
}
return out, nil
}

View File

@ -35,8 +35,17 @@ export function SupplierDeletionDialog({
const [error, setError] = useState("");
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [reload, setReload] = useState(0);
useEffect(() => {
let live = true;
const controller = new AbortController();
const timeout = setTimeout(() => {
controller.abort();
if (live) {
setError("删除范围核对超时,请稍后重试");
setLoading(false);
}
}, 25000);
setLoading(true);
setError("");
setName("");
@ -46,22 +55,25 @@ export function SupplierDeletionDialog({
void (async () => {
try {
let existing: SupplierDeletionTask | null = null;
if (existingDeletionId) existing = await supplierDeletionsApi.get(existingDeletionId);
const next = existing?.plan || (await supplierDeletionsApi.preview(supplierId));
if (existingDeletionId) existing = await supplierDeletionsApi.get(existingDeletionId, controller.signal);
const next = existing?.plan || (await supplierDeletionsApi.preview(supplierId, controller.signal));
if (live) {
setTask(existing);
setPlan(next);
}
} catch (e) {
if (live) setError(getActionErrorMessage(e, "加载删除范围失败"));
if (live && !controller.signal.aborted) setError(getActionErrorMessage(e, "加载删除范围失败"));
} finally {
clearTimeout(timeout);
if (live) setLoading(false);
}
})();
return () => {
live = false;
clearTimeout(timeout);
controller.abort();
};
}, [supplierId, existingDeletionId]);
}, [supplierId, existingDeletionId, reload]);
const active = task && !task.retryable && task.status !== "completed" && task.status !== "failed";
useEffect(() => {
if (!active || !task) return;
@ -145,6 +157,9 @@ export function SupplierDeletionDialog({
{error && (
<div role="alert" className="rounded-md border border-red-500/30 bg-red-500/10 p-3 text-sm text-red-600">
{error}
{!loading && !plan && (
<button className="ml-3 underline" onClick={() => setReload((value) => value + 1)}>重试加载</button>
)}
</div>
)}
{plan && (

View File

@ -39,10 +39,10 @@ export type SupplierDeletionTask = {
attempt: number;
};
export const supplierDeletionsApi = {
preview: (id: string) =>
apiRequest<SupplierDeletionPlan>(`/api/suppliers/${encodeURIComponent(id)}/deletion-preview`),
preview: (id: string, signal?: AbortSignal) =>
apiRequest<SupplierDeletionPlan>(`/api/suppliers/${encodeURIComponent(id)}/deletion-preview`, { signal }),
list: async () => (await apiRequest<{ items: SupplierDeletionTask[] }>("/api/supplier-deletions")).items,
get: (id: string) => apiRequest<SupplierDeletionTask>(`/api/supplier-deletions/${encodeURIComponent(id)}`),
get: (id: string, signal?: AbortSignal) => apiRequest<SupplierDeletionTask>(`/api/supplier-deletions/${encodeURIComponent(id)}`, { signal }),
start: (id: string, payload: { confirmationName: string; digest: string; stopRunning: boolean; reason: string }) =>
apiRequest<SupplierDeletionTask>(`/api/suppliers/${encodeURIComponent(id)}/deletions`, { method: "POST" }, payload),
};