From 2864d2aa1143bd794bf23f2e2d08df8593a18249 Mon Sep 17 00:00:00 2001 From: eust-w Date: Wed, 23 Sep 2026 04:30:13 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=20Process=20deletion=20targets=20conc?= =?UTF-8?q?urrently=20within=20each=20phase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit executeProjectDeletion processed every target (runtime, then workspace, then bos, then records) strictly one at a time — for a project with thousands of real runs, that meant thousands of sequential K8s/BOS/Mongo round trips. Observed live on Fanyu: BLF-data's runtime phase (stopping 4,641 runs) took tens of minutes at that pace. Replaced the per-target sequential loop with a bounded worker pool (PROJECT_DELETION_TARGET_CONCURRENCY, default 12) applied within each phase — targets of the same kind now run concurrently, but phases still run strictly in order (runtime, then workspace, then bos, then records), preserving the "stop/verify runtime, then remove files, then DB" ordering the plan is built with. The one-time phase-transition safety checks (wait for in-flight human work, re-verify runtime absence, revalidate references) still run exactly once, at the same point, between the runtime and workspace phases. Every read-modify-write of shared task state (a target's Status/Error, and every persist() call) is guarded by a mutex; the actual I/O (deleteProjectTarget) runs unlocked so workers genuinely overlap. Added TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes (30 runs) to lock in that concurrent execution reaches the same end state — every target done, every workspace directory removed, project record gone — as the old sequential loop. Full suite passes under -race with no warnings; 75 pre-existing unrelated failures unchanged. Co-Authored-By: Claude Sonnet 5 --- internal/api/project_deletion.go | 239 +++++++++++++++++--------- internal/api/project_deletion_test.go | 75 ++++++++ 2 files changed, 229 insertions(+), 85 deletions(-) diff --git a/internal/api/project_deletion.go b/internal/api/project_deletion.go index e31033cc..957d75b4 100644 --- a/internal/api/project_deletion.go +++ b/internal/api/project_deletion.go @@ -18,7 +18,9 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" + "sync" "time" ) @@ -646,107 +648,174 @@ func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner s } } } - runtimeDone := false - for index := range task.Plan.Targets { - target := &task.Plan.Targets[index] - if target.Status == "done" { - continue + targetConcurrency := 12 + if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" { + if n, e := strconv.Atoi(v); e == nil && n > 0 { + targetConcurrency = n } - if target.Kind != "runtime" && !runtimeDone { - deadline := time.Now().Add(150 * time.Second) - for { - items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID}) - if e != nil { - fail(e) - return - } - busy := false - for _, id := range task.Plan.RunIDs { - run, e := s.store.GetRun(ctx, id) - if e != nil { - fail(e) - return - } - if run != nil && run.ExecutionLeaseOwner != "" { - until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt) - if e != nil || until.After(time.Now()) { - busy = true - } - } - } - for i := range items { - item := &items[i] - if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil { - fail(e) - return - } - w := mapValue(item.Subject.Payload["sliceWorkflow"]) - if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" { - busy = true - } - } - if !busy { - break - } - if time.Now().After(deadline) { - fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试")) - return - } - select { - case <-ctx.Done(): - fail(ctx.Err()) - return - case <-time.After(2 * time.Second): - } + } + // processBatch runs deleteProjectTarget for the not-yet-done targets at + // indices with up to targetConcurrency in flight at once. Each target's + // Status/Error and every persist() call is guarded by mu, so concurrent + // workers never race on shared task state; deleteProjectTarget itself + // (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs + // unlocked. A single project's deletion used to process every target + // strictly one at a time, which made stopping thousands of real runs + // take as long as thousands of sequential K8s round trips. + var mu sync.Mutex + processBatch := func(indices []int) error { + sem := make(chan struct{}, targetConcurrency) + var wg sync.WaitGroup + var firstErr error + for _, index := range indices { + mu.Lock() + skip := task.Plan.Targets[index].Status == "done" || firstErr != nil + mu.Unlock() + if skip { + continue } - // Catch work admitted immediately before the project lock. - for _, step := range task.Plan.Targets { - if step.Kind == "runtime" { - if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil { - fail(e) - return - } + sem <- struct{}{} + wg.Add(1) + go func(index int) { + defer wg.Done() + defer func() { <-sem }() + mu.Lock() + if firstErr != nil { + mu.Unlock() + return } - } - // Scoped exactly as before (projectId match only, no workflowId - // fallback — see the loop below) but via an indexed query - // instead of an organization-wide scan. - fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil) + target := &task.Plan.Targets[index] + target.Error = "" + target.Status = "running" + persistErr := persist() + snapshot := *target + mu.Unlock() + if persistErr != nil { + mu.Lock() + if firstErr == nil { + firstErr = persistErr + } + mu.Unlock() + cancel() + return + } + runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot) + mu.Lock() + target = &task.Plan.Targets[index] + if runErr != nil { + target.Status = "failed" + target.Error = runErr.Error() + if firstErr == nil { + firstErr = runErr + } + } else { + target.Status = "done" + } + _ = persist() + mu.Unlock() + }(index) + } + wg.Wait() + return firstErr + } + indicesByKind := map[string][]int{} + nonRuntimePending := false + for index, t := range task.Plan.Targets { + indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index) + if t.Kind != "runtime" && t.Status != "done" { + nonRuntimePending = true + } + } + if e := processBatch(indicesByKind["runtime"]); e != nil { + fail(e) + return + } + // Re-run the phase-transition safety checks (wait for in-flight human + // work, re-verify runtime absence, revalidate references) exactly once + // per execution attempt, right before touching any file/storage target + // — the same point the original sequential loop ran them at. + if nonRuntimePending { + deadline := time.Now().Add(150 * time.Second) + for { + items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID}) if e != nil { fail(e) return } - known := map[string]bool{} + busy := false for _, id := range task.Plan.RunIDs { - known[id] = true + run, e := s.store.GetRun(ctx, id) + if e != nil { + fail(e) + return + } + if run != nil && run.ExecutionLeaseOwner != "" { + until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt) + if e != nil || until.After(time.Now()) { + busy = true + } + } } - for _, r := range fresh { - if r.ProjectID == task.Plan.ProjectID && !known[r.ID] { - fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID)) + for i := range items { + item := &items[i] + if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil { + fail(e) + return + } + w := mapValue(item.Subject.Payload["sliceWorkflow"]) + if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" { + busy = true + } + } + if !busy { + break + } + if time.Now().After(deadline) { + fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试")) + return + } + select { + case <-ctx.Done(): + fail(ctx.Err()) + return + case <-time.After(2 * time.Second): + } + } + // Catch work admitted immediately before the project lock. + for _, step := range task.Plan.Targets { + if step.Kind == "runtime" { + if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil { + fail(e) return } } - if e = s.validateDeletionReferences(ctx, task.Plan); e != nil { - fail(e) - return - } - runtimeDone = true } - target.Error = "" - target.Status = "running" - if e := persist(); e != nil { - cancel() - return - } - if e := s.deleteProjectTarget(ctx, task.Plan, *target); e != nil { - target.Status = "failed" - target.Error = e.Error() + // Scoped exactly as before (projectId match only, no workflowId + // fallback — see the loop below) but via an indexed query + // instead of an organization-wide scan. + fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil) + if e != nil { fail(e) return } - target.Status = "done" - if e := persist(); e != nil { - cancel() + known := map[string]bool{} + for _, id := range task.Plan.RunIDs { + known[id] = true + } + for _, r := range fresh { + if r.ProjectID == task.Plan.ProjectID && !known[r.ID] { + fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID)) + return + } + } + if e = s.validateDeletionReferences(ctx, task.Plan); e != nil { + fail(e) + return + } + } + for _, kind := range []string{"workspace", "bos", "records"} { + if e := processBatch(indicesByKind[kind]); e != nil { + fail(e) return } } diff --git a/internal/api/project_deletion_test.go b/internal/api/project_deletion_test.go index 62db2d0b..5ad95e9a 100644 --- a/internal/api/project_deletion_test.go +++ b/internal/api/project_deletion_test.go @@ -287,3 +287,78 @@ func TestProjectDeletionFailureRetryAndAudit(t *testing.T) { t.Fatal("raw data removed") } } + +// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the +// processBatch concurrency added to executeProjectDeletion: with many runs +// (many runtime + workspace targets processed by a worker pool instead of +// one at a time), the task must still end up "completed" with every target +// "done" and every run's workspace directory actually removed — i.e. the +// concurrent rewrite produces the same end state as the old sequential +// loop, just faster. +func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) { + ctx := context.Background() + db := store.NewMemoryStore() + p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"}) + if e != nil { + t.Fatal(e) + } + wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID}) + if e != nil { + t.Fatal(e) + } + s := &server{store: db, workspaceMountPath: t.TempDir()} + const runCount = 30 + dirs := make([]string, 0, runCount) + for i := 0; i < runCount; i++ { + runID := fmt.Sprintf("run-concurrent-fixture-%02d", i) + if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil { + t.Fatal(e) + } + dir, e := safeRunWorkspace(s.workspaceMountPath, runID) + if e != nil { + t.Fatal(e) + } + if e = os.MkdirAll(dir, 0700); e != nil { + t.Fatal(e) + } + dirs = append(dirs, dir) + } + plan, e := s.buildProjectDeletionPlan(ctx, p) + if e != nil { + t.Fatal(e) + } + if len(plan.Blockers) > 0 { + t.Fatal(plan.Blockers) + } + task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"}) + if e != nil { + t.Fatal(e) + } + old := projectDeletionRuntimeFactory + defer func() { projectDeletionRuntimeFactory = old }() + projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) { + return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil + } + claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute))) + if e != nil || !ok { + t.Fatal(e) + } + s.executeProjectDeletion(*claimed, "owner") + finished, _ := db.GetProjectDeletion(ctx, task.ID) + if finished.Status != "completed" { + t.Fatalf("%+v", finished) + } + for _, target := range finished.Plan.Targets { + if target.Status != "done" { + t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target) + } + } + for _, dir := range dirs { + if _, e = os.Stat(dir); !os.IsNotExist(e) { + t.Fatalf("workspace %s not removed", dir) + } + } + if existing, _ := db.GetProject(ctx, p.ID); existing != nil { + t.Fatal("project remains") + } +}