Merge pull request '⚡ Process deletion targets concurrently within each phase' (#14) from feat/project-deletion-parallel-target-execution into dev

This commit is contained in:
longtao.wu 2026-09-23 04:30:32 +08:00
commit 50683fc89d
2 changed files with 229 additions and 85 deletions

View File

@ -18,7 +18,9 @@ import (
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
)
@ -646,107 +648,174 @@ func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner s
}
}
}
runtimeDone := false
for index := range task.Plan.Targets {
target := &task.Plan.Targets[index]
if target.Status == "done" {
continue
targetConcurrency := 12
if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" {
if n, e := strconv.Atoi(v); e == nil && n > 0 {
targetConcurrency = n
}
if target.Kind != "runtime" && !runtimeDone {
deadline := time.Now().Add(150 * time.Second)
for {
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
if e != nil {
fail(e)
return
}
busy := false
for _, id := range task.Plan.RunIDs {
run, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if run != nil && run.ExecutionLeaseOwner != "" {
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
if e != nil || until.After(time.Now()) {
busy = true
}
}
}
for i := range items {
item := &items[i]
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
fail(e)
return
}
w := mapValue(item.Subject.Payload["sliceWorkflow"])
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
busy = true
}
}
if !busy {
break
}
if time.Now().After(deadline) {
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
return
}
select {
case <-ctx.Done():
fail(ctx.Err())
return
case <-time.After(2 * time.Second):
}
}
// processBatch runs deleteProjectTarget for the not-yet-done targets at
// indices with up to targetConcurrency in flight at once. Each target's
// Status/Error and every persist() call is guarded by mu, so concurrent
// workers never race on shared task state; deleteProjectTarget itself
// (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs
// unlocked. A single project's deletion used to process every target
// strictly one at a time, which made stopping thousands of real runs
// take as long as thousands of sequential K8s round trips.
var mu sync.Mutex
processBatch := func(indices []int) error {
sem := make(chan struct{}, targetConcurrency)
var wg sync.WaitGroup
var firstErr error
for _, index := range indices {
mu.Lock()
skip := task.Plan.Targets[index].Status == "done" || firstErr != nil
mu.Unlock()
if skip {
continue
}
// Catch work admitted immediately before the project lock.
for _, step := range task.Plan.Targets {
if step.Kind == "runtime" {
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
fail(e)
return
}
sem <- struct{}{}
wg.Add(1)
go func(index int) {
defer wg.Done()
defer func() { <-sem }()
mu.Lock()
if firstErr != nil {
mu.Unlock()
return
}
}
// Scoped exactly as before (projectId match only, no workflowId
// fallback — see the loop below) but via an indexed query
// instead of an organization-wide scan.
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
target := &task.Plan.Targets[index]
target.Error = ""
target.Status = "running"
persistErr := persist()
snapshot := *target
mu.Unlock()
if persistErr != nil {
mu.Lock()
if firstErr == nil {
firstErr = persistErr
}
mu.Unlock()
cancel()
return
}
runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot)
mu.Lock()
target = &task.Plan.Targets[index]
if runErr != nil {
target.Status = "failed"
target.Error = runErr.Error()
if firstErr == nil {
firstErr = runErr
}
} else {
target.Status = "done"
}
_ = persist()
mu.Unlock()
}(index)
}
wg.Wait()
return firstErr
}
indicesByKind := map[string][]int{}
nonRuntimePending := false
for index, t := range task.Plan.Targets {
indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index)
if t.Kind != "runtime" && t.Status != "done" {
nonRuntimePending = true
}
}
if e := processBatch(indicesByKind["runtime"]); e != nil {
fail(e)
return
}
// Re-run the phase-transition safety checks (wait for in-flight human
// work, re-verify runtime absence, revalidate references) exactly once
// per execution attempt, right before touching any file/storage target
// — the same point the original sequential loop ran them at.
if nonRuntimePending {
deadline := time.Now().Add(150 * time.Second)
for {
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
if e != nil {
fail(e)
return
}
known := map[string]bool{}
busy := false
for _, id := range task.Plan.RunIDs {
known[id] = true
run, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if run != nil && run.ExecutionLeaseOwner != "" {
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
if e != nil || until.After(time.Now()) {
busy = true
}
}
}
for _, r := range fresh {
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
for i := range items {
item := &items[i]
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
fail(e)
return
}
w := mapValue(item.Subject.Payload["sliceWorkflow"])
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
busy = true
}
}
if !busy {
break
}
if time.Now().After(deadline) {
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
return
}
select {
case <-ctx.Done():
fail(ctx.Err())
return
case <-time.After(2 * time.Second):
}
}
// Catch work admitted immediately before the project lock.
for _, step := range task.Plan.Targets {
if step.Kind == "runtime" {
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
fail(e)
return
}
}
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
fail(e)
return
}
runtimeDone = true
}
target.Error = ""
target.Status = "running"
if e := persist(); e != nil {
cancel()
return
}
if e := s.deleteProjectTarget(ctx, task.Plan, *target); e != nil {
target.Status = "failed"
target.Error = e.Error()
// Scoped exactly as before (projectId match only, no workflowId
// fallback — see the loop below) but via an indexed query
// instead of an organization-wide scan.
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
if e != nil {
fail(e)
return
}
target.Status = "done"
if e := persist(); e != nil {
cancel()
known := map[string]bool{}
for _, id := range task.Plan.RunIDs {
known[id] = true
}
for _, r := range fresh {
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
return
}
}
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
fail(e)
return
}
}
for _, kind := range []string{"workspace", "bos", "records"} {
if e := processBatch(indicesByKind[kind]); e != nil {
fail(e)
return
}
}

View File

@ -287,3 +287,78 @@ func TestProjectDeletionFailureRetryAndAudit(t *testing.T) {
t.Fatal("raw data removed")
}
}
// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the
// processBatch concurrency added to executeProjectDeletion: with many runs
// (many runtime + workspace targets processed by a worker pool instead of
// one at a time), the task must still end up "completed" with every target
// "done" and every run's workspace directory actually removed — i.e. the
// concurrent rewrite produces the same end state as the old sequential
// loop, just faster.
func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) {
ctx := context.Background()
db := store.NewMemoryStore()
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"})
if e != nil {
t.Fatal(e)
}
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
if e != nil {
t.Fatal(e)
}
s := &server{store: db, workspaceMountPath: t.TempDir()}
const runCount = 30
dirs := make([]string, 0, runCount)
for i := 0; i < runCount; i++ {
runID := fmt.Sprintf("run-concurrent-fixture-%02d", i)
if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
t.Fatal(e)
}
dir, e := safeRunWorkspace(s.workspaceMountPath, runID)
if e != nil {
t.Fatal(e)
}
if e = os.MkdirAll(dir, 0700); e != nil {
t.Fatal(e)
}
dirs = append(dirs, dir)
}
plan, e := s.buildProjectDeletionPlan(ctx, p)
if e != nil {
t.Fatal(e)
}
if len(plan.Blockers) > 0 {
t.Fatal(plan.Blockers)
}
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
if e != nil {
t.Fatal(e)
}
old := projectDeletionRuntimeFactory
defer func() { projectDeletionRuntimeFactory = old }()
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
}
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
if e != nil || !ok {
t.Fatal(e)
}
s.executeProjectDeletion(*claimed, "owner")
finished, _ := db.GetProjectDeletion(ctx, task.ID)
if finished.Status != "completed" {
t.Fatalf("%+v", finished)
}
for _, target := range finished.Plan.Targets {
if target.Status != "done" {
t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target)
}
}
for _, dir := range dirs {
if _, e = os.Stat(dir); !os.IsNotExist(e) {
t.Fatalf("workspace %s not removed", dir)
}
}
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
t.Fatal("project remains")
}
}