forked from longtao.wu/droboticflow
Merge pull request '⚡ Process deletion targets concurrently within each phase' (#14) from feat/project-deletion-parallel-target-execution into dev
This commit is contained in:
commit
50683fc89d
@ -18,7 +18,9 @@ import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@ -646,107 +648,174 @@ func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner s
|
||||
}
|
||||
}
|
||||
}
|
||||
runtimeDone := false
|
||||
for index := range task.Plan.Targets {
|
||||
target := &task.Plan.Targets[index]
|
||||
if target.Status == "done" {
|
||||
continue
|
||||
targetConcurrency := 12
|
||||
if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" {
|
||||
if n, e := strconv.Atoi(v); e == nil && n > 0 {
|
||||
targetConcurrency = n
|
||||
}
|
||||
if target.Kind != "runtime" && !runtimeDone {
|
||||
deadline := time.Now().Add(150 * time.Second)
|
||||
for {
|
||||
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
busy := false
|
||||
for _, id := range task.Plan.RunIDs {
|
||||
run, e := s.store.GetRun(ctx, id)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
if run != nil && run.ExecutionLeaseOwner != "" {
|
||||
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
|
||||
if e != nil || until.After(time.Now()) {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for i := range items {
|
||||
item := &items[i]
|
||||
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
w := mapValue(item.Subject.Payload["sliceWorkflow"])
|
||||
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
if !busy {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
fail(ctx.Err())
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
// processBatch runs deleteProjectTarget for the not-yet-done targets at
|
||||
// indices with up to targetConcurrency in flight at once. Each target's
|
||||
// Status/Error and every persist() call is guarded by mu, so concurrent
|
||||
// workers never race on shared task state; deleteProjectTarget itself
|
||||
// (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs
|
||||
// unlocked. A single project's deletion used to process every target
|
||||
// strictly one at a time, which made stopping thousands of real runs
|
||||
// take as long as thousands of sequential K8s round trips.
|
||||
var mu sync.Mutex
|
||||
processBatch := func(indices []int) error {
|
||||
sem := make(chan struct{}, targetConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
var firstErr error
|
||||
for _, index := range indices {
|
||||
mu.Lock()
|
||||
skip := task.Plan.Targets[index].Status == "done" || firstErr != nil
|
||||
mu.Unlock()
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
// Catch work admitted immediately before the project lock.
|
||||
for _, step := range task.Plan.Targets {
|
||||
if step.Kind == "runtime" {
|
||||
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
mu.Lock()
|
||||
if firstErr != nil {
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
// Scoped exactly as before (projectId match only, no workflowId
|
||||
// fallback — see the loop below) but via an indexed query
|
||||
// instead of an organization-wide scan.
|
||||
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
|
||||
target := &task.Plan.Targets[index]
|
||||
target.Error = ""
|
||||
target.Status = "running"
|
||||
persistErr := persist()
|
||||
snapshot := *target
|
||||
mu.Unlock()
|
||||
if persistErr != nil {
|
||||
mu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = persistErr
|
||||
}
|
||||
mu.Unlock()
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot)
|
||||
mu.Lock()
|
||||
target = &task.Plan.Targets[index]
|
||||
if runErr != nil {
|
||||
target.Status = "failed"
|
||||
target.Error = runErr.Error()
|
||||
if firstErr == nil {
|
||||
firstErr = runErr
|
||||
}
|
||||
} else {
|
||||
target.Status = "done"
|
||||
}
|
||||
_ = persist()
|
||||
mu.Unlock()
|
||||
}(index)
|
||||
}
|
||||
wg.Wait()
|
||||
return firstErr
|
||||
}
|
||||
indicesByKind := map[string][]int{}
|
||||
nonRuntimePending := false
|
||||
for index, t := range task.Plan.Targets {
|
||||
indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index)
|
||||
if t.Kind != "runtime" && t.Status != "done" {
|
||||
nonRuntimePending = true
|
||||
}
|
||||
}
|
||||
if e := processBatch(indicesByKind["runtime"]); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
// Re-run the phase-transition safety checks (wait for in-flight human
|
||||
// work, re-verify runtime absence, revalidate references) exactly once
|
||||
// per execution attempt, right before touching any file/storage target
|
||||
// — the same point the original sequential loop ran them at.
|
||||
if nonRuntimePending {
|
||||
deadline := time.Now().Add(150 * time.Second)
|
||||
for {
|
||||
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
known := map[string]bool{}
|
||||
busy := false
|
||||
for _, id := range task.Plan.RunIDs {
|
||||
known[id] = true
|
||||
run, e := s.store.GetRun(ctx, id)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
if run != nil && run.ExecutionLeaseOwner != "" {
|
||||
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
|
||||
if e != nil || until.After(time.Now()) {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, r := range fresh {
|
||||
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
|
||||
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
|
||||
for i := range items {
|
||||
item := &items[i]
|
||||
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
w := mapValue(item.Subject.Payload["sliceWorkflow"])
|
||||
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
|
||||
busy = true
|
||||
}
|
||||
}
|
||||
if !busy {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
fail(ctx.Err())
|
||||
return
|
||||
case <-time.After(2 * time.Second):
|
||||
}
|
||||
}
|
||||
// Catch work admitted immediately before the project lock.
|
||||
for _, step := range task.Plan.Targets {
|
||||
if step.Kind == "runtime" {
|
||||
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
runtimeDone = true
|
||||
}
|
||||
target.Error = ""
|
||||
target.Status = "running"
|
||||
if e := persist(); e != nil {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
if e := s.deleteProjectTarget(ctx, task.Plan, *target); e != nil {
|
||||
target.Status = "failed"
|
||||
target.Error = e.Error()
|
||||
// Scoped exactly as before (projectId match only, no workflowId
|
||||
// fallback — see the loop below) but via an indexed query
|
||||
// instead of an organization-wide scan.
|
||||
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
|
||||
if e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
target.Status = "done"
|
||||
if e := persist(); e != nil {
|
||||
cancel()
|
||||
known := map[string]bool{}
|
||||
for _, id := range task.Plan.RunIDs {
|
||||
known[id] = true
|
||||
}
|
||||
for _, r := range fresh {
|
||||
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
|
||||
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
|
||||
return
|
||||
}
|
||||
}
|
||||
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{"workspace", "bos", "records"} {
|
||||
if e := processBatch(indicesByKind[kind]); e != nil {
|
||||
fail(e)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@ -287,3 +287,78 @@ func TestProjectDeletionFailureRetryAndAudit(t *testing.T) {
|
||||
t.Fatal("raw data removed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the
|
||||
// processBatch concurrency added to executeProjectDeletion: with many runs
|
||||
// (many runtime + workspace targets processed by a worker pool instead of
|
||||
// one at a time), the task must still end up "completed" with every target
|
||||
// "done" and every run's workspace directory actually removed — i.e. the
|
||||
// concurrent rewrite produces the same end state as the old sequential
|
||||
// loop, just faster.
|
||||
func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := store.NewMemoryStore()
|
||||
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s := &server{store: db, workspaceMountPath: t.TempDir()}
|
||||
const runCount = 30
|
||||
dirs := make([]string, 0, runCount)
|
||||
for i := 0; i < runCount; i++ {
|
||||
runID := fmt.Sprintf("run-concurrent-fixture-%02d", i)
|
||||
if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
dir, e := safeRunWorkspace(s.workspaceMountPath, runID)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = os.MkdirAll(dir, 0700); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
plan, e := s.buildProjectDeletionPlan(ctx, p)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(plan.Blockers) > 0 {
|
||||
t.Fatal(plan.Blockers)
|
||||
}
|
||||
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
old := projectDeletionRuntimeFactory
|
||||
defer func() { projectDeletionRuntimeFactory = old }()
|
||||
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
|
||||
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
|
||||
}
|
||||
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.executeProjectDeletion(*claimed, "owner")
|
||||
finished, _ := db.GetProjectDeletion(ctx, task.ID)
|
||||
if finished.Status != "completed" {
|
||||
t.Fatalf("%+v", finished)
|
||||
}
|
||||
for _, target := range finished.Plan.Targets {
|
||||
if target.Status != "done" {
|
||||
t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target)
|
||||
}
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
if _, e = os.Stat(dir); !os.IsNotExist(e) {
|
||||
t.Fatalf("workspace %s not removed", dir)
|
||||
}
|
||||
}
|
||||
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
|
||||
t.Fatal("project remains")
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user