droboticflow/internal/api/project_deletion_test.go
eust-w 2864d2aa11 ⚡ Process deletion targets concurrently within each phase
executeProjectDeletion processed every target (runtime, then workspace,
then bos, then records) strictly one at a time — for a project with
thousands of real runs, that meant thousands of sequential K8s/BOS/Mongo
round trips. Observed live on Fanyu: BLF-data's runtime phase (stopping
4,641 runs) took tens of minutes at that pace.

Replaced the per-target sequential loop with a bounded worker pool
(PROJECT_DELETION_TARGET_CONCURRENCY, default 12) applied within each
phase — targets of the same kind now run concurrently, but phases still
run strictly in order (runtime, then workspace, then bos, then records),
preserving the "stop/verify runtime, then remove files, then DB"
ordering the plan is built with. The one-time phase-transition safety
checks (wait for in-flight human work, re-verify runtime absence,
revalidate references) still run exactly once, at the same point,
between the runtime and workspace phases.

Every read-modify-write of shared task state (a target's Status/Error,
and every persist() call) is guarded by a mutex; the actual I/O
(deleteProjectTarget) runs unlocked so workers genuinely overlap. Added
TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes (30 runs) to
lock in that concurrent execution reaches the same end state — every
target done, every workspace directory removed, project record gone —
as the old sequential loop. Full suite passes under -race with no
warnings; 75 pre-existing unrelated failures unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-23 04:30:13 +08:00

365 lines
14 KiB
Go

package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
type deniedDeletionBrowser struct{ objectStorageBrowser }
type deletionRuntimeFunc func(context.Context, string, string) error
func (f deletionRuntimeFunc) TerminateRunAndWait(ctx context.Context, namespace, runID string) error {
return f(ctx, namespace, runID)
}
func TestProjectDeletionRealClientUsesWorkspaceConfigAndMissingRuntimeIsIdempotent(t *testing.T) {
requests := 0
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
if r.Method != http.MethodGet || r.URL.Query().Get("labelSelector") != "runId=run-delete-fixture" || !strings.Contains(r.URL.Path, "/namespaces/delete-fixture/") {
t.Errorf("unexpected cleanup request: %s %s", r.Method, r.URL)
http.Error(w, "unexpected scope", 403)
return
}
w.Header().Set("Content-Type", "application/json")
if strings.HasSuffix(r.URL.Path, "/jobs") {
fmt.Fprint(w, `{"apiVersion":"batch/v1","kind":"JobList","items":[]}`)
} else if strings.HasSuffix(r.URL.Path, "/pods") {
fmt.Fprint(w, `{"apiVersion":"v1","kind":"PodList","items":[]}`)
} else {
t.Errorf("unexpected path: %s", r.URL.Path)
http.NotFound(w, r)
}
}))
defer api.Close()
t.Setenv("KUBERNETES_SERVICE_HOST", "")
t.Setenv("KUBERNETES_SERVICE_PORT", "")
config := filepath.Join(t.TempDir(), "config")
if err := os.WriteFile(config, []byte(fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: fixture\n cluster:\n server: %s\ncontexts:\n- name: fixture\n context:\n cluster: fixture\ncurrent-context: fixture\n", api.URL)), 0600); err != nil {
t.Fatal(err)
}
t.Setenv("KUBECONFIG", config)
s := &server{k8sNamespace: "delete-fixture", workspacePVCName: "actual-workspace", workspaceMountPath: "/workspace"}
for i := 0; i < 2; i++ {
if err := s.deleteProjectTarget(context.Background(), domain.ProjectDeletionPlan{}, domain.ProjectDeletionTarget{Kind: "runtime", RunID: "run-delete-fixture"}); err != nil {
t.Fatal(err)
}
}
if requests == 0 {
t.Fatal("real client never reached Kubernetes")
}
s.workspacePVCName = ""
if _, err := s.deletionRuntime(); err == nil || !strings.Contains(err.Error(), "workspace pvc name is required") {
t.Fatalf("missing PVC validation lost: %v", err)
}
}
func TestProjectDeletionPreflightFailurePreservesProjectAndListener(t *testing.T) {
ctx := context.Background()
db := store.NewMemoryStore()
p, _ := db.CreateProject(ctx, domain.CreateProjectInput{Name: "preflight", OrganizationID: "org-a"})
wf, err := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "listener", ProjectID: p.ID, OrganizationID: p.OrganizationID})
if err != nil {
t.Fatal(err)
}
wf.CaptureAutomation = &domain.CaptureAutomationConfig{Enabled: true}
if err = db.UpsertWorkflow(ctx, *wf); err != nil {
t.Fatal(err)
}
task, _ := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), OrganizationID: p.OrganizationID, Status: "prepared", Plan: domain.ProjectDeletionPlan{ProjectID: p.ID, WorkflowIDs: []string{wf.ID}, Targets: []domain.ProjectDeletionTarget{{Kind: "runtime", RunID: "run-fixture", Status: "pending"}}}})
claimed, ok, err := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(time.Minute)))
if err != nil || !ok {
t.Fatal(err)
}
s := &server{store: db, k8sNamespace: "delete-fixture"}
s.executeProjectDeletion(*claimed, "owner")
result, _ := db.GetProjectDeletion(ctx, task.ID)
if result.Status != "failed" || !strings.Contains(result.Error, "WORKSPACE_PVC_NAME") {
t.Fatalf("%+v", result)
}
if result.Plan.Targets[0].Status != "pending" {
t.Fatal("cleanup started before configuration validation")
}
project, _ := db.GetProject(ctx, p.ID)
if project == nil {
t.Fatal("project removed on preflight failure")
}
unchanged, _ := db.GetWorkflow(ctx, wf.ID)
if unchanged == nil || !unchanged.CaptureAutomation.Enabled {
t.Fatal("listener stopped before preflight")
}
}
func (b deniedDeletionBrowser) DeletePrefix(context.Context, string, string) error {
return fmt.Errorf("injected storage AccessDenied")
}
func TestProjectDeletionPropagatesStorageErrorAndProtectsRaw(t *testing.T) {
ctx := context.Background()
db := store.NewMemoryStore()
s := &server{store: db}
connection := domain.StorageConnectionRecord{ID: "storage-fixture", OrganizationID: "org-a", Provider: domain.StorageProviderBOS, Bucket: "fixture", Endpoint: "https://example.test"}
if e := db.UpsertStorageConnection(ctx, connection); e != nil {
t.Fatal(e)
}
original := storageConnectionBrowserResolver
defer func() { storageConnectionBrowserResolver = original }()
storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
return deniedDeletionBrowser{}, nil, nil
}
target := domain.ProjectDeletionTarget{Kind: "bos", ConnectionID: connection.ID, Bucket: "fixture", Prefix: "artifacts/project-a"}
plan := domain.ProjectDeletionPlan{ProjectID: "project-a", OrganizationID: "org-a", Targets: []domain.ProjectDeletionTarget{target}}
if e := s.deleteProjectTarget(ctx, plan, target); e == nil || e.Error() != "injected storage AccessDenied" {
t.Fatalf("storage error swallowed: %v", e)
}
if e := db.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "workflow-other", ProjectID: "project-other", OrganizationID: "org-a", CaptureAutomation: &domain.CaptureAutomationConfig{ConnectionID: connection.ID, Bucket: "fixture", Prefix: "artifacts/project-a/raw"}}); e != nil {
t.Fatal(e)
}
if e := s.validateDeletionReferences(ctx, plan); e == nil {
t.Fatal("raw root overlap allowed")
}
}
// TestProjectDeletionBlocksTargetReferencedByOtherRun locks in the
// referencedURIs fast path added to validateDeletionReferences: it must
// still catch a BOS target that some other run's artifact actually points
// at, and must still allow a target nothing else references, exactly as
// the original per-pair strings.Contains(raw, uri) scan did before it was
// replaced with a single combined-regexp pass over the other runs.
func TestProjectDeletionBlocksTargetReferencedByOtherRun(t *testing.T) {
ctx := context.Background()
db := store.NewMemoryStore()
s := &server{store: db}
referenced := domain.ProjectDeletionTarget{Kind: "bos", ConnectionID: "conn-a", Bucket: "fixture", Prefix: "artifacts/shared-output"}
unreferenced := domain.ProjectDeletionTarget{Kind: "bos", ConnectionID: "conn-a", Bucket: "fixture", Prefix: "artifacts/only-project-a"}
plan := domain.ProjectDeletionPlan{
ProjectID: "project-a",
OrganizationID: "org-a",
Targets: []domain.ProjectDeletionTarget{referenced, unreferenced},
}
if e := db.UpsertRun(ctx, domain.RunRecord{
ID: "run-other-project",
WorkflowID: "workflow-other",
ProjectID: "project-other",
OrganizationID: "org-a",
Status: domain.RunStatusCompleted,
Artifacts: []domain.ExecutionArtifact{
{ID: "artifact-other", RunID: "run-other-project", NodeID: "node-a", Name: "output.mp4", Path: "bos://fixture/artifacts/shared-output/output.mp4"},
},
}); e != nil {
t.Fatal(e)
}
if e := s.validateDeletionReferences(ctx, plan); e == nil || !strings.Contains(e.Error(), "其他运行引用") {
t.Fatalf("expected referenced target to be blocked, got: %v", e)
}
plan.Targets = []domain.ProjectDeletionTarget{unreferenced}
if e := s.validateDeletionReferences(ctx, plan); e != nil {
t.Fatalf("unreferenced target should not be blocked: %v", e)
}
}
func TestProjectDeletionWorkspaceBoundaries(t *testing.T) {
root := t.TempDir()
for _, id := range []string{"../x", "run-../x", "run-a/b", "", "run-"} {
if _, e := safeRunWorkspace(root, id); e == nil {
t.Fatalf("accepted %q", id)
}
}
if _, e := safeRunWorkspace("/", "run-a"); e == nil {
t.Fatal("root allowed")
}
outside := t.TempDir()
if e := os.Symlink(outside, filepath.Join(root, "system")); e != nil {
t.Fatal(e)
}
if _, e := safeRunWorkspace(root, "run-a"); e == nil {
t.Fatal("symlink parent allowed")
}
}
func TestProjectDeletionPrefixBoundaries(t *testing.T) {
for _, p := range []string{"", "/", "../raw", "system/../raw", "a//b", "a/{runId}"} {
if validDeletionPrefix(p) {
t.Fatalf("accepted %q", p)
}
}
if prefixOverlap("runs/run-a", "runs/run-ab") {
t.Fatal("prefix sibling overlap")
}
if !prefixOverlap("raw", "raw/capture") {
t.Fatal("missing ancestor overlap")
}
}
func TestProjectDeletionFailureRetryAndAudit(t *testing.T) {
ctx := context.Background()
db := store.NewMemoryStore()
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "delete fixture", OrganizationID: "org-a"})
if e != nil {
t.Fatal(e)
}
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
if e != nil {
t.Fatal(e)
}
if e = db.UpsertRun(ctx, domain.RunRecord{ID: "run-fixture", WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
t.Fatal(e)
}
s := &server{store: db, workspaceMountPath: t.TempDir()}
dir, e := safeRunWorkspace(s.workspaceMountPath, "run-fixture")
if e != nil {
t.Fatal(e)
}
if e = os.MkdirAll(dir, 0700); e != nil {
t.Fatal(e)
}
if e = os.WriteFile(filepath.Join(dir, "result.json"), []byte("{}"), 0600); e != nil {
t.Fatal(e)
}
raw := filepath.Join(s.workspaceMountPath, "raw")
if e = os.MkdirAll(raw, 0700); e != nil {
t.Fatal(e)
}
plan, e := s.buildProjectDeletionPlan(ctx, p)
if e != nil {
t.Fatal(e)
}
if len(plan.Blockers) > 0 {
t.Fatal(plan.Blockers)
}
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
if e != nil {
t.Fatal(e)
}
old := projectDeletionRuntimeFactory
defer func() { projectDeletionRuntimeFactory = old }()
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
return deletionRuntimeFunc(func(context.Context, string, string) error { return fmt.Errorf("injected runtime failure") }), nil
}
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(time.Minute)))
if e != nil || !ok {
t.Fatal(e)
}
s.executeProjectDeletion(*claimed, "owner")
failed, _ := db.GetProjectDeletion(ctx, task.ID)
if failed.Status != "failed" {
t.Fatalf("%+v", failed)
}
if existing, _ := db.GetProject(ctx, p.ID); existing == nil {
t.Fatal("project removed after resource failure")
}
if _, e = os.Stat(dir); e != nil {
t.Fatal("workspace removed after runtime failure")
}
if _, e = db.CreateRun(ctx, wf.ID, domain.RunTriggerManual); e == nil {
t.Fatal("deletion lock allowed admission")
}
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
}
claimed, ok, e = db.ClaimProjectDeletion(ctx, task.ID, "retry", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(time.Minute)))
if e != nil || !ok {
t.Fatal(e)
}
s.executeProjectDeletion(*claimed, "retry")
finished, _ := db.GetProjectDeletion(ctx, task.ID)
if finished.Status != "completed" {
t.Fatalf("%+v", finished)
}
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
t.Fatal("project remains")
}
if _, e = os.Stat(dir); !os.IsNotExist(e) {
t.Fatal("workspace remains")
}
if _, e = os.Stat(raw); e != nil {
t.Fatal("raw data removed")
}
}
// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the
// processBatch concurrency added to executeProjectDeletion: with many runs
// (many runtime + workspace targets processed by a worker pool instead of
// one at a time), the task must still end up "completed" with every target
// "done" and every run's workspace directory actually removed — i.e. the
// concurrent rewrite produces the same end state as the old sequential
// loop, just faster.
func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) {
ctx := context.Background()
db := store.NewMemoryStore()
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"})
if e != nil {
t.Fatal(e)
}
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
if e != nil {
t.Fatal(e)
}
s := &server{store: db, workspaceMountPath: t.TempDir()}
const runCount = 30
dirs := make([]string, 0, runCount)
for i := 0; i < runCount; i++ {
runID := fmt.Sprintf("run-concurrent-fixture-%02d", i)
if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
t.Fatal(e)
}
dir, e := safeRunWorkspace(s.workspaceMountPath, runID)
if e != nil {
t.Fatal(e)
}
if e = os.MkdirAll(dir, 0700); e != nil {
t.Fatal(e)
}
dirs = append(dirs, dir)
}
plan, e := s.buildProjectDeletionPlan(ctx, p)
if e != nil {
t.Fatal(e)
}
if len(plan.Blockers) > 0 {
t.Fatal(plan.Blockers)
}
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
if e != nil {
t.Fatal(e)
}
old := projectDeletionRuntimeFactory
defer func() { projectDeletionRuntimeFactory = old }()
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
}
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
if e != nil || !ok {
t.Fatal(e)
}
s.executeProjectDeletion(*claimed, "owner")
finished, _ := db.GetProjectDeletion(ctx, task.ID)
if finished.Status != "completed" {
t.Fatalf("%+v", finished)
}
for _, target := range finished.Plan.Targets {
if target.Status != "done" {
t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target)
}
}
for _, dir := range dirs {
if _, e = os.Stat(dir); !os.IsNotExist(e) {
t.Fatalf("workspace %s not removed", dir)
}
}
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
t.Fatal("project remains")
}
}