forked from longtao.wu/droboticflow
executeProjectDeletion processed every target (runtime, then workspace, then bos, then records) strictly one at a time — for a project with thousands of real runs, that meant thousands of sequential K8s/BOS/Mongo round trips. Observed live on Fanyu: BLF-data's runtime phase (stopping 4,641 runs) took tens of minutes at that pace. Replaced the per-target sequential loop with a bounded worker pool (PROJECT_DELETION_TARGET_CONCURRENCY, default 12) applied within each phase — targets of the same kind now run concurrently, but phases still run strictly in order (runtime, then workspace, then bos, then records), preserving the "stop/verify runtime, then remove files, then DB" ordering the plan is built with. The one-time phase-transition safety checks (wait for in-flight human work, re-verify runtime absence, revalidate references) still run exactly once, at the same point, between the runtime and workspace phases. Every read-modify-write of shared task state (a target's Status/Error, and every persist() call) is guarded by a mutex; the actual I/O (deleteProjectTarget) runs unlocked so workers genuinely overlap. Added TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes (30 runs) to lock in that concurrent execution reaches the same end state — every target done, every workspace directory removed, project record gone — as the old sequential loop. Full suite passes under -race with no warnings; 75 pre-existing unrelated failures unchanged. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
365 lines
14 KiB
Go
365 lines
14 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type deniedDeletionBrowser struct{ objectStorageBrowser }
|
|
|
|
type deletionRuntimeFunc func(context.Context, string, string) error
|
|
|
|
func (f deletionRuntimeFunc) TerminateRunAndWait(ctx context.Context, namespace, runID string) error {
|
|
return f(ctx, namespace, runID)
|
|
}
|
|
|
|
func TestProjectDeletionRealClientUsesWorkspaceConfigAndMissingRuntimeIsIdempotent(t *testing.T) {
|
|
requests := 0
|
|
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
requests++
|
|
if r.Method != http.MethodGet || r.URL.Query().Get("labelSelector") != "runId=run-delete-fixture" || !strings.Contains(r.URL.Path, "/namespaces/delete-fixture/") {
|
|
t.Errorf("unexpected cleanup request: %s %s", r.Method, r.URL)
|
|
http.Error(w, "unexpected scope", 403)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
if strings.HasSuffix(r.URL.Path, "/jobs") {
|
|
fmt.Fprint(w, `{"apiVersion":"batch/v1","kind":"JobList","items":[]}`)
|
|
} else if strings.HasSuffix(r.URL.Path, "/pods") {
|
|
fmt.Fprint(w, `{"apiVersion":"v1","kind":"PodList","items":[]}`)
|
|
} else {
|
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
defer api.Close()
|
|
t.Setenv("KUBERNETES_SERVICE_HOST", "")
|
|
t.Setenv("KUBERNETES_SERVICE_PORT", "")
|
|
config := filepath.Join(t.TempDir(), "config")
|
|
if err := os.WriteFile(config, []byte(fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: fixture\n cluster:\n server: %s\ncontexts:\n- name: fixture\n context:\n cluster: fixture\ncurrent-context: fixture\n", api.URL)), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("KUBECONFIG", config)
|
|
s := &server{k8sNamespace: "delete-fixture", workspacePVCName: "actual-workspace", workspaceMountPath: "/workspace"}
|
|
for i := 0; i < 2; i++ {
|
|
if err := s.deleteProjectTarget(context.Background(), domain.ProjectDeletionPlan{}, domain.ProjectDeletionTarget{Kind: "runtime", RunID: "run-delete-fixture"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if requests == 0 {
|
|
t.Fatal("real client never reached Kubernetes")
|
|
}
|
|
s.workspacePVCName = ""
|
|
if _, err := s.deletionRuntime(); err == nil || !strings.Contains(err.Error(), "workspace pvc name is required") {
|
|
t.Fatalf("missing PVC validation lost: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestProjectDeletionPreflightFailurePreservesProjectAndListener(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
p, _ := db.CreateProject(ctx, domain.CreateProjectInput{Name: "preflight", OrganizationID: "org-a"})
|
|
wf, err := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "listener", ProjectID: p.ID, OrganizationID: p.OrganizationID})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wf.CaptureAutomation = &domain.CaptureAutomationConfig{Enabled: true}
|
|
if err = db.UpsertWorkflow(ctx, *wf); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
task, _ := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), OrganizationID: p.OrganizationID, Status: "prepared", Plan: domain.ProjectDeletionPlan{ProjectID: p.ID, WorkflowIDs: []string{wf.ID}, Targets: []domain.ProjectDeletionTarget{{Kind: "runtime", RunID: "run-fixture", Status: "pending"}}}})
|
|
claimed, ok, err := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(time.Minute)))
|
|
if err != nil || !ok {
|
|
t.Fatal(err)
|
|
}
|
|
s := &server{store: db, k8sNamespace: "delete-fixture"}
|
|
s.executeProjectDeletion(*claimed, "owner")
|
|
result, _ := db.GetProjectDeletion(ctx, task.ID)
|
|
if result.Status != "failed" || !strings.Contains(result.Error, "WORKSPACE_PVC_NAME") {
|
|
t.Fatalf("%+v", result)
|
|
}
|
|
if result.Plan.Targets[0].Status != "pending" {
|
|
t.Fatal("cleanup started before configuration validation")
|
|
}
|
|
project, _ := db.GetProject(ctx, p.ID)
|
|
if project == nil {
|
|
t.Fatal("project removed on preflight failure")
|
|
}
|
|
unchanged, _ := db.GetWorkflow(ctx, wf.ID)
|
|
if unchanged == nil || !unchanged.CaptureAutomation.Enabled {
|
|
t.Fatal("listener stopped before preflight")
|
|
}
|
|
}
|
|
|
|
func (b deniedDeletionBrowser) DeletePrefix(context.Context, string, string) error {
|
|
return fmt.Errorf("injected storage AccessDenied")
|
|
}
|
|
func TestProjectDeletionPropagatesStorageErrorAndProtectsRaw(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
s := &server{store: db}
|
|
connection := domain.StorageConnectionRecord{ID: "storage-fixture", OrganizationID: "org-a", Provider: domain.StorageProviderBOS, Bucket: "fixture", Endpoint: "https://example.test"}
|
|
if e := db.UpsertStorageConnection(ctx, connection); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
original := storageConnectionBrowserResolver
|
|
defer func() { storageConnectionBrowserResolver = original }()
|
|
storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
|
return deniedDeletionBrowser{}, nil, nil
|
|
}
|
|
target := domain.ProjectDeletionTarget{Kind: "bos", ConnectionID: connection.ID, Bucket: "fixture", Prefix: "artifacts/project-a"}
|
|
plan := domain.ProjectDeletionPlan{ProjectID: "project-a", OrganizationID: "org-a", Targets: []domain.ProjectDeletionTarget{target}}
|
|
if e := s.deleteProjectTarget(ctx, plan, target); e == nil || e.Error() != "injected storage AccessDenied" {
|
|
t.Fatalf("storage error swallowed: %v", e)
|
|
}
|
|
if e := db.UpsertWorkflow(ctx, domain.WorkflowRecord{ID: "workflow-other", ProjectID: "project-other", OrganizationID: "org-a", CaptureAutomation: &domain.CaptureAutomationConfig{ConnectionID: connection.ID, Bucket: "fixture", Prefix: "artifacts/project-a/raw"}}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e := s.validateDeletionReferences(ctx, plan); e == nil {
|
|
t.Fatal("raw root overlap allowed")
|
|
}
|
|
}
|
|
|
|
// TestProjectDeletionBlocksTargetReferencedByOtherRun locks in the
|
|
// referencedURIs fast path added to validateDeletionReferences: it must
|
|
// still catch a BOS target that some other run's artifact actually points
|
|
// at, and must still allow a target nothing else references, exactly as
|
|
// the original per-pair strings.Contains(raw, uri) scan did before it was
|
|
// replaced with a single combined-regexp pass over the other runs.
|
|
func TestProjectDeletionBlocksTargetReferencedByOtherRun(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
s := &server{store: db}
|
|
|
|
referenced := domain.ProjectDeletionTarget{Kind: "bos", ConnectionID: "conn-a", Bucket: "fixture", Prefix: "artifacts/shared-output"}
|
|
unreferenced := domain.ProjectDeletionTarget{Kind: "bos", ConnectionID: "conn-a", Bucket: "fixture", Prefix: "artifacts/only-project-a"}
|
|
plan := domain.ProjectDeletionPlan{
|
|
ProjectID: "project-a",
|
|
OrganizationID: "org-a",
|
|
Targets: []domain.ProjectDeletionTarget{referenced, unreferenced},
|
|
}
|
|
|
|
if e := db.UpsertRun(ctx, domain.RunRecord{
|
|
ID: "run-other-project",
|
|
WorkflowID: "workflow-other",
|
|
ProjectID: "project-other",
|
|
OrganizationID: "org-a",
|
|
Status: domain.RunStatusCompleted,
|
|
Artifacts: []domain.ExecutionArtifact{
|
|
{ID: "artifact-other", RunID: "run-other-project", NodeID: "node-a", Name: "output.mp4", Path: "bos://fixture/artifacts/shared-output/output.mp4"},
|
|
},
|
|
}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
|
|
if e := s.validateDeletionReferences(ctx, plan); e == nil || !strings.Contains(e.Error(), "其他运行引用") {
|
|
t.Fatalf("expected referenced target to be blocked, got: %v", e)
|
|
}
|
|
|
|
plan.Targets = []domain.ProjectDeletionTarget{unreferenced}
|
|
if e := s.validateDeletionReferences(ctx, plan); e != nil {
|
|
t.Fatalf("unreferenced target should not be blocked: %v", e)
|
|
}
|
|
}
|
|
|
|
func TestProjectDeletionWorkspaceBoundaries(t *testing.T) {
|
|
root := t.TempDir()
|
|
for _, id := range []string{"../x", "run-../x", "run-a/b", "", "run-"} {
|
|
if _, e := safeRunWorkspace(root, id); e == nil {
|
|
t.Fatalf("accepted %q", id)
|
|
}
|
|
}
|
|
if _, e := safeRunWorkspace("/", "run-a"); e == nil {
|
|
t.Fatal("root allowed")
|
|
}
|
|
outside := t.TempDir()
|
|
if e := os.Symlink(outside, filepath.Join(root, "system")); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if _, e := safeRunWorkspace(root, "run-a"); e == nil {
|
|
t.Fatal("symlink parent allowed")
|
|
}
|
|
}
|
|
func TestProjectDeletionPrefixBoundaries(t *testing.T) {
|
|
for _, p := range []string{"", "/", "../raw", "system/../raw", "a//b", "a/{runId}"} {
|
|
if validDeletionPrefix(p) {
|
|
t.Fatalf("accepted %q", p)
|
|
}
|
|
}
|
|
if prefixOverlap("runs/run-a", "runs/run-ab") {
|
|
t.Fatal("prefix sibling overlap")
|
|
}
|
|
if !prefixOverlap("raw", "raw/capture") {
|
|
t.Fatal("missing ancestor overlap")
|
|
}
|
|
}
|
|
func TestProjectDeletionFailureRetryAndAudit(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "delete fixture", OrganizationID: "org-a"})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e = db.UpsertRun(ctx, domain.RunRecord{ID: "run-fixture", WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
s := &server{store: db, workspaceMountPath: t.TempDir()}
|
|
dir, e := safeRunWorkspace(s.workspaceMountPath, "run-fixture")
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e = os.MkdirAll(dir, 0700); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e = os.WriteFile(filepath.Join(dir, "result.json"), []byte("{}"), 0600); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
raw := filepath.Join(s.workspaceMountPath, "raw")
|
|
if e = os.MkdirAll(raw, 0700); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
plan, e := s.buildProjectDeletionPlan(ctx, p)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if len(plan.Blockers) > 0 {
|
|
t.Fatal(plan.Blockers)
|
|
}
|
|
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
old := projectDeletionRuntimeFactory
|
|
defer func() { projectDeletionRuntimeFactory = old }()
|
|
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
|
|
return deletionRuntimeFunc(func(context.Context, string, string) error { return fmt.Errorf("injected runtime failure") }), nil
|
|
}
|
|
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(time.Minute)))
|
|
if e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
s.executeProjectDeletion(*claimed, "owner")
|
|
failed, _ := db.GetProjectDeletion(ctx, task.ID)
|
|
if failed.Status != "failed" {
|
|
t.Fatalf("%+v", failed)
|
|
}
|
|
if existing, _ := db.GetProject(ctx, p.ID); existing == nil {
|
|
t.Fatal("project removed after resource failure")
|
|
}
|
|
if _, e = os.Stat(dir); e != nil {
|
|
t.Fatal("workspace removed after runtime failure")
|
|
}
|
|
if _, e = db.CreateRun(ctx, wf.ID, domain.RunTriggerManual); e == nil {
|
|
t.Fatal("deletion lock allowed admission")
|
|
}
|
|
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
|
|
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
|
|
}
|
|
claimed, ok, e = db.ClaimProjectDeletion(ctx, task.ID, "retry", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(time.Minute)))
|
|
if e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
s.executeProjectDeletion(*claimed, "retry")
|
|
finished, _ := db.GetProjectDeletion(ctx, task.ID)
|
|
if finished.Status != "completed" {
|
|
t.Fatalf("%+v", finished)
|
|
}
|
|
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
|
|
t.Fatal("project remains")
|
|
}
|
|
if _, e = os.Stat(dir); !os.IsNotExist(e) {
|
|
t.Fatal("workspace remains")
|
|
}
|
|
if _, e = os.Stat(raw); e != nil {
|
|
t.Fatal("raw data removed")
|
|
}
|
|
}
|
|
|
|
// TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes locks in the
|
|
// processBatch concurrency added to executeProjectDeletion: with many runs
|
|
// (many runtime + workspace targets processed by a worker pool instead of
|
|
// one at a time), the task must still end up "completed" with every target
|
|
// "done" and every run's workspace directory actually removed — i.e. the
|
|
// concurrent rewrite produces the same end state as the old sequential
|
|
// loop, just faster.
|
|
func TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes(t *testing.T) {
|
|
ctx := context.Background()
|
|
db := store.NewMemoryStore()
|
|
p, e := db.CreateProject(ctx, domain.CreateProjectInput{Name: "concurrent delete fixture", OrganizationID: "org-a"})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
wf, e := db.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "fixture", ProjectID: p.ID, OrganizationID: p.OrganizationID})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
s := &server{store: db, workspaceMountPath: t.TempDir()}
|
|
const runCount = 30
|
|
dirs := make([]string, 0, runCount)
|
|
for i := 0; i < runCount; i++ {
|
|
runID := fmt.Sprintf("run-concurrent-fixture-%02d", i)
|
|
if e = db.UpsertRun(ctx, domain.RunRecord{ID: runID, WorkflowID: wf.ID, ProjectID: p.ID, OrganizationID: p.OrganizationID, Status: domain.RunStatusCompleted}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
dir, e := safeRunWorkspace(s.workspaceMountPath, runID)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if e = os.MkdirAll(dir, 0700); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
dirs = append(dirs, dir)
|
|
}
|
|
plan, e := s.buildProjectDeletionPlan(ctx, p)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if len(plan.Blockers) > 0 {
|
|
t.Fatal(plan.Blockers)
|
|
}
|
|
task, e := db.CreateProjectDeletion(ctx, domain.ProjectDeletionTask{ID: deletionTaskID(p.ID), Plan: plan, OrganizationID: p.OrganizationID, Status: "prepared"})
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
old := projectDeletionRuntimeFactory
|
|
defer func() { projectDeletionRuntimeFactory = old }()
|
|
projectDeletionRuntimeFactory = func(string, string, string, string) (projectDeletionRuntime, error) {
|
|
return deletionRuntimeFunc(func(context.Context, string, string) error { return nil }), nil
|
|
}
|
|
claimed, ok, e := db.ClaimProjectDeletion(ctx, task.ID, "owner", deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
|
|
if e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
s.executeProjectDeletion(*claimed, "owner")
|
|
finished, _ := db.GetProjectDeletion(ctx, task.ID)
|
|
if finished.Status != "completed" {
|
|
t.Fatalf("%+v", finished)
|
|
}
|
|
for _, target := range finished.Plan.Targets {
|
|
if target.Status != "done" {
|
|
t.Fatalf("target %s (%s) not done: %+v", target.ID, target.Kind, target)
|
|
}
|
|
}
|
|
for _, dir := range dirs {
|
|
if _, e = os.Stat(dir); !os.IsNotExist(e) {
|
|
t.Fatalf("workspace %s not removed", dir)
|
|
}
|
|
}
|
|
if existing, _ := db.GetProject(ctx, p.ID); existing != nil {
|
|
t.Fatal("project remains")
|
|
}
|
|
}
|