droboticflow/internal/api/project_deletion.go
eust-w 2864d2aa11 ⚡ Process deletion targets concurrently within each phase
executeProjectDeletion processed every target (runtime, then workspace,
then bos, then records) strictly one at a time — for a project with
thousands of real runs, that meant thousands of sequential K8s/BOS/Mongo
round trips. Observed live on Fanyu: BLF-data's runtime phase (stopping
4,641 runs) took tens of minutes at that pace.

Replaced the per-target sequential loop with a bounded worker pool
(PROJECT_DELETION_TARGET_CONCURRENCY, default 12) applied within each
phase — targets of the same kind now run concurrently, but phases still
run strictly in order (runtime, then workspace, then bos, then records),
preserving the "stop/verify runtime, then remove files, then DB"
ordering the plan is built with. The one-time phase-transition safety
checks (wait for in-flight human work, re-verify runtime absence,
revalidate references) still run exactly once, at the same point,
between the runtime and workspace phases.

Every read-modify-write of shared task state (a target's Status/Error,
and every persist() call) is guarded by a mutex; the actual I/O
(deleteProjectTarget) runs unlocked so workers genuinely overlap. Added
TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes (30 runs) to
lock in that concurrent execution reaches the same end state — every
target done, every workspace directory removed, project record gone —
as the old sequential loop. Full suite passes under -race with no
warnings; 75 pre-existing unrelated failures unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-23 04:30:13 +08:00

1020 lines
33 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package api
import (
"bytes"
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"github.com/go-chi/chi/v5"
"io"
"net"
"net/http"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
)
const deletionLeaseLayout = "2006-01-02T15:04:05.000000000Z"
type deletionBatchResponse struct {
headers http.Header
status int
body bytes.Buffer
}
func (w *deletionBatchResponse) Header() http.Header {
if w.headers == nil {
w.headers = make(http.Header)
}
return w.headers
}
func (w *deletionBatchResponse) WriteHeader(status int) {
if w.status == 0 {
w.status = status
}
}
func (w *deletionBatchResponse) Write(b []byte) (int, error) {
if w.status == 0 {
w.status = http.StatusOK
}
return w.body.Write(b)
}
// Keep each project's authorization, preview digest, lease and deletion receipt.
// A bad project must not cancel successful admissions for the other projects.
func (s *server) handleProjectDeletionBatch(w http.ResponseWriter, r *http.Request) {
var input struct {
Projects []struct {
ProjectID string `json:"projectId"`
projectDeletionInput
} `json:"projects"`
}
if !decodeJSON(w, r, &input) {
return
}
if len(input.Projects) == 0 || len(input.Projects) > 20 {
renderValidationError(w, validationIssue{Field: "projects", Message: "每批请选择 1–20 个项目"})
return
}
seen := map[string]bool{}
for _, p := range input.Projects {
if strings.TrimSpace(p.ProjectID) == "" || seen[p.ProjectID] {
renderValidationError(w, validationIssue{Field: "projects", Message: "项目不能为空或重复"})
return
}
seen[p.ProjectID] = true
}
ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), time.Minute)
defer cancel()
results := make([]map[string]any, 0, len(input.Projects))
for _, p := range input.Projects {
payload, _ := json.Marshal(p.projectDeletionInput)
child := r.Clone(ctx)
child.Body = io.NopCloser(bytes.NewReader(payload))
child.ContentLength = int64(len(payload))
response := &deletionBatchResponse{}
s.startProjectDeletion(response, child, p.ProjectID, deletionTaskID(p.ProjectID))
var data any
if err := json.Unmarshal(response.body.Bytes(), &data); err != nil {
data = map[string]string{"error": "删除请求未返回有效结果,请在删除记录中核对"}
}
results = append(results, map[string]any{"projectId": p.ProjectID, "status": response.status, "data": data})
}
renderJSON(w, http.StatusOK, map[string]any{"results": results})
}
var deletionRunIDPattern = regexp.MustCompile(`^run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}$`)
type projectDeletionRuntime interface {
TerminateRunAndWait(context.Context, string, string) error
}
var projectDeletionRuntimeFactory = newProjectDeletionRuntime
func newProjectDeletionRuntime(namespace, pullSecret, pvc, mount string) (projectDeletionRuntime, error) {
if strings.TrimSpace(namespace) == "" {
return nil, fmt.Errorf("项目删除缺少 Kubernetes 命名空间配置")
}
return k8s.NewLauncher(namespace, pullSecret, pvc, mount, "", "", "", 0)
}
func (s *server) deletionRuntime() (projectDeletionRuntime, error) {
client, err := projectDeletionRuntimeFactory(s.k8sNamespace, s.k8sImagePullSecretName, s.workspacePVCName, s.workspaceMountPath)
if err != nil {
return nil, fmt.Errorf("初始化项目删除客户端失败,请检查 K8S_NAMESPACE、WORKSPACE_PVC_NAME 和 WORKSPACE_MOUNT_PATH:%w", err)
}
return client, nil
}
func (s *server) preflightProjectDeletion(plan domain.ProjectDeletionPlan) error {
for _, target := range plan.Targets {
if target.Kind == "runtime" {
_, err := s.deletionRuntime()
return err
}
}
return nil
}
func deletionTaskID(projectID string) string { return "project-deletion-" + projectID }
func deletionLeaseTime(t time.Time) string { return t.UTC().Format(deletionLeaseLayout) }
func deletionTaskResponse(t *domain.ProjectDeletionTask) *domain.ProjectDeletionTask {
if t == nil {
return nil
}
copy := *t
copy.Retryable = t.Status != "completed" && (t.LeaseOwner == "" || t.LeaseUntil <= deletionLeaseTime(time.Now()))
if t.Status == "running" && copy.Retryable {
copy.Status = "interrupted"
copy.Error = "执行已中断或租约到期,可重试未完成步骤"
}
return &copy
}
func prefixOverlap(a, b string) bool {
a = strings.Trim(a, "/")
b = strings.Trim(b, "/")
return a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/")
}
func validDeletionPrefix(prefix string) bool {
if prefix == "" || strings.HasPrefix(prefix, "/") || strings.Contains(prefix, "\\") {
return false
}
for _, part := range strings.Split(strings.TrimSuffix(prefix, "/"), "/") {
if part == "" || part == "." || part == ".." || strings.ContainsAny(part, "{}\x00") {
return false
}
}
return true
}
func (s *server) buildProjectDeletionPlan(ctx context.Context, p *domain.ProjectRecord) (domain.ProjectDeletionPlan, error) {
plan := domain.ProjectDeletionPlan{ProjectID: p.ID, ProjectName: p.Name, OrganizationID: p.OrganizationID, Counts: map[string]int{}, Targets: []domain.ProjectDeletionTarget{}, Preserved: []string{"用户原始输入、测试输入副本、账号、供应商、公共模板、模型、备份、共享服务及回收站"}, Blockers: []string{}}
workflows, err := s.store.ListWorkflowsByProject(ctx, p.ID)
if err != nil {
return plan, err
}
byWorkflow := map[string][]domain.RunRecord{}
knownWF := map[string]bool{}
for _, w := range workflows {
knownWF[w.ID] = true
plan.WorkflowIDs = append(plan.WorkflowIDs, w.ID)
}
// Scoped to this project's own runs (by projectId or by one of its
// workflows) instead of scanning every run in the organization — an
// org with substantial production history made the unscoped scan take
// minutes.
allRuns, err := s.store.ListRunArtifactsForProject(ctx, p.OrganizationID, p.ID, plan.WorkflowIDs)
if err != nil {
return plan, err
}
for _, r := range allRuns {
if r.ProjectID == p.ID || knownWF[r.WorkflowID] {
plan.RunIDs = append(plan.RunIDs, r.ID)
byWorkflow[r.WorkflowID] = append(byWorkflow[r.WorkflowID], r)
if workflowRunBlocksDeletion(r.Status) {
plan.BlockingRunCount++
}
}
}
sort.Strings(plan.WorkflowIDs)
sort.Strings(plan.RunIDs)
if len(plan.RunIDs) > 10000 {
plan.Blockers = append(plan.Blockers, "运行数量超过单个删除任务上限,请先分工作流清理")
}
add := func(kind, runID, connection, bucket, prefix string) {
sum := sha256.Sum256([]byte(kind + "|" + runID + "|" + connection + "|" + bucket + "|" + prefix))
plan.Targets = append(plan.Targets, domain.ProjectDeletionTarget{ID: hex.EncodeToString(sum[:12]), Kind: kind, RunID: runID, ConnectionID: connection, Bucket: bucket, Prefix: prefix, Status: "pending"})
}
for _, id := range plan.RunIDs {
if !deletionRunIDPattern.MatchString(id) {
plan.Blockers = append(plan.Blockers, "不能安全清理非标准运行编号:"+id)
continue
}
add("runtime", id, "", "", "")
add("workspace", id, "", "", "system/runs/"+id)
}
workspaceConn, workspaceBucket := os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET")
if (workspaceConn == "") != (workspaceBucket == "") {
plan.Blockers = append(plan.Blockers, "运行工作目录的 BOS 映射配置不完整")
}
if workspaceConn != "" && workspaceBucket != "" {
for _, id := range plan.RunIDs {
add("bos", id, workspaceConn, workspaceBucket, "system/runs/"+id)
}
}
for _, w := range workflows {
preview, _, e := s.buildWorkflowDeletionPreview(ctx, &w)
if e != nil {
return plan, e
}
plan.Counts["humanTasks"] += preview.HumanTaskCount
plan.Counts["deliveryPackages"] += preview.DeliveryPackageCount
plan.Counts["captures"] += preview.CaptureRecordCount
if w.CaptureAutomation != nil && w.CaptureAutomation.Prefix != "" {
plan.Preserved = append(plan.Preserved, "输入:bos://"+w.CaptureAutomation.Bucket+"/"+w.CaptureAutomation.Prefix)
if w.CaptureAutomation.DeliveryPrefix != "" {
plan.Preserved = append(plan.Preserved, "最终交付:bos://"+w.CaptureAutomation.Bucket+"/"+w.CaptureAutomation.DeliveryPrefix)
}
}
policy, e := s.effectiveArtifactStorage(ctx, &w)
if e != nil {
return plan, e
}
if policy == nil || len(byWorkflow[w.ID]) == 0 {
continue
}
prefixes, safe := workflowGeneratedArtifactPrefixes(&w, byWorkflow[w.ID], policy)
if !safe {
plan.Blockers = append(plan.Blockers, "产物路径不能安全限定到工作流或 Run:"+w.Name)
continue
}
for _, prefix := range prefixes {
if !validDeletionPrefix(prefix) {
plan.Blockers = append(plan.Blockers, "不安全的产物路径:"+prefix)
continue
}
add("bos", "", policy.ConnectionID, policy.Bucket, prefix)
}
}
for wid, runs := range byWorkflow {
if !knownWF[wid] && len(runs) > 0 {
plan.Blockers = append(plan.Blockers, "存在无工作流定义的运行,无法确认全部产物归属:"+wid)
}
}
plan.Counts["workflows"] = len(workflows)
plan.Counts["runs"] = len(plan.RunIDs)
members, e := s.store.ListProjectMembers(ctx, p.OrganizationID, p.ID)
if e != nil {
return plan, e
}
plan.Counts["projectMembers"] = len(members)
add("records", "", "", "", "")
// Resources are ordered: stop/verify every runtime, then remove files, then DB.
rank := map[string]int{"runtime": 0, "workspace": 1, "bos": 2, "records": 3}
sort.SliceStable(plan.Targets, func(i, j int) bool {
if rank[plan.Targets[i].Kind] != rank[plan.Targets[j].Kind] {
return rank[plan.Targets[i].Kind] < rank[plan.Targets[j].Kind]
}
return plan.Targets[i].ID < plan.Targets[j].ID
})
if err := s.validateDeletionReferences(ctx, plan); err != nil {
plan.Blockers = append(plan.Blockers, err.Error())
}
encoded, _ := json.Marshal(struct {
Project string
Workflows, Runs []string
Targets []domain.ProjectDeletionTarget
}{p.ID, plan.WorkflowIDs, plan.RunIDs, plan.Targets})
sum := sha256.Sum256(encoded)
plan.Digest = hex.EncodeToString(sum[:])
return plan, nil
}
func (s *server) validateDeletionReferences(ctx context.Context, plan domain.ProjectDeletionPlan) error {
// Scoped to this project's organization instead of scanning every
// workflow/run across every tenant — the org boundary is all this
// cross-reference check ever compares against (see the
// ListRunArtifactsForProject scoping in buildProjectDeletionPlan for
// the same class of fix).
workflows, err := s.store.ListAllWorkflowsByOrganization(ctx, plan.OrganizationID)
if err != nil {
return err
}
runs, err := s.store.ListRunArtifactsByOrganization(ctx, plan.OrganizationID, "")
if err != nil {
return err
}
captures, err := s.store.ListCaptureRecords(ctx, plan.OrganizationID, "", 0)
if err != nil {
return err
}
runsByWorkflow := map[string][]domain.RunRecord{}
otherRunJSON := []string{}
for _, run := range runs {
runsByWorkflow[run.WorkflowID] = append(runsByWorkflow[run.WorkflowID], run)
if run.ProjectID != plan.ProjectID {
raw, e := json.Marshal(run)
if e != nil {
return e
}
otherRunJSON = append(otherRunJSON, string(raw))
}
}
// Which BOS target URIs are actually referenced by some other run's
// JSON, computed with one linear pass per other-run through a single
// combined regexp instead of a strings.Contains scan per (target, run)
// pair. A data-heavy org can have thousands of BOS targets and
// thousands of other runs; the naive nested loop made every deletion
// preview take minutes (see the SL-data/BLF-data hangs this was
// tracked down from). Go's regexp is RE2-based, so matching a fixed
// alternation of literal (quoted) URIs against a text is linear in the
// text length regardless of how many alternatives there are — the
// boolean result for every (target, run) pair is identical to the
// original strings.Contains(raw, uri) check, just computed once per
// distinct URI/run instead of once per pair.
referencedURIs := map[string]bool{}
if len(otherRunJSON) > 0 {
uriSet := map[string]bool{}
for _, t := range plan.Targets {
if t.Kind != "bos" {
continue
}
uriSet["bos://"+t.Bucket+"/"+strings.Trim(t.Prefix, "/")] = true
}
if len(uriSet) > 0 {
patterns := make([]string, 0, len(uriSet))
for uri := range uriSet {
patterns = append(patterns, regexp.QuoteMeta(uri))
}
matcher := regexp.MustCompile(strings.Join(patterns, "|"))
for _, raw := range otherRunJSON {
for _, match := range matcher.FindAllString(raw, -1) {
referencedURIs[match] = true
}
}
}
}
type policyResult struct {
policy *domain.ArtifactStoragePolicy
prefixes []string
safe bool
}
policies := map[string]policyResult{}
for _, t := range plan.Targets {
if t.Kind != "bos" {
continue
}
if err := ctx.Err(); err != nil {
return err
}
for _, capture := range captures {
if d := capture.Delivery; d != nil && d.Bucket == t.Bucket && prefixOverlap(t.Prefix, d.Prefix) {
return fmt.Errorf("最终交付成品受保护,不能级联删除:%s", d.Prefix)
}
}
if !validDeletionPrefix(t.Prefix) {
return fmt.Errorf("拒绝不安全路径 %s", t.Prefix)
}
if strings.HasPrefix(t.Prefix, "models/") || strings.HasPrefix(t.Prefix, ".droboticflow-trash/") || t.Prefix == "models" || t.Prefix == "system" {
return fmt.Errorf("共享目录不能级联删除:%s", t.Prefix)
}
if strings.HasPrefix(t.Prefix, "system/") && !(t.RunID != "" && t.Prefix == "system/runs/"+t.RunID) {
return fmt.Errorf("系统共享路径不能级联删除:%s", t.Prefix)
}
for _, w := range workflows {
cfg := w.CaptureAutomation
if cfg != nil && cfg.Bucket == t.Bucket && cfg.DeliveryPrefix != "" && prefixOverlap(t.Prefix, cfg.DeliveryPrefix) {
return fmt.Errorf("最终交付目录受保护:%s", cfg.DeliveryPrefix)
}
if cfg != nil && cfg.ConnectionID == t.ConnectionID && cfg.Bucket == t.Bucket && cfg.Prefix != "" && prefixOverlap(t.Prefix, cfg.Prefix) {
return fmt.Errorf("目标与原始输入目录重叠,已保护:%s", t.Prefix)
}
if w.ProjectID != plan.ProjectID {
cached, ok := policies[w.ID]
if !ok {
policy, e := s.effectiveArtifactStorage(ctx, &w)
if e != nil {
return e
}
ps, safe := workflowGeneratedArtifactPrefixes(&w, runsByWorkflow[w.ID], policy)
cached = policyResult{policy, ps, safe}
policies[w.ID] = cached
}
policy := cached.policy
if policy != nil && policy.ConnectionID == t.ConnectionID && policy.Bucket == t.Bucket {
ps, safe := cached.prefixes, cached.safe
if !safe && len(runsByWorkflow[w.ID]) > 0 {
return fmt.Errorf("其他工作流产物范围不明确,无法证明独占")
}
for _, p := range ps {
if prefixOverlap(t.Prefix, p) {
return fmt.Errorf("目标被其他工作流使用:%s", t.Prefix)
}
}
}
}
}
uri := "bos://" + t.Bucket + "/" + strings.Trim(t.Prefix, "/")
if referencedURIs[uri] {
return fmt.Errorf("目标被其他运行引用:%s", t.Prefix)
}
}
return nil
}
func safeRunWorkspace(root, runID string) (string, error) {
if !deletionRunIDPattern.MatchString(runID) {
return "", fmt.Errorf("不安全的 Run ID")
}
root = filepath.Clean(root)
if !filepath.IsAbs(root) || root == "/" || root == "." {
return "", fmt.Errorf("不安全的工作区根目录")
}
for _, parent := range []string{root, filepath.Join(root, "system"), filepath.Join(root, "system", "runs")} {
info, err := os.Lstat(parent)
if os.IsNotExist(err) {
continue
}
if err != nil {
return "", err
}
if info.Mode()&os.ModeSymlink != 0 {
return "", fmt.Errorf("工作目录父路径是符号链接,拒绝清理")
}
}
return filepath.Join(root, "system", "runs", runID), nil
}
func (s *server) deleteProjectTarget(ctx context.Context, plan domain.ProjectDeletionPlan, t domain.ProjectDeletionTarget) error {
switch t.Kind {
case "runtime":
client, err := s.deletionRuntime()
if err != nil {
return err
}
return client.TerminateRunAndWait(ctx, s.k8sNamespace, t.RunID)
case "workspace":
path, err := safeRunWorkspace(s.workspaceMountPath, t.RunID)
if err != nil {
return err
}
if err = os.RemoveAll(path); err != nil {
return err
}
if _, err = os.Lstat(path); !os.IsNotExist(err) {
return fmt.Errorf("运行目录仍存在或无法核验:%s", path)
}
return nil
case "bos":
// validateDeletionReferences is already re-run once by
// executeProjectDeletion immediately before it starts processing
// non-runtime targets (the runtimeDone gate) — that is the right
// point to catch a run created between plan-build and execution
// referencing this path. Re-running the whole org-wide validation
// again here, per bos target, made a project with a few thousand
// BOS targets take hours (each call re-fetches and re-scans every
// other run in the organization).
browser, conn, cleanup, err := s.resolveObjectStorageBrowser(ctx, t.ConnectionID)
if err != nil {
return err
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || conn == nil || normalizedTenantID(conn.OrganizationID) != normalizedTenantID(plan.OrganizationID) {
return fmt.Errorf("存储连接不存在或归属不符")
}
bucket, err := resolveObjectStorageBucket(conn, t.Bucket)
if err != nil {
return err
}
if err = browser.DeletePrefix(ctx, bucket, t.Prefix); err != nil && !isBOSNoSuchKey(err) {
return err
}
remaining, err := browser.ListObjects(ctx, bucket, strings.TrimSuffix(t.Prefix, "/")+"/")
if err != nil && !isBOSNoSuchKey(err) {
return err
}
if len(remaining) > 0 {
return fmt.Errorf("产物目录仍有 %d 个对象", len(remaining))
}
return nil
case "records":
_, err := s.store.DeleteProject(ctx, plan.ProjectID)
if err != nil {
return err
}
p, err := s.store.GetProject(ctx, plan.ProjectID)
if err != nil {
return err
}
if p != nil {
return fmt.Errorf("项目元数据仍存在")
}
for _, id := range plan.RunIDs {
r, e := s.store.GetRun(ctx, id)
if e != nil {
return e
}
if r != nil {
return fmt.Errorf("Run 记录仍存在:%s", id)
}
}
workflows, e := s.store.ListWorkflowsByProject(ctx, plan.ProjectID)
if e != nil {
return e
}
if len(workflows) > 0 {
return fmt.Errorf("工作流记录仍存在")
}
items, e := s.store.ListHumanItems(ctx, plan.OrganizationID, domain.HumanItemQuery{ProjectID: plan.ProjectID})
if e != nil {
return e
}
if len(items) > 0 {
return fmt.Errorf("人工任务记录仍存在")
}
return nil
}
return fmt.Errorf("未知清理步骤 %s", t.Kind)
}
func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
done := make(chan struct{})
defer close(done)
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-ticker.C:
ok, e := s.store.RenewProjectDeletion(ctx, task.ID, owner, deletionLeaseTime(time.Now().Add(2*time.Minute)))
if e != nil || !ok {
cancel()
return
}
}
}
}()
persist := func() error {
task.UpdatedAt = nowISO()
saveCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer stop()
ok, e := s.store.SaveProjectDeletion(saveCtx, task, owner)
if e != nil {
return e
}
if !ok {
return fmt.Errorf("删除任务租约已失效")
}
return nil
}
fail := func(e error) { task.Status = "failed"; task.Error = e.Error(); _ = persist() }
// Validate the real client configuration before stopping writers or deleting
// files. Runtime targets are rechecked before storage cleanup even on retry.
if err := s.preflightProjectDeletion(task.Plan); err != nil {
fail(err)
return
}
if err := s.store.MarkProjectDeleting(ctx, task.Plan.ProjectID, task.ID); err != nil {
p, _ := s.store.GetProject(ctx, task.Plan.ProjectID)
if p != nil {
fail(err)
return
}
}
for _, id := range task.Plan.WorkflowIDs {
w, e := s.store.GetWorkflow(ctx, id)
if e != nil {
fail(e)
return
}
if w != nil && w.CaptureAutomation != nil && w.CaptureAutomation.Enabled {
w.CaptureAutomation.Enabled = false
if e = s.store.UpsertWorkflow(ctx, *w); e != nil {
fail(e)
return
}
}
}
// Freeze writers before touching any files. Existing user inputs are untouched.
for _, id := range task.Plan.RunIDs {
r, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if r == nil {
continue
}
if workflowRunBlocksDeletion(r.Status) && !task.StopRunning {
fail(fmt.Errorf("仍有运行中的任务,需确认停止后删除"))
return
}
if workflowRunBlocksDeletion(r.Status) || r.Status == domain.RunStatusWaitingForHuman || r.Status == domain.RunStatusWaitingForAudit {
if _, e = s.store.RequestRunStop(ctx, id, domain.RunStopModeForce); e != nil {
fail(e)
return
}
}
}
// Lock prevents new admissions; a pre-lock admission changes the manifest and
// must be explicitly reconfirmed, never silently added to destructive work.
recordsStarted := false
for _, t := range task.Plan.Targets {
if t.Kind == "records" && t.Status != "pending" {
recordsStarted = true
}
}
if !recordsStarted {
p, e := s.store.GetProject(ctx, task.Plan.ProjectID)
if e != nil {
fail(e)
return
}
if p != nil {
fresh, e := s.buildProjectDeletionPlan(ctx, p)
if e != nil {
fail(e)
return
}
if fresh.Digest != task.Plan.Digest {
task.Plan = fresh
fail(fmt.Errorf("删除范围发生变化,清单已更新,请重新核对并确认"))
return
}
}
}
// Re-verify runtime absence even when a previous attempt completed that step.
for _, t := range task.Plan.Targets {
if t.Kind == "runtime" && t.Status == "done" {
if e := s.deleteProjectTarget(ctx, task.Plan, t); e != nil {
fail(e)
return
}
}
}
targetConcurrency := 12
if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" {
if n, e := strconv.Atoi(v); e == nil && n > 0 {
targetConcurrency = n
}
}
// processBatch runs deleteProjectTarget for the not-yet-done targets at
// indices with up to targetConcurrency in flight at once. Each target's
// Status/Error and every persist() call is guarded by mu, so concurrent
// workers never race on shared task state; deleteProjectTarget itself
// (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs
// unlocked. A single project's deletion used to process every target
// strictly one at a time, which made stopping thousands of real runs
// take as long as thousands of sequential K8s round trips.
var mu sync.Mutex
processBatch := func(indices []int) error {
sem := make(chan struct{}, targetConcurrency)
var wg sync.WaitGroup
var firstErr error
for _, index := range indices {
mu.Lock()
skip := task.Plan.Targets[index].Status == "done" || firstErr != nil
mu.Unlock()
if skip {
continue
}
sem <- struct{}{}
wg.Add(1)
go func(index int) {
defer wg.Done()
defer func() { <-sem }()
mu.Lock()
if firstErr != nil {
mu.Unlock()
return
}
target := &task.Plan.Targets[index]
target.Error = ""
target.Status = "running"
persistErr := persist()
snapshot := *target
mu.Unlock()
if persistErr != nil {
mu.Lock()
if firstErr == nil {
firstErr = persistErr
}
mu.Unlock()
cancel()
return
}
runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot)
mu.Lock()
target = &task.Plan.Targets[index]
if runErr != nil {
target.Status = "failed"
target.Error = runErr.Error()
if firstErr == nil {
firstErr = runErr
}
} else {
target.Status = "done"
}
_ = persist()
mu.Unlock()
}(index)
}
wg.Wait()
return firstErr
}
indicesByKind := map[string][]int{}
nonRuntimePending := false
for index, t := range task.Plan.Targets {
indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index)
if t.Kind != "runtime" && t.Status != "done" {
nonRuntimePending = true
}
}
if e := processBatch(indicesByKind["runtime"]); e != nil {
fail(e)
return
}
// Re-run the phase-transition safety checks (wait for in-flight human
// work, re-verify runtime absence, revalidate references) exactly once
// per execution attempt, right before touching any file/storage target
// — the same point the original sequential loop ran them at.
if nonRuntimePending {
deadline := time.Now().Add(150 * time.Second)
for {
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
if e != nil {
fail(e)
return
}
busy := false
for _, id := range task.Plan.RunIDs {
run, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if run != nil && run.ExecutionLeaseOwner != "" {
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
if e != nil || until.After(time.Now()) {
busy = true
}
}
}
for i := range items {
item := &items[i]
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
fail(e)
return
}
w := mapValue(item.Subject.Payload["sliceWorkflow"])
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
busy = true
}
}
if !busy {
break
}
if time.Now().After(deadline) {
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
return
}
select {
case <-ctx.Done():
fail(ctx.Err())
return
case <-time.After(2 * time.Second):
}
}
// Catch work admitted immediately before the project lock.
for _, step := range task.Plan.Targets {
if step.Kind == "runtime" {
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
fail(e)
return
}
}
}
// Scoped exactly as before (projectId match only, no workflowId
// fallback — see the loop below) but via an indexed query
// instead of an organization-wide scan.
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
if e != nil {
fail(e)
return
}
known := map[string]bool{}
for _, id := range task.Plan.RunIDs {
known[id] = true
}
for _, r := range fresh {
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
return
}
}
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
fail(e)
return
}
}
for _, kind := range []string{"workspace", "bos", "records"} {
if e := processBatch(indicesByKind[kind]); e != nil {
fail(e)
return
}
}
task.Status = "completed"
task.Error = ""
_ = persist()
}
func (s *server) handleProjectDeletionPreview(w http.ResponseWriter, r *http.Request) {
p, e := s.store.GetProject(r.Context(), chi.URLParam(r, "id"))
if e != nil {
renderError(w, 500, e)
return
}
if p == nil {
renderJSON(w, 404, map[string]string{"error": "Project not found"})
return
}
if !s.ensureTenantAccess(w, r, p.OrganizationID, "Project") {
return
}
task, e := s.store.GetProjectDeletion(r.Context(), deletionTaskID(p.ID))
if e != nil {
renderError(w, 500, e)
return
}
if task != nil {
renderJSON(w, 200, map[string]any{"plan": task.Plan, "task": deletionTaskResponse(task)})
return
}
plan, e := s.buildProjectDeletionPlan(r.Context(), p)
if e != nil {
renderError(w, 500, e)
return
}
renderJSON(w, 200, map[string]any{"plan": plan})
}
func (s *server) handleProjectDeletionsList(w http.ResponseWriter, r *http.Request) {
items, e := s.store.ListProjectDeletions(r.Context(), s.requestOrganizationID(r))
if e != nil {
renderError(w, 500, e)
return
}
out := []*domain.ProjectDeletionTask{}
for i := range items {
out = append(out, deletionTaskResponse(&items[i]))
}
renderJSON(w, 200, map[string]any{"items": out})
}
func (s *server) handleProjectDeletionGet(w http.ResponseWriter, r *http.Request) {
t, e := s.store.GetProjectDeletion(r.Context(), chi.URLParam(r, "id"))
if e != nil {
renderError(w, 500, e)
return
}
if t == nil {
renderJSON(w, 404, map[string]string{"error": "Deletion task not found"})
return
}
if !s.ensureTenantAccess(w, r, t.OrganizationID, "Deletion task") {
return
}
renderJSON(w, 200, deletionTaskResponse(t))
}
type projectDeletionInput struct {
ConfirmationName string `json:"confirmationName"`
Digest string `json:"digest"`
Reason string `json:"reason"`
StopRunning bool `json:"stopRunning"`
}
func (s *server) startProjectDeletion(w http.ResponseWriter, r *http.Request, projectID, taskID string) {
var input projectDeletionInput
if !decodeProjectDeletionInput(w, r, &input) {
return
}
task, e := s.store.GetProjectDeletion(r.Context(), taskID)
if e != nil {
renderError(w, 500, e)
return
}
var plan domain.ProjectDeletionPlan
if task != nil {
if !s.ensureTenantAccess(w, r, task.OrganizationID, "Deletion task") {
return
}
plan = task.Plan
if task.Status == "completed" {
renderJSON(w, 200, deletionTaskResponse(task))
return
}
} else {
p, e := s.store.GetProject(r.Context(), projectID)
if e != nil {
renderError(w, 500, e)
return
}
if p == nil {
renderJSON(w, 404, map[string]string{"error": "Project not found"})
return
}
if !s.ensureTenantAccess(w, r, p.OrganizationID, "Project") {
return
}
plan, e = s.buildProjectDeletionPlan(r.Context(), p)
if e != nil {
renderError(w, 500, e)
return
}
}
if input.ConfirmationName != plan.ProjectName || input.Digest != plan.Digest || strings.TrimSpace(input.Reason) == "" {
renderValidationError(w, validationIssue{Field: "confirmation", Message: "请核对删除清单、输入完整项目名称及删除原因"})
return
}
if len(plan.Blockers) > 0 {
renderJSON(w, 409, map[string]any{"error": "存在无法安全清理的路径", "plan": plan})
return
}
if plan.BlockingRunCount > 0 && !input.StopRunning {
renderJSON(w, 409, map[string]string{"error": "请确认停止正在运行的任务后删除"})
return
}
if err := s.preflightProjectDeletion(plan); err != nil {
renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": err.Error(), "code": "project_deletion_configuration_invalid"})
return
}
if task == nil {
principal, _ := currentPrincipal(r)
actor := ""
if principal != nil {
actor = principal.User.ID
}
task, e = s.store.CreateProjectDeletion(r.Context(), domain.ProjectDeletionTask{ID: taskID, OrganizationID: plan.OrganizationID, Plan: plan, Status: "prepared", ActorUserID: actor, Reason: strings.TrimSpace(input.Reason), CreatedAt: nowISO(), UpdatedAt: nowISO()})
if e != nil {
renderError(w, 500, e)
return
}
}
owner := domain.NewID("deletion-lease")
claimed, ok, e := s.store.ClaimProjectDeletion(r.Context(), task.ID, owner, deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
if e != nil {
renderError(w, 500, e)
return
}
if !ok {
renderJSON(w, 202, deletionTaskResponse(claimed))
return
}
claimed.StopRunning = input.StopRunning
if e = s.store.MarkProjectDeleting(r.Context(), plan.ProjectID, task.ID); e != nil {
p, _ := s.store.GetProject(r.Context(), plan.ProjectID)
if p != nil {
claimed.Status = "failed"
claimed.Error = e.Error()
_, _ = s.store.SaveProjectDeletion(r.Context(), *claimed, owner)
renderJSON(w, 409, deletionTaskResponse(claimed))
return
}
}
if ok, e = s.store.SaveProjectDeletion(r.Context(), *claimed, owner); e != nil || !ok {
renderJSON(w, 500, map[string]string{"error": "删除任务状态保存失败,未开始资源清理"})
return
}
go s.executeProjectDeletion(*claimed, owner)
renderJSON(w, 202, deletionTaskResponse(claimed))
}
// Read the small confirmation body before expensive deletion-plan queries.
// Do not collapse transport timeouts into a misleading JSON syntax error.
func decodeProjectDeletionInput(w http.ResponseWriter, r *http.Request, input *projectDeletionInput) bool {
r.Body = http.MaxBytesReader(w, r.Body, 64<<10)
defer r.Body.Close()
decoder := json.NewDecoder(r.Body)
err := decoder.Decode(input)
if err == nil {
var extra any
err = decoder.Decode(&extra)
if err == io.EOF {
return true
}
if err == nil {
err = fmt.Errorf("multiple JSON values")
}
}
var timeout net.Error
var tooLarge *http.MaxBytesError
status, code, message := http.StatusBadRequest, "invalid_deletion_body", "删除请求格式无效,请重新打开删除窗口后重试"
if errors.As(err, &timeout) && timeout.Timeout() {
status, code, message = http.StatusRequestTimeout, "deletion_body_timeout", "读取删除请求超时,请重新打开删除窗口后重试"
} else if errors.As(err, &tooLarge) {
status, code, message = http.StatusRequestEntityTooLarge, "deletion_body_too_large", "删除确认请求过大"
} else if errors.Is(err, io.EOF) {
code, message = "empty_deletion_body", "删除请求缺少确认信息,请重新打开删除窗口后重试"
}
renderJSON(w, status, map[string]string{"error": message, "code": code})
return false
}
func (s *server) handleProjectDeletionRetry(w http.ResponseWriter, r *http.Request) {
s.startProjectDeletion(w, r, "", chi.URLParam(r, "id"))
}