executeProjectDeletion processed every target (runtime, then workspace, then bos, then records) strictly one at a time — for a project with thousands of real runs, that meant thousands of sequential K8s/BOS/Mongo round trips. Observed live on Fanyu: BLF-data's runtime phase (stopping 4,641 runs) took tens of minutes at that pace. Replaced the per-target sequential loop with a bounded worker pool (PROJECT_DELETION_TARGET_CONCURRENCY, default 12) applied within each phase — targets of the same kind now run concurrently, but phases still run strictly in order (runtime, then workspace, then bos, then records), preserving the "stop/verify runtime, then remove files, then DB" ordering the plan is built with. The one-time phase-transition safety checks (wait for in-flight human work, re-verify runtime absence, revalidate references) still run exactly once, at the same point, between the runtime and workspace phases. Every read-modify-write of shared task state (a target's Status/Error, and every persist() call) is guarded by a mutex; the actual I/O (deleteProjectTarget) runs unlocked so workers genuinely overlap. Added TestProjectDeletionExecutesTargetsConcurrentlyAndCompletes (30 runs) to lock in that concurrent execution reaches the same end state — every target done, every workspace directory removed, project record gone — as the old sequential loop. Full suite passes under -race with no warnings; 75 pre-existing unrelated failures unchanged. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1020 lines
33 KiB
Go
1020 lines
33 KiB
Go
package api
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/sha256"
|
||
"droboticflow/internal/domain"
|
||
"droboticflow/internal/k8s"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"github.com/go-chi/chi/v5"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
const deletionLeaseLayout = "2006-01-02T15:04:05.000000000Z"
|
||
|
||
type deletionBatchResponse struct {
|
||
headers http.Header
|
||
status int
|
||
body bytes.Buffer
|
||
}
|
||
|
||
func (w *deletionBatchResponse) Header() http.Header {
|
||
if w.headers == nil {
|
||
w.headers = make(http.Header)
|
||
}
|
||
return w.headers
|
||
}
|
||
func (w *deletionBatchResponse) WriteHeader(status int) {
|
||
if w.status == 0 {
|
||
w.status = status
|
||
}
|
||
}
|
||
func (w *deletionBatchResponse) Write(b []byte) (int, error) {
|
||
if w.status == 0 {
|
||
w.status = http.StatusOK
|
||
}
|
||
return w.body.Write(b)
|
||
}
|
||
|
||
// Keep each project's authorization, preview digest, lease and deletion receipt.
|
||
// A bad project must not cancel successful admissions for the other projects.
|
||
func (s *server) handleProjectDeletionBatch(w http.ResponseWriter, r *http.Request) {
|
||
var input struct {
|
||
Projects []struct {
|
||
ProjectID string `json:"projectId"`
|
||
projectDeletionInput
|
||
} `json:"projects"`
|
||
}
|
||
if !decodeJSON(w, r, &input) {
|
||
return
|
||
}
|
||
if len(input.Projects) == 0 || len(input.Projects) > 20 {
|
||
renderValidationError(w, validationIssue{Field: "projects", Message: "每批请选择 1–20 个项目"})
|
||
return
|
||
}
|
||
seen := map[string]bool{}
|
||
for _, p := range input.Projects {
|
||
if strings.TrimSpace(p.ProjectID) == "" || seen[p.ProjectID] {
|
||
renderValidationError(w, validationIssue{Field: "projects", Message: "项目不能为空或重复"})
|
||
return
|
||
}
|
||
seen[p.ProjectID] = true
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), time.Minute)
|
||
defer cancel()
|
||
results := make([]map[string]any, 0, len(input.Projects))
|
||
for _, p := range input.Projects {
|
||
payload, _ := json.Marshal(p.projectDeletionInput)
|
||
child := r.Clone(ctx)
|
||
child.Body = io.NopCloser(bytes.NewReader(payload))
|
||
child.ContentLength = int64(len(payload))
|
||
response := &deletionBatchResponse{}
|
||
s.startProjectDeletion(response, child, p.ProjectID, deletionTaskID(p.ProjectID))
|
||
var data any
|
||
if err := json.Unmarshal(response.body.Bytes(), &data); err != nil {
|
||
data = map[string]string{"error": "删除请求未返回有效结果,请在删除记录中核对"}
|
||
}
|
||
results = append(results, map[string]any{"projectId": p.ProjectID, "status": response.status, "data": data})
|
||
}
|
||
renderJSON(w, http.StatusOK, map[string]any{"results": results})
|
||
}
|
||
|
||
var deletionRunIDPattern = regexp.MustCompile(`^run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}$`)
|
||
|
||
type projectDeletionRuntime interface {
|
||
TerminateRunAndWait(context.Context, string, string) error
|
||
}
|
||
|
||
var projectDeletionRuntimeFactory = newProjectDeletionRuntime
|
||
|
||
func newProjectDeletionRuntime(namespace, pullSecret, pvc, mount string) (projectDeletionRuntime, error) {
|
||
if strings.TrimSpace(namespace) == "" {
|
||
return nil, fmt.Errorf("项目删除缺少 Kubernetes 命名空间配置")
|
||
}
|
||
return k8s.NewLauncher(namespace, pullSecret, pvc, mount, "", "", "", 0)
|
||
}
|
||
|
||
func (s *server) deletionRuntime() (projectDeletionRuntime, error) {
|
||
client, err := projectDeletionRuntimeFactory(s.k8sNamespace, s.k8sImagePullSecretName, s.workspacePVCName, s.workspaceMountPath)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("初始化项目删除客户端失败,请检查 K8S_NAMESPACE、WORKSPACE_PVC_NAME 和 WORKSPACE_MOUNT_PATH:%w", err)
|
||
}
|
||
return client, nil
|
||
}
|
||
|
||
func (s *server) preflightProjectDeletion(plan domain.ProjectDeletionPlan) error {
|
||
for _, target := range plan.Targets {
|
||
if target.Kind == "runtime" {
|
||
_, err := s.deletionRuntime()
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func deletionTaskID(projectID string) string { return "project-deletion-" + projectID }
|
||
func deletionLeaseTime(t time.Time) string { return t.UTC().Format(deletionLeaseLayout) }
|
||
func deletionTaskResponse(t *domain.ProjectDeletionTask) *domain.ProjectDeletionTask {
|
||
if t == nil {
|
||
return nil
|
||
}
|
||
copy := *t
|
||
copy.Retryable = t.Status != "completed" && (t.LeaseOwner == "" || t.LeaseUntil <= deletionLeaseTime(time.Now()))
|
||
if t.Status == "running" && copy.Retryable {
|
||
copy.Status = "interrupted"
|
||
copy.Error = "执行已中断或租约到期,可重试未完成步骤"
|
||
}
|
||
return ©
|
||
}
|
||
func prefixOverlap(a, b string) bool {
|
||
a = strings.Trim(a, "/")
|
||
b = strings.Trim(b, "/")
|
||
return a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/")
|
||
}
|
||
func validDeletionPrefix(prefix string) bool {
|
||
if prefix == "" || strings.HasPrefix(prefix, "/") || strings.Contains(prefix, "\\") {
|
||
return false
|
||
}
|
||
for _, part := range strings.Split(strings.TrimSuffix(prefix, "/"), "/") {
|
||
if part == "" || part == "." || part == ".." || strings.ContainsAny(part, "{}\x00") {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
func (s *server) buildProjectDeletionPlan(ctx context.Context, p *domain.ProjectRecord) (domain.ProjectDeletionPlan, error) {
|
||
plan := domain.ProjectDeletionPlan{ProjectID: p.ID, ProjectName: p.Name, OrganizationID: p.OrganizationID, Counts: map[string]int{}, Targets: []domain.ProjectDeletionTarget{}, Preserved: []string{"用户原始输入、测试输入副本、账号、供应商、公共模板、模型、备份、共享服务及回收站"}, Blockers: []string{}}
|
||
workflows, err := s.store.ListWorkflowsByProject(ctx, p.ID)
|
||
if err != nil {
|
||
return plan, err
|
||
}
|
||
byWorkflow := map[string][]domain.RunRecord{}
|
||
knownWF := map[string]bool{}
|
||
for _, w := range workflows {
|
||
knownWF[w.ID] = true
|
||
plan.WorkflowIDs = append(plan.WorkflowIDs, w.ID)
|
||
}
|
||
// Scoped to this project's own runs (by projectId or by one of its
|
||
// workflows) instead of scanning every run in the organization — an
|
||
// org with substantial production history made the unscoped scan take
|
||
// minutes.
|
||
allRuns, err := s.store.ListRunArtifactsForProject(ctx, p.OrganizationID, p.ID, plan.WorkflowIDs)
|
||
if err != nil {
|
||
return plan, err
|
||
}
|
||
for _, r := range allRuns {
|
||
if r.ProjectID == p.ID || knownWF[r.WorkflowID] {
|
||
plan.RunIDs = append(plan.RunIDs, r.ID)
|
||
byWorkflow[r.WorkflowID] = append(byWorkflow[r.WorkflowID], r)
|
||
if workflowRunBlocksDeletion(r.Status) {
|
||
plan.BlockingRunCount++
|
||
}
|
||
}
|
||
}
|
||
sort.Strings(plan.WorkflowIDs)
|
||
sort.Strings(plan.RunIDs)
|
||
if len(plan.RunIDs) > 10000 {
|
||
plan.Blockers = append(plan.Blockers, "运行数量超过单个删除任务上限,请先分工作流清理")
|
||
}
|
||
add := func(kind, runID, connection, bucket, prefix string) {
|
||
sum := sha256.Sum256([]byte(kind + "|" + runID + "|" + connection + "|" + bucket + "|" + prefix))
|
||
plan.Targets = append(plan.Targets, domain.ProjectDeletionTarget{ID: hex.EncodeToString(sum[:12]), Kind: kind, RunID: runID, ConnectionID: connection, Bucket: bucket, Prefix: prefix, Status: "pending"})
|
||
}
|
||
for _, id := range plan.RunIDs {
|
||
if !deletionRunIDPattern.MatchString(id) {
|
||
plan.Blockers = append(plan.Blockers, "不能安全清理非标准运行编号:"+id)
|
||
continue
|
||
}
|
||
add("runtime", id, "", "", "")
|
||
add("workspace", id, "", "", "system/runs/"+id)
|
||
}
|
||
workspaceConn, workspaceBucket := os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET")
|
||
if (workspaceConn == "") != (workspaceBucket == "") {
|
||
plan.Blockers = append(plan.Blockers, "运行工作目录的 BOS 映射配置不完整")
|
||
}
|
||
if workspaceConn != "" && workspaceBucket != "" {
|
||
for _, id := range plan.RunIDs {
|
||
add("bos", id, workspaceConn, workspaceBucket, "system/runs/"+id)
|
||
}
|
||
}
|
||
for _, w := range workflows {
|
||
preview, _, e := s.buildWorkflowDeletionPreview(ctx, &w)
|
||
if e != nil {
|
||
return plan, e
|
||
}
|
||
plan.Counts["humanTasks"] += preview.HumanTaskCount
|
||
plan.Counts["deliveryPackages"] += preview.DeliveryPackageCount
|
||
plan.Counts["captures"] += preview.CaptureRecordCount
|
||
if w.CaptureAutomation != nil && w.CaptureAutomation.Prefix != "" {
|
||
plan.Preserved = append(plan.Preserved, "输入:bos://"+w.CaptureAutomation.Bucket+"/"+w.CaptureAutomation.Prefix)
|
||
if w.CaptureAutomation.DeliveryPrefix != "" {
|
||
plan.Preserved = append(plan.Preserved, "最终交付:bos://"+w.CaptureAutomation.Bucket+"/"+w.CaptureAutomation.DeliveryPrefix)
|
||
}
|
||
}
|
||
policy, e := s.effectiveArtifactStorage(ctx, &w)
|
||
if e != nil {
|
||
return plan, e
|
||
}
|
||
if policy == nil || len(byWorkflow[w.ID]) == 0 {
|
||
continue
|
||
}
|
||
prefixes, safe := workflowGeneratedArtifactPrefixes(&w, byWorkflow[w.ID], policy)
|
||
if !safe {
|
||
plan.Blockers = append(plan.Blockers, "产物路径不能安全限定到工作流或 Run:"+w.Name)
|
||
continue
|
||
}
|
||
for _, prefix := range prefixes {
|
||
if !validDeletionPrefix(prefix) {
|
||
plan.Blockers = append(plan.Blockers, "不安全的产物路径:"+prefix)
|
||
continue
|
||
}
|
||
add("bos", "", policy.ConnectionID, policy.Bucket, prefix)
|
||
}
|
||
}
|
||
for wid, runs := range byWorkflow {
|
||
if !knownWF[wid] && len(runs) > 0 {
|
||
plan.Blockers = append(plan.Blockers, "存在无工作流定义的运行,无法确认全部产物归属:"+wid)
|
||
}
|
||
}
|
||
plan.Counts["workflows"] = len(workflows)
|
||
plan.Counts["runs"] = len(plan.RunIDs)
|
||
members, e := s.store.ListProjectMembers(ctx, p.OrganizationID, p.ID)
|
||
if e != nil {
|
||
return plan, e
|
||
}
|
||
plan.Counts["projectMembers"] = len(members)
|
||
add("records", "", "", "", "")
|
||
// Resources are ordered: stop/verify every runtime, then remove files, then DB.
|
||
rank := map[string]int{"runtime": 0, "workspace": 1, "bos": 2, "records": 3}
|
||
sort.SliceStable(plan.Targets, func(i, j int) bool {
|
||
if rank[plan.Targets[i].Kind] != rank[plan.Targets[j].Kind] {
|
||
return rank[plan.Targets[i].Kind] < rank[plan.Targets[j].Kind]
|
||
}
|
||
return plan.Targets[i].ID < plan.Targets[j].ID
|
||
})
|
||
if err := s.validateDeletionReferences(ctx, plan); err != nil {
|
||
plan.Blockers = append(plan.Blockers, err.Error())
|
||
}
|
||
encoded, _ := json.Marshal(struct {
|
||
Project string
|
||
Workflows, Runs []string
|
||
Targets []domain.ProjectDeletionTarget
|
||
}{p.ID, plan.WorkflowIDs, plan.RunIDs, plan.Targets})
|
||
sum := sha256.Sum256(encoded)
|
||
plan.Digest = hex.EncodeToString(sum[:])
|
||
return plan, nil
|
||
}
|
||
|
||
func (s *server) validateDeletionReferences(ctx context.Context, plan domain.ProjectDeletionPlan) error {
|
||
// Scoped to this project's organization instead of scanning every
|
||
// workflow/run across every tenant — the org boundary is all this
|
||
// cross-reference check ever compares against (see the
|
||
// ListRunArtifactsForProject scoping in buildProjectDeletionPlan for
|
||
// the same class of fix).
|
||
workflows, err := s.store.ListAllWorkflowsByOrganization(ctx, plan.OrganizationID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
runs, err := s.store.ListRunArtifactsByOrganization(ctx, plan.OrganizationID, "")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
captures, err := s.store.ListCaptureRecords(ctx, plan.OrganizationID, "", 0)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
runsByWorkflow := map[string][]domain.RunRecord{}
|
||
otherRunJSON := []string{}
|
||
for _, run := range runs {
|
||
runsByWorkflow[run.WorkflowID] = append(runsByWorkflow[run.WorkflowID], run)
|
||
if run.ProjectID != plan.ProjectID {
|
||
raw, e := json.Marshal(run)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
otherRunJSON = append(otherRunJSON, string(raw))
|
||
}
|
||
}
|
||
// Which BOS target URIs are actually referenced by some other run's
|
||
// JSON, computed with one linear pass per other-run through a single
|
||
// combined regexp instead of a strings.Contains scan per (target, run)
|
||
// pair. A data-heavy org can have thousands of BOS targets and
|
||
// thousands of other runs; the naive nested loop made every deletion
|
||
// preview take minutes (see the SL-data/BLF-data hangs this was
|
||
// tracked down from). Go's regexp is RE2-based, so matching a fixed
|
||
// alternation of literal (quoted) URIs against a text is linear in the
|
||
// text length regardless of how many alternatives there are — the
|
||
// boolean result for every (target, run) pair is identical to the
|
||
// original strings.Contains(raw, uri) check, just computed once per
|
||
// distinct URI/run instead of once per pair.
|
||
referencedURIs := map[string]bool{}
|
||
if len(otherRunJSON) > 0 {
|
||
uriSet := map[string]bool{}
|
||
for _, t := range plan.Targets {
|
||
if t.Kind != "bos" {
|
||
continue
|
||
}
|
||
uriSet["bos://"+t.Bucket+"/"+strings.Trim(t.Prefix, "/")] = true
|
||
}
|
||
if len(uriSet) > 0 {
|
||
patterns := make([]string, 0, len(uriSet))
|
||
for uri := range uriSet {
|
||
patterns = append(patterns, regexp.QuoteMeta(uri))
|
||
}
|
||
matcher := regexp.MustCompile(strings.Join(patterns, "|"))
|
||
for _, raw := range otherRunJSON {
|
||
for _, match := range matcher.FindAllString(raw, -1) {
|
||
referencedURIs[match] = true
|
||
}
|
||
}
|
||
}
|
||
}
|
||
type policyResult struct {
|
||
policy *domain.ArtifactStoragePolicy
|
||
prefixes []string
|
||
safe bool
|
||
}
|
||
policies := map[string]policyResult{}
|
||
for _, t := range plan.Targets {
|
||
if t.Kind != "bos" {
|
||
continue
|
||
}
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
for _, capture := range captures {
|
||
if d := capture.Delivery; d != nil && d.Bucket == t.Bucket && prefixOverlap(t.Prefix, d.Prefix) {
|
||
return fmt.Errorf("最终交付成品受保护,不能级联删除:%s", d.Prefix)
|
||
}
|
||
}
|
||
if !validDeletionPrefix(t.Prefix) {
|
||
return fmt.Errorf("拒绝不安全路径 %s", t.Prefix)
|
||
}
|
||
if strings.HasPrefix(t.Prefix, "models/") || strings.HasPrefix(t.Prefix, ".droboticflow-trash/") || t.Prefix == "models" || t.Prefix == "system" {
|
||
return fmt.Errorf("共享目录不能级联删除:%s", t.Prefix)
|
||
}
|
||
if strings.HasPrefix(t.Prefix, "system/") && !(t.RunID != "" && t.Prefix == "system/runs/"+t.RunID) {
|
||
return fmt.Errorf("系统共享路径不能级联删除:%s", t.Prefix)
|
||
}
|
||
for _, w := range workflows {
|
||
cfg := w.CaptureAutomation
|
||
if cfg != nil && cfg.Bucket == t.Bucket && cfg.DeliveryPrefix != "" && prefixOverlap(t.Prefix, cfg.DeliveryPrefix) {
|
||
return fmt.Errorf("最终交付目录受保护:%s", cfg.DeliveryPrefix)
|
||
}
|
||
if cfg != nil && cfg.ConnectionID == t.ConnectionID && cfg.Bucket == t.Bucket && cfg.Prefix != "" && prefixOverlap(t.Prefix, cfg.Prefix) {
|
||
return fmt.Errorf("目标与原始输入目录重叠,已保护:%s", t.Prefix)
|
||
}
|
||
if w.ProjectID != plan.ProjectID {
|
||
cached, ok := policies[w.ID]
|
||
if !ok {
|
||
policy, e := s.effectiveArtifactStorage(ctx, &w)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
ps, safe := workflowGeneratedArtifactPrefixes(&w, runsByWorkflow[w.ID], policy)
|
||
cached = policyResult{policy, ps, safe}
|
||
policies[w.ID] = cached
|
||
}
|
||
policy := cached.policy
|
||
if policy != nil && policy.ConnectionID == t.ConnectionID && policy.Bucket == t.Bucket {
|
||
ps, safe := cached.prefixes, cached.safe
|
||
if !safe && len(runsByWorkflow[w.ID]) > 0 {
|
||
return fmt.Errorf("其他工作流产物范围不明确,无法证明独占")
|
||
}
|
||
for _, p := range ps {
|
||
if prefixOverlap(t.Prefix, p) {
|
||
return fmt.Errorf("目标被其他工作流使用:%s", t.Prefix)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
uri := "bos://" + t.Bucket + "/" + strings.Trim(t.Prefix, "/")
|
||
if referencedURIs[uri] {
|
||
return fmt.Errorf("目标被其他运行引用:%s", t.Prefix)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func safeRunWorkspace(root, runID string) (string, error) {
|
||
if !deletionRunIDPattern.MatchString(runID) {
|
||
return "", fmt.Errorf("不安全的 Run ID")
|
||
}
|
||
root = filepath.Clean(root)
|
||
if !filepath.IsAbs(root) || root == "/" || root == "." {
|
||
return "", fmt.Errorf("不安全的工作区根目录")
|
||
}
|
||
for _, parent := range []string{root, filepath.Join(root, "system"), filepath.Join(root, "system", "runs")} {
|
||
info, err := os.Lstat(parent)
|
||
if os.IsNotExist(err) {
|
||
continue
|
||
}
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if info.Mode()&os.ModeSymlink != 0 {
|
||
return "", fmt.Errorf("工作目录父路径是符号链接,拒绝清理")
|
||
}
|
||
}
|
||
return filepath.Join(root, "system", "runs", runID), nil
|
||
}
|
||
func (s *server) deleteProjectTarget(ctx context.Context, plan domain.ProjectDeletionPlan, t domain.ProjectDeletionTarget) error {
|
||
switch t.Kind {
|
||
case "runtime":
|
||
client, err := s.deletionRuntime()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return client.TerminateRunAndWait(ctx, s.k8sNamespace, t.RunID)
|
||
case "workspace":
|
||
path, err := safeRunWorkspace(s.workspaceMountPath, t.RunID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err = os.RemoveAll(path); err != nil {
|
||
return err
|
||
}
|
||
if _, err = os.Lstat(path); !os.IsNotExist(err) {
|
||
return fmt.Errorf("运行目录仍存在或无法核验:%s", path)
|
||
}
|
||
return nil
|
||
case "bos":
|
||
// validateDeletionReferences is already re-run once by
|
||
// executeProjectDeletion immediately before it starts processing
|
||
// non-runtime targets (the runtimeDone gate) — that is the right
|
||
// point to catch a run created between plan-build and execution
|
||
// referencing this path. Re-running the whole org-wide validation
|
||
// again here, per bos target, made a project with a few thousand
|
||
// BOS targets take hours (each call re-fetches and re-scans every
|
||
// other run in the organization).
|
||
browser, conn, cleanup, err := s.resolveObjectStorageBrowser(ctx, t.ConnectionID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if cleanup != nil {
|
||
defer cleanup()
|
||
}
|
||
if browser == nil || conn == nil || normalizedTenantID(conn.OrganizationID) != normalizedTenantID(plan.OrganizationID) {
|
||
return fmt.Errorf("存储连接不存在或归属不符")
|
||
}
|
||
bucket, err := resolveObjectStorageBucket(conn, t.Bucket)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if err = browser.DeletePrefix(ctx, bucket, t.Prefix); err != nil && !isBOSNoSuchKey(err) {
|
||
return err
|
||
}
|
||
remaining, err := browser.ListObjects(ctx, bucket, strings.TrimSuffix(t.Prefix, "/")+"/")
|
||
if err != nil && !isBOSNoSuchKey(err) {
|
||
return err
|
||
}
|
||
if len(remaining) > 0 {
|
||
return fmt.Errorf("产物目录仍有 %d 个对象", len(remaining))
|
||
}
|
||
return nil
|
||
case "records":
|
||
_, err := s.store.DeleteProject(ctx, plan.ProjectID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
p, err := s.store.GetProject(ctx, plan.ProjectID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if p != nil {
|
||
return fmt.Errorf("项目元数据仍存在")
|
||
}
|
||
for _, id := range plan.RunIDs {
|
||
r, e := s.store.GetRun(ctx, id)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if r != nil {
|
||
return fmt.Errorf("Run 记录仍存在:%s", id)
|
||
}
|
||
}
|
||
workflows, e := s.store.ListWorkflowsByProject(ctx, plan.ProjectID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if len(workflows) > 0 {
|
||
return fmt.Errorf("工作流记录仍存在")
|
||
}
|
||
items, e := s.store.ListHumanItems(ctx, plan.OrganizationID, domain.HumanItemQuery{ProjectID: plan.ProjectID})
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if len(items) > 0 {
|
||
return fmt.Errorf("人工任务记录仍存在")
|
||
}
|
||
return nil
|
||
}
|
||
return fmt.Errorf("未知清理步骤 %s", t.Kind)
|
||
}
|
||
|
||
func (s *server) executeProjectDeletion(task domain.ProjectDeletionTask, owner string) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||
defer cancel()
|
||
done := make(chan struct{})
|
||
defer close(done)
|
||
go func() {
|
||
ticker := time.NewTicker(15 * time.Second)
|
||
defer ticker.Stop()
|
||
for {
|
||
select {
|
||
case <-done:
|
||
return
|
||
case <-ctx.Done():
|
||
return
|
||
case <-ticker.C:
|
||
ok, e := s.store.RenewProjectDeletion(ctx, task.ID, owner, deletionLeaseTime(time.Now().Add(2*time.Minute)))
|
||
if e != nil || !ok {
|
||
cancel()
|
||
return
|
||
}
|
||
}
|
||
}
|
||
}()
|
||
persist := func() error {
|
||
task.UpdatedAt = nowISO()
|
||
saveCtx, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||
defer stop()
|
||
ok, e := s.store.SaveProjectDeletion(saveCtx, task, owner)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if !ok {
|
||
return fmt.Errorf("删除任务租约已失效")
|
||
}
|
||
return nil
|
||
}
|
||
fail := func(e error) { task.Status = "failed"; task.Error = e.Error(); _ = persist() }
|
||
// Validate the real client configuration before stopping writers or deleting
|
||
// files. Runtime targets are rechecked before storage cleanup even on retry.
|
||
if err := s.preflightProjectDeletion(task.Plan); err != nil {
|
||
fail(err)
|
||
return
|
||
}
|
||
if err := s.store.MarkProjectDeleting(ctx, task.Plan.ProjectID, task.ID); err != nil {
|
||
p, _ := s.store.GetProject(ctx, task.Plan.ProjectID)
|
||
if p != nil {
|
||
fail(err)
|
||
return
|
||
}
|
||
}
|
||
for _, id := range task.Plan.WorkflowIDs {
|
||
w, e := s.store.GetWorkflow(ctx, id)
|
||
if e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
if w != nil && w.CaptureAutomation != nil && w.CaptureAutomation.Enabled {
|
||
w.CaptureAutomation.Enabled = false
|
||
if e = s.store.UpsertWorkflow(ctx, *w); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
}
|
||
}
|
||
// Freeze writers before touching any files. Existing user inputs are untouched.
|
||
for _, id := range task.Plan.RunIDs {
|
||
r, e := s.store.GetRun(ctx, id)
|
||
if e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
if r == nil {
|
||
continue
|
||
}
|
||
if workflowRunBlocksDeletion(r.Status) && !task.StopRunning {
|
||
fail(fmt.Errorf("仍有运行中的任务,需确认停止后删除"))
|
||
return
|
||
}
|
||
if workflowRunBlocksDeletion(r.Status) || r.Status == domain.RunStatusWaitingForHuman || r.Status == domain.RunStatusWaitingForAudit {
|
||
if _, e = s.store.RequestRunStop(ctx, id, domain.RunStopModeForce); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
}
|
||
}
|
||
// Lock prevents new admissions; a pre-lock admission changes the manifest and
|
||
// must be explicitly reconfirmed, never silently added to destructive work.
|
||
recordsStarted := false
|
||
for _, t := range task.Plan.Targets {
|
||
if t.Kind == "records" && t.Status != "pending" {
|
||
recordsStarted = true
|
||
}
|
||
}
|
||
if !recordsStarted {
|
||
p, e := s.store.GetProject(ctx, task.Plan.ProjectID)
|
||
if e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
if p != nil {
|
||
fresh, e := s.buildProjectDeletionPlan(ctx, p)
|
||
if e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
if fresh.Digest != task.Plan.Digest {
|
||
task.Plan = fresh
|
||
fail(fmt.Errorf("删除范围发生变化,清单已更新,请重新核对并确认"))
|
||
return
|
||
}
|
||
}
|
||
}
|
||
// Re-verify runtime absence even when a previous attempt completed that step.
|
||
for _, t := range task.Plan.Targets {
|
||
if t.Kind == "runtime" && t.Status == "done" {
|
||
if e := s.deleteProjectTarget(ctx, task.Plan, t); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
}
|
||
}
|
||
targetConcurrency := 12
|
||
if v := strings.TrimSpace(os.Getenv("PROJECT_DELETION_TARGET_CONCURRENCY")); v != "" {
|
||
if n, e := strconv.Atoi(v); e == nil && n > 0 {
|
||
targetConcurrency = n
|
||
}
|
||
}
|
||
// processBatch runs deleteProjectTarget for the not-yet-done targets at
|
||
// indices with up to targetConcurrency in flight at once. Each target's
|
||
// Status/Error and every persist() call is guarded by mu, so concurrent
|
||
// workers never race on shared task state; deleteProjectTarget itself
|
||
// (the actual K8s/BOS/Mongo I/O — the part worth overlapping) runs
|
||
// unlocked. A single project's deletion used to process every target
|
||
// strictly one at a time, which made stopping thousands of real runs
|
||
// take as long as thousands of sequential K8s round trips.
|
||
var mu sync.Mutex
|
||
processBatch := func(indices []int) error {
|
||
sem := make(chan struct{}, targetConcurrency)
|
||
var wg sync.WaitGroup
|
||
var firstErr error
|
||
for _, index := range indices {
|
||
mu.Lock()
|
||
skip := task.Plan.Targets[index].Status == "done" || firstErr != nil
|
||
mu.Unlock()
|
||
if skip {
|
||
continue
|
||
}
|
||
sem <- struct{}{}
|
||
wg.Add(1)
|
||
go func(index int) {
|
||
defer wg.Done()
|
||
defer func() { <-sem }()
|
||
mu.Lock()
|
||
if firstErr != nil {
|
||
mu.Unlock()
|
||
return
|
||
}
|
||
target := &task.Plan.Targets[index]
|
||
target.Error = ""
|
||
target.Status = "running"
|
||
persistErr := persist()
|
||
snapshot := *target
|
||
mu.Unlock()
|
||
if persistErr != nil {
|
||
mu.Lock()
|
||
if firstErr == nil {
|
||
firstErr = persistErr
|
||
}
|
||
mu.Unlock()
|
||
cancel()
|
||
return
|
||
}
|
||
runErr := s.deleteProjectTarget(ctx, task.Plan, snapshot)
|
||
mu.Lock()
|
||
target = &task.Plan.Targets[index]
|
||
if runErr != nil {
|
||
target.Status = "failed"
|
||
target.Error = runErr.Error()
|
||
if firstErr == nil {
|
||
firstErr = runErr
|
||
}
|
||
} else {
|
||
target.Status = "done"
|
||
}
|
||
_ = persist()
|
||
mu.Unlock()
|
||
}(index)
|
||
}
|
||
wg.Wait()
|
||
return firstErr
|
||
}
|
||
indicesByKind := map[string][]int{}
|
||
nonRuntimePending := false
|
||
for index, t := range task.Plan.Targets {
|
||
indicesByKind[t.Kind] = append(indicesByKind[t.Kind], index)
|
||
if t.Kind != "runtime" && t.Status != "done" {
|
||
nonRuntimePending = true
|
||
}
|
||
}
|
||
if e := processBatch(indicesByKind["runtime"]); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
// Re-run the phase-transition safety checks (wait for in-flight human
|
||
// work, re-verify runtime absence, revalidate references) exactly once
|
||
// per execution attempt, right before touching any file/storage target
|
||
// — the same point the original sequential loop ran them at.
|
||
if nonRuntimePending {
|
||
deadline := time.Now().Add(150 * time.Second)
|
||
for {
|
||
items, e := s.store.ListHumanItems(ctx, task.OrganizationID, domain.HumanItemQuery{ProjectID: task.Plan.ProjectID})
|
||
if e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
busy := false
|
||
for _, id := range task.Plan.RunIDs {
|
||
run, e := s.store.GetRun(ctx, id)
|
||
if e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
if run != nil && run.ExecutionLeaseOwner != "" {
|
||
until, e := time.Parse(time.RFC3339Nano, run.ExecutionLeaseExpiresAt)
|
||
if e != nil || until.After(time.Now()) {
|
||
busy = true
|
||
}
|
||
}
|
||
}
|
||
for i := range items {
|
||
item := &items[i]
|
||
if e = s.recoverInterruptedSlice(ctx, item, time.Now()); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
w := mapValue(item.Subject.Payload["sliceWorkflow"])
|
||
if stringValue(w["status"]) == "processing" || item.PendingCombinedSubmissionID != "" {
|
||
busy = true
|
||
}
|
||
}
|
||
if !busy {
|
||
break
|
||
}
|
||
if time.Now().After(deadline) {
|
||
fail(fmt.Errorf("切片或提交仍在写入,未删除任何存储文件,请稍后重试"))
|
||
return
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
fail(ctx.Err())
|
||
return
|
||
case <-time.After(2 * time.Second):
|
||
}
|
||
}
|
||
// Catch work admitted immediately before the project lock.
|
||
for _, step := range task.Plan.Targets {
|
||
if step.Kind == "runtime" {
|
||
if e := s.deleteProjectTarget(ctx, task.Plan, step); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
}
|
||
}
|
||
// Scoped exactly as before (projectId match only, no workflowId
|
||
// fallback — see the loop below) but via an indexed query
|
||
// instead of an organization-wide scan.
|
||
fresh, e := s.store.ListRunArtifactsForProject(ctx, task.OrganizationID, task.Plan.ProjectID, nil)
|
||
if e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
known := map[string]bool{}
|
||
for _, id := range task.Plan.RunIDs {
|
||
known[id] = true
|
||
}
|
||
for _, r := range fresh {
|
||
if r.ProjectID == task.Plan.ProjectID && !known[r.ID] {
|
||
fail(fmt.Errorf("删除清单变化:出现新运行 %s,请重新核对", r.ID))
|
||
return
|
||
}
|
||
}
|
||
if e = s.validateDeletionReferences(ctx, task.Plan); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
}
|
||
for _, kind := range []string{"workspace", "bos", "records"} {
|
||
if e := processBatch(indicesByKind[kind]); e != nil {
|
||
fail(e)
|
||
return
|
||
}
|
||
}
|
||
task.Status = "completed"
|
||
task.Error = ""
|
||
_ = persist()
|
||
}
|
||
|
||
func (s *server) handleProjectDeletionPreview(w http.ResponseWriter, r *http.Request) {
|
||
p, e := s.store.GetProject(r.Context(), chi.URLParam(r, "id"))
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
if p == nil {
|
||
renderJSON(w, 404, map[string]string{"error": "Project not found"})
|
||
return
|
||
}
|
||
if !s.ensureTenantAccess(w, r, p.OrganizationID, "Project") {
|
||
return
|
||
}
|
||
task, e := s.store.GetProjectDeletion(r.Context(), deletionTaskID(p.ID))
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
if task != nil {
|
||
renderJSON(w, 200, map[string]any{"plan": task.Plan, "task": deletionTaskResponse(task)})
|
||
return
|
||
}
|
||
plan, e := s.buildProjectDeletionPlan(r.Context(), p)
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
renderJSON(w, 200, map[string]any{"plan": plan})
|
||
}
|
||
func (s *server) handleProjectDeletionsList(w http.ResponseWriter, r *http.Request) {
|
||
items, e := s.store.ListProjectDeletions(r.Context(), s.requestOrganizationID(r))
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
out := []*domain.ProjectDeletionTask{}
|
||
for i := range items {
|
||
out = append(out, deletionTaskResponse(&items[i]))
|
||
}
|
||
renderJSON(w, 200, map[string]any{"items": out})
|
||
}
|
||
func (s *server) handleProjectDeletionGet(w http.ResponseWriter, r *http.Request) {
|
||
t, e := s.store.GetProjectDeletion(r.Context(), chi.URLParam(r, "id"))
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
if t == nil {
|
||
renderJSON(w, 404, map[string]string{"error": "Deletion task not found"})
|
||
return
|
||
}
|
||
if !s.ensureTenantAccess(w, r, t.OrganizationID, "Deletion task") {
|
||
return
|
||
}
|
||
renderJSON(w, 200, deletionTaskResponse(t))
|
||
}
|
||
|
||
type projectDeletionInput struct {
|
||
ConfirmationName string `json:"confirmationName"`
|
||
Digest string `json:"digest"`
|
||
Reason string `json:"reason"`
|
||
StopRunning bool `json:"stopRunning"`
|
||
}
|
||
|
||
func (s *server) startProjectDeletion(w http.ResponseWriter, r *http.Request, projectID, taskID string) {
|
||
var input projectDeletionInput
|
||
if !decodeProjectDeletionInput(w, r, &input) {
|
||
return
|
||
}
|
||
task, e := s.store.GetProjectDeletion(r.Context(), taskID)
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
var plan domain.ProjectDeletionPlan
|
||
if task != nil {
|
||
if !s.ensureTenantAccess(w, r, task.OrganizationID, "Deletion task") {
|
||
return
|
||
}
|
||
plan = task.Plan
|
||
if task.Status == "completed" {
|
||
renderJSON(w, 200, deletionTaskResponse(task))
|
||
return
|
||
}
|
||
} else {
|
||
p, e := s.store.GetProject(r.Context(), projectID)
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
if p == nil {
|
||
renderJSON(w, 404, map[string]string{"error": "Project not found"})
|
||
return
|
||
}
|
||
if !s.ensureTenantAccess(w, r, p.OrganizationID, "Project") {
|
||
return
|
||
}
|
||
plan, e = s.buildProjectDeletionPlan(r.Context(), p)
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
}
|
||
if input.ConfirmationName != plan.ProjectName || input.Digest != plan.Digest || strings.TrimSpace(input.Reason) == "" {
|
||
renderValidationError(w, validationIssue{Field: "confirmation", Message: "请核对删除清单、输入完整项目名称及删除原因"})
|
||
return
|
||
}
|
||
if len(plan.Blockers) > 0 {
|
||
renderJSON(w, 409, map[string]any{"error": "存在无法安全清理的路径", "plan": plan})
|
||
return
|
||
}
|
||
if plan.BlockingRunCount > 0 && !input.StopRunning {
|
||
renderJSON(w, 409, map[string]string{"error": "请确认停止正在运行的任务后删除"})
|
||
return
|
||
}
|
||
if err := s.preflightProjectDeletion(plan); err != nil {
|
||
renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": err.Error(), "code": "project_deletion_configuration_invalid"})
|
||
return
|
||
}
|
||
if task == nil {
|
||
principal, _ := currentPrincipal(r)
|
||
actor := ""
|
||
if principal != nil {
|
||
actor = principal.User.ID
|
||
}
|
||
task, e = s.store.CreateProjectDeletion(r.Context(), domain.ProjectDeletionTask{ID: taskID, OrganizationID: plan.OrganizationID, Plan: plan, Status: "prepared", ActorUserID: actor, Reason: strings.TrimSpace(input.Reason), CreatedAt: nowISO(), UpdatedAt: nowISO()})
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
}
|
||
owner := domain.NewID("deletion-lease")
|
||
claimed, ok, e := s.store.ClaimProjectDeletion(r.Context(), task.ID, owner, deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
|
||
if e != nil {
|
||
renderError(w, 500, e)
|
||
return
|
||
}
|
||
if !ok {
|
||
renderJSON(w, 202, deletionTaskResponse(claimed))
|
||
return
|
||
}
|
||
claimed.StopRunning = input.StopRunning
|
||
if e = s.store.MarkProjectDeleting(r.Context(), plan.ProjectID, task.ID); e != nil {
|
||
p, _ := s.store.GetProject(r.Context(), plan.ProjectID)
|
||
if p != nil {
|
||
claimed.Status = "failed"
|
||
claimed.Error = e.Error()
|
||
_, _ = s.store.SaveProjectDeletion(r.Context(), *claimed, owner)
|
||
renderJSON(w, 409, deletionTaskResponse(claimed))
|
||
return
|
||
}
|
||
}
|
||
if ok, e = s.store.SaveProjectDeletion(r.Context(), *claimed, owner); e != nil || !ok {
|
||
renderJSON(w, 500, map[string]string{"error": "删除任务状态保存失败,未开始资源清理"})
|
||
return
|
||
}
|
||
go s.executeProjectDeletion(*claimed, owner)
|
||
renderJSON(w, 202, deletionTaskResponse(claimed))
|
||
}
|
||
|
||
// Read the small confirmation body before expensive deletion-plan queries.
|
||
// Do not collapse transport timeouts into a misleading JSON syntax error.
|
||
func decodeProjectDeletionInput(w http.ResponseWriter, r *http.Request, input *projectDeletionInput) bool {
|
||
r.Body = http.MaxBytesReader(w, r.Body, 64<<10)
|
||
defer r.Body.Close()
|
||
decoder := json.NewDecoder(r.Body)
|
||
err := decoder.Decode(input)
|
||
if err == nil {
|
||
var extra any
|
||
err = decoder.Decode(&extra)
|
||
if err == io.EOF {
|
||
return true
|
||
}
|
||
if err == nil {
|
||
err = fmt.Errorf("multiple JSON values")
|
||
}
|
||
}
|
||
var timeout net.Error
|
||
var tooLarge *http.MaxBytesError
|
||
status, code, message := http.StatusBadRequest, "invalid_deletion_body", "删除请求格式无效,请重新打开删除窗口后重试"
|
||
if errors.As(err, &timeout) && timeout.Timeout() {
|
||
status, code, message = http.StatusRequestTimeout, "deletion_body_timeout", "读取删除请求超时,请重新打开删除窗口后重试"
|
||
} else if errors.As(err, &tooLarge) {
|
||
status, code, message = http.StatusRequestEntityTooLarge, "deletion_body_too_large", "删除确认请求过大"
|
||
} else if errors.Is(err, io.EOF) {
|
||
code, message = "empty_deletion_body", "删除请求缺少确认信息,请重新打开删除窗口后重试"
|
||
}
|
||
renderJSON(w, status, map[string]string{"error": message, "code": code})
|
||
return false
|
||
}
|
||
func (s *server) handleProjectDeletionRetry(w http.ResponseWriter, r *http.Request) {
|
||
s.startProjectDeletion(w, r, "", chi.URLParam(r, "id"))
|
||
}
|