forked from longtao.wu/droboticflow
- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix - C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown - C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation - C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache - C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback - S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed - Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush - Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500 Co-authored-by: AgentTeams slow-page-fix
694 lines
20 KiB
Go
694 lines
20 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
"github.com/go-chi/chi/v5"
|
|
)
|
|
|
|
func supplierDeletionID(id string) string { return "supplier-deletion-" + id }
|
|
func supplierDeletionResponse(t *domain.SupplierDeletionTask) *domain.SupplierDeletionTask {
|
|
if t == nil {
|
|
return nil
|
|
}
|
|
v := *t
|
|
v.Retryable = t.Status != "completed" && (t.LeaseOwner == "" || t.LeaseUntil <= deletionLeaseTime(time.Now()))
|
|
if v.Status == "running" && v.Retryable {
|
|
v.Status = "interrupted"
|
|
}
|
|
return &v
|
|
}
|
|
func (s *server) supplierDeletionAdmin(w http.ResponseWriter, r *http.Request) bool {
|
|
p, _ := currentPrincipal(r)
|
|
if !isOrganizationAdmin(p) {
|
|
renderJSON(w, 403, map[string]string{"error": "仅管理员可级联删除供应商"})
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Bound concurrent scans and release all work when the requesting browser leaves.
|
|
var supplierPreviewSlots = make(chan struct{}, 2)
|
|
var supplierRunReferencePattern = regexp.MustCompile(`run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}`)
|
|
|
|
func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
|
defer cancel()
|
|
ctx = store.WithSupplierDeletionReadScope(ctx)
|
|
select {
|
|
case supplierPreviewSlots <- struct{}{}:
|
|
defer func() { <-supplierPreviewSlots }()
|
|
case <-ctx.Done():
|
|
return domain.SupplierDeletionPlan{}, ctx.Err()
|
|
}
|
|
|
|
p, e := s.store.PreviewSupplierDeletion(ctx, org, id)
|
|
if e != nil {
|
|
return p, e
|
|
}
|
|
p.ScopeDigest = p.Digest
|
|
for _, pid := range p.ProjectIDs {
|
|
project, e := s.store.GetProject(ctx, pid)
|
|
if e != nil {
|
|
return p, e
|
|
}
|
|
if project != nil {
|
|
p.ProjectNames = append(p.ProjectNames, project.Name)
|
|
}
|
|
}
|
|
if len(p.RunIDs) == 0 {
|
|
p.RunIDs = []string{}
|
|
p.Counts["fileTargets"], p.Counts["runningTasks"] = 0, 0
|
|
store.FinalizeSupplierDeletionPlan(&p)
|
|
return p, ctx.Err()
|
|
}
|
|
runs, e := s.store.ListSupplierDeletionRunReferences(ctx)
|
|
if e != nil {
|
|
return p, e
|
|
}
|
|
workflows, e := s.store.ListAllWorkflows(ctx)
|
|
if e != nil {
|
|
return p, e
|
|
}
|
|
captures, e := s.store.ListCaptureRecords(ctx, org, "", 0)
|
|
if e != nil {
|
|
return p, e
|
|
}
|
|
owns := map[string]bool{}
|
|
for _, id := range p.RunIDs {
|
|
owns[id] = true
|
|
}
|
|
byID := map[string]domain.RunRecord{}
|
|
runJSON := map[string]string{}
|
|
runsByWorkflow := map[string][]domain.RunRecord{}
|
|
for _, r := range runs {
|
|
if err := ctx.Err(); err != nil {
|
|
return p, err
|
|
}
|
|
byID[r.ID] = r
|
|
raw, _ := json.Marshal(r)
|
|
runJSON[r.ID] = string(raw)
|
|
runsByWorkflow[r.WorkflowID] = append(runsByWorkflow[r.WorkflowID], r)
|
|
}
|
|
// Index foreign references once, including across projects/organizations.
|
|
// A newly shared producer is queued so transitive sharing is preserved.
|
|
queue := []string{}
|
|
for _, r := range runs {
|
|
if !owns[r.ID] {
|
|
queue = append(queue, r.ID)
|
|
}
|
|
}
|
|
for head := 0; head < len(queue); head++ {
|
|
if err := ctx.Err(); err != nil {
|
|
return p, err
|
|
}
|
|
for _, id := range supplierRunReferencePattern.FindAllString(runJSON[queue[head]], -1) {
|
|
if owns[id] {
|
|
delete(owns, id)
|
|
p.SharedRunIDs = append(p.SharedRunIDs, id)
|
|
p.SharedRecords["runs"] = append(p.SharedRecords["runs"], id)
|
|
p.Preserved = append(p.Preserved, "其他运行引用的产物及运行记录:"+id)
|
|
queue = append(queue, id)
|
|
}
|
|
}
|
|
}
|
|
p.RunIDs = []string{}
|
|
for id := range owns {
|
|
p.RunIDs = append(p.RunIDs, id)
|
|
}
|
|
sort.Strings(p.RunIDs)
|
|
p.Records["runs"] = p.RunIDs
|
|
p.Counts["runs"] = len(p.RunIDs)
|
|
add := func(kind, run, conn, bucket, prefix string) {
|
|
sum := sha256.Sum256([]byte(kind + "|" + run + "|" + conn + "|" + bucket + "|" + prefix))
|
|
for _, old := range p.Targets {
|
|
if old.ID == hex.EncodeToString(sum[:12]) {
|
|
return
|
|
}
|
|
}
|
|
p.Targets = append(p.Targets, domain.ProjectDeletionTarget{ID: hex.EncodeToString(sum[:12]), Kind: kind, RunID: run, ConnectionID: conn, Bucket: bucket, Prefix: prefix, Status: "pending"})
|
|
}
|
|
type retainedPath struct{ connection, bucket, prefix string }
|
|
retainedPaths := []retainedPath{}
|
|
uncertainStores := map[string]bool{}
|
|
for _, w := range workflows {
|
|
if err := ctx.Err(); err != nil {
|
|
return p, err
|
|
}
|
|
retained := []domain.RunRecord{}
|
|
for _, r := range runsByWorkflow[w.ID] {
|
|
if !owns[r.ID] {
|
|
retained = append(retained, r)
|
|
}
|
|
}
|
|
if len(retained) == 0 {
|
|
continue
|
|
}
|
|
policy, e := s.effectiveArtifactStorage(ctx, &w)
|
|
if e != nil {
|
|
return p, e
|
|
}
|
|
if policy == nil {
|
|
continue
|
|
}
|
|
prefixes, safe := workflowGeneratedArtifactPrefixes(&w, retained, policy)
|
|
if !safe {
|
|
uncertainStores[policy.ConnectionID+"|"+policy.Bucket] = true
|
|
}
|
|
for _, prefix := range prefixes {
|
|
retainedPaths = append(retainedPaths, retainedPath{policy.ConnectionID, policy.Bucket, prefix})
|
|
}
|
|
}
|
|
addBOS := func(run, connection, bucket, prefix string) {
|
|
if uncertainStores[connection+"|"+bucket] {
|
|
p.Preserved = append(p.Preserved, "其他运行产物范围不明确,保留目录:"+prefix)
|
|
return
|
|
}
|
|
for _, other := range retainedPaths {
|
|
if other.connection == connection && other.bucket == bucket && prefixOverlap(prefix, other.prefix) {
|
|
p.Preserved = append(p.Preserved, "其他供应商使用的产物目录:"+prefix)
|
|
return
|
|
}
|
|
}
|
|
|
|
if !validDeletionPrefix(prefix) || strings.HasPrefix(prefix, "models") || strings.HasPrefix(prefix, ".droboticflow-trash") || (strings.HasPrefix(prefix, "system/") && prefix != "system/runs/"+run) {
|
|
p.Preserved = append(p.Preserved, "无法确认独占的存储路径:"+prefix)
|
|
return
|
|
}
|
|
for _, c := range captures {
|
|
if c.Bucket == bucket && c.Prefix != "" && prefixOverlap(prefix, c.Prefix) {
|
|
p.Preserved = append(p.Preserved, "原始采集目录:"+prefix)
|
|
return
|
|
}
|
|
if c.Delivery != nil && c.Delivery.Bucket == bucket && !owns[c.RunID] && prefixOverlap(prefix, c.Delivery.Prefix) {
|
|
p.Preserved = append(p.Preserved, "共享交付目录:"+prefix)
|
|
return
|
|
}
|
|
}
|
|
for _, w := range workflows {
|
|
if cfg := w.CaptureAutomation; cfg != nil && cfg.Bucket == bucket && cfg.Prefix != "" && prefixOverlap(prefix, cfg.Prefix) {
|
|
p.Preserved = append(p.Preserved, "原始监听目录:"+prefix)
|
|
return
|
|
}
|
|
}
|
|
for _, r := range runs {
|
|
if owns[r.ID] {
|
|
continue
|
|
}
|
|
raw := runJSON[r.ID]
|
|
if strings.Contains(string(raw), "bos://"+bucket+"/"+strings.Trim(prefix, "/")) {
|
|
p.Preserved = append(p.Preserved, "其他运行引用的文件:"+prefix)
|
|
return
|
|
}
|
|
}
|
|
add("bos", run, connection, bucket, prefix)
|
|
}
|
|
p.Counts["runningTasks"] = 0
|
|
for _, id := range p.RunIDs {
|
|
if !deletionRunIDPattern.MatchString(id) {
|
|
p.Blockers = append(p.Blockers, "无法安全限定运行目录:"+id)
|
|
continue
|
|
}
|
|
add("runtime", id, "", "", "")
|
|
add("workspace", id, "", "", "system/runs/"+id)
|
|
if workflowRunBlocksDeletion(byID[id].Status) {
|
|
p.Counts["runningTasks"]++
|
|
}
|
|
if conn, bucket := os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET"); conn != "" && bucket != "" {
|
|
addBOS(id, conn, bucket, "system/runs/"+id)
|
|
}
|
|
}
|
|
for _, w := range workflows {
|
|
ownedRuns := []domain.RunRecord{}
|
|
foreignRun := false
|
|
for _, r := range runsByWorkflow[w.ID] {
|
|
if r.WorkflowID == w.ID {
|
|
if owns[r.ID] {
|
|
ownedRuns = append(ownedRuns, r)
|
|
} else {
|
|
foreignRun = true
|
|
}
|
|
}
|
|
}
|
|
if len(ownedRuns) == 0 {
|
|
continue
|
|
}
|
|
policy, e := s.effectiveArtifactStorage(ctx, &w)
|
|
if e != nil {
|
|
return p, e
|
|
}
|
|
if policy == nil {
|
|
continue
|
|
}
|
|
prefixes, safe := workflowGeneratedArtifactPrefixes(&w, ownedRuns, policy)
|
|
if !safe {
|
|
p.Preserved = append(p.Preserved, "产物路径归属不明确,保留工作流产物:"+w.ID)
|
|
continue
|
|
}
|
|
for _, prefix := range prefixes {
|
|
runID := ""
|
|
for _, r := range ownedRuns {
|
|
if strings.Contains(prefix, r.ID) {
|
|
runID = r.ID
|
|
}
|
|
}
|
|
if foreignRun && runID == "" {
|
|
p.Preserved = append(p.Preserved, "同项目其他供应商共用目录:"+prefix)
|
|
continue
|
|
}
|
|
addBOS(runID, policy.ConnectionID, policy.Bucket, prefix)
|
|
}
|
|
}
|
|
for _, c := range captures {
|
|
if owns[c.RunID] && c.Delivery != nil {
|
|
d := c.Delivery
|
|
addBOS(c.RunID, d.ConnectionID, d.Bucket, d.Prefix)
|
|
}
|
|
}
|
|
rank := map[string]int{"runtime": 0, "workspace": 1, "bos": 2}
|
|
sort.Slice(p.Targets, func(i, j int) bool {
|
|
if p.Targets[i].Kind != p.Targets[j].Kind {
|
|
return rank[p.Targets[i].Kind] < rank[p.Targets[j].Kind]
|
|
}
|
|
return p.Targets[i].ID < p.Targets[j].ID
|
|
})
|
|
p.Counts["fileTargets"] = 0
|
|
for _, t := range p.Targets {
|
|
if t.Kind != "runtime" {
|
|
p.Counts["fileTargets"]++
|
|
}
|
|
}
|
|
store.FinalizeSupplierDeletionPlan(&p)
|
|
return p, ctx.Err()
|
|
}
|
|
func (s *server) handleSupplierDeletionPreview(w http.ResponseWriter, r *http.Request) {
|
|
if !s.supplierDeletionAdmin(w, r) {
|
|
return
|
|
}
|
|
id := chi.URLParam(r, "supplierId")
|
|
p, e := s.buildSupplierDeletionPreview(r.Context(), s.requestOrganizationID(r), id)
|
|
if errors.Is(e, context.DeadlineExceeded) {
|
|
renderJSON(w, http.StatusRequestTimeout, map[string]string{"error": "删除范围核对超时,请稍后重试", "code": "supplier_preview_timeout"})
|
|
return
|
|
}
|
|
if errors.Is(e, context.Canceled) {
|
|
return
|
|
}
|
|
if e != nil {
|
|
renderSupplierMutationError(w, e)
|
|
return
|
|
}
|
|
renderJSON(w, 200, p)
|
|
}
|
|
func (s *server) handleSupplierDeletionsList(w http.ResponseWriter, r *http.Request) {
|
|
if !s.supplierDeletionAdmin(w, r) {
|
|
return
|
|
}
|
|
items, e := s.store.ListSupplierDeletions(r.Context(), s.requestOrganizationID(r))
|
|
if e != nil {
|
|
renderError(w, 500, e)
|
|
return
|
|
}
|
|
out := []*domain.SupplierDeletionTask{}
|
|
for _, v := range items {
|
|
out = append(out, supplierDeletionResponse(&v))
|
|
}
|
|
renderJSON(w, 200, map[string]any{"items": out})
|
|
}
|
|
func (s *server) handleSupplierDeletionGet(w http.ResponseWriter, r *http.Request) {
|
|
if !s.supplierDeletionAdmin(w, r) {
|
|
return
|
|
}
|
|
t, e := s.store.GetSupplierDeletion(r.Context(), chi.URLParam(r, "deletionId"))
|
|
if e != nil {
|
|
renderError(w, 500, e)
|
|
return
|
|
}
|
|
if t == nil || !s.filterByTenant(r, t.OrganizationID) {
|
|
renderJSON(w, 404, map[string]string{"error": "删除记录不存在"})
|
|
return
|
|
}
|
|
renderJSON(w, 200, supplierDeletionResponse(t))
|
|
}
|
|
func (s *server) handleSupplierDeletionStart(w http.ResponseWriter, r *http.Request) {
|
|
if !s.supplierDeletionAdmin(w, r) {
|
|
return
|
|
}
|
|
var input struct {
|
|
ConfirmationName string `json:"confirmationName"`
|
|
Digest string `json:"digest"`
|
|
StopRunning bool `json:"stopRunning"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
if !decodeJSON(w, r, &input) {
|
|
return
|
|
}
|
|
org, id := s.requestOrganizationID(r), chi.URLParam(r, "supplierId")
|
|
task, e := s.store.GetSupplierDeletion(r.Context(), supplierDeletionID(id))
|
|
if e != nil {
|
|
renderError(w, 500, e)
|
|
return
|
|
}
|
|
if task != nil && !s.filterByTenant(r, task.OrganizationID) {
|
|
renderJSON(w, 404, map[string]string{"error": "供应商不存在"})
|
|
return
|
|
}
|
|
if task != nil && task.Status == "completed" {
|
|
renderJSON(w, 200, supplierDeletionResponse(task))
|
|
return
|
|
}
|
|
var plan domain.SupplierDeletionPlan
|
|
if task == nil {
|
|
plan, e = s.buildSupplierDeletionPreview(r.Context(), org, id)
|
|
} else {
|
|
plan = task.Plan
|
|
}
|
|
if e != nil {
|
|
renderSupplierMutationError(w, e)
|
|
return
|
|
}
|
|
if input.ConfirmationName != plan.SupplierName || input.Digest != plan.Digest {
|
|
renderJSON(w, 409, map[string]any{"error": "删除范围已变化或确认名称不正确,请刷新预览后确认", "plan": plan})
|
|
return
|
|
}
|
|
if len(plan.Blockers) > 0 {
|
|
renderJSON(w, 409, map[string]any{"error": "存在不能安全删除的共享关联", "plan": plan})
|
|
return
|
|
}
|
|
if len(plan.RunIDs) > 0 && !input.StopRunning {
|
|
renderJSON(w, 400, map[string]string{"error": "请确认停止相关运行任务后删除"})
|
|
return
|
|
}
|
|
if len(plan.RunIDs) > 0 {
|
|
if _, e = s.deletionRuntime(); e != nil {
|
|
renderError(w, 503, e)
|
|
return
|
|
}
|
|
}
|
|
if task == nil {
|
|
principal, _ := currentPrincipal(r)
|
|
task = &domain.SupplierDeletionTask{ID: supplierDeletionID(id), OrganizationID: org, Plan: plan, ActorUserID: principal.User.ID, Reason: strings.TrimSpace(input.Reason), StopRunning: input.StopRunning, Status: "prepared", Phase: "freezing", CreatedAt: nowISO(), UpdatedAt: nowISO()}
|
|
}
|
|
task, e = s.store.BeginSupplierDeletion(r.Context(), *task)
|
|
if e != nil {
|
|
renderJSON(w, 409, map[string]string{"error": e.Error()})
|
|
return
|
|
}
|
|
owner := domain.NewID("supplier-delete-owner")
|
|
claimed, ok, e := s.store.ClaimSupplierDeletion(r.Context(), task.ID, owner, deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
|
|
if e != nil {
|
|
renderError(w, 500, e)
|
|
return
|
|
}
|
|
if ok {
|
|
claimed.StopRunning = input.StopRunning
|
|
go s.executeSupplierDeletion(*claimed, owner)
|
|
}
|
|
renderJSON(w, 202, supplierDeletionResponse(claimed))
|
|
}
|
|
|
|
// The platform workspace connection may be registered under the platform's
|
|
// tenant. Its configured bucket is shared infrastructure, but only an approved
|
|
// exclusive run's exact generated directory is eligible for this exception.
|
|
func supplierDeletionPlatformWorkspace(p domain.SupplierDeletionPlan, t domain.ProjectDeletionTarget) bool {
|
|
connection, bucket := os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET")
|
|
if connection == "" || bucket == "" || t.ConnectionID != connection || t.Bucket != bucket ||
|
|
!deletionRunIDPattern.MatchString(t.RunID) || t.Prefix != "system/runs/"+t.RunID {
|
|
return false
|
|
}
|
|
for _, id := range p.RunIDs {
|
|
if id == t.RunID {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func (s *server) deleteSupplierFile(ctx context.Context, p domain.SupplierDeletionPlan, t domain.ProjectDeletionTarget) error {
|
|
if t.Kind != "bos" {
|
|
return s.deleteProjectTarget(ctx, domain.ProjectDeletionPlan{}, t)
|
|
}
|
|
browser, conn, cleanup, e := s.resolveObjectStorageBrowser(ctx, t.ConnectionID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if cleanup != nil {
|
|
defer cleanup()
|
|
}
|
|
if browser == nil || conn == nil || (!annotationOrganizationEqual(conn.OrganizationID, p.OrganizationID) && !supplierDeletionPlatformWorkspace(p, t)) {
|
|
return fmt.Errorf("文件存储连接归属不匹配")
|
|
}
|
|
bucket, e := resolveObjectStorageBucket(conn, t.Bucket)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if e = browser.DeletePrefix(ctx, bucket, t.Prefix); e != nil && !isBOSNoSuchKey(e) {
|
|
return e
|
|
}
|
|
objects, e := browser.ListObjects(ctx, bucket, strings.TrimSuffix(t.Prefix, "/")+"/")
|
|
if e != nil && !isBOSNoSuchKey(e) {
|
|
return e
|
|
}
|
|
if len(objects) > 0 {
|
|
return fmt.Errorf("目录仍有文件,稍后重试")
|
|
}
|
|
return nil
|
|
}
|
|
func (s *server) executeSupplierDeletion(task domain.SupplierDeletionTask, owner string) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
|
defer cancel()
|
|
done := make(chan struct{})
|
|
defer close(done)
|
|
var saveMu sync.Mutex
|
|
go func() {
|
|
ticker := time.NewTicker(15 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
ok, e := s.store.RenewSupplierDeletion(ctx, task.ID, owner, deletionLeaseTime(time.Now().Add(2*time.Minute)))
|
|
if e != nil || !ok {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
persist := func() error {
|
|
saveMu.Lock()
|
|
defer saveMu.Unlock()
|
|
task.UpdatedAt = nowISO()
|
|
task.LeaseUntil = deletionLeaseTime(time.Now().Add(2 * time.Minute))
|
|
ok, e := s.store.SaveSupplierDeletion(context.WithoutCancel(ctx), task, owner)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if !ok {
|
|
return fmt.Errorf("删除执行权已失效")
|
|
}
|
|
return nil
|
|
}
|
|
fail := func(e error) { task.Status = "failed"; task.Error = e.Error(); _ = persist() }
|
|
if !task.WritesClosed {
|
|
task.Phase = "stopping"
|
|
}
|
|
if e := persist(); e != nil {
|
|
return
|
|
}
|
|
for _, id := range task.Plan.WorkflowIDs {
|
|
w, e := s.store.GetWorkflow(ctx, id)
|
|
if e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
if w != nil && w.CaptureAutomation != nil {
|
|
w.CaptureAutomation.Enabled = false
|
|
if e = s.store.UpsertWorkflow(ctx, *w); e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
for _, id := range task.Plan.RunIDs {
|
|
r, e := s.store.GetRun(ctx, id)
|
|
if e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
if r != nil && (workflowRunBlocksDeletion(r.Status) || r.Status == domain.RunStatusWaitingForHuman || r.Status == domain.RunStatusWaitingForAudit) {
|
|
if !task.StopRunning {
|
|
fail(fmt.Errorf("未确认停止运行任务"))
|
|
return
|
|
}
|
|
if _, e = s.store.RequestRunStop(ctx, id, domain.RunStopModeForce); e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
for i := range task.Plan.Targets {
|
|
target := &task.Plan.Targets[i]
|
|
if target.Kind != "runtime" {
|
|
continue
|
|
}
|
|
if e := s.deleteSupplierFile(ctx, task.Plan, *target); e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
target.Status = "done"
|
|
if e := persist(); e != nil {
|
|
return
|
|
}
|
|
}
|
|
if e := s.store.CancelSupplierQueuedProcessing(ctx, task.Plan); e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
deadline := time.Now().Add(3 * time.Minute)
|
|
for {
|
|
busy, e := s.store.SupplierIndexingActive(ctx, task.Plan)
|
|
if e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
for _, id := range task.Plan.Records["humanItems"] {
|
|
h, e := s.store.GetHumanItem(ctx, id)
|
|
if e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
if h == nil {
|
|
continue
|
|
}
|
|
if j := h.ProcessingJob; j != nil && (j.Status == "running" || j.Status == "queued") {
|
|
busy = true
|
|
} else if j == nil && (h.PendingCombinedSubmissionID != "" || stringValue(mapValue(h.Subject.Payload["sliceWorkflow"])["status"]) == "processing") {
|
|
busy = true
|
|
}
|
|
}
|
|
for _, id := range task.Plan.RunIDs {
|
|
r, e := s.store.GetRun(ctx, id)
|
|
if e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
if r != nil && r.ExecutionLeaseOwner != "" {
|
|
until, e := time.Parse(time.RFC3339Nano, r.ExecutionLeaseExpiresAt)
|
|
if e != nil || until.After(time.Now()) {
|
|
busy = true
|
|
}
|
|
}
|
|
}
|
|
if !busy {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
fail(fmt.Errorf("相关处理仍未退出,未开始文件删除,请稍后重试"))
|
|
return
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
fail(ctx.Err())
|
|
return
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
// Re-evaluate after writers drain. New roots require a fresh confirmation;
|
|
// retries keep the original tombstone ID set to prevent resurrection.
|
|
fresh, e := s.buildSupplierDeletionPreview(ctx, task.OrganizationID, task.Plan.SupplierID)
|
|
if e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
if task.Phase != "records" {
|
|
for col, ids := range fresh.Records {
|
|
for _, id := range ids {
|
|
found := false
|
|
for _, old := range task.Plan.Records[col] {
|
|
if old == id {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
task.Plan = fresh
|
|
fail(fmt.Errorf("停止期间删除范围发生变化,请核对新清单并重新确认"))
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
task.WritesClosed = true
|
|
task.Phase = "files"
|
|
if e = persist(); e != nil {
|
|
return
|
|
}
|
|
for i := range task.Plan.Targets {
|
|
target := &task.Plan.Targets[i]
|
|
if target.Kind == "runtime" || target.Status == "done" {
|
|
continue
|
|
}
|
|
if e = ctx.Err(); e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
// Every retry checks for newly shared files again before touching storage.
|
|
safe := false
|
|
for _, f := range fresh.Targets {
|
|
if f.ID == target.ID {
|
|
safe = true
|
|
}
|
|
}
|
|
if !safe {
|
|
fail(fmt.Errorf("文件引用发生变化,已停止删除:%s", target.Prefix))
|
|
return
|
|
}
|
|
target.Status = "running"
|
|
if e = persist(); e != nil {
|
|
return
|
|
}
|
|
if e = s.deleteSupplierFile(ctx, task.Plan, *target); e != nil {
|
|
target.Error = e.Error()
|
|
fail(e)
|
|
return
|
|
}
|
|
target.Status = "done"
|
|
target.Error = ""
|
|
if e = persist(); e != nil {
|
|
return
|
|
}
|
|
}
|
|
task.Phase = "records"
|
|
if e = persist(); e != nil {
|
|
return
|
|
}
|
|
if e = s.store.ExecuteSupplierDeletionRecords(ctx, task, owner); e != nil {
|
|
fail(e)
|
|
return
|
|
}
|
|
// The records phase deletes employee accounts and their sessions directly
|
|
// in the store; flush cached principals so revoked access is immediate.
|
|
invalidateAuthCache()
|
|
task.Phase = "completed"
|
|
task.Status = "completed"
|
|
task.Error = ""
|
|
_ = persist()
|
|
}
|