droboticflow/internal/api/supplier_deletion.go
d-robotics da446d13b5 ⚡ Fix slow page loading: static gzip, list pagination, auth cache, dashboard aggregation, video direct link
- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix
- C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown
- C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation
- C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache
- C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback
- S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed
- Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush
- Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500

Co-authored-by: AgentTeams slow-page-fix
2026-10-09 17:45:27 -07:00

694 lines
20 KiB
Go

package api
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"regexp"
"sort"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"github.com/go-chi/chi/v5"
)
func supplierDeletionID(id string) string { return "supplier-deletion-" + id }
func supplierDeletionResponse(t *domain.SupplierDeletionTask) *domain.SupplierDeletionTask {
if t == nil {
return nil
}
v := *t
v.Retryable = t.Status != "completed" && (t.LeaseOwner == "" || t.LeaseUntil <= deletionLeaseTime(time.Now()))
if v.Status == "running" && v.Retryable {
v.Status = "interrupted"
}
return &v
}
func (s *server) supplierDeletionAdmin(w http.ResponseWriter, r *http.Request) bool {
p, _ := currentPrincipal(r)
if !isOrganizationAdmin(p) {
renderJSON(w, 403, map[string]string{"error": "仅管理员可级联删除供应商"})
return false
}
return true
}
// Bound concurrent scans and release all work when the requesting browser leaves.
var supplierPreviewSlots = make(chan struct{}, 2)
var supplierRunReferencePattern = regexp.MustCompile(`run-[A-Za-z0-9][A-Za-z0-9_-]{0,190}`)
func (s *server) buildSupplierDeletionPreview(ctx context.Context, org, id string) (domain.SupplierDeletionPlan, error) {
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
ctx = store.WithSupplierDeletionReadScope(ctx)
select {
case supplierPreviewSlots <- struct{}{}:
defer func() { <-supplierPreviewSlots }()
case <-ctx.Done():
return domain.SupplierDeletionPlan{}, ctx.Err()
}
p, e := s.store.PreviewSupplierDeletion(ctx, org, id)
if e != nil {
return p, e
}
p.ScopeDigest = p.Digest
for _, pid := range p.ProjectIDs {
project, e := s.store.GetProject(ctx, pid)
if e != nil {
return p, e
}
if project != nil {
p.ProjectNames = append(p.ProjectNames, project.Name)
}
}
if len(p.RunIDs) == 0 {
p.RunIDs = []string{}
p.Counts["fileTargets"], p.Counts["runningTasks"] = 0, 0
store.FinalizeSupplierDeletionPlan(&p)
return p, ctx.Err()
}
runs, e := s.store.ListSupplierDeletionRunReferences(ctx)
if e != nil {
return p, e
}
workflows, e := s.store.ListAllWorkflows(ctx)
if e != nil {
return p, e
}
captures, e := s.store.ListCaptureRecords(ctx, org, "", 0)
if e != nil {
return p, e
}
owns := map[string]bool{}
for _, id := range p.RunIDs {
owns[id] = true
}
byID := map[string]domain.RunRecord{}
runJSON := map[string]string{}
runsByWorkflow := map[string][]domain.RunRecord{}
for _, r := range runs {
if err := ctx.Err(); err != nil {
return p, err
}
byID[r.ID] = r
raw, _ := json.Marshal(r)
runJSON[r.ID] = string(raw)
runsByWorkflow[r.WorkflowID] = append(runsByWorkflow[r.WorkflowID], r)
}
// Index foreign references once, including across projects/organizations.
// A newly shared producer is queued so transitive sharing is preserved.
queue := []string{}
for _, r := range runs {
if !owns[r.ID] {
queue = append(queue, r.ID)
}
}
for head := 0; head < len(queue); head++ {
if err := ctx.Err(); err != nil {
return p, err
}
for _, id := range supplierRunReferencePattern.FindAllString(runJSON[queue[head]], -1) {
if owns[id] {
delete(owns, id)
p.SharedRunIDs = append(p.SharedRunIDs, id)
p.SharedRecords["runs"] = append(p.SharedRecords["runs"], id)
p.Preserved = append(p.Preserved, "其他运行引用的产物及运行记录:"+id)
queue = append(queue, id)
}
}
}
p.RunIDs = []string{}
for id := range owns {
p.RunIDs = append(p.RunIDs, id)
}
sort.Strings(p.RunIDs)
p.Records["runs"] = p.RunIDs
p.Counts["runs"] = len(p.RunIDs)
add := func(kind, run, conn, bucket, prefix string) {
sum := sha256.Sum256([]byte(kind + "|" + run + "|" + conn + "|" + bucket + "|" + prefix))
for _, old := range p.Targets {
if old.ID == hex.EncodeToString(sum[:12]) {
return
}
}
p.Targets = append(p.Targets, domain.ProjectDeletionTarget{ID: hex.EncodeToString(sum[:12]), Kind: kind, RunID: run, ConnectionID: conn, Bucket: bucket, Prefix: prefix, Status: "pending"})
}
type retainedPath struct{ connection, bucket, prefix string }
retainedPaths := []retainedPath{}
uncertainStores := map[string]bool{}
for _, w := range workflows {
if err := ctx.Err(); err != nil {
return p, err
}
retained := []domain.RunRecord{}
for _, r := range runsByWorkflow[w.ID] {
if !owns[r.ID] {
retained = append(retained, r)
}
}
if len(retained) == 0 {
continue
}
policy, e := s.effectiveArtifactStorage(ctx, &w)
if e != nil {
return p, e
}
if policy == nil {
continue
}
prefixes, safe := workflowGeneratedArtifactPrefixes(&w, retained, policy)
if !safe {
uncertainStores[policy.ConnectionID+"|"+policy.Bucket] = true
}
for _, prefix := range prefixes {
retainedPaths = append(retainedPaths, retainedPath{policy.ConnectionID, policy.Bucket, prefix})
}
}
addBOS := func(run, connection, bucket, prefix string) {
if uncertainStores[connection+"|"+bucket] {
p.Preserved = append(p.Preserved, "其他运行产物范围不明确,保留目录:"+prefix)
return
}
for _, other := range retainedPaths {
if other.connection == connection && other.bucket == bucket && prefixOverlap(prefix, other.prefix) {
p.Preserved = append(p.Preserved, "其他供应商使用的产物目录:"+prefix)
return
}
}
if !validDeletionPrefix(prefix) || strings.HasPrefix(prefix, "models") || strings.HasPrefix(prefix, ".droboticflow-trash") || (strings.HasPrefix(prefix, "system/") && prefix != "system/runs/"+run) {
p.Preserved = append(p.Preserved, "无法确认独占的存储路径:"+prefix)
return
}
for _, c := range captures {
if c.Bucket == bucket && c.Prefix != "" && prefixOverlap(prefix, c.Prefix) {
p.Preserved = append(p.Preserved, "原始采集目录:"+prefix)
return
}
if c.Delivery != nil && c.Delivery.Bucket == bucket && !owns[c.RunID] && prefixOverlap(prefix, c.Delivery.Prefix) {
p.Preserved = append(p.Preserved, "共享交付目录:"+prefix)
return
}
}
for _, w := range workflows {
if cfg := w.CaptureAutomation; cfg != nil && cfg.Bucket == bucket && cfg.Prefix != "" && prefixOverlap(prefix, cfg.Prefix) {
p.Preserved = append(p.Preserved, "原始监听目录:"+prefix)
return
}
}
for _, r := range runs {
if owns[r.ID] {
continue
}
raw := runJSON[r.ID]
if strings.Contains(string(raw), "bos://"+bucket+"/"+strings.Trim(prefix, "/")) {
p.Preserved = append(p.Preserved, "其他运行引用的文件:"+prefix)
return
}
}
add("bos", run, connection, bucket, prefix)
}
p.Counts["runningTasks"] = 0
for _, id := range p.RunIDs {
if !deletionRunIDPattern.MatchString(id) {
p.Blockers = append(p.Blockers, "无法安全限定运行目录:"+id)
continue
}
add("runtime", id, "", "", "")
add("workspace", id, "", "", "system/runs/"+id)
if workflowRunBlocksDeletion(byID[id].Status) {
p.Counts["runningTasks"]++
}
if conn, bucket := os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET"); conn != "" && bucket != "" {
addBOS(id, conn, bucket, "system/runs/"+id)
}
}
for _, w := range workflows {
ownedRuns := []domain.RunRecord{}
foreignRun := false
for _, r := range runsByWorkflow[w.ID] {
if r.WorkflowID == w.ID {
if owns[r.ID] {
ownedRuns = append(ownedRuns, r)
} else {
foreignRun = true
}
}
}
if len(ownedRuns) == 0 {
continue
}
policy, e := s.effectiveArtifactStorage(ctx, &w)
if e != nil {
return p, e
}
if policy == nil {
continue
}
prefixes, safe := workflowGeneratedArtifactPrefixes(&w, ownedRuns, policy)
if !safe {
p.Preserved = append(p.Preserved, "产物路径归属不明确,保留工作流产物:"+w.ID)
continue
}
for _, prefix := range prefixes {
runID := ""
for _, r := range ownedRuns {
if strings.Contains(prefix, r.ID) {
runID = r.ID
}
}
if foreignRun && runID == "" {
p.Preserved = append(p.Preserved, "同项目其他供应商共用目录:"+prefix)
continue
}
addBOS(runID, policy.ConnectionID, policy.Bucket, prefix)
}
}
for _, c := range captures {
if owns[c.RunID] && c.Delivery != nil {
d := c.Delivery
addBOS(c.RunID, d.ConnectionID, d.Bucket, d.Prefix)
}
}
rank := map[string]int{"runtime": 0, "workspace": 1, "bos": 2}
sort.Slice(p.Targets, func(i, j int) bool {
if p.Targets[i].Kind != p.Targets[j].Kind {
return rank[p.Targets[i].Kind] < rank[p.Targets[j].Kind]
}
return p.Targets[i].ID < p.Targets[j].ID
})
p.Counts["fileTargets"] = 0
for _, t := range p.Targets {
if t.Kind != "runtime" {
p.Counts["fileTargets"]++
}
}
store.FinalizeSupplierDeletionPlan(&p)
return p, ctx.Err()
}
func (s *server) handleSupplierDeletionPreview(w http.ResponseWriter, r *http.Request) {
if !s.supplierDeletionAdmin(w, r) {
return
}
id := chi.URLParam(r, "supplierId")
p, e := s.buildSupplierDeletionPreview(r.Context(), s.requestOrganizationID(r), id)
if errors.Is(e, context.DeadlineExceeded) {
renderJSON(w, http.StatusRequestTimeout, map[string]string{"error": "删除范围核对超时,请稍后重试", "code": "supplier_preview_timeout"})
return
}
if errors.Is(e, context.Canceled) {
return
}
if e != nil {
renderSupplierMutationError(w, e)
return
}
renderJSON(w, 200, p)
}
func (s *server) handleSupplierDeletionsList(w http.ResponseWriter, r *http.Request) {
if !s.supplierDeletionAdmin(w, r) {
return
}
items, e := s.store.ListSupplierDeletions(r.Context(), s.requestOrganizationID(r))
if e != nil {
renderError(w, 500, e)
return
}
out := []*domain.SupplierDeletionTask{}
for _, v := range items {
out = append(out, supplierDeletionResponse(&v))
}
renderJSON(w, 200, map[string]any{"items": out})
}
func (s *server) handleSupplierDeletionGet(w http.ResponseWriter, r *http.Request) {
if !s.supplierDeletionAdmin(w, r) {
return
}
t, e := s.store.GetSupplierDeletion(r.Context(), chi.URLParam(r, "deletionId"))
if e != nil {
renderError(w, 500, e)
return
}
if t == nil || !s.filterByTenant(r, t.OrganizationID) {
renderJSON(w, 404, map[string]string{"error": "删除记录不存在"})
return
}
renderJSON(w, 200, supplierDeletionResponse(t))
}
func (s *server) handleSupplierDeletionStart(w http.ResponseWriter, r *http.Request) {
if !s.supplierDeletionAdmin(w, r) {
return
}
var input struct {
ConfirmationName string `json:"confirmationName"`
Digest string `json:"digest"`
StopRunning bool `json:"stopRunning"`
Reason string `json:"reason"`
}
if !decodeJSON(w, r, &input) {
return
}
org, id := s.requestOrganizationID(r), chi.URLParam(r, "supplierId")
task, e := s.store.GetSupplierDeletion(r.Context(), supplierDeletionID(id))
if e != nil {
renderError(w, 500, e)
return
}
if task != nil && !s.filterByTenant(r, task.OrganizationID) {
renderJSON(w, 404, map[string]string{"error": "供应商不存在"})
return
}
if task != nil && task.Status == "completed" {
renderJSON(w, 200, supplierDeletionResponse(task))
return
}
var plan domain.SupplierDeletionPlan
if task == nil {
plan, e = s.buildSupplierDeletionPreview(r.Context(), org, id)
} else {
plan = task.Plan
}
if e != nil {
renderSupplierMutationError(w, e)
return
}
if input.ConfirmationName != plan.SupplierName || input.Digest != plan.Digest {
renderJSON(w, 409, map[string]any{"error": "删除范围已变化或确认名称不正确,请刷新预览后确认", "plan": plan})
return
}
if len(plan.Blockers) > 0 {
renderJSON(w, 409, map[string]any{"error": "存在不能安全删除的共享关联", "plan": plan})
return
}
if len(plan.RunIDs) > 0 && !input.StopRunning {
renderJSON(w, 400, map[string]string{"error": "请确认停止相关运行任务后删除"})
return
}
if len(plan.RunIDs) > 0 {
if _, e = s.deletionRuntime(); e != nil {
renderError(w, 503, e)
return
}
}
if task == nil {
principal, _ := currentPrincipal(r)
task = &domain.SupplierDeletionTask{ID: supplierDeletionID(id), OrganizationID: org, Plan: plan, ActorUserID: principal.User.ID, Reason: strings.TrimSpace(input.Reason), StopRunning: input.StopRunning, Status: "prepared", Phase: "freezing", CreatedAt: nowISO(), UpdatedAt: nowISO()}
}
task, e = s.store.BeginSupplierDeletion(r.Context(), *task)
if e != nil {
renderJSON(w, 409, map[string]string{"error": e.Error()})
return
}
owner := domain.NewID("supplier-delete-owner")
claimed, ok, e := s.store.ClaimSupplierDeletion(r.Context(), task.ID, owner, deletionLeaseTime(time.Now()), deletionLeaseTime(time.Now().Add(2*time.Minute)))
if e != nil {
renderError(w, 500, e)
return
}
if ok {
claimed.StopRunning = input.StopRunning
go s.executeSupplierDeletion(*claimed, owner)
}
renderJSON(w, 202, supplierDeletionResponse(claimed))
}
// The platform workspace connection may be registered under the platform's
// tenant. Its configured bucket is shared infrastructure, but only an approved
// exclusive run's exact generated directory is eligible for this exception.
func supplierDeletionPlatformWorkspace(p domain.SupplierDeletionPlan, t domain.ProjectDeletionTarget) bool {
connection, bucket := os.Getenv("PROJECT_DELETION_WORKSPACE_CONNECTION_ID"), os.Getenv("PROJECT_DELETION_WORKSPACE_BUCKET")
if connection == "" || bucket == "" || t.ConnectionID != connection || t.Bucket != bucket ||
!deletionRunIDPattern.MatchString(t.RunID) || t.Prefix != "system/runs/"+t.RunID {
return false
}
for _, id := range p.RunIDs {
if id == t.RunID {
return true
}
}
return false
}
func (s *server) deleteSupplierFile(ctx context.Context, p domain.SupplierDeletionPlan, t domain.ProjectDeletionTarget) error {
if t.Kind != "bos" {
return s.deleteProjectTarget(ctx, domain.ProjectDeletionPlan{}, t)
}
browser, conn, cleanup, e := s.resolveObjectStorageBrowser(ctx, t.ConnectionID)
if e != nil {
return e
}
if cleanup != nil {
defer cleanup()
}
if browser == nil || conn == nil || (!annotationOrganizationEqual(conn.OrganizationID, p.OrganizationID) && !supplierDeletionPlatformWorkspace(p, t)) {
return fmt.Errorf("文件存储连接归属不匹配")
}
bucket, e := resolveObjectStorageBucket(conn, t.Bucket)
if e != nil {
return e
}
if e = browser.DeletePrefix(ctx, bucket, t.Prefix); e != nil && !isBOSNoSuchKey(e) {
return e
}
objects, e := browser.ListObjects(ctx, bucket, strings.TrimSuffix(t.Prefix, "/")+"/")
if e != nil && !isBOSNoSuchKey(e) {
return e
}
if len(objects) > 0 {
return fmt.Errorf("目录仍有文件,稍后重试")
}
return nil
}
func (s *server) executeSupplierDeletion(task domain.SupplierDeletionTask, owner string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
done := make(chan struct{})
defer close(done)
var saveMu sync.Mutex
go func() {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case <-ticker.C:
ok, e := s.store.RenewSupplierDeletion(ctx, task.ID, owner, deletionLeaseTime(time.Now().Add(2*time.Minute)))
if e != nil || !ok {
cancel()
return
}
}
}
}()
persist := func() error {
saveMu.Lock()
defer saveMu.Unlock()
task.UpdatedAt = nowISO()
task.LeaseUntil = deletionLeaseTime(time.Now().Add(2 * time.Minute))
ok, e := s.store.SaveSupplierDeletion(context.WithoutCancel(ctx), task, owner)
if e != nil {
return e
}
if !ok {
return fmt.Errorf("删除执行权已失效")
}
return nil
}
fail := func(e error) { task.Status = "failed"; task.Error = e.Error(); _ = persist() }
if !task.WritesClosed {
task.Phase = "stopping"
}
if e := persist(); e != nil {
return
}
for _, id := range task.Plan.WorkflowIDs {
w, e := s.store.GetWorkflow(ctx, id)
if e != nil {
fail(e)
return
}
if w != nil && w.CaptureAutomation != nil {
w.CaptureAutomation.Enabled = false
if e = s.store.UpsertWorkflow(ctx, *w); e != nil {
fail(e)
return
}
}
}
for _, id := range task.Plan.RunIDs {
r, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if r != nil && (workflowRunBlocksDeletion(r.Status) || r.Status == domain.RunStatusWaitingForHuman || r.Status == domain.RunStatusWaitingForAudit) {
if !task.StopRunning {
fail(fmt.Errorf("未确认停止运行任务"))
return
}
if _, e = s.store.RequestRunStop(ctx, id, domain.RunStopModeForce); e != nil {
fail(e)
return
}
}
}
for i := range task.Plan.Targets {
target := &task.Plan.Targets[i]
if target.Kind != "runtime" {
continue
}
if e := s.deleteSupplierFile(ctx, task.Plan, *target); e != nil {
fail(e)
return
}
target.Status = "done"
if e := persist(); e != nil {
return
}
}
if e := s.store.CancelSupplierQueuedProcessing(ctx, task.Plan); e != nil {
fail(e)
return
}
deadline := time.Now().Add(3 * time.Minute)
for {
busy, e := s.store.SupplierIndexingActive(ctx, task.Plan)
if e != nil {
fail(e)
return
}
for _, id := range task.Plan.Records["humanItems"] {
h, e := s.store.GetHumanItem(ctx, id)
if e != nil {
fail(e)
return
}
if h == nil {
continue
}
if j := h.ProcessingJob; j != nil && (j.Status == "running" || j.Status == "queued") {
busy = true
} else if j == nil && (h.PendingCombinedSubmissionID != "" || stringValue(mapValue(h.Subject.Payload["sliceWorkflow"])["status"]) == "processing") {
busy = true
}
}
for _, id := range task.Plan.RunIDs {
r, e := s.store.GetRun(ctx, id)
if e != nil {
fail(e)
return
}
if r != nil && r.ExecutionLeaseOwner != "" {
until, e := time.Parse(time.RFC3339Nano, r.ExecutionLeaseExpiresAt)
if e != nil || until.After(time.Now()) {
busy = true
}
}
}
if !busy {
break
}
if time.Now().After(deadline) {
fail(fmt.Errorf("相关处理仍未退出,未开始文件删除,请稍后重试"))
return
}
select {
case <-ctx.Done():
fail(ctx.Err())
return
case <-time.After(time.Second):
}
}
// Re-evaluate after writers drain. New roots require a fresh confirmation;
// retries keep the original tombstone ID set to prevent resurrection.
fresh, e := s.buildSupplierDeletionPreview(ctx, task.OrganizationID, task.Plan.SupplierID)
if e != nil {
fail(e)
return
}
if task.Phase != "records" {
for col, ids := range fresh.Records {
for _, id := range ids {
found := false
for _, old := range task.Plan.Records[col] {
if old == id {
found = true
break
}
}
if !found {
task.Plan = fresh
fail(fmt.Errorf("停止期间删除范围发生变化,请核对新清单并重新确认"))
return
}
}
}
}
task.WritesClosed = true
task.Phase = "files"
if e = persist(); e != nil {
return
}
for i := range task.Plan.Targets {
target := &task.Plan.Targets[i]
if target.Kind == "runtime" || target.Status == "done" {
continue
}
if e = ctx.Err(); e != nil {
fail(e)
return
}
// Every retry checks for newly shared files again before touching storage.
safe := false
for _, f := range fresh.Targets {
if f.ID == target.ID {
safe = true
}
}
if !safe {
fail(fmt.Errorf("文件引用发生变化,已停止删除:%s", target.Prefix))
return
}
target.Status = "running"
if e = persist(); e != nil {
return
}
if e = s.deleteSupplierFile(ctx, task.Plan, *target); e != nil {
target.Error = e.Error()
fail(e)
return
}
target.Status = "done"
target.Error = ""
if e = persist(); e != nil {
return
}
}
task.Phase = "records"
if e = persist(); e != nil {
return
}
if e = s.store.ExecuteSupplierDeletionRecords(ctx, task, owner); e != nil {
fail(e)
return
}
// The records phase deletes employee accounts and their sessions directly
// in the store; flush cached principals so revoked access is immediate.
invalidateAuthCache()
task.Phase = "completed"
task.Status = "completed"
task.Error = ""
_ = persist()
}