package api import ( "context" "fmt" "net/http" "sort" "strings" "sync" "time" "droboticflow/internal/domain" "github.com/go-chi/chi/v5" ) type objectStorageCaptureAutomationConflict struct { WorkflowID string `json:"workflowId"` WorkflowName string `json:"workflowName"` Prefix string `json:"prefix"` } type objectStorageMoveJob struct { mu sync.Mutex id string organizationID string connectionID string bucket string items []domain.ObjectStorageObjectMoveItem force bool status string totalObjects int copiedObjects int skippedObjects int deletedObjects int currentKey string error string result *domain.ObjectStorageObjectSummary createdAt time.Time updatedAt time.Time } func (job *objectStorageMoveJob) snapshot() map[string]any { job.mu.Lock() defer job.mu.Unlock() snapshot := map[string]any{ "moveId": job.id, "connectionId": job.connectionID, "bucket": job.bucket, "status": job.status, "totalObjects": job.totalObjects, "copiedObjects": job.copiedObjects, "skippedObjects": job.skippedObjects, "deletedObjects": job.deletedObjects, "createdAt": job.createdAt.Format(time.RFC3339), "updatedAt": job.updatedAt.Format(time.RFC3339), } if job.currentKey != "" { snapshot["currentKey"] = job.currentKey } if job.error != "" { snapshot["error"] = job.error } if job.result != nil { snapshot["result"] = *job.result } return snapshot } func (job *objectStorageMoveJob) update(mutate func(*objectStorageMoveJob)) { job.mu.Lock() defer job.mu.Unlock() mutate(job) job.updatedAt = time.Now().UTC() } type objectStorageMoveJobRegistry struct { mu sync.Mutex jobs map[string]*objectStorageMoveJob ttl time.Duration } func (registry *objectStorageMoveJobRegistry) create(job *objectStorageMoveJob) { registry.mu.Lock() defer registry.mu.Unlock() now := time.Now().UTC() for id, existing := range registry.jobs { if now.Sub(existing.createdAt) > registry.ttl { delete(registry.jobs, id) } } registry.jobs[job.id] = job } func (registry *objectStorageMoveJobRegistry) get(id string) *objectStorageMoveJob { registry.mu.Lock() defer registry.mu.Unlock() job := registry.jobs[id] return job } var objectStorageMoveJobs = &objectStorageMoveJobRegistry{jobs: map[string]*objectStorageMoveJob{}, ttl: 2 * time.Hour} // objectStorageMovePrecheckIssues validates a batch move before any storage // call: it rejects no-op moves (destination identical to the source key) and // selections that contain parent/child overlaps, where moving the parent would // also move the child and the child move would then fail with NoSuchKey. func objectStorageMovePrecheckIssues(items []domain.ObjectStorageObjectMoveItem, destinationPrefix string) []validationIssue { destinationPrefix = strings.Trim(strings.TrimSpace(destinationPrefix), "/") type plannedMove struct { sourceKey string destinationKey string } planned := make([]plannedMove, 0, len(items)) issues := make([]validationIssue, 0) for index, item := range items { sourceKey := strings.Trim(strings.TrimSpace(item.Key), "/") if sourceKey == "" { issues = append(issues, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "key is required"}) continue } destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix) if destinationKey == sourceKey { issues = append(issues, validationIssue{ Field: fmt.Sprintf("items[%d].key", index), Code: "move_destination_same_as_source", Message: fmt.Sprintf("%s 已经位于目标前缀 %s 下,该移动不会产生任何变化;请输入其他目标前缀", sourceKey, destinationPrefix), }) continue } if destinationPrefix != "" && strings.HasPrefix(destinationPrefix+"/", sourceKey+"/") { issues = append(issues, validationIssue{ Field: fmt.Sprintf("items[%d].key", index), Code: "move_destination_inside_source", Message: fmt.Sprintf("目标前缀 %s 位于源 %s 内部,不能把目录移动到它自己的子目录下", destinationPrefix, sourceKey), }) continue } planned = append(planned, plannedMove{sourceKey: sourceKey, destinationKey: destinationKey}) } sort.Slice(planned, func(i, j int) bool { return planned[i].sourceKey < planned[j].sourceKey }) for i := 1; i < len(planned); i++ { if strings.HasPrefix(planned[i].sourceKey+"/", planned[i-1].sourceKey+"/") { issues = append(issues, validationIssue{ Field: "items", Code: "move_items_overlap", Message: fmt.Sprintf("勾选项之间存在父子重叠:%s 包含 %s,请分开移动", planned[i-1].sourceKey, planned[i].sourceKey), }) break } } return issues } func objectStoragePrefixOverlaps(prefix string, key string) bool { prefix = strings.Trim(strings.TrimSpace(prefix), "/") key = strings.Trim(strings.TrimSpace(key), "/") if prefix == "" || key == "" { return false } return prefix == key || strings.HasPrefix(key, prefix+"/") || strings.HasPrefix(prefix, key+"/") } func (server *server) objectStorageCaptureAutomationConflicts(ctx context.Context, organizationID string, connectionID string, bucket string, prefixes []string) []objectStorageCaptureAutomationConflict { workflows, err := server.store.ListEnabledCaptureAutomations(ctx, organizationID) if err != nil { return nil } conflicts := make([]objectStorageCaptureAutomationConflict, 0) for _, workflow := range workflows { automation := workflow.CaptureAutomation if automation == nil || !automation.Enabled { continue } if strings.TrimSpace(automation.ConnectionID) != strings.TrimSpace(connectionID) || strings.TrimSpace(automation.Bucket) != strings.TrimSpace(bucket) { continue } for _, prefix := range prefixes { if objectStoragePrefixOverlaps(automation.Prefix, prefix) { conflicts = append(conflicts, objectStorageCaptureAutomationConflict{ WorkflowID: workflow.ID, WorkflowName: workflow.Name, Prefix: strings.Trim(strings.TrimSpace(automation.Prefix), "/"), }) break } } } return conflicts } func renderObjectStorageCaptureAutomationConflict(writer http.ResponseWriter, conflicts []objectStorageCaptureAutomationConflict) { names := make([]string, 0, len(conflicts)) for _, conflict := range conflicts { names = append(names, conflict.WorkflowName) } renderJSON(writer, http.StatusConflict, map[string]any{ "error": fmt.Sprintf("以下持续接入(Capture)工作流正在监听所涉及的目录:%s。移动或删除这些数据会影响在跑的采集,请先暂停对应工作流的持续接入,或确认后强制执行。", strings.Join(names, "、")), "code": "capture_automation_conflict", "conflicts": conflicts, "forceRequired": true, }) } func (server *server) handleObjectStorageMoveJobCreate(writer http.ResponseWriter, request *http.Request) { input, err := decodeObjectStorageObjectRenameInput(request) if err != nil { renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"}) return } connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if browser == nil || connection == nil { renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"}) return } asyncOwnsBrowser := false defer func() { if !asyncOwnsBrowser { cleanup() } }() bucket, err := resolveObjectStorageBucket(connection, input.Bucket) if err != nil { renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) return } destinationPrefix := strings.Trim(strings.TrimSpace(input.DestinationPrefix), "/") if len(input.Items) == 0 || destinationPrefix == "" { renderValidationError(writer, validationIssue{Field: "destinationPrefix", Message: "destinationPrefix and at least one item are required"}) return } if issues := objectStorageMovePrecheckIssues(input.Items, destinationPrefix); len(issues) > 0 { renderValidationError(writer, issues...) return } sourcePrefixes := make([]string, 0, len(input.Items)+1) for _, item := range input.Items { sourcePrefixes = append(sourcePrefixes, strings.Trim(strings.TrimSpace(item.Key), "/")) } sourcePrefixes = append(sourcePrefixes, destinationPrefix) force := input.Force || strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("force")), "true") if !force { conflicts := server.objectStorageCaptureAutomationConflicts(request.Context(), connection.OrganizationID, connectionID, bucket, sourcePrefixes) if len(conflicts) > 0 { names := make([]string, 0, len(conflicts)) for _, conflict := range conflicts { names = append(names, conflict.WorkflowName) } renderObjectStorageCaptureAutomationConflict(writer, conflicts) return } } moveID := domain.NewID("object-storage-move") job := &objectStorageMoveJob{ id: moveID, organizationID: normalizedOrganizationID(connection.OrganizationID), connectionID: connectionID, bucket: bucket, items: input.Items, force: input.Force, status: "running", totalObjects: len(input.Items), createdAt: time.Now().UTC(), } objectStorageMoveJobs.create(job) asyncOwnsBrowser = true go func() { defer cleanup() defer objectStorageMoveJobs.removeAfterTTL(job.id, 2*time.Hour) moveContext, cancel := context.WithTimeout(context.Background(), 6*time.Hour) defer cancel() var moveErr error var summary domain.ObjectStorageObjectSummary copiedObjects := 0 skippedObjects := 0 deletedObjects := 0 totalObjects := 0 lastProgress := objectStorageMoveProgress{} progress := func(state objectStorageMoveProgress) { lastProgress = state job.update(func(current *objectStorageMoveJob) { current.totalObjects = totalObjects + state.TotalObjects current.copiedObjects = copiedObjects + state.CopiedObjects current.skippedObjects = skippedObjects + state.SkippedObjects current.deletedObjects = deletedObjects + state.DeletedObjects current.currentKey = state.CurrentKey }) } progressiveMover, hasProgressMover := browser.(interface { MovePrefixWithProgress(ctx context.Context, bucket string, prefix string, newPrefix string, progress func(objectStorageMoveProgress)) (domain.ObjectStorageObjectSummary, error) }) for _, item := range input.Items { lastProgress = objectStorageMoveProgress{} sourceKey := strings.Trim(strings.TrimSpace(item.Key), "/") currentKey := sourceKey job.update(func(current *objectStorageMoveJob) { current.currentKey = currentKey }) destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix) if strings.EqualFold(strings.TrimSpace(item.Kind), "directory") || item.Recursive { if hasProgressMover { summary, moveErr = progressiveMover.MovePrefixWithProgress(moveContext, bucket, sourceKey, destinationKey, progress) } else { summary, moveErr = browser.MovePrefix(moveContext, bucket, sourceKey, destinationKey) } } else { summary, moveErr = browser.RenameObject(moveContext, bucket, sourceKey, destinationKey) } if moveErr != nil { if isBOSNoSuchKey(moveErr) { continue } break } if lastProgress.TotalObjects > 0 { totalObjects += lastProgress.TotalObjects copiedObjects += lastProgress.CopiedObjects skippedObjects += lastProgress.SkippedObjects deletedObjects += lastProgress.DeletedObjects } else { totalObjects++ copiedObjects++ deletedObjects++ } } job.update(func(current *objectStorageMoveJob) { current.currentKey = "" if moveErr != nil { current.status = "failed" current.error = moveErr.Error() } else { current.totalObjects = totalObjects current.copiedObjects = copiedObjects current.skippedObjects = skippedObjects current.deletedObjects = deletedObjects current.status = "completed" current.result = &summary } current.updatedAt = time.Now().UTC() }) }() renderJSON(writer, http.StatusAccepted, map[string]any{"moveId": moveID, "status": "running"}) } func (server *server) handleObjectStorageMoveJobGet(writer http.ResponseWriter, request *http.Request) { moveID := strings.TrimSpace(chi.URLParam(request, "moveId")) job := objectStorageMoveJobs.get(moveID) if job == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "move job not found or expired"}) return } connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) if connectionID != "" && job.connectionID != connectionID { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "move job not found or expired"}) return } renderJSON(writer, http.StatusOK, job.snapshot()) } func (registry *objectStorageMoveJobRegistry) save(job *objectStorageMoveJob) { registry.mu.Lock() defer registry.mu.Unlock() registry.jobs[job.id] = job } func (registry *objectStorageMoveJobRegistry) removeAfterTTL(id string, ttl time.Duration) { registry.mu.Lock() defer registry.mu.Unlock() existing, ok := registry.jobs[id] if !ok { return } if time.Now().UTC().Sub(existing.updatedAt) < ttl { return } delete(registry.jobs, id) }