droboticflow/internal/api/object_storage_move.go

381 lines
13 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package api
import (
"context"
"fmt"
"net/http"
"sort"
"strings"
"sync"
"time"
"droboticflow/internal/domain"
"github.com/go-chi/chi/v5"
)
type objectStorageCaptureAutomationConflict struct {
WorkflowID string `json:"workflowId"`
WorkflowName string `json:"workflowName"`
Prefix string `json:"prefix"`
}
type objectStorageMoveJob struct {
mu sync.Mutex
id string
organizationID string
connectionID string
bucket string
items []domain.ObjectStorageObjectMoveItem
force bool
status string
totalObjects int
copiedObjects int
skippedObjects int
deletedObjects int
currentKey string
error string
result *domain.ObjectStorageObjectSummary
createdAt time.Time
updatedAt time.Time
}
func (job *objectStorageMoveJob) snapshot() map[string]any {
job.mu.Lock()
defer job.mu.Unlock()
snapshot := map[string]any{
"moveId": job.id,
"connectionId": job.connectionID,
"bucket": job.bucket,
"status": job.status,
"totalObjects": job.totalObjects,
"copiedObjects": job.copiedObjects,
"skippedObjects": job.skippedObjects,
"deletedObjects": job.deletedObjects,
"createdAt": job.createdAt.Format(time.RFC3339),
"updatedAt": job.updatedAt.Format(time.RFC3339),
}
if job.currentKey != "" {
snapshot["currentKey"] = job.currentKey
}
if job.error != "" {
snapshot["error"] = job.error
}
if job.result != nil {
snapshot["result"] = *job.result
}
return snapshot
}
func (job *objectStorageMoveJob) update(mutate func(*objectStorageMoveJob)) {
job.mu.Lock()
defer job.mu.Unlock()
mutate(job)
job.updatedAt = time.Now().UTC()
}
type objectStorageMoveJobRegistry struct {
mu sync.Mutex
jobs map[string]*objectStorageMoveJob
ttl time.Duration
}
func (registry *objectStorageMoveJobRegistry) create(job *objectStorageMoveJob) {
registry.mu.Lock()
defer registry.mu.Unlock()
now := time.Now().UTC()
for id, existing := range registry.jobs {
if now.Sub(existing.createdAt) > registry.ttl {
delete(registry.jobs, id)
}
}
registry.jobs[job.id] = job
}
func (registry *objectStorageMoveJobRegistry) get(id string) *objectStorageMoveJob {
registry.mu.Lock()
defer registry.mu.Unlock()
job := registry.jobs[id]
return job
}
var objectStorageMoveJobs = &objectStorageMoveJobRegistry{jobs: map[string]*objectStorageMoveJob{}, ttl: 2 * time.Hour}
// objectStorageMovePrecheckIssues validates a batch move before any storage
// call: it rejects no-op moves (destination identical to the source key) and
// selections that contain parent/child overlaps, where moving the parent would
// also move the child and the child move would then fail with NoSuchKey.
func objectStorageMovePrecheckIssues(items []domain.ObjectStorageObjectMoveItem, destinationPrefix string) []validationIssue {
destinationPrefix = strings.Trim(strings.TrimSpace(destinationPrefix), "/")
type plannedMove struct {
sourceKey string
destinationKey string
}
planned := make([]plannedMove, 0, len(items))
issues := make([]validationIssue, 0)
for index, item := range items {
sourceKey := strings.Trim(strings.TrimSpace(item.Key), "/")
if sourceKey == "" {
issues = append(issues, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "key is required"})
continue
}
destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix)
if destinationKey == sourceKey {
issues = append(issues, validationIssue{
Field: fmt.Sprintf("items[%d].key", index),
Code: "move_destination_same_as_source",
Message: fmt.Sprintf("%s 已经位于目标前缀 %s 下,该移动不会产生任何变化;请输入其他目标前缀", sourceKey, destinationPrefix),
})
continue
}
if destinationPrefix != "" && strings.HasPrefix(destinationPrefix+"/", sourceKey+"/") {
issues = append(issues, validationIssue{
Field: fmt.Sprintf("items[%d].key", index),
Code: "move_destination_inside_source",
Message: fmt.Sprintf("目标前缀 %s 位于源 %s 内部,不能把目录移动到它自己的子目录下", destinationPrefix, sourceKey),
})
continue
}
planned = append(planned, plannedMove{sourceKey: sourceKey, destinationKey: destinationKey})
}
sort.Slice(planned, func(i, j int) bool { return planned[i].sourceKey < planned[j].sourceKey })
for i := 1; i < len(planned); i++ {
if strings.HasPrefix(planned[i].sourceKey+"/", planned[i-1].sourceKey+"/") {
issues = append(issues, validationIssue{
Field: "items",
Code: "move_items_overlap",
Message: fmt.Sprintf("勾选项之间存在父子重叠:%s 包含 %s,请分开移动", planned[i-1].sourceKey, planned[i].sourceKey),
})
break
}
}
return issues
}
func objectStoragePrefixOverlaps(prefix string, key string) bool {
prefix = strings.Trim(strings.TrimSpace(prefix), "/")
key = strings.Trim(strings.TrimSpace(key), "/")
if prefix == "" || key == "" {
return false
}
return prefix == key || strings.HasPrefix(key, prefix+"/") || strings.HasPrefix(prefix, key+"/")
}
func (server *server) objectStorageCaptureAutomationConflicts(ctx context.Context, organizationID string, connectionID string, bucket string, prefixes []string) []objectStorageCaptureAutomationConflict {
workflows, err := server.store.ListEnabledCaptureAutomations(ctx, organizationID)
if err != nil {
return nil
}
conflicts := make([]objectStorageCaptureAutomationConflict, 0)
for _, workflow := range workflows {
automation := workflow.CaptureAutomation
if automation == nil || !automation.Enabled {
continue
}
if strings.TrimSpace(automation.ConnectionID) != strings.TrimSpace(connectionID) || strings.TrimSpace(automation.Bucket) != strings.TrimSpace(bucket) {
continue
}
for _, prefix := range prefixes {
if objectStoragePrefixOverlaps(automation.Prefix, prefix) {
conflicts = append(conflicts, objectStorageCaptureAutomationConflict{
WorkflowID: workflow.ID,
WorkflowName: workflow.Name,
Prefix: strings.Trim(strings.TrimSpace(automation.Prefix), "/"),
})
break
}
}
}
return conflicts
}
func renderObjectStorageCaptureAutomationConflict(writer http.ResponseWriter, conflicts []objectStorageCaptureAutomationConflict) {
names := make([]string, 0, len(conflicts))
for _, conflict := range conflicts {
names = append(names, conflict.WorkflowName)
}
renderJSON(writer, http.StatusConflict, map[string]any{
"error": fmt.Sprintf("以下持续接入(Capture)工作流正在监听所涉及的目录:%s。移动或删除这些数据会影响在跑的采集,请先暂停对应工作流的持续接入,或确认后强制执行。", strings.Join(names, "、")),
"code": "capture_automation_conflict",
"conflicts": conflicts,
"forceRequired": true,
})
}
func (server *server) handleObjectStorageMoveJobCreate(writer http.ResponseWriter, request *http.Request) {
input, err := decodeObjectStorageObjectRenameInput(request)
if err != nil {
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if browser == nil || connection == nil {
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
return
}
asyncOwnsBrowser := false
defer func() {
if !asyncOwnsBrowser {
cleanup()
}
}()
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
if err != nil {
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
return
}
destinationPrefix := strings.Trim(strings.TrimSpace(input.DestinationPrefix), "/")
if len(input.Items) == 0 || destinationPrefix == "" {
renderValidationError(writer, validationIssue{Field: "destinationPrefix", Message: "destinationPrefix and at least one item are required"})
return
}
if issues := objectStorageMovePrecheckIssues(input.Items, destinationPrefix); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
sourcePrefixes := make([]string, 0, len(input.Items)+1)
for _, item := range input.Items {
sourcePrefixes = append(sourcePrefixes, strings.Trim(strings.TrimSpace(item.Key), "/"))
}
sourcePrefixes = append(sourcePrefixes, destinationPrefix)
force := input.Force || strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("force")), "true")
if !force {
conflicts := server.objectStorageCaptureAutomationConflicts(request.Context(), connection.OrganizationID, connectionID, bucket, sourcePrefixes)
if len(conflicts) > 0 {
names := make([]string, 0, len(conflicts))
for _, conflict := range conflicts {
names = append(names, conflict.WorkflowName)
}
renderObjectStorageCaptureAutomationConflict(writer, conflicts)
return
}
}
moveID := domain.NewID("object-storage-move")
job := &objectStorageMoveJob{
id: moveID,
organizationID: normalizedOrganizationID(connection.OrganizationID),
connectionID: connectionID,
bucket: bucket,
items: input.Items,
force: input.Force,
status: "running",
totalObjects: len(input.Items),
createdAt: time.Now().UTC(),
}
objectStorageMoveJobs.create(job)
asyncOwnsBrowser = true
go func() {
defer cleanup()
defer objectStorageMoveJobs.removeAfterTTL(job.id, 2*time.Hour)
moveContext, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
defer cancel()
var moveErr error
var summary domain.ObjectStorageObjectSummary
copiedObjects := 0
skippedObjects := 0
deletedObjects := 0
totalObjects := 0
lastProgress := objectStorageMoveProgress{}
progress := func(state objectStorageMoveProgress) {
lastProgress = state
job.update(func(current *objectStorageMoveJob) {
current.totalObjects = totalObjects + state.TotalObjects
current.copiedObjects = copiedObjects + state.CopiedObjects
current.skippedObjects = skippedObjects + state.SkippedObjects
current.deletedObjects = deletedObjects + state.DeletedObjects
current.currentKey = state.CurrentKey
})
}
progressiveMover, hasProgressMover := browser.(interface {
MovePrefixWithProgress(ctx context.Context, bucket string, prefix string, newPrefix string, progress func(objectStorageMoveProgress)) (domain.ObjectStorageObjectSummary, error)
})
for _, item := range input.Items {
lastProgress = objectStorageMoveProgress{}
sourceKey := strings.Trim(strings.TrimSpace(item.Key), "/")
currentKey := sourceKey
job.update(func(current *objectStorageMoveJob) { current.currentKey = currentKey })
destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix)
if strings.EqualFold(strings.TrimSpace(item.Kind), "directory") || item.Recursive {
if hasProgressMover {
summary, moveErr = progressiveMover.MovePrefixWithProgress(moveContext, bucket, sourceKey, destinationKey, progress)
} else {
summary, moveErr = browser.MovePrefix(moveContext, bucket, sourceKey, destinationKey)
}
} else {
summary, moveErr = browser.RenameObject(moveContext, bucket, sourceKey, destinationKey)
}
if moveErr != nil {
if isBOSNoSuchKey(moveErr) {
continue
}
break
}
if lastProgress.TotalObjects > 0 {
totalObjects += lastProgress.TotalObjects
copiedObjects += lastProgress.CopiedObjects
skippedObjects += lastProgress.SkippedObjects
deletedObjects += lastProgress.DeletedObjects
} else {
totalObjects++
copiedObjects++
deletedObjects++
}
}
job.update(func(current *objectStorageMoveJob) {
current.currentKey = ""
if moveErr != nil {
current.status = "failed"
current.error = moveErr.Error()
} else {
current.totalObjects = totalObjects
current.copiedObjects = copiedObjects
current.skippedObjects = skippedObjects
current.deletedObjects = deletedObjects
current.status = "completed"
current.result = &summary
}
current.updatedAt = time.Now().UTC()
})
}()
renderJSON(writer, http.StatusAccepted, map[string]any{"moveId": moveID, "status": "running"})
}
func (server *server) handleObjectStorageMoveJobGet(writer http.ResponseWriter, request *http.Request) {
moveID := strings.TrimSpace(chi.URLParam(request, "moveId"))
job := objectStorageMoveJobs.get(moveID)
if job == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "move job not found or expired"})
return
}
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
if connectionID != "" && job.connectionID != connectionID {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "move job not found or expired"})
return
}
renderJSON(writer, http.StatusOK, job.snapshot())
}
func (registry *objectStorageMoveJobRegistry) save(job *objectStorageMoveJob) {
registry.mu.Lock()
defer registry.mu.Unlock()
registry.jobs[job.id] = job
}
func (registry *objectStorageMoveJobRegistry) removeAfterTTL(id string, ttl time.Duration) {
registry.mu.Lock()
defer registry.mu.Unlock()
existing, ok := registry.jobs[id]
if !ok {
return
}
if time.Now().UTC().Sub(existing.updatedAt) < ttl {
return
}
delete(registry.jobs, id)
}