381 lines
13 KiB
Go
381 lines
13 KiB
Go
package api
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"net/http"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"droboticflow/internal/domain"
|
||
|
||
"github.com/go-chi/chi/v5"
|
||
)
|
||
|
||
type objectStorageCaptureAutomationConflict struct {
|
||
WorkflowID string `json:"workflowId"`
|
||
WorkflowName string `json:"workflowName"`
|
||
Prefix string `json:"prefix"`
|
||
}
|
||
|
||
type objectStorageMoveJob struct {
|
||
mu sync.Mutex
|
||
id string
|
||
organizationID string
|
||
connectionID string
|
||
bucket string
|
||
items []domain.ObjectStorageObjectMoveItem
|
||
force bool
|
||
status string
|
||
totalObjects int
|
||
copiedObjects int
|
||
skippedObjects int
|
||
deletedObjects int
|
||
currentKey string
|
||
error string
|
||
result *domain.ObjectStorageObjectSummary
|
||
createdAt time.Time
|
||
updatedAt time.Time
|
||
}
|
||
|
||
func (job *objectStorageMoveJob) snapshot() map[string]any {
|
||
job.mu.Lock()
|
||
defer job.mu.Unlock()
|
||
snapshot := map[string]any{
|
||
"moveId": job.id,
|
||
"connectionId": job.connectionID,
|
||
"bucket": job.bucket,
|
||
"status": job.status,
|
||
"totalObjects": job.totalObjects,
|
||
"copiedObjects": job.copiedObjects,
|
||
"skippedObjects": job.skippedObjects,
|
||
"deletedObjects": job.deletedObjects,
|
||
"createdAt": job.createdAt.Format(time.RFC3339),
|
||
"updatedAt": job.updatedAt.Format(time.RFC3339),
|
||
}
|
||
if job.currentKey != "" {
|
||
snapshot["currentKey"] = job.currentKey
|
||
}
|
||
if job.error != "" {
|
||
snapshot["error"] = job.error
|
||
}
|
||
if job.result != nil {
|
||
snapshot["result"] = *job.result
|
||
}
|
||
return snapshot
|
||
}
|
||
|
||
func (job *objectStorageMoveJob) update(mutate func(*objectStorageMoveJob)) {
|
||
job.mu.Lock()
|
||
defer job.mu.Unlock()
|
||
mutate(job)
|
||
job.updatedAt = time.Now().UTC()
|
||
}
|
||
|
||
type objectStorageMoveJobRegistry struct {
|
||
mu sync.Mutex
|
||
jobs map[string]*objectStorageMoveJob
|
||
ttl time.Duration
|
||
}
|
||
|
||
func (registry *objectStorageMoveJobRegistry) create(job *objectStorageMoveJob) {
|
||
registry.mu.Lock()
|
||
defer registry.mu.Unlock()
|
||
now := time.Now().UTC()
|
||
for id, existing := range registry.jobs {
|
||
if now.Sub(existing.createdAt) > registry.ttl {
|
||
delete(registry.jobs, id)
|
||
}
|
||
}
|
||
registry.jobs[job.id] = job
|
||
}
|
||
|
||
func (registry *objectStorageMoveJobRegistry) get(id string) *objectStorageMoveJob {
|
||
registry.mu.Lock()
|
||
defer registry.mu.Unlock()
|
||
job := registry.jobs[id]
|
||
return job
|
||
}
|
||
|
||
var objectStorageMoveJobs = &objectStorageMoveJobRegistry{jobs: map[string]*objectStorageMoveJob{}, ttl: 2 * time.Hour}
|
||
|
||
// objectStorageMovePrecheckIssues validates a batch move before any storage
|
||
// call: it rejects no-op moves (destination identical to the source key) and
|
||
// selections that contain parent/child overlaps, where moving the parent would
|
||
// also move the child and the child move would then fail with NoSuchKey.
|
||
func objectStorageMovePrecheckIssues(items []domain.ObjectStorageObjectMoveItem, destinationPrefix string) []validationIssue {
|
||
destinationPrefix = strings.Trim(strings.TrimSpace(destinationPrefix), "/")
|
||
type plannedMove struct {
|
||
sourceKey string
|
||
destinationKey string
|
||
}
|
||
planned := make([]plannedMove, 0, len(items))
|
||
issues := make([]validationIssue, 0)
|
||
for index, item := range items {
|
||
sourceKey := strings.Trim(strings.TrimSpace(item.Key), "/")
|
||
if sourceKey == "" {
|
||
issues = append(issues, validationIssue{Field: fmt.Sprintf("items[%d].key", index), Message: "key is required"})
|
||
continue
|
||
}
|
||
destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix)
|
||
if destinationKey == sourceKey {
|
||
issues = append(issues, validationIssue{
|
||
Field: fmt.Sprintf("items[%d].key", index),
|
||
Code: "move_destination_same_as_source",
|
||
Message: fmt.Sprintf("%s 已经位于目标前缀 %s 下,该移动不会产生任何变化;请输入其他目标前缀", sourceKey, destinationPrefix),
|
||
})
|
||
continue
|
||
}
|
||
if destinationPrefix != "" && strings.HasPrefix(destinationPrefix+"/", sourceKey+"/") {
|
||
issues = append(issues, validationIssue{
|
||
Field: fmt.Sprintf("items[%d].key", index),
|
||
Code: "move_destination_inside_source",
|
||
Message: fmt.Sprintf("目标前缀 %s 位于源 %s 内部,不能把目录移动到它自己的子目录下", destinationPrefix, sourceKey),
|
||
})
|
||
continue
|
||
}
|
||
planned = append(planned, plannedMove{sourceKey: sourceKey, destinationKey: destinationKey})
|
||
}
|
||
sort.Slice(planned, func(i, j int) bool { return planned[i].sourceKey < planned[j].sourceKey })
|
||
for i := 1; i < len(planned); i++ {
|
||
if strings.HasPrefix(planned[i].sourceKey+"/", planned[i-1].sourceKey+"/") {
|
||
issues = append(issues, validationIssue{
|
||
Field: "items",
|
||
Code: "move_items_overlap",
|
||
Message: fmt.Sprintf("勾选项之间存在父子重叠:%s 包含 %s,请分开移动", planned[i-1].sourceKey, planned[i].sourceKey),
|
||
})
|
||
break
|
||
}
|
||
}
|
||
return issues
|
||
}
|
||
|
||
func objectStoragePrefixOverlaps(prefix string, key string) bool {
|
||
prefix = strings.Trim(strings.TrimSpace(prefix), "/")
|
||
key = strings.Trim(strings.TrimSpace(key), "/")
|
||
if prefix == "" || key == "" {
|
||
return false
|
||
}
|
||
return prefix == key || strings.HasPrefix(key, prefix+"/") || strings.HasPrefix(prefix, key+"/")
|
||
}
|
||
|
||
func (server *server) objectStorageCaptureAutomationConflicts(ctx context.Context, organizationID string, connectionID string, bucket string, prefixes []string) []objectStorageCaptureAutomationConflict {
|
||
workflows, err := server.store.ListEnabledCaptureAutomations(ctx, organizationID)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
conflicts := make([]objectStorageCaptureAutomationConflict, 0)
|
||
for _, workflow := range workflows {
|
||
automation := workflow.CaptureAutomation
|
||
if automation == nil || !automation.Enabled {
|
||
continue
|
||
}
|
||
if strings.TrimSpace(automation.ConnectionID) != strings.TrimSpace(connectionID) || strings.TrimSpace(automation.Bucket) != strings.TrimSpace(bucket) {
|
||
continue
|
||
}
|
||
for _, prefix := range prefixes {
|
||
if objectStoragePrefixOverlaps(automation.Prefix, prefix) {
|
||
conflicts = append(conflicts, objectStorageCaptureAutomationConflict{
|
||
WorkflowID: workflow.ID,
|
||
WorkflowName: workflow.Name,
|
||
Prefix: strings.Trim(strings.TrimSpace(automation.Prefix), "/"),
|
||
})
|
||
break
|
||
}
|
||
}
|
||
}
|
||
return conflicts
|
||
}
|
||
|
||
func renderObjectStorageCaptureAutomationConflict(writer http.ResponseWriter, conflicts []objectStorageCaptureAutomationConflict) {
|
||
names := make([]string, 0, len(conflicts))
|
||
for _, conflict := range conflicts {
|
||
names = append(names, conflict.WorkflowName)
|
||
}
|
||
renderJSON(writer, http.StatusConflict, map[string]any{
|
||
"error": fmt.Sprintf("以下持续接入(Capture)工作流正在监听所涉及的目录:%s。移动或删除这些数据会影响在跑的采集,请先暂停对应工作流的持续接入,或确认后强制执行。", strings.Join(names, "、")),
|
||
"code": "capture_automation_conflict",
|
||
"conflicts": conflicts,
|
||
"forceRequired": true,
|
||
})
|
||
}
|
||
|
||
func (server *server) handleObjectStorageMoveJobCreate(writer http.ResponseWriter, request *http.Request) {
|
||
input, err := decodeObjectStorageObjectRenameInput(request)
|
||
if err != nil {
|
||
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
|
||
return
|
||
}
|
||
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
||
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if browser == nil || connection == nil {
|
||
renderValidationError(writer, validationIssue{Field: "connectionId", Message: "connectionId is required"})
|
||
return
|
||
}
|
||
asyncOwnsBrowser := false
|
||
defer func() {
|
||
if !asyncOwnsBrowser {
|
||
cleanup()
|
||
}
|
||
}()
|
||
bucket, err := resolveObjectStorageBucket(connection, input.Bucket)
|
||
if err != nil {
|
||
renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()})
|
||
return
|
||
}
|
||
destinationPrefix := strings.Trim(strings.TrimSpace(input.DestinationPrefix), "/")
|
||
if len(input.Items) == 0 || destinationPrefix == "" {
|
||
renderValidationError(writer, validationIssue{Field: "destinationPrefix", Message: "destinationPrefix and at least one item are required"})
|
||
return
|
||
}
|
||
if issues := objectStorageMovePrecheckIssues(input.Items, destinationPrefix); len(issues) > 0 {
|
||
renderValidationError(writer, issues...)
|
||
return
|
||
}
|
||
sourcePrefixes := make([]string, 0, len(input.Items)+1)
|
||
for _, item := range input.Items {
|
||
sourcePrefixes = append(sourcePrefixes, strings.Trim(strings.TrimSpace(item.Key), "/"))
|
||
}
|
||
sourcePrefixes = append(sourcePrefixes, destinationPrefix)
|
||
force := input.Force || strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("force")), "true")
|
||
if !force {
|
||
conflicts := server.objectStorageCaptureAutomationConflicts(request.Context(), connection.OrganizationID, connectionID, bucket, sourcePrefixes)
|
||
if len(conflicts) > 0 {
|
||
names := make([]string, 0, len(conflicts))
|
||
for _, conflict := range conflicts {
|
||
names = append(names, conflict.WorkflowName)
|
||
}
|
||
renderObjectStorageCaptureAutomationConflict(writer, conflicts)
|
||
return
|
||
}
|
||
}
|
||
moveID := domain.NewID("object-storage-move")
|
||
job := &objectStorageMoveJob{
|
||
id: moveID,
|
||
organizationID: normalizedOrganizationID(connection.OrganizationID),
|
||
connectionID: connectionID,
|
||
bucket: bucket,
|
||
items: input.Items,
|
||
force: input.Force,
|
||
status: "running",
|
||
totalObjects: len(input.Items),
|
||
createdAt: time.Now().UTC(),
|
||
}
|
||
objectStorageMoveJobs.create(job)
|
||
asyncOwnsBrowser = true
|
||
go func() {
|
||
defer cleanup()
|
||
defer objectStorageMoveJobs.removeAfterTTL(job.id, 2*time.Hour)
|
||
moveContext, cancel := context.WithTimeout(context.Background(), 6*time.Hour)
|
||
defer cancel()
|
||
var moveErr error
|
||
var summary domain.ObjectStorageObjectSummary
|
||
copiedObjects := 0
|
||
skippedObjects := 0
|
||
deletedObjects := 0
|
||
totalObjects := 0
|
||
lastProgress := objectStorageMoveProgress{}
|
||
progress := func(state objectStorageMoveProgress) {
|
||
lastProgress = state
|
||
job.update(func(current *objectStorageMoveJob) {
|
||
current.totalObjects = totalObjects + state.TotalObjects
|
||
current.copiedObjects = copiedObjects + state.CopiedObjects
|
||
current.skippedObjects = skippedObjects + state.SkippedObjects
|
||
current.deletedObjects = deletedObjects + state.DeletedObjects
|
||
current.currentKey = state.CurrentKey
|
||
})
|
||
}
|
||
progressiveMover, hasProgressMover := browser.(interface {
|
||
MovePrefixWithProgress(ctx context.Context, bucket string, prefix string, newPrefix string, progress func(objectStorageMoveProgress)) (domain.ObjectStorageObjectSummary, error)
|
||
})
|
||
for _, item := range input.Items {
|
||
lastProgress = objectStorageMoveProgress{}
|
||
sourceKey := strings.Trim(strings.TrimSpace(item.Key), "/")
|
||
currentKey := sourceKey
|
||
job.update(func(current *objectStorageMoveJob) { current.currentKey = currentKey })
|
||
destinationKey := objectStorageMoveDestination(sourceKey, destinationPrefix)
|
||
if strings.EqualFold(strings.TrimSpace(item.Kind), "directory") || item.Recursive {
|
||
if hasProgressMover {
|
||
summary, moveErr = progressiveMover.MovePrefixWithProgress(moveContext, bucket, sourceKey, destinationKey, progress)
|
||
} else {
|
||
summary, moveErr = browser.MovePrefix(moveContext, bucket, sourceKey, destinationKey)
|
||
}
|
||
} else {
|
||
summary, moveErr = browser.RenameObject(moveContext, bucket, sourceKey, destinationKey)
|
||
}
|
||
if moveErr != nil {
|
||
if isBOSNoSuchKey(moveErr) {
|
||
continue
|
||
}
|
||
break
|
||
}
|
||
if lastProgress.TotalObjects > 0 {
|
||
totalObjects += lastProgress.TotalObjects
|
||
copiedObjects += lastProgress.CopiedObjects
|
||
skippedObjects += lastProgress.SkippedObjects
|
||
deletedObjects += lastProgress.DeletedObjects
|
||
} else {
|
||
totalObjects++
|
||
copiedObjects++
|
||
deletedObjects++
|
||
}
|
||
}
|
||
job.update(func(current *objectStorageMoveJob) {
|
||
current.currentKey = ""
|
||
if moveErr != nil {
|
||
current.status = "failed"
|
||
current.error = moveErr.Error()
|
||
} else {
|
||
current.totalObjects = totalObjects
|
||
current.copiedObjects = copiedObjects
|
||
current.skippedObjects = skippedObjects
|
||
current.deletedObjects = deletedObjects
|
||
current.status = "completed"
|
||
current.result = &summary
|
||
}
|
||
current.updatedAt = time.Now().UTC()
|
||
})
|
||
}()
|
||
renderJSON(writer, http.StatusAccepted, map[string]any{"moveId": moveID, "status": "running"})
|
||
}
|
||
|
||
func (server *server) handleObjectStorageMoveJobGet(writer http.ResponseWriter, request *http.Request) {
|
||
moveID := strings.TrimSpace(chi.URLParam(request, "moveId"))
|
||
job := objectStorageMoveJobs.get(moveID)
|
||
if job == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "move job not found or expired"})
|
||
return
|
||
}
|
||
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
||
if connectionID != "" && job.connectionID != connectionID {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "move job not found or expired"})
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusOK, job.snapshot())
|
||
}
|
||
|
||
func (registry *objectStorageMoveJobRegistry) save(job *objectStorageMoveJob) {
|
||
registry.mu.Lock()
|
||
defer registry.mu.Unlock()
|
||
registry.jobs[job.id] = job
|
||
}
|
||
|
||
func (registry *objectStorageMoveJobRegistry) removeAfterTTL(id string, ttl time.Duration) {
|
||
registry.mu.Lock()
|
||
defer registry.mu.Unlock()
|
||
existing, ok := registry.jobs[id]
|
||
if !ok {
|
||
return
|
||
}
|
||
if time.Now().UTC().Sub(existing.updatedAt) < ttl {
|
||
return
|
||
}
|
||
delete(registry.jobs, id)
|
||
}
|