droboticflow/internal/worker/materialization_cleanup.go

196 lines
6.7 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package worker
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"droboticflow/internal/domain"
)
type materializationCleanupScope struct {
organizationID string
status domain.HumanItemStatus
}
// The maintenance lane is serial. Cursors are local to this Runtime; a restart
// safely rescans directories, since removal remains idempotent.
type materializationCleanupProgress struct {
nextScope int
cursors map[materializationCleanupScope]*domain.HumanItemCursor
}
// maintenance-backlog#01(B06 §11.1):切片物化产物代际清理。
// 保留最近 K=2 代(当前轮 + 上一轮对比);更旧代在任务终态(accepted/rejected)后由
// 本低频任务回收磁盘。退回基线(B02)内嵌结构化数据、正式记录、源视频工件(leftVideos/
// rightVideos 等非 materializedSliceDirectory 端口)零影响;终态任务不可能再产生新代际或返工,
// 清理窗口安全。数据库工件记录保留为墓碑(spec §7“数据库留引用不等于媒体仍可访问”,
// 超出 K 窗口的代际本就无人读取,对比视图对缺失历史有既定降级);重复扫描对已删目录是
// 幂等 no-op,限流批次保证不挤占标注/审核流量。
const (
materializationRetentionGenerations = 2
materializationCleanupBatchPerTick = 3
materializationCleanupScanLimit = 500
)
func materializationRevision(value any) int {
switch v := value.(type) {
case int:
return v
case int64:
return int(v)
case float64:
return int(v)
case string:
parsed, _ := strconv.Atoi(strings.TrimSpace(v))
return parsed
default:
return 0
}
}
// staleMaterializationDirs 返回任务可回收的旧代物化目录(绝对路径)。
// 路径由受控分量重建并校验落在 workspace 的 human-materializations 根之下,
// 不读取工件记录里的历史路径,杜绝目录穿越。
func staleMaterializationDirs(workspaceRoot string, item domain.HumanItemRecord, task domain.AnnotationTaskRecord, run domain.RunRecord) []string {
// 标识符防穿越:item/task ID 出现路径分量即整体放弃(Join 会清洗 ../,前缀校验防不住)。
for _, id := range []string{item.ID, task.ID} {
if id == "" || id == "." || id == ".." || strings.ContainsAny(id, "/\\") {
return nil
}
}
workflow := mapValue(item.Subject.Payload["sliceWorkflow"])
current := materializationRevision(mapValue(workflow["materialization"])["revision"])
keep := current - materializationRetentionGenerations
if current <= 0 || keep < 1 {
return nil
}
materializationsRoot := filepath.Join(filepath.Clean(workspaceRoot), "system", "runs", task.RunID, "human-materializations", item.ID, task.ID)
// 工件归属过滤:只认本任务 ID 命名的 materialized-revision 工件,
// 防止同 Run 其他任务的代际工件混入(评审轮测试抓出)。
ownerMarker := fmt.Sprintf("-%s-materialized-revision-", task.ID)
var dirs []string
for _, artifact := range run.Artifacts {
if artifact.PortID != "materializedSliceDirectory" || artifact.RunID != task.RunID || !strings.Contains(artifact.ID, ownerMarker) {
continue
}
revision := materializationRevision(artifact.Data["revision"])
if revision <= 0 || revision > keep {
continue
}
dir := filepath.Join(materializationsRoot, fmt.Sprintf("revision-%06d", revision))
if dir == materializationsRoot || !strings.HasPrefix(dir, materializationsRoot+string(os.PathSeparator)) {
continue
}
dirs = append(dirs, dir)
}
return dirs
}
// mapValue 与 internal/api 的同名助手保持语义一致(bson.M/map[string]any 归一化)。
func mapValue(value any) map[string]any {
switch v := value.(type) {
case map[string]any:
return v
case map[string]string:
out := make(map[string]any, len(v))
for key, item := range v {
out[key] = item
}
return out
default:
return map[string]any{}
}
}
func (runtime *Runtime) cleanupStaleMaterializationGenerations(ctx context.Context) error {
organizations, err := runtime.store.ListOrganizations(ctx)
if err != nil {
return err
}
var scopes []materializationCleanupScope
for _, organization := range organizations {
for _, status := range []domain.HumanItemStatus{domain.HumanItemAccepted, domain.HumanItemRejected} {
scopes = append(scopes, materializationCleanupScope{organization.ID, status})
}
}
if len(scopes) == 0 {
return nil
}
sort.Slice(scopes, func(i, j int) bool {
if scopes[i].organizationID != scopes[j].organizationID {
return scopes[i].organizationID < scopes[j].organizationID
}
return scopes[i].status < scopes[j].status
})
progress := &runtime.materializationCleanup
if progress.cursors == nil {
progress.cursors = make(map[materializationCleanupScope]*domain.HumanItemCursor)
}
active := make(map[materializationCleanupScope]bool, len(scopes))
for _, scope := range scopes {
active[scope] = true
}
for scope := range progress.cursors {
if !active[scope] {
delete(progress.cursors, scope)
}
}
start := progress.nextScope % len(scopes)
cleaned, scanned := 0, 0
for offset := 0; offset < len(scopes); offset++ {
if err := ctx.Err(); err != nil {
return err
}
index := (start + offset) % len(scopes)
scope := scopes[index]
// Yield to another organization/status next tick even if this queue
// consumes the complete scan or cleanup budget.
progress.nextScope = (index + 1) % len(scopes)
limit := materializationCleanupScanLimit - scanned
items, err := runtime.store.ListHumanItems(ctx, scope.organizationID, domain.HumanItemQuery{
Status: scope.status, Limit: limit, Cursor: progress.cursors[scope],
})
if err != nil {
return err
}
for _, item := range items {
if err := ctx.Err(); err != nil {
return err
}
progress.cursors[scope] = &domain.HumanItemCursor{CreatedAt: item.CreatedAt, ID: item.ID}
scanned++
if item.LegacyTaskID != "" {
task, taskErr := runtime.store.GetAnnotationTask(ctx, item.LegacyTaskID)
if taskErr == nil && task != nil && task.RunID != "" {
run, runErr := runtime.store.GetRun(ctx, task.RunID)
if runErr == nil && run != nil && run.OrganizationID == item.OrganizationID {
dirs := staleMaterializationDirs(runtime.workspaceRoot, item, *task, *run)
for _, dir := range dirs {
if err := os.RemoveAll(dir); err != nil {
fmt.Printf("[worker] materialization cleanup %s failed: %v\n", dir, err)
}
}
if len(dirs) > 0 {
cleaned++
}
}
}
}
if cleaned >= materializationCleanupBatchPerTick || scanned >= materializationCleanupScanLimit {
return nil
}
}
// Exhausted this queue: next pass starts at its newest rows, including
// tasks that changed state or became eligible while we were scanning.
if len(items) < limit {
delete(progress.cursors, scope)
}
}
return nil
}