forked from longtao.wu/droboticflow
196 lines
6.7 KiB
Go
196 lines
6.7 KiB
Go
package worker
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
|
||
"droboticflow/internal/domain"
|
||
)
|
||
|
||
type materializationCleanupScope struct {
|
||
organizationID string
|
||
status domain.HumanItemStatus
|
||
}
|
||
|
||
// The maintenance lane is serial. Cursors are local to this Runtime; a restart
|
||
// safely rescans directories, since removal remains idempotent.
|
||
type materializationCleanupProgress struct {
|
||
nextScope int
|
||
cursors map[materializationCleanupScope]*domain.HumanItemCursor
|
||
}
|
||
|
||
// maintenance-backlog#01(B06 §11.1):切片物化产物代际清理。
|
||
// 保留最近 K=2 代(当前轮 + 上一轮对比);更旧代在任务终态(accepted/rejected)后由
|
||
// 本低频任务回收磁盘。退回基线(B02)内嵌结构化数据、正式记录、源视频工件(leftVideos/
|
||
// rightVideos 等非 materializedSliceDirectory 端口)零影响;终态任务不可能再产生新代际或返工,
|
||
// 清理窗口安全。数据库工件记录保留为墓碑(spec §7“数据库留引用不等于媒体仍可访问”,
|
||
// 超出 K 窗口的代际本就无人读取,对比视图对缺失历史有既定降级);重复扫描对已删目录是
|
||
// 幂等 no-op,限流批次保证不挤占标注/审核流量。
|
||
const (
|
||
materializationRetentionGenerations = 2
|
||
materializationCleanupBatchPerTick = 3
|
||
materializationCleanupScanLimit = 500
|
||
)
|
||
|
||
func materializationRevision(value any) int {
|
||
switch v := value.(type) {
|
||
case int:
|
||
return v
|
||
case int64:
|
||
return int(v)
|
||
case float64:
|
||
return int(v)
|
||
case string:
|
||
parsed, _ := strconv.Atoi(strings.TrimSpace(v))
|
||
return parsed
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
// staleMaterializationDirs 返回任务可回收的旧代物化目录(绝对路径)。
|
||
// 路径由受控分量重建并校验落在 workspace 的 human-materializations 根之下,
|
||
// 不读取工件记录里的历史路径,杜绝目录穿越。
|
||
func staleMaterializationDirs(workspaceRoot string, item domain.HumanItemRecord, task domain.AnnotationTaskRecord, run domain.RunRecord) []string {
|
||
// 标识符防穿越:item/task ID 出现路径分量即整体放弃(Join 会清洗 ../,前缀校验防不住)。
|
||
for _, id := range []string{item.ID, task.ID} {
|
||
if id == "" || id == "." || id == ".." || strings.ContainsAny(id, "/\\") {
|
||
return nil
|
||
}
|
||
}
|
||
workflow := mapValue(item.Subject.Payload["sliceWorkflow"])
|
||
current := materializationRevision(mapValue(workflow["materialization"])["revision"])
|
||
keep := current - materializationRetentionGenerations
|
||
if current <= 0 || keep < 1 {
|
||
return nil
|
||
}
|
||
materializationsRoot := filepath.Join(filepath.Clean(workspaceRoot), "system", "runs", task.RunID, "human-materializations", item.ID, task.ID)
|
||
// 工件归属过滤:只认本任务 ID 命名的 materialized-revision 工件,
|
||
// 防止同 Run 其他任务的代际工件混入(评审轮测试抓出)。
|
||
ownerMarker := fmt.Sprintf("-%s-materialized-revision-", task.ID)
|
||
var dirs []string
|
||
for _, artifact := range run.Artifacts {
|
||
if artifact.PortID != "materializedSliceDirectory" || artifact.RunID != task.RunID || !strings.Contains(artifact.ID, ownerMarker) {
|
||
continue
|
||
}
|
||
revision := materializationRevision(artifact.Data["revision"])
|
||
if revision <= 0 || revision > keep {
|
||
continue
|
||
}
|
||
dir := filepath.Join(materializationsRoot, fmt.Sprintf("revision-%06d", revision))
|
||
if dir == materializationsRoot || !strings.HasPrefix(dir, materializationsRoot+string(os.PathSeparator)) {
|
||
continue
|
||
}
|
||
dirs = append(dirs, dir)
|
||
}
|
||
return dirs
|
||
}
|
||
|
||
// mapValue 与 internal/api 的同名助手保持语义一致(bson.M/map[string]any 归一化)。
|
||
func mapValue(value any) map[string]any {
|
||
switch v := value.(type) {
|
||
case map[string]any:
|
||
return v
|
||
case map[string]string:
|
||
out := make(map[string]any, len(v))
|
||
for key, item := range v {
|
||
out[key] = item
|
||
}
|
||
return out
|
||
default:
|
||
return map[string]any{}
|
||
}
|
||
}
|
||
|
||
func (runtime *Runtime) cleanupStaleMaterializationGenerations(ctx context.Context) error {
|
||
organizations, err := runtime.store.ListOrganizations(ctx)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
var scopes []materializationCleanupScope
|
||
for _, organization := range organizations {
|
||
for _, status := range []domain.HumanItemStatus{domain.HumanItemAccepted, domain.HumanItemRejected} {
|
||
scopes = append(scopes, materializationCleanupScope{organization.ID, status})
|
||
}
|
||
}
|
||
if len(scopes) == 0 {
|
||
return nil
|
||
}
|
||
sort.Slice(scopes, func(i, j int) bool {
|
||
if scopes[i].organizationID != scopes[j].organizationID {
|
||
return scopes[i].organizationID < scopes[j].organizationID
|
||
}
|
||
return scopes[i].status < scopes[j].status
|
||
})
|
||
progress := &runtime.materializationCleanup
|
||
if progress.cursors == nil {
|
||
progress.cursors = make(map[materializationCleanupScope]*domain.HumanItemCursor)
|
||
}
|
||
active := make(map[materializationCleanupScope]bool, len(scopes))
|
||
for _, scope := range scopes {
|
||
active[scope] = true
|
||
}
|
||
for scope := range progress.cursors {
|
||
if !active[scope] {
|
||
delete(progress.cursors, scope)
|
||
}
|
||
}
|
||
start := progress.nextScope % len(scopes)
|
||
cleaned, scanned := 0, 0
|
||
for offset := 0; offset < len(scopes); offset++ {
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
index := (start + offset) % len(scopes)
|
||
scope := scopes[index]
|
||
// Yield to another organization/status next tick even if this queue
|
||
// consumes the complete scan or cleanup budget.
|
||
progress.nextScope = (index + 1) % len(scopes)
|
||
limit := materializationCleanupScanLimit - scanned
|
||
items, err := runtime.store.ListHumanItems(ctx, scope.organizationID, domain.HumanItemQuery{
|
||
Status: scope.status, Limit: limit, Cursor: progress.cursors[scope],
|
||
})
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, item := range items {
|
||
if err := ctx.Err(); err != nil {
|
||
return err
|
||
}
|
||
progress.cursors[scope] = &domain.HumanItemCursor{CreatedAt: item.CreatedAt, ID: item.ID}
|
||
scanned++
|
||
if item.LegacyTaskID != "" {
|
||
task, taskErr := runtime.store.GetAnnotationTask(ctx, item.LegacyTaskID)
|
||
if taskErr == nil && task != nil && task.RunID != "" {
|
||
run, runErr := runtime.store.GetRun(ctx, task.RunID)
|
||
if runErr == nil && run != nil && run.OrganizationID == item.OrganizationID {
|
||
dirs := staleMaterializationDirs(runtime.workspaceRoot, item, *task, *run)
|
||
for _, dir := range dirs {
|
||
if err := os.RemoveAll(dir); err != nil {
|
||
fmt.Printf("[worker] materialization cleanup %s failed: %v\n", dir, err)
|
||
}
|
||
}
|
||
if len(dirs) > 0 {
|
||
cleaned++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if cleaned >= materializationCleanupBatchPerTick || scanned >= materializationCleanupScanLimit {
|
||
return nil
|
||
}
|
||
}
|
||
// Exhausted this queue: next pass starts at its newest rows, including
|
||
// tasks that changed state or became eligible while we were scanning.
|
||
if len(items) < limit {
|
||
delete(progress.cursors, scope)
|
||
}
|
||
}
|
||
return nil
|
||
}
|