droboticflow/internal/worker/materialization_cleanup_test.go

272 lines
11 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package worker
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func seedCleanupQueueTask(t *testing.T, runtime *Runtime, organizationID, id string, status domain.HumanItemStatus, oldDirectory bool) string {
t.Helper()
ctx := context.Background()
if err := runtime.store.UpsertOrganization(ctx, domain.OrganizationRecord{ID: organizationID, Name: organizationID}); err != nil {
t.Fatal(err)
}
task := domain.AnnotationTaskRecord{ID: "task-" + id, RunID: "run-" + id, OrganizationID: organizationID}
item := domain.HumanItemRecord{ID: id, OrganizationID: organizationID, Status: status, LegacyTaskID: task.ID,
CreatedAt: "2026-10-15T00:00:00Z", Subject: domain.HumanSubjectRef{Payload: map[string]any{
"sliceWorkflow": map[string]any{"materialization": map[string]any{"revision": 5}},
}}}
run := domain.RunRecord{ID: task.RunID, OrganizationID: organizationID, Artifacts: []domain.ExecutionArtifact{{
ID: "run-" + id + "-" + task.ID + "-materialized-revision-000001", RunID: task.RunID,
PortID: "materializedSliceDirectory", Data: map[string]any{"revision": 1},
}}}
for _, err := range []error{runtime.store.UpsertAnnotationTask(ctx, task), runtime.store.UpsertHumanItem(ctx, item), runtime.store.UpsertRun(ctx, run)} {
if err != nil {
t.Fatal(err)
}
}
dir := filepath.Join(runtime.workspaceRoot, "system", "runs", task.RunID, "human-materializations", item.ID, task.ID, "revision-000001")
if oldDirectory {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
return dir
}
func TestCleanupAdvancesPastAlreadyCleanedTasks(t *testing.T) {
runtime, _, _, _ := cleanupFixture(t)
var tail []string
for i := 0; i < 3; i++ {
seedCleanupQueueTask(t, runtime, domain.DefaultOrganizationID, fmt.Sprintf("z-cleaned-%d", i), domain.HumanItemAccepted, false)
tail = append(tail, seedCleanupQueueTask(t, runtime, domain.DefaultOrganizationID, fmt.Sprintf("a-stale-%d", i), domain.HumanItemAccepted, true))
}
for tick := 0; tick < 8; tick++ {
if err := runtime.cleanupStaleMaterializationGenerations(context.Background()); err != nil {
t.Fatal(err)
}
}
for _, dir := range tail {
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Fatalf("later task starved behind already-cleaned head: %s (%v)", dir, err)
}
}
}
func TestCleanupPaginatesAndRotatesOrganizationsAndStatuses(t *testing.T) {
runtime, _, _, _ := cleanupFixture(t)
ctx := context.Background()
// More than a complete scan window of rows without a task must not hide
// an older valid row, or monopolize the other organization's status queues.
for i := 0; i < materializationCleanupScanLimit+5; i++ {
item := domain.HumanItemRecord{ID: fmt.Sprintf("z-no-task-%04d", i), OrganizationID: domain.DefaultOrganizationID,
Status: domain.HumanItemAccepted, CreatedAt: "2026-10-15T00:00:00Z"}
if err := runtime.store.UpsertHumanItem(ctx, item); err != nil {
t.Fatal(err)
}
}
var directories []string
for _, org := range []string{domain.DefaultOrganizationID, "org-cleanup-other"} {
for _, status := range []domain.HumanItemStatus{domain.HumanItemAccepted, domain.HumanItemRejected} {
directories = append(directories, seedCleanupQueueTask(t, runtime, org, "a-"+org+"-"+string(status), status, true))
}
}
for tick := 0; tick < 12; tick++ {
if err := runtime.cleanupStaleMaterializationGenerations(ctx); err != nil {
t.Fatal(err)
}
}
for _, dir := range directories {
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Fatalf("organization/status queue did not advance beyond the first window: %s (%v)", dir, err)
}
}
}
func TestCleanupRevisitsNoopTasksAfterRuntimeRestart(t *testing.T) {
runtime, root, _, _ := cleanupFixture(t)
dir := seedCleanupQueueTask(t, runtime, domain.DefaultOrganizationID, "z-late-directory", domain.HumanItemAccepted, false)
if err := runtime.cleanupStaleMaterializationGenerations(context.Background()); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
restarted := &Runtime{store: runtime.store, workspaceRoot: root}
for tick := 0; tick < 4; tick++ {
if err := restarted.cleanupStaleMaterializationGenerations(context.Background()); err != nil {
t.Fatal(err)
}
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Fatalf("restart must safely revisit a formerly empty directory: %v", err)
}
}
// maintenance-backlog#01:K=2 保留、终态才清、路径防逃逸、源媒体与基线零影响。
func cleanupFixture(t *testing.T) (runtime *Runtime, root string, task domain.AnnotationTaskRecord, run domain.RunRecord) {
t.Helper()
root = t.TempDir()
dataStore := store.NewMemoryStore()
runtime = &Runtime{store: dataStore, workspaceRoot: root}
ctx := context.Background()
now := "2026-10-13T00:00:00Z"
item := domain.HumanItemRecord{ID: "item-mc", OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa",
BatchID: "batch-mc", ProjectID: "project-mc", CurrentStageID: domain.FanyuV58StageAnnotationCombined,
Status: domain.HumanItemAccepted, LegacyTaskID: "task-mc", ReviewRound: 4, Version: 9,
Subject: domain.HumanSubjectRef{ArtifactID: "art", ItemKey: "mc/key",
Payload: map[string]any{"sliceWorkflow": map[string]any{"materialization": map[string]any{"revision": 5}}}},
CreatedAt: now, UpdatedAt: now}
if err := dataStore.UpsertHumanItem(ctx, item); err != nil {
t.Fatal(err)
}
task = domain.AnnotationTaskRecord{ID: "task-mc", OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-mc",
RunID: "run-mc", Status: domain.AnnotationTaskStatusAccepted, CreatedAt: now, UpdatedAt: now}
if err := dataStore.UpsertAnnotationTask(ctx, task); err != nil {
t.Fatal(err)
}
artifacts := []domain.ExecutionArtifact{}
for revision := 1; revision <= 5; revision++ {
artifacts = append(artifacts, domain.ExecutionArtifact{
ID: "run-mc-node-mc-task-mc-materialized-revision-" + padRevision(revision), RunID: "run-mc", NodeID: "node-mc",
PortID: "materializedSliceDirectory", Type: domain.PortTypeFile,
Data: map[string]any{"revision": revision},
})
}
// 源视频工件与无关端口不得被清理触碰。
artifacts = append(artifacts,
domain.ExecutionArtifact{ID: "source-left", RunID: "run-mc", PortID: "fullLeftVideos", Data: map[string]any{"revision": 1}},
domain.ExecutionArtifact{ID: "run-mc-node-mc-other-task-materialized-revision-000001", RunID: "run-mc", PortID: "materializedSliceDirectory", Data: map[string]any{"revision": 1}},
)
run = domain.RunRecord{ID: "run-mc", OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-mc", Status: "completed",
Artifacts: artifacts, CreatedAt: now, UpdatedAt: now}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatal(err)
}
// 磁盘布局:revision-1..5 + 源视频目录 + 其他任务目录。
for revision := 1; revision <= 5; revision++ {
dir := filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", "revision-"+padRevision(revision))
if err := os.MkdirAll(filepath.Join(dir, "slices"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "slices", "left.mp4"), []byte("x"), 0o644); err != nil {
t.Fatal(err)
}
}
sourceDir := filepath.Join(root, "system", "runs", "run-mc", "source-videos")
if err := os.MkdirAll(sourceDir, 0o755); err != nil {
t.Fatal(err)
}
otherTaskDir := filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "other-task", "revision-000001")
if err := os.MkdirAll(otherTaskDir, 0o755); err != nil {
t.Fatal(err)
}
return runtime, root, task, run
}
func padRevision(revision int) string {
return fmtRevision(revision)
}
func fmtRevision(revision int) string {
s := ""
for v := revision; v > 0; v /= 10 {
s = string(rune('0'+v%10)) + s
}
for len(s) < 6 {
s = "0" + s
}
return s
}
func TestCleanupKeepsLastTwoGenerationsAndNeverTouchesSources(t *testing.T) {
runtime, root, _, _ := cleanupFixture(t)
ctx := context.Background()
item, _ := runtime.store.GetHumanItem(ctx, "item-mc")
task, _ := runtime.store.GetAnnotationTask(ctx, "task-mc")
run, _ := runtime.store.GetRun(ctx, "run-mc")
dirs := staleMaterializationDirs(root, *item, *task, *run)
if len(dirs) != 3 { // revision 1..3(5-2=3 为保留下界,≤3 回收)
t.Fatalf("expected revisions 1-3 stale, got %v", dirs)
}
for _, d := range dirs {
if !containsRevision(d, 1) && !containsRevision(d, 2) && !containsRevision(d, 3) {
t.Fatalf("unexpected dir %s", d)
}
}
if err := runtime.cleanupStaleMaterializationGenerations(ctx); err != nil {
t.Fatal(err)
}
kept := []string{"revision-000004", "revision-000005"}
for _, k := range kept {
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", k)); err != nil {
t.Fatalf("generation %s must survive: %v", k, err)
}
}
for _, gone := range []string{"revision-000001", "revision-000002", "revision-000003"} {
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", gone)); !os.IsNotExist(err) {
t.Fatalf("generation %s must be removed", gone)
}
}
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "source-videos")); err != nil {
t.Fatal("source videos must never be touched")
}
// 其他任务的目录不受本任务清理影响(按 item+task 定位)。
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "other-task", "revision-000001")); err != nil {
t.Fatal("other task generations must not be removed by this task's cleanup")
}
}
func TestCleanupSkipsNonTerminalAndShallowTasks(t *testing.T) {
runtime, root, _, _ := cleanupFixture(t)
ctx := context.Background()
// 非终态(claimed)不清理。
item, _ := runtime.store.GetHumanItem(ctx, "item-mc")
item.Status = domain.HumanItemClaimed
if err := runtime.store.UpsertHumanItem(ctx, *item); err != nil {
t.Fatal(err)
}
if err := runtime.cleanupStaleMaterializationGenerations(ctx); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", "revision-000001")); err != nil {
t.Fatal("non-terminal item generations must stay")
}
// 浅历史(revision ≤ K)无清理目标。
item.Status = domain.HumanItemAccepted
item.Subject.Payload["sliceWorkflow"] = map[string]any{"materialization": map[string]any{"revision": 2}}
if err := runtime.store.UpsertHumanItem(ctx, *item); err != nil {
t.Fatal(err)
}
task, _ := runtime.store.GetAnnotationTask(ctx, "task-mc")
run, _ := runtime.store.GetRun(ctx, "run-mc")
if dirs := staleMaterializationDirs(root, *item, *task, *run); len(dirs) != 0 {
t.Fatalf("revision 2 keeps everything, got %v", dirs)
}
}
func TestCleanupPathContainment(t *testing.T) {
runtime, root, task, run := cleanupFixture(t)
item, _ := runtime.store.GetHumanItem(context.Background(), "item-mc")
// 构造逃逸尝试:任务 ID 携带路径穿越。
task.ID = "../../escape"
dirs := staleMaterializationDirs(root, *item, task, run)
for _, dir := range dirs {
cleanedRoot := filepath.Join(filepath.Clean(root), "system", "runs", "run-mc", "human-materializations")
if !filepath.HasPrefix(dir, cleanedRoot) {
t.Fatalf("path escaped materializations root: %s", dir)
}
}
}
func containsRevision(dir string, revision int) bool {
return filepath.Base(dir) == "revision-"+fmtRevision(revision)
}