forked from longtao.wu/droboticflow
272 lines
11 KiB
Go
272 lines
11 KiB
Go
package worker
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"os"
|
||
"path/filepath"
|
||
"testing"
|
||
|
||
"droboticflow/internal/domain"
|
||
"droboticflow/internal/store"
|
||
)
|
||
|
||
func seedCleanupQueueTask(t *testing.T, runtime *Runtime, organizationID, id string, status domain.HumanItemStatus, oldDirectory bool) string {
|
||
t.Helper()
|
||
ctx := context.Background()
|
||
if err := runtime.store.UpsertOrganization(ctx, domain.OrganizationRecord{ID: organizationID, Name: organizationID}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
task := domain.AnnotationTaskRecord{ID: "task-" + id, RunID: "run-" + id, OrganizationID: organizationID}
|
||
item := domain.HumanItemRecord{ID: id, OrganizationID: organizationID, Status: status, LegacyTaskID: task.ID,
|
||
CreatedAt: "2026-10-15T00:00:00Z", Subject: domain.HumanSubjectRef{Payload: map[string]any{
|
||
"sliceWorkflow": map[string]any{"materialization": map[string]any{"revision": 5}},
|
||
}}}
|
||
run := domain.RunRecord{ID: task.RunID, OrganizationID: organizationID, Artifacts: []domain.ExecutionArtifact{{
|
||
ID: "run-" + id + "-" + task.ID + "-materialized-revision-000001", RunID: task.RunID,
|
||
PortID: "materializedSliceDirectory", Data: map[string]any{"revision": 1},
|
||
}}}
|
||
for _, err := range []error{runtime.store.UpsertAnnotationTask(ctx, task), runtime.store.UpsertHumanItem(ctx, item), runtime.store.UpsertRun(ctx, run)} {
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
dir := filepath.Join(runtime.workspaceRoot, "system", "runs", task.RunID, "human-materializations", item.ID, task.ID, "revision-000001")
|
||
if oldDirectory {
|
||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
return dir
|
||
}
|
||
|
||
func TestCleanupAdvancesPastAlreadyCleanedTasks(t *testing.T) {
|
||
runtime, _, _, _ := cleanupFixture(t)
|
||
var tail []string
|
||
for i := 0; i < 3; i++ {
|
||
seedCleanupQueueTask(t, runtime, domain.DefaultOrganizationID, fmt.Sprintf("z-cleaned-%d", i), domain.HumanItemAccepted, false)
|
||
tail = append(tail, seedCleanupQueueTask(t, runtime, domain.DefaultOrganizationID, fmt.Sprintf("a-stale-%d", i), domain.HumanItemAccepted, true))
|
||
}
|
||
for tick := 0; tick < 8; tick++ {
|
||
if err := runtime.cleanupStaleMaterializationGenerations(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
for _, dir := range tail {
|
||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||
t.Fatalf("later task starved behind already-cleaned head: %s (%v)", dir, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestCleanupPaginatesAndRotatesOrganizationsAndStatuses(t *testing.T) {
|
||
runtime, _, _, _ := cleanupFixture(t)
|
||
ctx := context.Background()
|
||
// More than a complete scan window of rows without a task must not hide
|
||
// an older valid row, or monopolize the other organization's status queues.
|
||
for i := 0; i < materializationCleanupScanLimit+5; i++ {
|
||
item := domain.HumanItemRecord{ID: fmt.Sprintf("z-no-task-%04d", i), OrganizationID: domain.DefaultOrganizationID,
|
||
Status: domain.HumanItemAccepted, CreatedAt: "2026-10-15T00:00:00Z"}
|
||
if err := runtime.store.UpsertHumanItem(ctx, item); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
var directories []string
|
||
for _, org := range []string{domain.DefaultOrganizationID, "org-cleanup-other"} {
|
||
for _, status := range []domain.HumanItemStatus{domain.HumanItemAccepted, domain.HumanItemRejected} {
|
||
directories = append(directories, seedCleanupQueueTask(t, runtime, org, "a-"+org+"-"+string(status), status, true))
|
||
}
|
||
}
|
||
for tick := 0; tick < 12; tick++ {
|
||
if err := runtime.cleanupStaleMaterializationGenerations(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
for _, dir := range directories {
|
||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||
t.Fatalf("organization/status queue did not advance beyond the first window: %s (%v)", dir, err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestCleanupRevisitsNoopTasksAfterRuntimeRestart(t *testing.T) {
|
||
runtime, root, _, _ := cleanupFixture(t)
|
||
dir := seedCleanupQueueTask(t, runtime, domain.DefaultOrganizationID, "z-late-directory", domain.HumanItemAccepted, false)
|
||
if err := runtime.cleanupStaleMaterializationGenerations(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
restarted := &Runtime{store: runtime.store, workspaceRoot: root}
|
||
for tick := 0; tick < 4; tick++ {
|
||
if err := restarted.cleanupStaleMaterializationGenerations(context.Background()); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||
t.Fatalf("restart must safely revisit a formerly empty directory: %v", err)
|
||
}
|
||
}
|
||
|
||
// maintenance-backlog#01:K=2 保留、终态才清、路径防逃逸、源媒体与基线零影响。
|
||
|
||
func cleanupFixture(t *testing.T) (runtime *Runtime, root string, task domain.AnnotationTaskRecord, run domain.RunRecord) {
|
||
t.Helper()
|
||
root = t.TempDir()
|
||
dataStore := store.NewMemoryStore()
|
||
runtime = &Runtime{store: dataStore, workspaceRoot: root}
|
||
ctx := context.Background()
|
||
now := "2026-10-13T00:00:00Z"
|
||
item := domain.HumanItemRecord{ID: "item-mc", OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa",
|
||
BatchID: "batch-mc", ProjectID: "project-mc", CurrentStageID: domain.FanyuV58StageAnnotationCombined,
|
||
Status: domain.HumanItemAccepted, LegacyTaskID: "task-mc", ReviewRound: 4, Version: 9,
|
||
Subject: domain.HumanSubjectRef{ArtifactID: "art", ItemKey: "mc/key",
|
||
Payload: map[string]any{"sliceWorkflow": map[string]any{"materialization": map[string]any{"revision": 5}}}},
|
||
CreatedAt: now, UpdatedAt: now}
|
||
if err := dataStore.UpsertHumanItem(ctx, item); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
task = domain.AnnotationTaskRecord{ID: "task-mc", OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-mc",
|
||
RunID: "run-mc", Status: domain.AnnotationTaskStatusAccepted, CreatedAt: now, UpdatedAt: now}
|
||
if err := dataStore.UpsertAnnotationTask(ctx, task); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
artifacts := []domain.ExecutionArtifact{}
|
||
for revision := 1; revision <= 5; revision++ {
|
||
artifacts = append(artifacts, domain.ExecutionArtifact{
|
||
ID: "run-mc-node-mc-task-mc-materialized-revision-" + padRevision(revision), RunID: "run-mc", NodeID: "node-mc",
|
||
PortID: "materializedSliceDirectory", Type: domain.PortTypeFile,
|
||
Data: map[string]any{"revision": revision},
|
||
})
|
||
}
|
||
// 源视频工件与无关端口不得被清理触碰。
|
||
artifacts = append(artifacts,
|
||
domain.ExecutionArtifact{ID: "source-left", RunID: "run-mc", PortID: "fullLeftVideos", Data: map[string]any{"revision": 1}},
|
||
domain.ExecutionArtifact{ID: "run-mc-node-mc-other-task-materialized-revision-000001", RunID: "run-mc", PortID: "materializedSliceDirectory", Data: map[string]any{"revision": 1}},
|
||
)
|
||
run = domain.RunRecord{ID: "run-mc", OrganizationID: domain.DefaultOrganizationID, ProjectID: "project-mc", Status: "completed",
|
||
Artifacts: artifacts, CreatedAt: now, UpdatedAt: now}
|
||
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// 磁盘布局:revision-1..5 + 源视频目录 + 其他任务目录。
|
||
for revision := 1; revision <= 5; revision++ {
|
||
dir := filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", "revision-"+padRevision(revision))
|
||
if err := os.MkdirAll(filepath.Join(dir, "slices"), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(filepath.Join(dir, "slices", "left.mp4"), []byte("x"), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
sourceDir := filepath.Join(root, "system", "runs", "run-mc", "source-videos")
|
||
if err := os.MkdirAll(sourceDir, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
otherTaskDir := filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "other-task", "revision-000001")
|
||
if err := os.MkdirAll(otherTaskDir, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return runtime, root, task, run
|
||
}
|
||
|
||
func padRevision(revision int) string {
|
||
return fmtRevision(revision)
|
||
}
|
||
|
||
func fmtRevision(revision int) string {
|
||
s := ""
|
||
for v := revision; v > 0; v /= 10 {
|
||
s = string(rune('0'+v%10)) + s
|
||
}
|
||
for len(s) < 6 {
|
||
s = "0" + s
|
||
}
|
||
return s
|
||
}
|
||
|
||
func TestCleanupKeepsLastTwoGenerationsAndNeverTouchesSources(t *testing.T) {
|
||
runtime, root, _, _ := cleanupFixture(t)
|
||
ctx := context.Background()
|
||
item, _ := runtime.store.GetHumanItem(ctx, "item-mc")
|
||
task, _ := runtime.store.GetAnnotationTask(ctx, "task-mc")
|
||
run, _ := runtime.store.GetRun(ctx, "run-mc")
|
||
dirs := staleMaterializationDirs(root, *item, *task, *run)
|
||
if len(dirs) != 3 { // revision 1..3(5-2=3 为保留下界,≤3 回收)
|
||
t.Fatalf("expected revisions 1-3 stale, got %v", dirs)
|
||
}
|
||
for _, d := range dirs {
|
||
if !containsRevision(d, 1) && !containsRevision(d, 2) && !containsRevision(d, 3) {
|
||
t.Fatalf("unexpected dir %s", d)
|
||
}
|
||
}
|
||
if err := runtime.cleanupStaleMaterializationGenerations(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
kept := []string{"revision-000004", "revision-000005"}
|
||
for _, k := range kept {
|
||
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", k)); err != nil {
|
||
t.Fatalf("generation %s must survive: %v", k, err)
|
||
}
|
||
}
|
||
for _, gone := range []string{"revision-000001", "revision-000002", "revision-000003"} {
|
||
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", gone)); !os.IsNotExist(err) {
|
||
t.Fatalf("generation %s must be removed", gone)
|
||
}
|
||
}
|
||
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "source-videos")); err != nil {
|
||
t.Fatal("source videos must never be touched")
|
||
}
|
||
// 其他任务的目录不受本任务清理影响(按 item+task 定位)。
|
||
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "other-task", "revision-000001")); err != nil {
|
||
t.Fatal("other task generations must not be removed by this task's cleanup")
|
||
}
|
||
}
|
||
|
||
func TestCleanupSkipsNonTerminalAndShallowTasks(t *testing.T) {
|
||
runtime, root, _, _ := cleanupFixture(t)
|
||
ctx := context.Background()
|
||
// 非终态(claimed)不清理。
|
||
item, _ := runtime.store.GetHumanItem(ctx, "item-mc")
|
||
item.Status = domain.HumanItemClaimed
|
||
if err := runtime.store.UpsertHumanItem(ctx, *item); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := runtime.cleanupStaleMaterializationGenerations(ctx); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := os.Stat(filepath.Join(root, "system", "runs", "run-mc", "human-materializations", "item-mc", "task-mc", "revision-000001")); err != nil {
|
||
t.Fatal("non-terminal item generations must stay")
|
||
}
|
||
// 浅历史(revision ≤ K)无清理目标。
|
||
item.Status = domain.HumanItemAccepted
|
||
item.Subject.Payload["sliceWorkflow"] = map[string]any{"materialization": map[string]any{"revision": 2}}
|
||
if err := runtime.store.UpsertHumanItem(ctx, *item); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
task, _ := runtime.store.GetAnnotationTask(ctx, "task-mc")
|
||
run, _ := runtime.store.GetRun(ctx, "run-mc")
|
||
if dirs := staleMaterializationDirs(root, *item, *task, *run); len(dirs) != 0 {
|
||
t.Fatalf("revision 2 keeps everything, got %v", dirs)
|
||
}
|
||
}
|
||
|
||
func TestCleanupPathContainment(t *testing.T) {
|
||
runtime, root, task, run := cleanupFixture(t)
|
||
item, _ := runtime.store.GetHumanItem(context.Background(), "item-mc")
|
||
// 构造逃逸尝试:任务 ID 携带路径穿越。
|
||
task.ID = "../../escape"
|
||
dirs := staleMaterializationDirs(root, *item, task, run)
|
||
for _, dir := range dirs {
|
||
cleanedRoot := filepath.Join(filepath.Clean(root), "system", "runs", "run-mc", "human-materializations")
|
||
if !filepath.HasPrefix(dir, cleanedRoot) {
|
||
t.Fatalf("path escaped materializations root: %s", dir)
|
||
}
|
||
}
|
||
}
|
||
|
||
func containsRevision(dir string, revision int) bool {
|
||
return filepath.Base(dir) == "revision-"+fmtRevision(revision)
|
||
}
|