droboticflow/internal/store/memory_test.go

916 lines
32 KiB
Go

package store_test
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func TestMemoryStoreCRUD(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
storageRoot := t.TempDir()
for _, bucket := range []string{"incoming", "processed"} {
if err := os.Mkdir(filepath.Join(storageRoot, bucket), 0o755); err != nil {
t.Fatalf("create bucket %s: %v", bucket, err)
}
}
settings, err := dataStore.GetSettings(ctx)
if err != nil {
t.Fatalf("get settings: %v", err)
}
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
t.Fatalf("update settings: %v", err)
}
if err := os.MkdirAll(filepath.Join(storageRoot, "incoming", "clips"), 0o755); err != nil {
t.Fatalf("create nested prefix: %v", err)
}
if err := os.WriteFile(filepath.Join(storageRoot, "incoming", "clips", "sample.mp4"), []byte("data"), 0o644); err != nil {
t.Fatalf("write sample object: %v", err)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "新项目",
Description: "用于仓储测试",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
if project.ID == "" {
t.Fatalf("expected created project id")
}
operator, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
Name: "真实数据源",
Type: "builtin",
Category: "IO",
Description: "Emit a manifest artifact from source text",
Inputs: []domain.PortDefinition{},
Outputs: []domain.PortDefinition{
{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON},
},
})
if err != nil {
t.Fatalf("create operator: %v", err)
}
if operator.ID == "" {
t.Fatalf("expected created operator id")
}
template, err := dataStore.CreateTemplate(ctx, domain.CreateTemplateInput{
Name: "基础模板",
Description: "用于模板列表测试",
Nodes: []domain.WorkflowNode{
{
ID: "node-1",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Outputs: []domain.PortDefinition{
{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create template: %v", err)
}
if template.ID == "" {
t.Fatalf("expected created template id")
}
user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
Name: "测试用户",
Email: "tester@example.com",
Role: domain.UserRoleAdmin,
})
if err != nil {
t.Fatalf("create user: %v", err)
}
if user.ID == "" {
t.Fatalf("expected created user id")
}
projects, err := dataStore.ListProjects(ctx)
if err != nil {
t.Fatalf("list projects: %v", err)
}
if len(projects) != 1 {
t.Fatalf("expected 1 project, got %d", len(projects))
}
operators, err := dataStore.ListOperators(ctx)
if err != nil {
t.Fatalf("list operators: %v", err)
}
if len(operators) != 1 {
t.Fatalf("expected 1 operator, got %d", len(operators))
}
templates, err := dataStore.ListTemplates(ctx)
if err != nil {
t.Fatalf("list templates: %v", err)
}
if len(templates) != 1 {
t.Fatalf("expected 1 template, got %d", len(templates))
}
users, err := dataStore.ListUsers(ctx)
if err != nil {
t.Fatalf("list users: %v", err)
}
if len(users) != 1 {
t.Fatalf("expected 1 user, got %d", len(users))
}
buckets, err := dataStore.ListStorageBuckets(ctx, storageRoot)
if err != nil {
t.Fatalf("list storage buckets: %v", err)
}
if len(buckets) != 2 {
t.Fatalf("expected 2 buckets, got %d", len(buckets))
}
objects, err := dataStore.ListStorageObjects(ctx, storageRoot, "incoming", "clips")
if err != nil {
t.Fatalf("list storage objects: %v", err)
}
if len(objects) != 1 || objects[0] != "clips/sample.mp4" {
t.Fatalf("unexpected storage objects: %#v", objects)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "测试工作流",
Description: "测试 DAG",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "n1",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
Inputs: []domain.PortDefinition{},
Outputs: []domain.PortDefinition{
{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON},
},
},
Position: domain.Position{X: 0, Y: 0},
},
},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if workflow.ProjectID != project.ID {
t.Fatalf("expected workflow project %s, got %s", project.ID, workflow.ProjectID)
}
projectWorkflows, err := dataStore.ListWorkflowsByProject(ctx, project.ID)
if err != nil {
t.Fatalf("list project workflows: %v", err)
}
if len(projectWorkflows) != 1 {
t.Fatalf("expected workflow in project listing")
}
project.Description = "已更新描述"
project.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
if err := dataStore.UpsertProject(ctx, *project); err != nil {
t.Fatalf("update project: %v", err)
}
updatedProject, err := dataStore.GetProject(ctx, project.ID)
if err != nil {
t.Fatalf("get updated project: %v", err)
}
if updatedProject == nil || updatedProject.Description != "已更新描述" {
t.Fatalf("expected updated description")
}
deleted, err := dataStore.DeleteProject(ctx, project.ID)
if err != nil {
t.Fatalf("delete project: %v", err)
}
if !deleted {
t.Fatalf("expected project deleted")
}
deletedProject, err := dataStore.GetProject(ctx, project.ID)
if err != nil {
t.Fatalf("get deleted project: %v", err)
}
if deletedProject != nil {
t.Fatalf("expected deleted project to be nil")
}
}
func TestRequestRunStopCancelsActiveNodeExecutions(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
run := domain.RunRecord{
ID: "run-stop-1",
WorkflowID: "workflow-1",
Status: domain.RunStatusRunning,
StartedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
NodeExecutions: []domain.NodeExecutionRecord{
{
NodeID: "sleep",
NodeLabel: "Shell 命令执行",
Status: domain.NodeExecutionStatusRunning,
StartedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
},
{
NodeID: "upload",
NodeLabel: "结果上传",
Status: domain.NodeExecutionStatusQueued,
},
},
Logs: []domain.LogEntryRecord{},
ResourceSeries: []domain.ResourceSample{},
Artifacts: []domain.ExecutionArtifact{},
K8sRefs: []domain.K8sRef{},
CreatedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
UpdatedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("UpsertRun: %v", err)
}
stopped, err := dataStore.RequestRunStop(ctx, run.ID, domain.RunStopModeForce)
if err != nil {
t.Fatalf("RequestRunStop: %v", err)
}
if stopped.Status != domain.RunStatusCancelled {
t.Fatalf("expected cancelled run, got %s", stopped.Status)
}
if len(stopped.NodeExecutions) != 2 {
t.Fatalf("expected 2 node executions, got %d", len(stopped.NodeExecutions))
}
for _, execution := range stopped.NodeExecutions {
if execution.Status != domain.NodeExecutionStatusCancelled {
t.Fatalf("expected cancelled execution, got %#v", execution)
}
if execution.EndedAt == "" {
t.Fatalf("expected endedAt for cancelled execution %#v", execution)
}
}
}
// AppendNodeExecutionResourceSample lets the worker stream live GPU samples onto a
// running node's resourceSeries in place (the worker's progress reporter uses it so
// the resource monitor reflects custom-container GPU usage during execution).
func TestAppendNodeExecutionResourceSampleAppendsInPlace(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
run := domain.RunRecord{
ID: "run-progress-1",
WorkflowID: "workflow-1",
Status: domain.RunStatusRunning,
NodeExecutions: []domain.NodeExecutionRecord{
{
NodeID: "sam3",
NodeLabel: "SAM3 视频分割",
Status: domain.NodeExecutionStatusRunning,
ResourceSeries: []domain.ResourceSample{{Timestamp: "t0", GPU: 10, GPUMemoryUsedMiB: 1000}},
},
{
NodeID: "source",
NodeLabel: "数据源",
Status: domain.NodeExecutionStatusCompleted,
ResourceSeries: []domain.ResourceSample{{Timestamp: "t1"}},
},
},
}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("UpsertRun: %v", err)
}
sample := domain.ResourceSample{Timestamp: "t2", GPU: 80, GPUMemoryUsedMiB: 28000, GPUMemoryTotalMiB: 81920, GPUName: "A800"}
if err := dataStore.AppendNodeExecutionResourceSample(ctx, run.ID, "sam3", sample); err != nil {
t.Fatalf("AppendNodeExecutionResourceSample: %v", err)
}
loaded, err := dataStore.GetRun(ctx, run.ID)
if err != nil || loaded == nil {
t.Fatalf("GetRun: %v %v", loaded, err)
}
var sam3 *domain.NodeExecutionRecord
for i := range loaded.NodeExecutions {
if loaded.NodeExecutions[i].NodeID == "sam3" {
sam3 = &loaded.NodeExecutions[i]
}
}
if sam3 == nil {
t.Fatalf("sam3 node execution missing")
}
if len(sam3.ResourceSeries) != 2 {
t.Fatalf("expected 2 samples after append, got %d", len(sam3.ResourceSeries))
}
appended := sam3.ResourceSeries[1]
if appended.Timestamp != "t2" || appended.GPU != 80 || appended.GPUMemoryUsedMiB != 28000 || appended.GPUName != "A800" {
t.Fatalf("appended sample mismatch: %#v", appended)
}
// Missing run / node are silent no-ops, and other nodes are untouched.
if err := dataStore.AppendNodeExecutionResourceSample(ctx, "no-such-run", "sam3", sample); err != nil {
t.Fatalf("append to missing run should be a silent no-op, got %v", err)
}
if err := dataStore.AppendNodeExecutionResourceSample(ctx, run.ID, "no-such-node", sample); err != nil {
t.Fatalf("append to missing node should be a silent no-op, got %v", err)
}
loaded2, _ := dataStore.GetRun(ctx, run.ID)
for i := range loaded2.NodeExecutions {
if loaded2.NodeExecutions[i].NodeID == "source" && len(loaded2.NodeExecutions[i].ResourceSeries) != 1 {
t.Fatalf("source node series should be unchanged")
}
}
}
func TestMemoryStoreOrganizationAndSessionSupport(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
defaultOrg, err := dataStore.EnsureDefaultOrganization(ctx)
if err != nil {
t.Fatalf("ensure default organization: %v", err)
}
if defaultOrg.ID != domain.DefaultOrganizationID {
t.Fatalf("expected default organization id, got %q", defaultOrg.ID)
}
customOrg := domain.OrganizationRecord{
ID: "org-a",
Name: "Org A",
Slug: "org-a",
Status: domain.OrganizationStatusActive,
}
if err := dataStore.UpsertOrganization(ctx, customOrg); err != nil {
t.Fatalf("upsert organization: %v", err)
}
user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
OrganizationID: customOrg.ID,
Name: "Org User",
Email: "USER@EXAMPLE.COM",
Role: domain.UserRoleOperator,
PasswordHash: "hashed",
})
if err != nil {
t.Fatalf("create user: %v", err)
}
found, err := dataStore.FindUserByEmail(ctx, "user@example.com")
if err != nil {
t.Fatalf("find user: %v", err)
}
if found == nil || found.ID != user.ID {
t.Fatalf("expected to find normalized email user, got %#v", found)
}
session, err := dataStore.CreateSession(ctx, domain.SessionRecord{
UserID: user.ID,
OrganizationID: customOrg.ID,
TokenHash: "hash-1",
ExpiresAt: time.Now().Add(time.Hour).Format(time.RFC3339),
})
if err != nil {
t.Fatalf("create session: %v", err)
}
foundSession, err := dataStore.GetSessionByTokenHash(ctx, "hash-1")
if err != nil {
t.Fatalf("find session: %v", err)
}
if foundSession == nil || foundSession.ID != session.ID {
t.Fatalf("expected session by hash, got %#v", foundSession)
}
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
OrganizationID: customOrg.ID,
Name: "Org Project",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
deleted, err := dataStore.DeleteOrganizationCascade(ctx, customOrg.ID)
if err != nil {
t.Fatalf("delete organization: %v", err)
}
if !deleted {
t.Fatalf("expected organization to be deleted")
}
if remainingUser, _ := dataStore.GetUser(ctx, user.ID); remainingUser != nil {
t.Fatalf("expected org user to be removed, got %#v", remainingUser)
}
if remainingProject, _ := dataStore.GetProject(ctx, project.ID); remainingProject != nil {
t.Fatalf("expected org project to be removed, got %#v", remainingProject)
}
}
func TestMemoryStoreEnforcesUniqueOrganizationSlugsAndUserEmails(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
org := domain.OrganizationRecord{
ID: "org-a",
Name: "Org A",
Slug: "Org-A",
Status: domain.OrganizationStatusActive,
}
if err := dataStore.UpsertOrganization(ctx, org); err != nil {
t.Fatalf("upsert organization: %v", err)
}
org.Slug = "ORG-A"
if err := dataStore.UpsertOrganization(ctx, org); err != nil {
t.Fatalf("same organization slug update should be allowed: %v", err)
}
if err := dataStore.UpsertOrganization(ctx, domain.OrganizationRecord{
ID: "org-b",
Name: "Org B",
Slug: "org-a",
Status: domain.OrganizationStatusActive,
}); !errors.Is(err, store.ErrDuplicateOrganizationSlug) {
t.Fatalf("expected duplicate organization slug error, got %v", err)
}
if err := dataStore.UpsertOrganization(ctx, domain.OrganizationRecord{
ID: "org-default-duplicate",
Name: "Default Duplicate",
Slug: "default",
Status: domain.OrganizationStatusActive,
}); !errors.Is(err, store.ErrDuplicateOrganizationSlug) {
t.Fatalf("expected duplicate default organization slug error, got %v", err)
}
user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
OrganizationID: org.ID,
Name: "Org User",
Email: "USER@EXAMPLE.COM",
Role: domain.UserRoleOperator,
})
if err != nil {
t.Fatalf("create user: %v", err)
}
if _, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
OrganizationID: org.ID,
Name: "Duplicate User",
Email: "user@example.com",
Role: domain.UserRoleOperator,
}); !errors.Is(err, store.ErrDuplicateUserEmail) {
t.Fatalf("expected duplicate user email create error, got %v", err)
}
user.Email = "USER@example.com"
if err := dataStore.UpsertUser(ctx, *user); err != nil {
t.Fatalf("same user email update should be allowed: %v", err)
}
otherUser, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
OrganizationID: org.ID,
Name: "Other User",
Email: "other@example.com",
Role: domain.UserRoleOperator,
})
if err != nil {
t.Fatalf("create other user: %v", err)
}
otherUser.Email = "user@example.com"
if err := dataStore.UpsertUser(ctx, *otherUser); !errors.Is(err, store.ErrDuplicateUserEmail) {
t.Fatalf("expected duplicate user email update error, got %v", err)
}
}
func TestMemoryStoreSourceStateCRUD(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
state := domain.SourceStateRecord{
ID: "workflow-1:node-1:fingerprint-1",
WorkflowID: "workflow-1",
NodeID: "node-1",
SourceFingerprint: "fingerprint-1",
DataSource: "BOS",
SourcePath: "bos://datastorage/algo/X-platform-test",
Bucket: "datastorage",
Prefix: "algo/X-platform-test",
ScanMode: "INCREMENTAL",
SeenFingerprints: []string{"entry-a", "entry-b"},
LastRunID: "run-1",
LastListedCount: 2,
LastNewCount: 1,
UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano),
}
if err := dataStore.UpsertSourceState(ctx, state); err != nil {
t.Fatalf("upsert source state: %v", err)
}
stored, err := dataStore.GetSourceState(ctx, "workflow-1", "node-1", "fingerprint-1")
if err != nil {
t.Fatalf("get source state: %v", err)
}
if stored == nil {
t.Fatalf("expected source state")
}
if stored.LastRunID != "run-1" || len(stored.SeenFingerprints) != 2 {
t.Fatalf("unexpected source state payload: %#v", stored)
}
}
func TestMemoryStoreStorageConnectionCRUD(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
Name: "BOS 主连接",
Description: "用于扫描",
Provider: domain.StorageProviderBOS,
Endpoint: "https://bos.example.test",
AccessKeyID: "ak-memory-1234",
SecretAccessKey: "sk-memory-5678",
})
if err != nil {
t.Fatalf("create storage connection: %v", err)
}
if connection.ID == "" {
t.Fatalf("expected storage connection id")
}
stored, err := dataStore.GetStorageConnection(ctx, connection.ID)
if err != nil {
t.Fatalf("get storage connection: %v", err)
}
if stored == nil {
t.Fatalf("expected stored connection")
}
if stored.AccessKeyID != "ak-memory-1234" || stored.SecretAccessKey != "sk-memory-5678" {
t.Fatalf("unexpected stored secrets: %#v", stored)
}
items, err := dataStore.ListStorageConnections(ctx)
if err != nil {
t.Fatalf("list storage connections: %v", err)
}
if len(items) != 1 || items[0].ID != connection.ID {
t.Fatalf("unexpected storage connection list: %#v", items)
}
stored.Description = "已更新"
if err := dataStore.UpsertStorageConnection(ctx, *stored); err != nil {
t.Fatalf("upsert storage connection: %v", err)
}
updated, err := dataStore.GetStorageConnection(ctx, connection.ID)
if err != nil {
t.Fatalf("get updated storage connection: %v", err)
}
if updated == nil || updated.Description != "已更新" {
t.Fatalf("expected updated description, got %#v", updated)
}
}
func TestCompletedWorkflowCanBeRunAgainManuallyWhenInactive(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "只读项目",
Description: "completed workflow guard",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "已完成工作流",
Description: "completed",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{
{
ID: "node-1",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "真实数据源",
Category: "IO",
Description: "source",
},
Position: domain.Position{X: 0, Y: 0},
},
},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
workflow.Status = domain.WorkflowStatusCompleted
if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil {
t.Fatalf("mark workflow completed: %v", err)
}
manualRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("expected completed workflow rerun to be allowed, got %v", err)
}
if manualRun == nil || manualRun.Status != domain.RunStatusQueued {
t.Fatalf("expected manual rerun queued, got %#v", manualRun)
}
}
func TestHasActiveRunRecognizesBlockingStatuses(t *testing.T) {
ctx := context.Background()
for _, status := range []domain.RunStatus{
domain.RunStatusQueued,
domain.RunStatusRunning,
domain.RunStatusStopping,
domain.RunStatusWaitingForAudit,
} {
t.Run(string(status), func(t *testing.T) {
dataStore := store.NewMemoryStore()
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "active guard"})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
if err := dataStore.UpsertRun(ctx, domain.RunRecord{
ID: "run-active-" + string(status),
WorkflowID: workflow.ID,
Status: status,
}); err != nil {
t.Fatalf("upsert active run: %v", err)
}
active, err := dataStore.HasActiveRun(ctx, workflow.ID)
if err != nil {
t.Fatalf("check active run: %v", err)
}
if !active {
t.Fatalf("expected %s to count as active", status)
}
})
}
dataStore := store.NewMemoryStore()
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "scheduled concurrency"})
if err != nil {
t.Fatalf("create scheduled workflow: %v", err)
}
if err := dataStore.UpsertRun(ctx, domain.RunRecord{ID: "run-active-manual", WorkflowID: workflow.ID, Status: domain.RunStatusRunning}); err != nil {
t.Fatalf("upsert active scheduled run: %v", err)
}
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
t.Fatalf("internal queue creation remains governed by worker concurrency limits, got %v", err)
}
}
func TestCreateRunTracksPreviousAndCurrentRunTimestamps(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
Name: "时间戳项目",
Description: "run timestamp tracking",
})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "时间戳工作流",
Description: "timestamps",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
firstRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
if err != nil {
t.Fatalf("create first run: %v", err)
}
if firstRun.PreviousRunAt != "" {
t.Fatalf("expected first previousRunAt empty, got %q", firstRun.PreviousRunAt)
}
if firstRun.CurrentRunAt == "" {
t.Fatalf("expected first currentRunAt populated")
}
firstRun.Status = domain.RunStatusCompleted
if err := dataStore.UpsertRun(ctx, *firstRun); err != nil {
t.Fatalf("complete first run: %v", err)
}
secondRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled)
if err != nil {
t.Fatalf("create second run: %v", err)
}
if secondRun.PreviousRunAt != firstRun.CurrentRunAt {
t.Fatalf("expected second previousRunAt=%q, got %q", firstRun.CurrentRunAt, secondRun.PreviousRunAt)
}
if secondRun.CurrentRunAt == "" {
t.Fatalf("expected second currentRunAt populated")
}
}
func TestListRunSummariesPageFiltersCountsAndAdvancesCursor(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
runs := []domain.RunRecord{
{ID: "run-5", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusFailed, Error: strings.Repeat("E", 5000), CreatedAt: "2026-07-10T05:00:00Z", UpdatedAt: "2026-07-10T05:00:00Z", Failure: &domain.RunFailure{Code: "gpu_failed", Message: strings.Repeat("M", 5000), Suggestion: strings.Repeat("S", 5000)}},
{ID: "run-4", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T04:00:00Z", UpdatedAt: "2026-07-10T04:00:00Z", Progress: &domain.RunProgress{Stage: "train", Step: 4, MaxStep: 10, UpdatedAt: "2026-07-10T04:00:00Z"}},
{ID: "run-3", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"},
{ID: "run-2", OrganizationID: "org-a", WorkflowID: "workflow-audio", WorkflowName: "Audio Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusCancelled, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"},
{ID: "run-1", OrganizationID: "org-b", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusFailed, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"},
}
for _, run := range runs {
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run %s: %v", run.ID, err)
}
}
query := domain.RunSummaryQuery{Limit: 2, Search: "vision", ProjectID: "project-a"}
first, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query)
if err != nil {
t.Fatalf("list first page: %v", err)
}
if first.Total != 3 || len(first.Items) != 2 || !first.HasMore || first.NextCursor == nil {
t.Fatalf("unexpected first page: %#v", first)
}
if first.Items[0].ID != "run-5" || first.Items[1].ID != "run-4" {
t.Fatalf("unexpected first page order: %#v", first.Items)
}
if first.StatusCounts[domain.RunStatusFailed] != 1 || first.StatusCounts[domain.RunStatusRunning] != 1 || first.StatusCounts[domain.RunStatusCompleted] != 1 {
t.Fatalf("unexpected status counts: %#v", first.StatusCounts)
}
if first.Items[0].Failure == nil || first.Items[0].Failure.Code != "gpu_failed" || first.Items[1].Progress == nil || first.Items[1].Progress.Step != 4 {
t.Fatalf("structured failure/progress missing from summaries: %#v", first.Items)
}
if len([]rune(first.Items[0].Error)) != 1024 || len([]rune(first.Items[0].Failure.Message)) != 1024 || len([]rune(first.Items[0].Failure.Suggestion)) != 1024 {
t.Fatalf("paged run summaries must bound diagnostic text: %#v", first.Items[0])
}
legacy, err := dataStore.ListRunSummariesByOrganization(ctx, "org-a", string(domain.RunStatusFailed))
if err != nil {
t.Fatalf("list legacy summaries: %v", err)
}
if len(legacy) != 1 || len(legacy[0].Error) != 5000 {
t.Fatalf("legacy summaries must retain compatibility for one version: %#v", legacy)
}
query.Cursor = first.NextCursor
second, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query)
if err != nil {
t.Fatalf("list second page: %v", err)
}
if second.Total != 3 || len(second.Items) != 1 || second.Items[0].ID != "run-3" || second.HasMore || second.NextCursor != nil {
t.Fatalf("unexpected second page: %#v", second)
}
query.Cursor = nil
query.Status = domain.RunStatusFailed
failed, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query)
if err != nil {
t.Fatalf("list failed page: %v", err)
}
if failed.Total != 3 || len(failed.Items) != 1 || failed.Items[0].ID != "run-5" {
t.Fatalf("unexpected failed page: %#v", failed)
}
if failed.StatusCounts[domain.RunStatusRunning] != 1 {
t.Fatalf("status counts should remain global to non-status filters: %#v", failed.StatusCounts)
}
}
func TestWorkerQueriesReturnOnlyDispatchAndReconcileState(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
scheduled := domain.WorkflowRecord{
ID: "workflow-scheduled",
Type: domain.WorkflowTypeStreaming,
Schedule: &domain.WorkflowSchedule{Mode: domain.WorkflowScheduleModeIntervalSeconds, Interval: 60},
CreatedAt: "2026-07-10T01:00:00Z",
UpdatedAt: "2026-07-10T01:00:00Z",
}
requested := domain.WorkflowRecord{ID: "workflow-requested", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"}
unrelated := domain.WorkflowRecord{ID: "workflow-unrelated", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"}
for _, workflow := range []domain.WorkflowRecord{scheduled, requested, unrelated} {
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
t.Fatalf("upsert workflow %s: %v", workflow.ID, err)
}
}
workflows, err := dataStore.ListWorkerWorkflows(ctx, []string{requested.ID}, 20)
if err != nil {
t.Fatalf("list worker workflows: %v", err)
}
workflowIDs := map[string]bool{}
for _, workflow := range workflows {
workflowIDs[workflow.ID] = true
}
if !workflowIDs[scheduled.ID] || !workflowIDs[requested.ID] || workflowIDs[unrelated.ID] {
t.Fatalf("unexpected worker workflows: %#v", workflowIDs)
}
runs := []domain.RunRecord{
{ID: "run-queued", WorkflowID: requested.ID, Status: domain.RunStatusQueued, UpdatedAt: "2026-07-10T01:00:00Z"},
{ID: "run-claimed", WorkflowID: requested.ID, Status: domain.RunStatusQueued, CurrentNodeID: "node-a", UpdatedAt: "2026-07-10T02:00:00Z"},
{ID: "run-running", WorkflowID: scheduled.ID, Status: domain.RunStatusRunning, UpdatedAt: "2026-07-10T03:00:00Z"},
{ID: "run-audit", WorkflowID: scheduled.ID, Status: domain.RunStatusWaitingForAudit, UpdatedAt: "2026-07-10T04:00:00Z"},
{ID: "run-completed", WorkflowID: scheduled.ID, Status: domain.RunStatusCompleted, UpdatedAt: "2026-07-10T05:00:00Z"},
}
for _, run := range runs {
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run %s: %v", run.ID, err)
}
}
reconcile, err := dataStore.ListReconcileRunSummaries(ctx, 20)
if err != nil {
t.Fatalf("list reconcile runs: %v", err)
}
if len(reconcile) != 2 || reconcile[0].ID != "run-claimed" || reconcile[1].ID != "run-running" {
t.Fatalf("unexpected reconcile runs: %#v", reconcile)
}
counts, err := dataStore.CountWorkerRuns(ctx)
if err != nil {
t.Fatalf("count worker runs: %v", err)
}
if counts.QueuedTotal != 2 || counts.ActiveTotal != 1 {
t.Fatalf("unexpected worker totals: %#v", counts)
}
if counts.ActiveByWorkflow[scheduled.ID] != 2 || counts.InflightByWorkflow[requested.ID] != 2 {
t.Fatalf("unexpected worker workflow counts: %#v", counts)
}
}
func TestUpdateRunProgressAndFailureUseStructuredFields(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
run := domain.RunRecord{ID: "run-structured", Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"}
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run: %v", err)
}
progress := &domain.RunProgress{Stage: "train", Epoch: 2, Step: 40, MaxStep: 100, SamplesPerSecond: 8.5, Checkpoint: "checkpoint-40", UpdatedAt: "2026-07-10T01:05:00Z"}
if err := dataStore.UpdateRunProgress(ctx, run.ID, progress); err != nil {
t.Fatalf("update progress: %v", err)
}
failure := &domain.RunFailure{Code: "dtype_mismatch", Stage: "preflight", NodeID: "train", Message: "dtype mismatch", Retryable: false, Suggestion: "align model and input dtype"}
if err := dataStore.UpdateRunFailure(ctx, run.ID, failure); err != nil {
t.Fatalf("update failure: %v", err)
}
stored, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if stored.Progress == nil || stored.Progress.Step != 40 || stored.Progress.Checkpoint != "checkpoint-40" {
t.Fatalf("unexpected progress: %#v", stored.Progress)
}
if stored.Failure == nil || stored.Failure.Code != "dtype_mismatch" || stored.Failure.Suggestion == "" {
t.Fatalf("unexpected failure: %#v", stored.Failure)
}
if err := dataStore.UpdateRunProgress(ctx, run.ID, nil); err != nil {
t.Fatalf("clear progress: %v", err)
}
if err := dataStore.UpdateRunFailure(ctx, run.ID, nil); err != nil {
t.Fatalf("clear failure: %v", err)
}
cleared, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get cleared run: %v", err)
}
if cleared.Progress != nil || cleared.Failure != nil {
t.Fatalf("expected structured fields cleared, got progress=%#v failure=%#v", cleared.Progress, cleared.Failure)
}
}
func TestBuildDashboardSummaryCountsTodayInConfiguredTimezone(t *testing.T) {
now := time.Date(2026, time.July, 10, 16, 30, 0, 0, time.UTC) // 2026-07-11 00:30 in Asia/Shanghai.
runs := []domain.RunSummary{
{ID: "previous-local-day", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T15:59:59Z"},
{ID: "local-midnight", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T16:00:00Z"},
{ID: "local-late", Status: domain.RunStatusFailed, CreatedAt: "2026-07-11T15:59:59Z"},
{ID: "next-local-day", Status: domain.RunStatusFailed, CreatedAt: "2026-07-11T16:00:00Z"},
{ID: "invalid-time", Status: domain.RunStatusRunning, CreatedAt: "not-a-time"},
}
summary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "Asia/Shanghai", now)
if summary.Jobs.Today != 2 {
t.Fatalf("Asia/Shanghai jobs.today = %d, want 2", summary.Jobs.Today)
}
utcSummary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "UTC", now)
if utcSummary.Jobs.Today != 2 {
t.Fatalf("UTC jobs.today = %d, want 2", utcSummary.Jobs.Today)
}
fallbackSummary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "Mars/Base", now)
if fallbackSummary.Jobs.Today != summary.Jobs.Today {
t.Fatalf("invalid timezone should fall back to %s: got %d want %d", domain.DefaultWorkflowScheduleTimezone, fallbackSummary.Jobs.Today, summary.Jobs.Today)
}
}