916 lines
32 KiB
Go
916 lines
32 KiB
Go
package store_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
func TestMemoryStoreCRUD(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
storageRoot := t.TempDir()
|
|
for _, bucket := range []string{"incoming", "processed"} {
|
|
if err := os.Mkdir(filepath.Join(storageRoot, bucket), 0o755); err != nil {
|
|
t.Fatalf("create bucket %s: %v", bucket, err)
|
|
}
|
|
}
|
|
|
|
settings, err := dataStore.GetSettings(ctx)
|
|
if err != nil {
|
|
t.Fatalf("get settings: %v", err)
|
|
}
|
|
if err := dataStore.UpsertSettings(ctx, *settings); err != nil {
|
|
t.Fatalf("update settings: %v", err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(storageRoot, "incoming", "clips"), 0o755); err != nil {
|
|
t.Fatalf("create nested prefix: %v", err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(storageRoot, "incoming", "clips", "sample.mp4"), []byte("data"), 0o644); err != nil {
|
|
t.Fatalf("write sample object: %v", err)
|
|
}
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "新项目",
|
|
Description: "用于仓储测试",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
if project.ID == "" {
|
|
t.Fatalf("expected created project id")
|
|
}
|
|
|
|
operator, err := dataStore.CreateOperator(ctx, domain.CreateOperatorInput{
|
|
Name: "真实数据源",
|
|
Type: "builtin",
|
|
Category: "IO",
|
|
Description: "Emit a manifest artifact from source text",
|
|
Inputs: []domain.PortDefinition{},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create operator: %v", err)
|
|
}
|
|
if operator.ID == "" {
|
|
t.Fatalf("expected created operator id")
|
|
}
|
|
|
|
template, err := dataStore.CreateTemplate(ctx, domain.CreateTemplateInput{
|
|
Name: "基础模板",
|
|
Description: "用于模板列表测试",
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "node-1",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create template: %v", err)
|
|
}
|
|
if template.ID == "" {
|
|
t.Fatalf("expected created template id")
|
|
}
|
|
|
|
user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
|
|
Name: "测试用户",
|
|
Email: "tester@example.com",
|
|
Role: domain.UserRoleAdmin,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create user: %v", err)
|
|
}
|
|
if user.ID == "" {
|
|
t.Fatalf("expected created user id")
|
|
}
|
|
|
|
projects, err := dataStore.ListProjects(ctx)
|
|
if err != nil {
|
|
t.Fatalf("list projects: %v", err)
|
|
}
|
|
if len(projects) != 1 {
|
|
t.Fatalf("expected 1 project, got %d", len(projects))
|
|
}
|
|
|
|
operators, err := dataStore.ListOperators(ctx)
|
|
if err != nil {
|
|
t.Fatalf("list operators: %v", err)
|
|
}
|
|
if len(operators) != 1 {
|
|
t.Fatalf("expected 1 operator, got %d", len(operators))
|
|
}
|
|
|
|
templates, err := dataStore.ListTemplates(ctx)
|
|
if err != nil {
|
|
t.Fatalf("list templates: %v", err)
|
|
}
|
|
if len(templates) != 1 {
|
|
t.Fatalf("expected 1 template, got %d", len(templates))
|
|
}
|
|
|
|
users, err := dataStore.ListUsers(ctx)
|
|
if err != nil {
|
|
t.Fatalf("list users: %v", err)
|
|
}
|
|
if len(users) != 1 {
|
|
t.Fatalf("expected 1 user, got %d", len(users))
|
|
}
|
|
|
|
buckets, err := dataStore.ListStorageBuckets(ctx, storageRoot)
|
|
if err != nil {
|
|
t.Fatalf("list storage buckets: %v", err)
|
|
}
|
|
if len(buckets) != 2 {
|
|
t.Fatalf("expected 2 buckets, got %d", len(buckets))
|
|
}
|
|
|
|
objects, err := dataStore.ListStorageObjects(ctx, storageRoot, "incoming", "clips")
|
|
if err != nil {
|
|
t.Fatalf("list storage objects: %v", err)
|
|
}
|
|
if len(objects) != 1 || objects[0] != "clips/sample.mp4" {
|
|
t.Fatalf("unexpected storage objects: %#v", objects)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "测试工作流",
|
|
Description: "测试 DAG",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "n1",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
Inputs: []domain.PortDefinition{},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out-1", Name: "manifest", Type: domain.PortTypeJSON},
|
|
},
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if workflow.ProjectID != project.ID {
|
|
t.Fatalf("expected workflow project %s, got %s", project.ID, workflow.ProjectID)
|
|
}
|
|
|
|
projectWorkflows, err := dataStore.ListWorkflowsByProject(ctx, project.ID)
|
|
if err != nil {
|
|
t.Fatalf("list project workflows: %v", err)
|
|
}
|
|
if len(projectWorkflows) != 1 {
|
|
t.Fatalf("expected workflow in project listing")
|
|
}
|
|
|
|
project.Description = "已更新描述"
|
|
project.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
|
if err := dataStore.UpsertProject(ctx, *project); err != nil {
|
|
t.Fatalf("update project: %v", err)
|
|
}
|
|
|
|
updatedProject, err := dataStore.GetProject(ctx, project.ID)
|
|
if err != nil {
|
|
t.Fatalf("get updated project: %v", err)
|
|
}
|
|
if updatedProject == nil || updatedProject.Description != "已更新描述" {
|
|
t.Fatalf("expected updated description")
|
|
}
|
|
|
|
deleted, err := dataStore.DeleteProject(ctx, project.ID)
|
|
if err != nil {
|
|
t.Fatalf("delete project: %v", err)
|
|
}
|
|
if !deleted {
|
|
t.Fatalf("expected project deleted")
|
|
}
|
|
|
|
deletedProject, err := dataStore.GetProject(ctx, project.ID)
|
|
if err != nil {
|
|
t.Fatalf("get deleted project: %v", err)
|
|
}
|
|
if deletedProject != nil {
|
|
t.Fatalf("expected deleted project to be nil")
|
|
}
|
|
}
|
|
|
|
func TestRequestRunStopCancelsActiveNodeExecutions(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
run := domain.RunRecord{
|
|
ID: "run-stop-1",
|
|
WorkflowID: "workflow-1",
|
|
Status: domain.RunStatusRunning,
|
|
StartedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
|
|
NodeExecutions: []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "sleep",
|
|
NodeLabel: "Shell 命令执行",
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
StartedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
|
|
},
|
|
{
|
|
NodeID: "upload",
|
|
NodeLabel: "结果上传",
|
|
Status: domain.NodeExecutionStatusQueued,
|
|
},
|
|
},
|
|
Logs: []domain.LogEntryRecord{},
|
|
ResourceSeries: []domain.ResourceSample{},
|
|
Artifacts: []domain.ExecutionArtifact{},
|
|
K8sRefs: []domain.K8sRef{},
|
|
CreatedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
|
|
UpdatedAt: time.Now().UTC().Add(-5 * time.Second).Format(time.RFC3339Nano),
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("UpsertRun: %v", err)
|
|
}
|
|
|
|
stopped, err := dataStore.RequestRunStop(ctx, run.ID, domain.RunStopModeForce)
|
|
if err != nil {
|
|
t.Fatalf("RequestRunStop: %v", err)
|
|
}
|
|
if stopped.Status != domain.RunStatusCancelled {
|
|
t.Fatalf("expected cancelled run, got %s", stopped.Status)
|
|
}
|
|
if len(stopped.NodeExecutions) != 2 {
|
|
t.Fatalf("expected 2 node executions, got %d", len(stopped.NodeExecutions))
|
|
}
|
|
for _, execution := range stopped.NodeExecutions {
|
|
if execution.Status != domain.NodeExecutionStatusCancelled {
|
|
t.Fatalf("expected cancelled execution, got %#v", execution)
|
|
}
|
|
if execution.EndedAt == "" {
|
|
t.Fatalf("expected endedAt for cancelled execution %#v", execution)
|
|
}
|
|
}
|
|
}
|
|
|
|
// AppendNodeExecutionResourceSample lets the worker stream live GPU samples onto a
|
|
// running node's resourceSeries in place (the worker's progress reporter uses it so
|
|
// the resource monitor reflects custom-container GPU usage during execution).
|
|
func TestAppendNodeExecutionResourceSampleAppendsInPlace(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
run := domain.RunRecord{
|
|
ID: "run-progress-1",
|
|
WorkflowID: "workflow-1",
|
|
Status: domain.RunStatusRunning,
|
|
NodeExecutions: []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "sam3",
|
|
NodeLabel: "SAM3 视频分割",
|
|
Status: domain.NodeExecutionStatusRunning,
|
|
ResourceSeries: []domain.ResourceSample{{Timestamp: "t0", GPU: 10, GPUMemoryUsedMiB: 1000}},
|
|
},
|
|
{
|
|
NodeID: "source",
|
|
NodeLabel: "数据源",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
ResourceSeries: []domain.ResourceSample{{Timestamp: "t1"}},
|
|
},
|
|
},
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("UpsertRun: %v", err)
|
|
}
|
|
|
|
sample := domain.ResourceSample{Timestamp: "t2", GPU: 80, GPUMemoryUsedMiB: 28000, GPUMemoryTotalMiB: 81920, GPUName: "A800"}
|
|
if err := dataStore.AppendNodeExecutionResourceSample(ctx, run.ID, "sam3", sample); err != nil {
|
|
t.Fatalf("AppendNodeExecutionResourceSample: %v", err)
|
|
}
|
|
|
|
loaded, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil || loaded == nil {
|
|
t.Fatalf("GetRun: %v %v", loaded, err)
|
|
}
|
|
var sam3 *domain.NodeExecutionRecord
|
|
for i := range loaded.NodeExecutions {
|
|
if loaded.NodeExecutions[i].NodeID == "sam3" {
|
|
sam3 = &loaded.NodeExecutions[i]
|
|
}
|
|
}
|
|
if sam3 == nil {
|
|
t.Fatalf("sam3 node execution missing")
|
|
}
|
|
if len(sam3.ResourceSeries) != 2 {
|
|
t.Fatalf("expected 2 samples after append, got %d", len(sam3.ResourceSeries))
|
|
}
|
|
appended := sam3.ResourceSeries[1]
|
|
if appended.Timestamp != "t2" || appended.GPU != 80 || appended.GPUMemoryUsedMiB != 28000 || appended.GPUName != "A800" {
|
|
t.Fatalf("appended sample mismatch: %#v", appended)
|
|
}
|
|
|
|
// Missing run / node are silent no-ops, and other nodes are untouched.
|
|
if err := dataStore.AppendNodeExecutionResourceSample(ctx, "no-such-run", "sam3", sample); err != nil {
|
|
t.Fatalf("append to missing run should be a silent no-op, got %v", err)
|
|
}
|
|
if err := dataStore.AppendNodeExecutionResourceSample(ctx, run.ID, "no-such-node", sample); err != nil {
|
|
t.Fatalf("append to missing node should be a silent no-op, got %v", err)
|
|
}
|
|
loaded2, _ := dataStore.GetRun(ctx, run.ID)
|
|
for i := range loaded2.NodeExecutions {
|
|
if loaded2.NodeExecutions[i].NodeID == "source" && len(loaded2.NodeExecutions[i].ResourceSeries) != 1 {
|
|
t.Fatalf("source node series should be unchanged")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestMemoryStoreOrganizationAndSessionSupport(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
defaultOrg, err := dataStore.EnsureDefaultOrganization(ctx)
|
|
if err != nil {
|
|
t.Fatalf("ensure default organization: %v", err)
|
|
}
|
|
if defaultOrg.ID != domain.DefaultOrganizationID {
|
|
t.Fatalf("expected default organization id, got %q", defaultOrg.ID)
|
|
}
|
|
|
|
customOrg := domain.OrganizationRecord{
|
|
ID: "org-a",
|
|
Name: "Org A",
|
|
Slug: "org-a",
|
|
Status: domain.OrganizationStatusActive,
|
|
}
|
|
if err := dataStore.UpsertOrganization(ctx, customOrg); err != nil {
|
|
t.Fatalf("upsert organization: %v", err)
|
|
}
|
|
user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
|
|
OrganizationID: customOrg.ID,
|
|
Name: "Org User",
|
|
Email: "USER@EXAMPLE.COM",
|
|
Role: domain.UserRoleOperator,
|
|
PasswordHash: "hashed",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create user: %v", err)
|
|
}
|
|
found, err := dataStore.FindUserByEmail(ctx, "user@example.com")
|
|
if err != nil {
|
|
t.Fatalf("find user: %v", err)
|
|
}
|
|
if found == nil || found.ID != user.ID {
|
|
t.Fatalf("expected to find normalized email user, got %#v", found)
|
|
}
|
|
session, err := dataStore.CreateSession(ctx, domain.SessionRecord{
|
|
UserID: user.ID,
|
|
OrganizationID: customOrg.ID,
|
|
TokenHash: "hash-1",
|
|
ExpiresAt: time.Now().Add(time.Hour).Format(time.RFC3339),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create session: %v", err)
|
|
}
|
|
foundSession, err := dataStore.GetSessionByTokenHash(ctx, "hash-1")
|
|
if err != nil {
|
|
t.Fatalf("find session: %v", err)
|
|
}
|
|
if foundSession == nil || foundSession.ID != session.ID {
|
|
t.Fatalf("expected session by hash, got %#v", foundSession)
|
|
}
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
OrganizationID: customOrg.ID,
|
|
Name: "Org Project",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
deleted, err := dataStore.DeleteOrganizationCascade(ctx, customOrg.ID)
|
|
if err != nil {
|
|
t.Fatalf("delete organization: %v", err)
|
|
}
|
|
if !deleted {
|
|
t.Fatalf("expected organization to be deleted")
|
|
}
|
|
if remainingUser, _ := dataStore.GetUser(ctx, user.ID); remainingUser != nil {
|
|
t.Fatalf("expected org user to be removed, got %#v", remainingUser)
|
|
}
|
|
if remainingProject, _ := dataStore.GetProject(ctx, project.ID); remainingProject != nil {
|
|
t.Fatalf("expected org project to be removed, got %#v", remainingProject)
|
|
}
|
|
}
|
|
|
|
func TestMemoryStoreEnforcesUniqueOrganizationSlugsAndUserEmails(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
org := domain.OrganizationRecord{
|
|
ID: "org-a",
|
|
Name: "Org A",
|
|
Slug: "Org-A",
|
|
Status: domain.OrganizationStatusActive,
|
|
}
|
|
if err := dataStore.UpsertOrganization(ctx, org); err != nil {
|
|
t.Fatalf("upsert organization: %v", err)
|
|
}
|
|
org.Slug = "ORG-A"
|
|
if err := dataStore.UpsertOrganization(ctx, org); err != nil {
|
|
t.Fatalf("same organization slug update should be allowed: %v", err)
|
|
}
|
|
if err := dataStore.UpsertOrganization(ctx, domain.OrganizationRecord{
|
|
ID: "org-b",
|
|
Name: "Org B",
|
|
Slug: "org-a",
|
|
Status: domain.OrganizationStatusActive,
|
|
}); !errors.Is(err, store.ErrDuplicateOrganizationSlug) {
|
|
t.Fatalf("expected duplicate organization slug error, got %v", err)
|
|
}
|
|
if err := dataStore.UpsertOrganization(ctx, domain.OrganizationRecord{
|
|
ID: "org-default-duplicate",
|
|
Name: "Default Duplicate",
|
|
Slug: "default",
|
|
Status: domain.OrganizationStatusActive,
|
|
}); !errors.Is(err, store.ErrDuplicateOrganizationSlug) {
|
|
t.Fatalf("expected duplicate default organization slug error, got %v", err)
|
|
}
|
|
|
|
user, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
|
|
OrganizationID: org.ID,
|
|
Name: "Org User",
|
|
Email: "USER@EXAMPLE.COM",
|
|
Role: domain.UserRoleOperator,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create user: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
|
|
OrganizationID: org.ID,
|
|
Name: "Duplicate User",
|
|
Email: "user@example.com",
|
|
Role: domain.UserRoleOperator,
|
|
}); !errors.Is(err, store.ErrDuplicateUserEmail) {
|
|
t.Fatalf("expected duplicate user email create error, got %v", err)
|
|
}
|
|
user.Email = "USER@example.com"
|
|
if err := dataStore.UpsertUser(ctx, *user); err != nil {
|
|
t.Fatalf("same user email update should be allowed: %v", err)
|
|
}
|
|
otherUser, err := dataStore.CreateUser(ctx, domain.CreateUserInput{
|
|
OrganizationID: org.ID,
|
|
Name: "Other User",
|
|
Email: "other@example.com",
|
|
Role: domain.UserRoleOperator,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create other user: %v", err)
|
|
}
|
|
otherUser.Email = "user@example.com"
|
|
if err := dataStore.UpsertUser(ctx, *otherUser); !errors.Is(err, store.ErrDuplicateUserEmail) {
|
|
t.Fatalf("expected duplicate user email update error, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestMemoryStoreSourceStateCRUD(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
state := domain.SourceStateRecord{
|
|
ID: "workflow-1:node-1:fingerprint-1",
|
|
WorkflowID: "workflow-1",
|
|
NodeID: "node-1",
|
|
SourceFingerprint: "fingerprint-1",
|
|
DataSource: "BOS",
|
|
SourcePath: "bos://datastorage/algo/X-platform-test",
|
|
Bucket: "datastorage",
|
|
Prefix: "algo/X-platform-test",
|
|
ScanMode: "INCREMENTAL",
|
|
SeenFingerprints: []string{"entry-a", "entry-b"},
|
|
LastRunID: "run-1",
|
|
LastListedCount: 2,
|
|
LastNewCount: 1,
|
|
UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
|
}
|
|
if err := dataStore.UpsertSourceState(ctx, state); err != nil {
|
|
t.Fatalf("upsert source state: %v", err)
|
|
}
|
|
|
|
stored, err := dataStore.GetSourceState(ctx, "workflow-1", "node-1", "fingerprint-1")
|
|
if err != nil {
|
|
t.Fatalf("get source state: %v", err)
|
|
}
|
|
if stored == nil {
|
|
t.Fatalf("expected source state")
|
|
}
|
|
if stored.LastRunID != "run-1" || len(stored.SeenFingerprints) != 2 {
|
|
t.Fatalf("unexpected source state payload: %#v", stored)
|
|
}
|
|
}
|
|
|
|
func TestMemoryStoreStorageConnectionCRUD(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
connection, err := dataStore.CreateStorageConnection(ctx, domain.CreateStorageConnectionInput{
|
|
Name: "BOS 主连接",
|
|
Description: "用于扫描",
|
|
Provider: domain.StorageProviderBOS,
|
|
Endpoint: "https://bos.example.test",
|
|
AccessKeyID: "ak-memory-1234",
|
|
SecretAccessKey: "sk-memory-5678",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create storage connection: %v", err)
|
|
}
|
|
if connection.ID == "" {
|
|
t.Fatalf("expected storage connection id")
|
|
}
|
|
|
|
stored, err := dataStore.GetStorageConnection(ctx, connection.ID)
|
|
if err != nil {
|
|
t.Fatalf("get storage connection: %v", err)
|
|
}
|
|
if stored == nil {
|
|
t.Fatalf("expected stored connection")
|
|
}
|
|
if stored.AccessKeyID != "ak-memory-1234" || stored.SecretAccessKey != "sk-memory-5678" {
|
|
t.Fatalf("unexpected stored secrets: %#v", stored)
|
|
}
|
|
|
|
items, err := dataStore.ListStorageConnections(ctx)
|
|
if err != nil {
|
|
t.Fatalf("list storage connections: %v", err)
|
|
}
|
|
if len(items) != 1 || items[0].ID != connection.ID {
|
|
t.Fatalf("unexpected storage connection list: %#v", items)
|
|
}
|
|
|
|
stored.Description = "已更新"
|
|
if err := dataStore.UpsertStorageConnection(ctx, *stored); err != nil {
|
|
t.Fatalf("upsert storage connection: %v", err)
|
|
}
|
|
|
|
updated, err := dataStore.GetStorageConnection(ctx, connection.ID)
|
|
if err != nil {
|
|
t.Fatalf("get updated storage connection: %v", err)
|
|
}
|
|
if updated == nil || updated.Description != "已更新" {
|
|
t.Fatalf("expected updated description, got %#v", updated)
|
|
}
|
|
}
|
|
|
|
func TestCompletedWorkflowCanBeRunAgainManuallyWhenInactive(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "只读项目",
|
|
Description: "completed workflow guard",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "已完成工作流",
|
|
Description: "completed",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{
|
|
{
|
|
ID: "node-1",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "真实数据源",
|
|
Category: "IO",
|
|
Description: "source",
|
|
},
|
|
Position: domain.Position{X: 0, Y: 0},
|
|
},
|
|
},
|
|
Edges: []domain.WorkflowEdge{},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
workflow.Status = domain.WorkflowStatusCompleted
|
|
if err := dataStore.UpsertWorkflow(ctx, *workflow); err != nil {
|
|
t.Fatalf("mark workflow completed: %v", err)
|
|
}
|
|
|
|
manualRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("expected completed workflow rerun to be allowed, got %v", err)
|
|
}
|
|
if manualRun == nil || manualRun.Status != domain.RunStatusQueued {
|
|
t.Fatalf("expected manual rerun queued, got %#v", manualRun)
|
|
}
|
|
}
|
|
|
|
func TestHasActiveRunRecognizesBlockingStatuses(t *testing.T) {
|
|
ctx := context.Background()
|
|
for _, status := range []domain.RunStatus{
|
|
domain.RunStatusQueued,
|
|
domain.RunStatusRunning,
|
|
domain.RunStatusStopping,
|
|
domain.RunStatusWaitingForAudit,
|
|
} {
|
|
t.Run(string(status), func(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "active guard"})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, domain.RunRecord{
|
|
ID: "run-active-" + string(status),
|
|
WorkflowID: workflow.ID,
|
|
Status: status,
|
|
}); err != nil {
|
|
t.Fatalf("upsert active run: %v", err)
|
|
}
|
|
active, err := dataStore.HasActiveRun(ctx, workflow.ID)
|
|
if err != nil {
|
|
t.Fatalf("check active run: %v", err)
|
|
}
|
|
if !active {
|
|
t.Fatalf("expected %s to count as active", status)
|
|
}
|
|
})
|
|
}
|
|
|
|
dataStore := store.NewMemoryStore()
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "scheduled concurrency"})
|
|
if err != nil {
|
|
t.Fatalf("create scheduled workflow: %v", err)
|
|
}
|
|
if err := dataStore.UpsertRun(ctx, domain.RunRecord{ID: "run-active-manual", WorkflowID: workflow.ID, Status: domain.RunStatusRunning}); err != nil {
|
|
t.Fatalf("upsert active scheduled run: %v", err)
|
|
}
|
|
if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil {
|
|
t.Fatalf("internal queue creation remains governed by worker concurrency limits, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateRunTracksPreviousAndCurrentRunTimestamps(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
|
|
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{
|
|
Name: "时间戳项目",
|
|
Description: "run timestamp tracking",
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create project: %v", err)
|
|
}
|
|
|
|
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
ProjectID: project.ID,
|
|
Name: "时间戳工作流",
|
|
Description: "timestamps",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: []domain.WorkflowNode{},
|
|
Edges: []domain.WorkflowEdge{},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("create workflow: %v", err)
|
|
}
|
|
|
|
firstRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual)
|
|
if err != nil {
|
|
t.Fatalf("create first run: %v", err)
|
|
}
|
|
if firstRun.PreviousRunAt != "" {
|
|
t.Fatalf("expected first previousRunAt empty, got %q", firstRun.PreviousRunAt)
|
|
}
|
|
if firstRun.CurrentRunAt == "" {
|
|
t.Fatalf("expected first currentRunAt populated")
|
|
}
|
|
firstRun.Status = domain.RunStatusCompleted
|
|
if err := dataStore.UpsertRun(ctx, *firstRun); err != nil {
|
|
t.Fatalf("complete first run: %v", err)
|
|
}
|
|
|
|
secondRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled)
|
|
if err != nil {
|
|
t.Fatalf("create second run: %v", err)
|
|
}
|
|
if secondRun.PreviousRunAt != firstRun.CurrentRunAt {
|
|
t.Fatalf("expected second previousRunAt=%q, got %q", firstRun.CurrentRunAt, secondRun.PreviousRunAt)
|
|
}
|
|
if secondRun.CurrentRunAt == "" {
|
|
t.Fatalf("expected second currentRunAt populated")
|
|
}
|
|
}
|
|
|
|
func TestListRunSummariesPageFiltersCountsAndAdvancesCursor(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
runs := []domain.RunRecord{
|
|
{ID: "run-5", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusFailed, Error: strings.Repeat("E", 5000), CreatedAt: "2026-07-10T05:00:00Z", UpdatedAt: "2026-07-10T05:00:00Z", Failure: &domain.RunFailure{Code: "gpu_failed", Message: strings.Repeat("M", 5000), Suggestion: strings.Repeat("S", 5000)}},
|
|
{ID: "run-4", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T04:00:00Z", UpdatedAt: "2026-07-10T04:00:00Z", Progress: &domain.RunProgress{Stage: "train", Step: 4, MaxStep: 10, UpdatedAt: "2026-07-10T04:00:00Z"}},
|
|
{ID: "run-3", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"},
|
|
{ID: "run-2", OrganizationID: "org-a", WorkflowID: "workflow-audio", WorkflowName: "Audio Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusCancelled, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"},
|
|
{ID: "run-1", OrganizationID: "org-b", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusFailed, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"},
|
|
}
|
|
for _, run := range runs {
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run %s: %v", run.ID, err)
|
|
}
|
|
}
|
|
|
|
query := domain.RunSummaryQuery{Limit: 2, Search: "vision", ProjectID: "project-a"}
|
|
first, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query)
|
|
if err != nil {
|
|
t.Fatalf("list first page: %v", err)
|
|
}
|
|
if first.Total != 3 || len(first.Items) != 2 || !first.HasMore || first.NextCursor == nil {
|
|
t.Fatalf("unexpected first page: %#v", first)
|
|
}
|
|
if first.Items[0].ID != "run-5" || first.Items[1].ID != "run-4" {
|
|
t.Fatalf("unexpected first page order: %#v", first.Items)
|
|
}
|
|
if first.StatusCounts[domain.RunStatusFailed] != 1 || first.StatusCounts[domain.RunStatusRunning] != 1 || first.StatusCounts[domain.RunStatusCompleted] != 1 {
|
|
t.Fatalf("unexpected status counts: %#v", first.StatusCounts)
|
|
}
|
|
if first.Items[0].Failure == nil || first.Items[0].Failure.Code != "gpu_failed" || first.Items[1].Progress == nil || first.Items[1].Progress.Step != 4 {
|
|
t.Fatalf("structured failure/progress missing from summaries: %#v", first.Items)
|
|
}
|
|
if len([]rune(first.Items[0].Error)) != 1024 || len([]rune(first.Items[0].Failure.Message)) != 1024 || len([]rune(first.Items[0].Failure.Suggestion)) != 1024 {
|
|
t.Fatalf("paged run summaries must bound diagnostic text: %#v", first.Items[0])
|
|
}
|
|
legacy, err := dataStore.ListRunSummariesByOrganization(ctx, "org-a", string(domain.RunStatusFailed))
|
|
if err != nil {
|
|
t.Fatalf("list legacy summaries: %v", err)
|
|
}
|
|
if len(legacy) != 1 || len(legacy[0].Error) != 5000 {
|
|
t.Fatalf("legacy summaries must retain compatibility for one version: %#v", legacy)
|
|
}
|
|
|
|
query.Cursor = first.NextCursor
|
|
second, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query)
|
|
if err != nil {
|
|
t.Fatalf("list second page: %v", err)
|
|
}
|
|
if second.Total != 3 || len(second.Items) != 1 || second.Items[0].ID != "run-3" || second.HasMore || second.NextCursor != nil {
|
|
t.Fatalf("unexpected second page: %#v", second)
|
|
}
|
|
|
|
query.Cursor = nil
|
|
query.Status = domain.RunStatusFailed
|
|
failed, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query)
|
|
if err != nil {
|
|
t.Fatalf("list failed page: %v", err)
|
|
}
|
|
if failed.Total != 3 || len(failed.Items) != 1 || failed.Items[0].ID != "run-5" {
|
|
t.Fatalf("unexpected failed page: %#v", failed)
|
|
}
|
|
if failed.StatusCounts[domain.RunStatusRunning] != 1 {
|
|
t.Fatalf("status counts should remain global to non-status filters: %#v", failed.StatusCounts)
|
|
}
|
|
}
|
|
|
|
func TestWorkerQueriesReturnOnlyDispatchAndReconcileState(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
scheduled := domain.WorkflowRecord{
|
|
ID: "workflow-scheduled",
|
|
Type: domain.WorkflowTypeStreaming,
|
|
Schedule: &domain.WorkflowSchedule{Mode: domain.WorkflowScheduleModeIntervalSeconds, Interval: 60},
|
|
CreatedAt: "2026-07-10T01:00:00Z",
|
|
UpdatedAt: "2026-07-10T01:00:00Z",
|
|
}
|
|
requested := domain.WorkflowRecord{ID: "workflow-requested", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"}
|
|
unrelated := domain.WorkflowRecord{ID: "workflow-unrelated", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"}
|
|
for _, workflow := range []domain.WorkflowRecord{scheduled, requested, unrelated} {
|
|
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
|
|
t.Fatalf("upsert workflow %s: %v", workflow.ID, err)
|
|
}
|
|
}
|
|
workflows, err := dataStore.ListWorkerWorkflows(ctx, []string{requested.ID}, 20)
|
|
if err != nil {
|
|
t.Fatalf("list worker workflows: %v", err)
|
|
}
|
|
workflowIDs := map[string]bool{}
|
|
for _, workflow := range workflows {
|
|
workflowIDs[workflow.ID] = true
|
|
}
|
|
if !workflowIDs[scheduled.ID] || !workflowIDs[requested.ID] || workflowIDs[unrelated.ID] {
|
|
t.Fatalf("unexpected worker workflows: %#v", workflowIDs)
|
|
}
|
|
|
|
runs := []domain.RunRecord{
|
|
{ID: "run-queued", WorkflowID: requested.ID, Status: domain.RunStatusQueued, UpdatedAt: "2026-07-10T01:00:00Z"},
|
|
{ID: "run-claimed", WorkflowID: requested.ID, Status: domain.RunStatusQueued, CurrentNodeID: "node-a", UpdatedAt: "2026-07-10T02:00:00Z"},
|
|
{ID: "run-running", WorkflowID: scheduled.ID, Status: domain.RunStatusRunning, UpdatedAt: "2026-07-10T03:00:00Z"},
|
|
{ID: "run-audit", WorkflowID: scheduled.ID, Status: domain.RunStatusWaitingForAudit, UpdatedAt: "2026-07-10T04:00:00Z"},
|
|
{ID: "run-completed", WorkflowID: scheduled.ID, Status: domain.RunStatusCompleted, UpdatedAt: "2026-07-10T05:00:00Z"},
|
|
}
|
|
for _, run := range runs {
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run %s: %v", run.ID, err)
|
|
}
|
|
}
|
|
reconcile, err := dataStore.ListReconcileRunSummaries(ctx, 20)
|
|
if err != nil {
|
|
t.Fatalf("list reconcile runs: %v", err)
|
|
}
|
|
if len(reconcile) != 2 || reconcile[0].ID != "run-claimed" || reconcile[1].ID != "run-running" {
|
|
t.Fatalf("unexpected reconcile runs: %#v", reconcile)
|
|
}
|
|
counts, err := dataStore.CountWorkerRuns(ctx)
|
|
if err != nil {
|
|
t.Fatalf("count worker runs: %v", err)
|
|
}
|
|
if counts.QueuedTotal != 2 || counts.ActiveTotal != 1 {
|
|
t.Fatalf("unexpected worker totals: %#v", counts)
|
|
}
|
|
if counts.ActiveByWorkflow[scheduled.ID] != 2 || counts.InflightByWorkflow[requested.ID] != 2 {
|
|
t.Fatalf("unexpected worker workflow counts: %#v", counts)
|
|
}
|
|
}
|
|
|
|
func TestUpdateRunProgressAndFailureUseStructuredFields(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
run := domain.RunRecord{ID: "run-structured", Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"}
|
|
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
progress := &domain.RunProgress{Stage: "train", Epoch: 2, Step: 40, MaxStep: 100, SamplesPerSecond: 8.5, Checkpoint: "checkpoint-40", UpdatedAt: "2026-07-10T01:05:00Z"}
|
|
if err := dataStore.UpdateRunProgress(ctx, run.ID, progress); err != nil {
|
|
t.Fatalf("update progress: %v", err)
|
|
}
|
|
failure := &domain.RunFailure{Code: "dtype_mismatch", Stage: "preflight", NodeID: "train", Message: "dtype mismatch", Retryable: false, Suggestion: "align model and input dtype"}
|
|
if err := dataStore.UpdateRunFailure(ctx, run.ID, failure); err != nil {
|
|
t.Fatalf("update failure: %v", err)
|
|
}
|
|
stored, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get run: %v", err)
|
|
}
|
|
if stored.Progress == nil || stored.Progress.Step != 40 || stored.Progress.Checkpoint != "checkpoint-40" {
|
|
t.Fatalf("unexpected progress: %#v", stored.Progress)
|
|
}
|
|
if stored.Failure == nil || stored.Failure.Code != "dtype_mismatch" || stored.Failure.Suggestion == "" {
|
|
t.Fatalf("unexpected failure: %#v", stored.Failure)
|
|
}
|
|
if err := dataStore.UpdateRunProgress(ctx, run.ID, nil); err != nil {
|
|
t.Fatalf("clear progress: %v", err)
|
|
}
|
|
if err := dataStore.UpdateRunFailure(ctx, run.ID, nil); err != nil {
|
|
t.Fatalf("clear failure: %v", err)
|
|
}
|
|
cleared, err := dataStore.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatalf("get cleared run: %v", err)
|
|
}
|
|
if cleared.Progress != nil || cleared.Failure != nil {
|
|
t.Fatalf("expected structured fields cleared, got progress=%#v failure=%#v", cleared.Progress, cleared.Failure)
|
|
}
|
|
}
|
|
|
|
func TestBuildDashboardSummaryCountsTodayInConfiguredTimezone(t *testing.T) {
|
|
now := time.Date(2026, time.July, 10, 16, 30, 0, 0, time.UTC) // 2026-07-11 00:30 in Asia/Shanghai.
|
|
runs := []domain.RunSummary{
|
|
{ID: "previous-local-day", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T15:59:59Z"},
|
|
{ID: "local-midnight", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T16:00:00Z"},
|
|
{ID: "local-late", Status: domain.RunStatusFailed, CreatedAt: "2026-07-11T15:59:59Z"},
|
|
{ID: "next-local-day", Status: domain.RunStatusFailed, CreatedAt: "2026-07-11T16:00:00Z"},
|
|
{ID: "invalid-time", Status: domain.RunStatusRunning, CreatedAt: "not-a-time"},
|
|
}
|
|
summary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "Asia/Shanghai", now)
|
|
if summary.Jobs.Today != 2 {
|
|
t.Fatalf("Asia/Shanghai jobs.today = %d, want 2", summary.Jobs.Today)
|
|
}
|
|
|
|
utcSummary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "UTC", now)
|
|
if utcSummary.Jobs.Today != 2 {
|
|
t.Fatalf("UTC jobs.today = %d, want 2", utcSummary.Jobs.Today)
|
|
}
|
|
|
|
fallbackSummary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "Mars/Base", now)
|
|
if fallbackSummary.Jobs.Today != summary.Jobs.Today {
|
|
t.Fatalf("invalid timezone should fall back to %s: got %d want %d", domain.DefaultWorkflowScheduleTimezone, fallbackSummary.Jobs.Today, summary.Jobs.Today)
|
|
}
|
|
}
|