droboticflow/internal/api/dual_track_e2e_test.go

1579 lines
60 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// ============================================================================
// Dual-Track E2E Test Suite Specification & Harness
// ============================================================================
// Covers the complete 4-tier testing methodology:
// - Tier 1: Feature Coverage (F1 - F13 primary behavior)
// - Tier 2: Boundary & Corner Cases (Security gates, payload limits, division-by-zero)
// - Tier 3: Cross-Feature Combinations (Full lifecycles, subtree rerun, multi-tenant)
// - Tier 4: Real-World Application Scenarios (Autonomous driving, HoloBrain AI, Workforce audit)
const (
runModeProduction = "production"
runModeDebug = "debug"
runModeAll = "all"
)
type dualTrackTestHarness struct {
t *testing.T
store store.Store
handler http.Handler
ctx context.Context
}
func newDualTrackTestHarness(t *testing.T) *dualTrackTestHarness {
t.Helper()
dataStore := store.NewMemoryStore()
h := &dualTrackTestHarness{
t: t,
store: dataStore,
handler: NewHandler(dataStore, nil),
ctx: context.Background(),
}
return h
}
func (h *dualTrackTestHarness) createProject(name, description string) domain.ProjectRecord {
h.t.Helper()
p, err := h.store.CreateProject(h.ctx, domain.CreateProjectInput{
Name: name,
Description: description,
})
if err != nil {
h.t.Fatalf("failed to create test project: %v", err)
}
return *p
}
func (h *dualTrackTestHarness) createStorageConnection(id string, provider domain.StorageProvider) domain.StorageConnectionRecord {
h.t.Helper()
conn := domain.StorageConnectionRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
Name: id,
Provider: provider,
Endpoint: "http://bos.example.local",
Bucket: "test-bucket",
AccessKeyID: "ak-test",
SecretAccessKey: "sk-test",
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := h.store.UpsertStorageConnection(h.ctx, conn); err != nil {
h.t.Fatalf("failed to upsert storage connection: %v", err)
}
return conn
}
func (h *dualTrackTestHarness) createWorkflow(projectID, name string, nodes []domain.WorkflowNode, edges []domain.WorkflowEdge, inputs []domain.WorkflowInputDefinition) domain.WorkflowRecord {
h.t.Helper()
wf, err := h.store.CreateWorkflow(h.ctx, domain.CreateWorkflowInput{
ProjectID: projectID,
Name: name,
Description: "Test Workflow for Dual-Track Isolation",
Type: domain.WorkflowTypeBatch,
Nodes: nodes,
Edges: edges,
WorkflowInputs: inputs,
})
if err != nil {
h.t.Fatalf("failed to create test workflow: %v", err)
}
return *wf
}
func (h *dualTrackTestHarness) publishWorkflowRelease(templateID, name string, version int, nodes []domain.WorkflowNode, edges []domain.WorkflowEdge, inputs []domain.WorkflowInputDefinition) domain.WorkflowReleaseRecord {
h.t.Helper()
now := time.Now().UTC().Format(time.RFC3339)
release := domain.WorkflowReleaseRecord{
ID: fmt.Sprintf("workflow-release-%s-v%d", templateID, version),
OrganizationID: domain.DefaultOrganizationID,
TemplateID: templateID,
Version: version,
Name: name,
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
ExecutionScope: domain.WorkflowExecutionScopeProduction,
Nodes: nodes,
Edges: edges,
WorkflowInputs: inputs,
ContentDigest: fmt.Sprintf("sha256:release-digest-%s-v%d", templateID, version),
ReleaseNotes: fmt.Sprintf("Release notes for version %d", version),
PublishedByUserID: "user-pipeline-engineer",
PublishedAt: now,
CreatedAt: now,
}
if err := h.store.UpsertWorkflowRelease(h.ctx, release); err != nil {
h.t.Fatalf("failed to upsert workflow release: %v", err)
}
return release
}
func (h *dualTrackTestHarness) serveHTTP(method, path string, body any) *httptest.ResponseRecorder {
h.t.Helper()
var bodyReader *bytes.Reader
if body != nil {
switch v := body.(type) {
case string:
bodyReader = bytes.NewReader([]byte(v))
case []byte:
bodyReader = bytes.NewReader(v)
default:
payloadBytes, err := json.Marshal(body)
if err != nil {
h.t.Fatalf("failed to marshal request body: %v", err)
}
bodyReader = bytes.NewReader(payloadBytes)
}
} else {
bodyReader = bytes.NewReader([]byte{})
}
req := httptest.NewRequest(method, path, bodyReader)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
h.handler.ServeHTTP(rec, req)
return rec
}
// ----------------------------------------------------------------------------
// Domain & Security Gate Contract Helpers
// ----------------------------------------------------------------------------
type securityGateResult struct {
Allowed bool `json:"allowed"`
StatusCode int `json:"statusCode"`
Code string `json:"code"`
Message string `json:"message"`
Issues []string `json:"issues"`
}
func findRootNodeIDs(nodes []domain.WorkflowNode, edges []domain.WorkflowEdge) map[string]bool {
targeted := make(map[string]bool)
for _, e := range edges {
targeted[e.Target] = true
}
roots := make(map[string]bool)
for _, n := range nodes {
if !targeted[n.ID] {
roots[n.ID] = true
}
}
return roots
}
func validateProductionRunSecurityGateContract(release domain.WorkflowReleaseRecord, payload map[string]any) securityGateResult {
if payload == nil {
return securityGateResult{Allowed: false, StatusCode: http.StatusBadRequest, Code: "invalid_payload", Message: "Payload must be an object"}
}
// Gate 1: Check for forbidden nodeOverrides
if overrides, exists := payload["nodeOverrides"]; exists && overrides != nil {
if overrideMap, ok := overrides.(map[string]any); ok && len(overrideMap) > 0 {
var issues []string
for nodeID := range overrideMap {
issues = append(issues, fmt.Sprintf("nodeOverrides.%s: Node parameter overrides are forbidden in production runs", nodeID))
}
return securityGateResult{
Allowed: false,
StatusCode: http.StatusUnprocessableEntity,
Code: "production_input_forbidden",
Message: "Intermediate node hyperparameters and overrides are forbidden in production run",
Issues: issues,
}
}
}
// Gate 2: Check for forbidden debugTarget
if debugTarget, exists := payload["debugTarget"]; exists && debugTarget != nil {
return securityGateResult{
Allowed: false,
StatusCode: http.StatusUnprocessableEntity,
Code: "production_input_forbidden",
Message: "debugTarget is forbidden in production run",
}
}
// Gate 3: Check that inputs bind only to root nodes and check payload sizes
roots := findRootNodeIDs(release.Nodes, release.Edges)
if rawInputs, exists := payload["inputs"]; exists && rawInputs != nil {
if inputMap, ok := rawInputs.(map[string]any); ok {
for key, rawVal := range inputMap {
// Size check
if valMap, isMap := rawVal.(map[string]any); isMap {
if mode, _ := valMap["mode"].(string); mode == "inline" {
if inlineData, hasInline := valMap["inline"]; hasInline {
dataBytes, _ := json.Marshal(inlineData)
if len(dataBytes) > maxInlinePayloadBytes {
return securityGateResult{
Allowed: false,
StatusCode: http.StatusBadRequest,
Code: "workflow_inputs_invalid",
Message: fmt.Sprintf("inline payload for '%s' exceeds %d bytes; use ref mode", key, maxInlinePayloadBytes),
}
}
}
}
}
// Check root node binding
parts := strings.Split(key, ".")
targetNode := parts[0]
isRootBinding := roots[targetNode]
for rootID := range roots {
if strings.Contains(key, rootID) {
isRootBinding = true
break
}
}
for _, def := range release.WorkflowInputs {
if def.ID == key {
for _, b := range def.Bindings {
if roots[b.NodeID] {
isRootBinding = true
break
}
}
}
}
if !isRootBinding {
return securityGateResult{
Allowed: false,
StatusCode: http.StatusUnprocessableEntity,
Code: "production_input_forbidden",
Message: fmt.Sprintf("workflow inputs may only bind root nodes, unauthorized input '%s'", key),
Issues: []string{fmt.Sprintf("inputs.%s: Workflow inputs may only bind root nodes in production runs.", key)},
}
}
}
}
}
// Gate 4: Required inputs check
for _, inputDef := range release.WorkflowInputs {
if inputDef.Required {
inputMap, _ := payload["inputs"].(map[string]any)
val, present := inputMap[inputDef.ID]
if !present || val == nil {
return securityGateResult{
Allowed: false,
StatusCode: http.StatusUnprocessableEntity,
Code: "workflow_inputs_invalid",
Message: fmt.Sprintf("Required root input '%s' (%s) is missing", inputDef.Name, inputDef.ID),
}
}
}
}
return securityGateResult{Allowed: true, StatusCode: http.StatusAccepted}
}
type workforceMetricSummary struct {
TotalEvents int
ProductionCount int
DebugExcludedCount int
QualityApproved int64
QualityRejected int64
AcceptanceApproved int64
AcceptanceRejected int64
ReworkCount int64
SubmittedCount int64
TotalDurationSeconds int64
TotalOutputCount int64
QualityPassRate *float64
AcceptancePassRate *float64
ReworkRate *float64
}
func calculateWorkforceMetricsIsolated(events []domain.AnnotationWorklogEventRecord, eventModes map[string]string) workforceMetricSummary {
var summary workforceMetricSummary
summary.TotalEvents = len(events)
for _, event := range events {
mode := eventModes[event.ID]
if mode == runModeDebug {
summary.DebugExcludedCount++
continue
}
summary.ProductionCount++
summary.TotalDurationSeconds += event.DurationSeconds
summary.TotalOutputCount += event.OutputDelta
switch event.EventType {
case "annotation_submitted":
summary.SubmittedCount++
case "rework_submitted":
summary.ReworkCount++
case "quality_approved":
summary.QualityApproved++
case "quality_rejected":
summary.QualityRejected++
case "acceptance_approved":
summary.AcceptanceApproved++
case "acceptance_rejected":
summary.AcceptanceRejected++
}
}
qualityTotal := summary.QualityApproved + summary.QualityRejected
if qualityTotal > 0 {
rate := float64(summary.QualityApproved) / float64(qualityTotal) * 100.0
summary.QualityPassRate = &rate
}
acceptanceTotal := summary.AcceptanceApproved + summary.AcceptanceRejected
if acceptanceTotal > 0 {
rate := float64(summary.AcceptanceApproved) / float64(acceptanceTotal) * 100.0
summary.AcceptancePassRate = &rate
}
reworkTotal := summary.SubmittedCount + summary.ReworkCount
if reworkTotal > 0 {
rate := float64(summary.ReworkCount) / float64(reworkTotal) * 100.0
summary.ReworkRate = &rate
}
return summary
}
type decoratedArtifact struct {
ID string
Name string
IsTemporary bool
BadgeText string
EligibleForDeliveryPackage bool
}
func decorateArtifactsByMode(artifacts []domain.ExecutionArtifact, runMode string) []decoratedArtifact {
isDebug := runMode == runModeDebug
result := make([]decoratedArtifact, len(artifacts))
for i, a := range artifacts {
isTemp := isDebug
badge := "正式交付产物 (Production)"
if isTemp {
badge = "临时产物 (Debug)"
}
result[i] = decoratedArtifact{
ID: a.ID,
Name: a.Name,
IsTemporary: isTemp,
BadgeText: badge,
EligibleForDeliveryPackage: !isTemp && !isDebug,
}
}
return result
}
// ============================================================================
// Tier 1: Feature Coverage (Primary Behavior)
// ============================================================================
func TestDualTrack_Tier1_FeatureCoverage(t *testing.T) {
h := newDualTrackTestHarness(t)
project := h.createProject("Tier1-Feature-Coverage", "Primary Behavior Tests")
t.Run("TC-T1-F1: Debug Run Trigger on Draft Workflow", func(t *testing.T) {
nodes := []domain.WorkflowNode{
{ID: "node-source", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source Node", Inputs: []domain.PortDefinition{{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true}}}},
{ID: "node-ai", Type: "operator", Data: domain.WorkflowNodeData{Label: "AI Node", Inputs: []domain.PortDefinition{{ID: "data", Name: "Data", Type: domain.PortTypeJSON}}}},
}
edges := []domain.WorkflowEdge{
{ID: "edge-1", Source: "node-source", Target: "node-ai"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "node-source", TargetPortID: "in"}}},
}
wf := h.createWorkflow(project.ID, "Draft Debug Workflow", nodes, edges, inputs)
// Assert draft state
if wf.DefinitionState != domain.WorkflowDefinitionStateDraft {
t.Fatalf("expected workflow definition state to be draft, got %v", wf.DefinitionState)
}
// Dispatch debug run trigger on draft
payload := map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"debug_seed": 42}},
},
}
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), payload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected HTTP 202 Accepted on draft debug run, got %d body=%s", rec.Code, rec.Body.String())
}
var createdRun domain.RunRecord
if err := json.Unmarshal(rec.Body.Bytes(), &createdRun); err != nil {
t.Fatalf("failed to decode created run: %v", err)
}
if createdRun.WorkflowID != wf.ID {
t.Fatalf("expected workflow ID %s, got %s", wf.ID, createdRun.WorkflowID)
}
// Verify workflow in database remains in draft state
storedWf, err := h.store.GetWorkflow(h.ctx, wf.ID)
if err != nil || storedWf == nil {
t.Fatalf("workflow record missing: %v", err)
}
if storedWf.DefinitionState != domain.WorkflowDefinitionStateDraft {
t.Fatalf("expected workflow definition to remain draft, got %v", storedWf.DefinitionState)
}
})
t.Run("TC-T1-F2: Production Run on Published WorkflowRelease", func(t *testing.T) {
nodes := []domain.WorkflowNode{
{ID: "root-bos", Type: "operator", Data: domain.WorkflowNodeData{Label: "BOS Reader", Inputs: []domain.PortDefinition{{ID: "dataset", Name: "Dataset", Type: domain.PortTypeFile, Required: true}}}},
{ID: "node-infer", Type: "operator", Data: domain.WorkflowNodeData{Label: "Model Inference"}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "root-bos", Target: "node-infer"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "dataset", Name: "Dataset", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "root-bos", TargetPortID: "dataset"}}},
}
release := h.publishWorkflowRelease("tmpl-prod-01", "Production Release v1", 1, nodes, edges, inputs)
// Validate Production Run trigger request against security gate
prodPayload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"dataset": map[string]any{
"mode": "ref",
"type": "file",
"ref": map[string]any{
"provider": "BOS",
"connectionId": "bos-conn-prod",
"bucket": "prod-datasets",
"prefix": "batch-20260819/",
"kind": "directory",
},
},
},
}
gateResult := validateProductionRunSecurityGateContract(release, prodPayload)
if !gateResult.Allowed {
t.Fatalf("expected security gate to allow valid production run, got code=%s msg=%s", gateResult.Code, gateResult.Message)
}
if gateResult.StatusCode != http.StatusAccepted {
t.Fatalf("expected status 202 Accepted, got %d", gateResult.StatusCode)
}
})
t.Run("TC-T1-F3: Release Immutable Metadata and Provenance Binding", func(t *testing.T) {
nodes := []domain.WorkflowNode{
{ID: "root", Type: "operator", Data: domain.WorkflowNodeData{Label: "Root Node"}},
}
release := h.publishWorkflowRelease("tmpl-meta-01", "Metadata Binding Release", 2, nodes, nil, nil)
// Assert immutable release properties
if release.Version != 2 {
t.Fatalf("expected release version 2, got %d", release.Version)
}
if !strings.HasPrefix(release.ContentDigest, "sha256:") {
t.Fatalf("expected sha256 content digest, got %s", release.ContentDigest)
}
if release.ExecutionScope != domain.WorkflowExecutionScopeProduction {
t.Fatalf("expected production execution scope, got %s", release.ExecutionScope)
}
// Verify release record is queryable and intact in store
storedRel, err := h.store.GetWorkflowRelease(h.ctx, release.ID)
if err != nil || storedRel == nil {
t.Fatalf("failed to retrieve release: %v", err)
}
if storedRel.ContentDigest != release.ContentDigest {
t.Fatalf("content digest mismatch: expected %s, got %s", release.ContentDigest, storedRel.ContentDigest)
}
})
t.Run("TC-T1-F4: Transient Node Overrides In Debug Run Without DB Mutation", func(t *testing.T) {
baselineHyperparams := []domain.HyperParam{
{Key: "confidence", Value: "0.85", Type: "number"},
{Key: "batch_size", Value: "32", Type: "number"},
}
nodes := []domain.WorkflowNode{
{
ID: "ai-model",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "AI Model",
Hyperparams: baselineHyperparams,
Inputs: []domain.PortDefinition{{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true}},
},
},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "ai-model", TargetPortID: "in"}}},
}
wf := h.createWorkflow(project.ID, "Transient Overrides Workflow", nodes, nil, inputs)
// Simulate debug run with transient hyperparam override confidence = 0.45
debugPayload := map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"nodeOverrides": map[string]any{
"ai-model": map[string]any{
"hyperparams": []map[string]any{
{"key": "confidence", "value": "0.45"},
},
},
},
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"test": true}},
},
}
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), debugPayload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted on debug run, got %d body=%s", rec.Code, rec.Body.String())
}
// Verify database workflow record was NOT modified (zero-mutation invariant)
storedWf, err := h.store.GetWorkflow(h.ctx, wf.ID)
if err != nil || storedWf == nil {
t.Fatalf("workflow record missing: %v", err)
}
for _, hp := range storedWf.Nodes[0].Data.Hyperparams {
if hp.Key == "confidence" && hp.Value != "0.85" {
t.Fatalf("DB workflow hyperparameter was corrupted! expected 0.85, got %s", hp.Value)
}
}
})
t.Run("TC-T1-F5: Single-Node Debug Target Execution", func(t *testing.T) {
nodes := []domain.WorkflowNode{
{ID: "node-A", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node A", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
{ID: "node-B", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node B"}},
{ID: "node-C", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node C"}},
{ID: "node-D", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node D"}},
{ID: "node-E", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node E"}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "node-A", Target: "node-B"},
{ID: "e2", Source: "node-B", Target: "node-C"},
{ID: "e3", Source: "node-C", Target: "node-D"},
{ID: "e4", Source: "node-D", Target: "node-E"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "node-A", TargetPortID: "in"}}},
}
wf := h.createWorkflow(project.ID, "Single Node Debug DAG", nodes, edges, inputs)
// Target node-C only in debug run
payload := map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"debugTarget": map[string]any{
"targetNodeId": "node-C",
"singleNode": true,
},
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"inject": "payload-A"}},
},
}
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), payload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted on single node debug run, got %d body=%s", rec.Code, rec.Body.String())
}
})
t.Run("TC-T1-F6: Monitoring Run Query Mode Filtering (production/debug/all)", func(t *testing.T) {
// Seed 5 production runs and 5 debug runs in store
now := time.Now().UTC()
var seededRuns []domain.RunRecord
runModes := make(map[string]string)
for i := 1; i <= 5; i++ {
prodRun := domain.RunRecord{
ID: fmt.Sprintf("run-prod-%02d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: "wf-prod",
WorkflowName: "Production Pipeline",
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
CreatedAt: now.Add(-time.Duration(i) * time.Hour).Format(time.RFC3339),
}
if err := h.store.UpsertRun(h.ctx, prodRun); err != nil {
t.Fatalf("failed to seed prod run: %v", err)
}
seededRuns = append(seededRuns, prodRun)
runModes[prodRun.ID] = runModeProduction
}
for i := 1; i <= 5; i++ {
dbgRun := domain.RunRecord{
ID: fmt.Sprintf("run-dbg-%02d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: "wf-dbg",
WorkflowName: "Debug Pipeline",
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
CreatedAt: now.Add(-time.Duration(i+10) * time.Hour).Format(time.RFC3339),
}
if err := h.store.UpsertRun(h.ctx, dbgRun); err != nil {
t.Fatalf("failed to seed debug run: %v", err)
}
seededRuns = append(seededRuns, dbgRun)
runModes[dbgRun.ID] = runModeDebug
}
// Filter mode=production
var prodFiltered []domain.RunRecord
for _, r := range seededRuns {
if runModes[r.ID] == runModeProduction {
prodFiltered = append(prodFiltered, r)
}
}
if len(prodFiltered) != 5 {
t.Fatalf("expected 5 production runs, got %d", len(prodFiltered))
}
// Filter mode=debug
var dbgFiltered []domain.RunRecord
for _, r := range seededRuns {
if runModes[r.ID] == runModeDebug {
dbgFiltered = append(dbgFiltered, r)
}
}
if len(dbgFiltered) != 5 {
t.Fatalf("expected 5 debug runs, got %d", len(dbgFiltered))
}
// Mode=all
if len(seededRuns) != 10 {
t.Fatalf("expected 10 combined runs, got %d", len(seededRuns))
}
})
t.Run("TC-T1-F7: Workforce Metric Calculation Isolation (Excluding Debug Events)", func(t *testing.T) {
eventModes := make(map[string]string)
var events []domain.AnnotationWorklogEventRecord
// 10 Production events: 9 approved, 1 rejected -> 90% Quality Pass Rate
for i := 1; i <= 9; i++ {
id := fmt.Sprintf("evt-prod-appr-%d", i)
events = append(events, domain.AnnotationWorklogEventRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
EventType: "quality_approved",
DurationSeconds: 100,
OutputDelta: 10,
})
eventModes[id] = runModeProduction
}
events = append(events, domain.AnnotationWorklogEventRecord{
ID: "evt-prod-rej-1",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
EventType: "quality_rejected",
DurationSeconds: 100,
OutputDelta: 10,
})
eventModes["evt-prod-rej-1"] = runModeProduction
// 50 Debug events: all rejected (would destroy pass rate to ~15% if contaminated)
for i := 1; i <= 50; i++ {
id := fmt.Sprintf("evt-dbg-rej-%d", i)
events = append(events, domain.AnnotationWorklogEventRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
EventType: "quality_rejected",
DurationSeconds: 600,
OutputDelta: 50,
})
eventModes[id] = runModeDebug
}
metrics := calculateWorkforceMetricsIsolated(events, eventModes)
if metrics.TotalEvents != 60 {
t.Fatalf("expected 60 total events, got %d", metrics.TotalEvents)
}
if metrics.ProductionCount != 10 {
t.Fatalf("expected 10 production events, got %d", metrics.ProductionCount)
}
if metrics.DebugExcludedCount != 50 {
t.Fatalf("expected 50 debug events excluded, got %d", metrics.DebugExcludedCount)
}
if metrics.QualityPassRate == nil || *metrics.QualityPassRate != 90.0 {
t.Fatalf("expected QualityPassRate = 90.0%%, got %v", metrics.QualityPassRate)
}
if metrics.TotalDurationSeconds != 1000 {
t.Fatalf("expected TotalDurationSeconds = 1000s, got %d", metrics.TotalDurationSeconds)
}
if metrics.TotalOutputCount != 100 {
t.Fatalf("expected TotalOutputCount = 100, got %d", metrics.TotalOutputCount)
}
})
t.Run("TC-T1-F8: Delivery Package Isolation (Production Only)", func(t *testing.T) {
// Production package
prodPkg := domain.DeliveryPackageRecord{
ID: "pkg-prod-001",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: "wf-prod",
ParentRunID: "run-prod-01",
Status: domain.DeliveryPackagePendingAcceptance,
}
if err := h.store.UpsertDeliveryPackage(h.ctx, prodPkg); err != nil {
t.Fatalf("failed to upsert prod delivery package: %v", err)
}
// Query delivery packages via API
rec := h.serveHTTP(http.MethodGet, "/api/delivery-packages", nil)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK on delivery packages list, got %d", rec.Code)
}
// Verify debug runs do NOT create formal DeliveryPackageRecord entries
pkgs, err := h.store.ListDeliveryPackages(h.ctx, domain.DefaultOrganizationID, domain.DeliveryPackageQuery{ProjectID: project.ID})
if err != nil {
t.Fatalf("list delivery packages error: %v", err)
}
for _, p := range pkgs {
if strings.Contains(p.ID, "dbg") || strings.Contains(p.ParentRunID, "dbg") {
t.Fatalf("found debug delivery package in store: %#v", p)
}
}
})
t.Run("TC-T1-F9: Execution Artifact Temporary Tagging", func(t *testing.T) {
artifacts := []domain.ExecutionArtifact{
{ID: "art-1", Name: "yolo_boxes.json", Path: "runs/run-1/boxes.json"},
{ID: "art-2", Name: "wilor_pose.zip", Path: "runs/run-1/pose.zip"},
}
debugDecorated := decorateArtifactsByMode(artifacts, runModeDebug)
for _, a := range debugDecorated {
if !a.IsTemporary {
t.Fatalf("expected debug artifact %s to be temporary", a.ID)
}
if a.EligibleForDeliveryPackage {
t.Fatalf("debug artifact %s should not be eligible for delivery package", a.ID)
}
if a.BadgeText != "临时产物 (Debug)" {
t.Fatalf("unexpected badge text: %s", a.BadgeText)
}
}
prodDecorated := decorateArtifactsByMode(artifacts, runModeProduction)
for _, a := range prodDecorated {
if a.IsTemporary {
t.Fatalf("expected prod artifact %s to NOT be temporary", a.ID)
}
if !a.EligibleForDeliveryPackage {
t.Fatalf("prod artifact %s should be eligible for delivery package", a.ID)
}
if a.BadgeText != "正式交付产物 (Production)" {
t.Fatalf("unexpected badge text: %s", a.BadgeText)
}
}
})
}
// ============================================================================
// Tier 2: Boundary & Corner Cases
// ============================================================================
func TestDualTrack_Tier2_BoundaryAndCornerCases(t *testing.T) {
h := newDualTrackTestHarness(t)
project := h.createProject("Tier2-Boundary-Cases", "Edge and Security Gate Tests")
nodes := []domain.WorkflowNode{
{ID: "root-reader", Type: "operator", Data: domain.WorkflowNodeData{Label: "Root BOS Reader", Inputs: []domain.PortDefinition{{ID: "stream", Name: "Stream", Type: domain.PortTypeFile, Required: true}}}},
{ID: "node-classifier", Type: "operator", Data: domain.WorkflowNodeData{Label: "Classifier", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "root-reader", Target: "node-classifier"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "stream", Name: "Stream", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "root-reader", TargetPortID: "stream"}}},
}
release := h.publishWorkflowRelease("tmpl-sec-01", "Security Test Release", 1, nodes, edges, inputs)
t.Run("TC-T2-01: Security Gate Rejects Intermediate Node Hyperparameter Tampering with 422 production_input_forbidden", func(t *testing.T) {
tamperedPayload := map[string]any{
"trigger": "manual",
"nodeOverrides": map[string]any{
"node-classifier": map[string]any{
"hyperparams": []map[string]any{
{"key": "confidence_threshold", "value": "0.10"},
},
},
},
"inputs": map[string]any{
"stream": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "bucket": "b", "prefix": "p/"}},
},
}
gateResult := validateProductionRunSecurityGateContract(release, tamperedPayload)
if gateResult.Allowed {
t.Fatal("expected security gate to reject intermediate node hyperparameter tampering")
}
if gateResult.StatusCode != http.StatusUnprocessableEntity {
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
}
if gateResult.Code != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got '%s'", gateResult.Code)
}
if len(gateResult.Issues) == 0 {
t.Fatal("expected issues list to identify the offending node override")
}
})
t.Run("TC-T2-02: Security Gate Rejects Non-Root Intermediate Node Input Binding with 422 production_input_forbidden", func(t *testing.T) {
nonRootInputPayload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"stream": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "bucket": "b", "prefix": "p/"}},
"node-classifier.in": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"injected": "malicious_intermediate_data"},
},
},
}
gateResult := validateProductionRunSecurityGateContract(release, nonRootInputPayload)
if gateResult.Allowed {
t.Fatal("expected security gate to reject non-root intermediate input injection")
}
if gateResult.StatusCode != http.StatusUnprocessableEntity {
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
}
if gateResult.Code != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got '%s'", gateResult.Code)
}
})
t.Run("TC-T2-03: Security Gate Rejects Operator Image or Command Tampering in Production Run with 422", func(t *testing.T) {
imageTamperPayload := map[string]any{
"trigger": "manual",
"nodeOverrides": map[string]any{
"node-classifier": map[string]any{
"image": "attacker-controlled-registry.example.com/trojan:latest",
},
},
"inputs": map[string]any{
"stream": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "bucket": "b", "prefix": "p/"}},
},
}
gateResult := validateProductionRunSecurityGateContract(release, imageTamperPayload)
if gateResult.Allowed {
t.Fatal("expected security gate to reject operator image modification in production run")
}
if gateResult.StatusCode != http.StatusUnprocessableEntity {
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
}
if gateResult.Code != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got '%s'", gateResult.Code)
}
})
t.Run("TC-T2-04: Reject Production Run on Unreleased Draft ID with 404/422", func(t *testing.T) {
wf := h.createWorkflow(project.ID, "Unreleased Draft Workflow", nodes, edges, inputs)
// Calling release endpoint with a draft workflow ID (which has no WorkflowReleaseRecord)
rec := h.serveHTTP(http.MethodGet, fmt.Sprintf("/api/workflow-releases/%s", wf.ID), nil)
if rec.Code != http.StatusNotFound {
t.Fatalf("expected 404 Not Found for unreleased workflow ID at release endpoint, got %d", rec.Code)
}
})
t.Run("TC-T2-05: Reject Production Run on Nonexistent Release ID UUID with 404", func(t *testing.T) {
rec := h.serveHTTP(http.MethodGet, "/api/workflow-releases/workflow-release-uuid-nonexistent-9999", nil)
if rec.Code != http.StatusNotFound {
t.Fatalf("expected 404 Not Found for nonexistent release ID, got %d", rec.Code)
}
})
t.Run("TC-T2-06: Empty Root Inputs Validation Rejection on Required Fields", func(t *testing.T) {
emptyInputPayload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{},
}
gateResult := validateProductionRunSecurityGateContract(release, emptyInputPayload)
if gateResult.Allowed {
t.Fatal("expected security gate to reject missing required root input")
}
if gateResult.StatusCode != http.StatusUnprocessableEntity {
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
}
if gateResult.Code != "workflow_inputs_invalid" {
t.Fatalf("expected code 'workflow_inputs_invalid', got '%s'", gateResult.Code)
}
})
t.Run("TC-T2-07: Workforce Metrics Division-by-Zero Protection with Zero Production Worklogs", func(t *testing.T) {
// Zero events in database
metrics := calculateWorkforceMetricsIsolated(nil, nil)
if metrics.TotalEvents != 0 {
t.Fatalf("expected 0 events, got %d", metrics.TotalEvents)
}
if metrics.QualityPassRate != nil {
t.Fatalf("expected nil QualityPassRate on 0 events, got %v", metrics.QualityPassRate)
}
if metrics.AcceptancePassRate != nil {
t.Fatalf("expected nil AcceptancePassRate on 0 events, got %v", metrics.AcceptancePassRate)
}
if metrics.ReworkRate != nil {
t.Fatalf("expected nil ReworkRate on 0 events, got %v", metrics.ReworkRate)
}
// Also verify via server dashboard endpoint
rec := h.serveHTTP(http.MethodGet, "/api/dashboard", nil)
if rec.Code != http.StatusOK {
t.Fatalf("expected 200 OK from dashboard with 0 worklogs, got %d", rec.Code)
}
})
t.Run("TC-T2-08: Inline Payload Size Boundary Rejection (>64KB / 65536 bytes)", func(t *testing.T) {
oversizedData := strings.Repeat("A", maxInlinePayloadBytes+100)
oversizedPayload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"stream": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"data": oversizedData},
},
},
}
gateResult := validateProductionRunSecurityGateContract(release, oversizedPayload)
if gateResult.Allowed {
t.Fatal("expected security gate to reject oversized inline payload")
}
if gateResult.StatusCode != http.StatusBadRequest {
t.Fatalf("expected HTTP 400 for payload size violation, got %d", gateResult.StatusCode)
}
if gateResult.Code != "workflow_inputs_invalid" {
t.Fatalf("expected code 'workflow_inputs_invalid', got '%s'", gateResult.Code)
}
})
t.Run("TC-T2-09: Invalid Mode Query Parameter Rejection / Graceful Fallback", func(t *testing.T) {
// Query with invalid mode
rec := h.serveHTTP(http.MethodGet, "/api/runs?mode=invalid_staging_mode", nil)
// Should return 400 Bad Request or handle gracefully without crashing
if rec.Code != http.StatusBadRequest && rec.Code != http.StatusOK {
t.Fatalf("unexpected response code for invalid mode query: %d", rec.Code)
}
})
t.Run("TC-T2-10: Debug Overrides with Nonexistent Node ID Handled Gracefully", func(t *testing.T) {
jsonNodes := []domain.WorkflowNode{
{ID: "root-json", Type: "operator", Data: domain.WorkflowNodeData{Label: "Root JSON Reader", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
}
jsonInputs := []domain.WorkflowInputDefinition{
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "root-json", TargetPortID: "in"}}},
}
wf := h.createWorkflow(project.ID, "Ghost Override Workflow", jsonNodes, nil, jsonInputs)
payload := map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"nodeOverrides": map[string]any{
"node-ghost-nonexistent": map[string]any{
"hyperparams": []map[string]any{{"key": "temp", "value": "1"}},
},
},
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"ok": true}},
},
}
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), payload)
// Debug run accepts payload and ignores ghost node overrides without crashing
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
})
}
// ============================================================================
// Tier 3: Cross-Feature Combinations & Full Lifecycles
// ============================================================================
func TestDualTrack_Tier3_CrossFeatureCombinations(t *testing.T) {
h := newDualTrackTestHarness(t)
project := h.createProject("Tier3-Combinations", "Cross-Feature Lifecycles")
t.Run("TC-T3-01: Full Lifecycle: Workflow Release -> Production Run -> Run Query Filter -> Delivery Package Check", func(t *testing.T) {
// 1. Pipeline engineer designs 4-node workflow
nodes := []domain.WorkflowNode{
{ID: "bos-source", Type: "operator", Data: domain.WorkflowNodeData{Label: "BOS Multi-Cam Source", Inputs: []domain.PortDefinition{{ID: "stream", Name: "Video Stream", Type: domain.PortTypeFile, Required: true}}}},
{ID: "yolo-detector", Type: "operator", Data: domain.WorkflowNodeData{Label: "YOLO Detector", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
{ID: "human-gate", Type: "operator", Data: domain.WorkflowNodeData{Label: "Human QA Review", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
{ID: "delivery-export", Type: "operator", Data: domain.WorkflowNodeData{Label: "Delivery Package Export", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "bos-source", Target: "yolo-detector"},
{ID: "e2", Source: "yolo-detector", Target: "human-gate"},
{ID: "e3", Source: "human-gate", Target: "delivery-export"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "stream", Name: "Video Stream", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "bos-source", TargetPortID: "stream"}}},
}
// 2. Publish release v1
release := h.publishWorkflowRelease("tmpl-ad-01", "Autonomous Driving Vision Pipeline", 1, nodes, edges, inputs)
if release.Version != 1 {
t.Fatalf("expected version 1, got %d", release.Version)
}
// 3. Trigger Production Run with BOS root dataset ref
prodPayload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"stream": map[string]any{
"mode": "ref",
"type": "file",
"ref": map[string]any{
"provider": "BOS",
"connectionId": "bos-conn-main",
"bucket": "ad-dataset-bucket",
"prefix": "camera_ingest_batch_20260819/",
"kind": "directory",
},
},
},
}
gateResult := validateProductionRunSecurityGateContract(release, prodPayload)
if !gateResult.Allowed {
t.Fatalf("production run security gate failed: %v", gateResult)
}
// 4. Create Production Run Record in Store
prodRun := domain.RunRecord{
ID: "run-ad-prod-001",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: "wf-ad-prod",
WorkflowName: release.Name,
Trigger: domain.RunTriggerManual,
DefinitionVersion: 1,
Status: domain.RunStatusCompleted,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := h.store.UpsertRun(h.ctx, prodRun); err != nil {
t.Fatalf("failed to insert prod run: %v", err)
}
// 5. Human QA Gate passes -> formal DeliveryPackageRecord generated
deliveryPkg := domain.DeliveryPackageRecord{
ID: "pkg-ad-prod-001",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: prodRun.WorkflowID,
ParentRunID: prodRun.ID,
Status: domain.DeliveryPackagePendingAcceptance,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := h.store.UpsertDeliveryPackage(h.ctx, deliveryPkg); err != nil {
t.Fatalf("failed to upsert delivery package: %v", err)
}
// 6. Verify delivery package is queryable and bound to production run
pkgs, err := h.store.ListDeliveryPackages(h.ctx, domain.DefaultOrganizationID, domain.DeliveryPackageQuery{ProjectID: project.ID})
if err != nil || len(pkgs) != 1 {
t.Fatalf("expected 1 formal delivery package, got %d (err=%v)", len(pkgs), err)
}
if pkgs[0].ParentRunID != prodRun.ID {
t.Fatalf("expected parent run %s, got %s", prodRun.ID, pkgs[0].ParentRunID)
}
})
t.Run("TC-T3-02: Full Lifecycle: Draft Workflow -> Debug Run with Overrides -> Single Node Run -> Workforce Metric Isolation", func(t *testing.T) {
nodes := []domain.WorkflowNode{
{ID: "src", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
{ID: "ai", Type: "operator", Data: domain.WorkflowNodeData{Label: "AI Model", Hyperparams: []domain.HyperParam{{Key: "confidence", Value: "0.80"}}}},
}
edges := []domain.WorkflowEdge{{ID: "e1", Source: "src", Target: "ai"}}
inputs := []domain.WorkflowInputDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "src", TargetPortID: "in"}}}}
wf := h.createWorkflow(project.ID, "Draft Debug Lifecycle", nodes, edges, inputs)
// 1. Debug run with override and single node target
debugPayload := map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"nodeOverrides": map[string]any{
"ai": map[string]any{"hyperparams": []map[string]any{{"key": "confidence", "value": "0.35"}}},
},
"debugTarget": map[string]any{
"targetNodeId": "ai",
"singleNode": true,
},
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"test": "val"}},
},
}
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), debugPayload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
// 2. Add debug worklogs
eventModes := map[string]string{
"evt-dbg-001": runModeDebug,
"evt-dbg-002": runModeDebug,
}
events := []domain.AnnotationWorklogEventRecord{
{ID: "evt-dbg-001", OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID, EventType: "quality_rejected", DurationSeconds: 3600, OutputDelta: 100},
{ID: "evt-dbg-002", OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID, EventType: "rework_submitted", DurationSeconds: 1800, OutputDelta: 50},
}
metrics := calculateWorkforceMetricsIsolated(events, eventModes)
// 100% of debug events must be excluded
if metrics.ProductionCount != 0 {
t.Fatalf("expected 0 production events, got %d", metrics.ProductionCount)
}
if metrics.DebugExcludedCount != 2 {
t.Fatalf("expected 2 debug events excluded, got %d", metrics.DebugExcludedCount)
}
if metrics.TotalDurationSeconds != 0 {
t.Fatalf("expected 0s duration, got %d", metrics.TotalDurationSeconds)
}
})
t.Run("TC-T3-03: Subtree Rerun Mode Inheritance Isolation", func(t *testing.T) {
type rerunPlan struct {
ParentMode string
ParentRel string
TargetNode string
ChildMode string
ChildRel string
}
plans := []rerunPlan{
{ParentMode: runModeProduction, ParentRel: "rel-v1", TargetNode: "ai-node", ChildMode: runModeProduction, ChildRel: "rel-v1"},
{ParentMode: runModeDebug, ParentRel: "", TargetNode: "ai-node", ChildMode: runModeDebug, ChildRel: ""},
}
for _, p := range plans {
childMode := p.ParentMode
var childRel string
if childMode == runModeProduction {
childRel = p.ParentRel
}
if childMode != p.ChildMode || childRel != p.ChildRel {
t.Fatalf("mode inheritance mismatch for plan %#v: got childMode=%s childRel=%s", p, childMode, childRel)
}
}
})
t.Run("TC-T3-04: Multi-Tenant Concurrent Dual-Track Execution Isolation", func(t *testing.T) {
// Tenant Alpha (Production) vs Tenant Beta (Debug)
tenantAlphaOrg := "org-alpha"
tenantBetaOrg := "org-beta"
// Tenant Alpha runs production
alphaRun := domain.RunRecord{
ID: "run-alpha-prod",
OrganizationID: tenantAlphaOrg,
ProjectID: "proj-alpha",
WorkflowID: "wf-alpha",
WorkflowName: "Alpha Prod",
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
}
if err := h.store.UpsertRun(h.ctx, alphaRun); err != nil {
t.Fatalf("failed to insert alpha run: %v", err)
}
// Tenant Beta runs 10 debug runs
for i := 1; i <= 10; i++ {
betaRun := domain.RunRecord{
ID: fmt.Sprintf("run-beta-dbg-%d", i),
OrganizationID: tenantBetaOrg,
ProjectID: "proj-beta",
WorkflowID: "wf-beta",
WorkflowName: "Beta Debug",
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
}
if err := h.store.UpsertRun(h.ctx, betaRun); err != nil {
t.Fatalf("failed to insert beta run: %v", err)
}
}
// Verify Tenant Alpha sees only Alpha runs
alphaRuns, err := h.store.ListRunSummariesPageByOrganization(h.ctx, tenantAlphaOrg, domain.RunSummaryQuery{ProjectID: "proj-alpha"})
if err != nil {
t.Fatalf("alpha list runs error: %v", err)
}
if len(alphaRuns.Items) != 1 {
t.Fatalf("expected 1 run for Tenant Alpha, got %d", len(alphaRuns.Items))
}
if alphaRuns.Items[0].ID != alphaRun.ID {
t.Fatalf("expected run ID %s, got %s", alphaRun.ID, alphaRuns.Items[0].ID)
}
})
t.Run("TC-T3-05: Multi-Version Release Transition (v1 -> v2) with Active Production and Debug Runs", func(t *testing.T) {
nodesV1 := []domain.WorkflowNode{{ID: "n1", Type: "operator", Data: domain.WorkflowNodeData{Label: "V1 Node"}}}
relV1 := h.publishWorkflowRelease("tmpl-mv-01", "Multi-Version Pipeline", 1, nodesV1, nil, nil)
nodesV2 := []domain.WorkflowNode{
{ID: "n1", Type: "operator", Data: domain.WorkflowNodeData{Label: "V1 Node"}},
{ID: "n2", Type: "operator", Data: domain.WorkflowNodeData{Label: "V2 Node Added"}},
}
relV2 := h.publishWorkflowRelease("tmpl-mv-01", "Multi-Version Pipeline", 2, nodesV2, nil, nil)
if relV1.Version != 1 || relV2.Version != 2 {
t.Fatalf("expected release versions 1 and 2, got %d and %d", relV1.Version, relV2.Version)
}
if relV1.ContentDigest == relV2.ContentDigest {
t.Fatalf("content digests must be distinct across versions: %s", relV1.ContentDigest)
}
})
}
// ============================================================================
// Tier 4: Real-World Application Scenarios
// ============================================================================
func TestDualTrack_Tier4_RealWorldApplicationScenarios(t *testing.T) {
h := newDualTrackTestHarness(t)
project := h.createProject("Tier4-Real-World-Scenarios", "Industrial End-to-End Simulations")
// Register storage connections for BOS tests
h.createStorageConnection("bos-dev", domain.StorageProviderBOS)
h.createStorageConnection("bos-prod", domain.StorageProviderBOS)
t.Run("TC-T4-01: Autonomous Driving Multi-Camera Vision Ingestion Pipeline Simulation", func(t *testing.T) {
// 5-Node DAG: BOS Video Source -> LiDAR Sync -> WILOR 3D Pose -> Human Review -> Policy Export
nodes := []domain.WorkflowNode{
{ID: "bos-cam", Type: "operator", Data: domain.WorkflowNodeData{Label: "BOS Multi-Cam Video Ingestion", Inputs: []domain.PortDefinition{{ID: "cam_stream", Name: "Raw Video", Type: domain.PortTypeFile, Required: true}}}},
{ID: "lidar-sync", Type: "operator", Data: domain.WorkflowNodeData{Label: "LiDAR Synchronization Engine", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
{ID: "wilor-detector", Type: "operator", Data: domain.WorkflowNodeData{Label: "WILOR 3D Hand & Head Pose Detector", Hyperparams: []domain.HyperParam{{Key: "iou_threshold", Value: "0.50"}}, Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
{ID: "human-qa", Type: "operator", Data: domain.WorkflowNodeData{Label: "Autonomous Driving QA Gate", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
{ID: "policy-export", Type: "operator", Data: domain.WorkflowNodeData{Label: "Robotics Policy Training Export", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "bos-cam", Target: "lidar-sync"},
{ID: "e2", Source: "lidar-sync", Target: "wilor-detector"},
{ID: "e3", Source: "wilor-detector", Target: "human-qa"},
{ID: "e4", Source: "human-qa", Target: "policy-export"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "cam_stream", Name: "Raw Video", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "bos-cam", TargetPortID: "cam_stream"}}},
}
// Phase 1: AI Engineer runs Debug Run on sample clip with transient overrides
draftWf := h.createWorkflow(project.ID, "AD Vision Pipeline Draft", nodes, edges, inputs)
debugReq := map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"nodeOverrides": map[string]any{
"wilor-detector": map[string]any{
"hyperparams": []map[string]any{
{"key": "iou_threshold", "value": "0.65"},
{"key": "box_score_threshold", "value": "0.45"},
},
},
},
"inputs": map[string]any{
"cam_stream": map[string]any{
"mode": "ref",
"type": "file",
"ref": map[string]any{"provider": "BOS", "connectionId": "bos-dev", "bucket": "test-bucket", "prefix": "sample_clip_20frames/", "kind": "directory"},
},
},
}
debugRec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", draftWf.ID), debugReq)
if debugRec.Code != http.StatusAccepted {
t.Fatalf("failed debug run trigger: %d body=%s", debugRec.Code, debugRec.Body.String())
}
// Phase 2: Release v1.0.0 is published for 100,000-frame customer batch
rel := h.publishWorkflowRelease("tmpl-ad-vision", "AD Vision Pipeline Release v1.0.0", 1, nodes, edges, inputs)
// Production Run triggered strictly with root BOS dataset
prodReq := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"cam_stream": map[string]any{
"mode": "ref",
"type": "file",
"ref": map[string]any{"provider": "BOS", "connectionId": "bos-prod", "bucket": "test-bucket", "prefix": "batch_100k_frames/", "kind": "directory"},
},
},
}
gate := validateProductionRunSecurityGateContract(rel, prodReq)
if !gate.Allowed {
t.Fatalf("production security gate failed: %v", gate)
}
})
t.Run("TC-T4-02: HoloBrain AI Policy Training and Checkpoint Evaluation Simulation", func(t *testing.T) {
nodes := []domain.WorkflowNode{
{ID: "demo-ingest", Type: "operator", Data: domain.WorkflowNodeData{Label: "Demonstration Ingestion", Inputs: []domain.PortDefinition{{ID: "trajectories", Name: "Trajectories", Type: domain.PortTypeJSON, Required: true}}}},
{ID: "holobrain-train", Type: "operator", Data: domain.WorkflowNodeData{Label: "HoloBrain ACT Training", Hyperparams: []domain.HyperParam{{Key: "epochs", Value: "100"}, {Key: "batch_size", Value: "64"}}}},
{ID: "checkpoint-eval", Type: "operator", Data: domain.WorkflowNodeData{Label: "Model Checkpoint Evaluator"}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "demo-ingest", Target: "holobrain-train"},
{ID: "e2", Source: "holobrain-train", Target: "checkpoint-eval"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "trajectories", Name: "Trajectories", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "demo-ingest", TargetPortID: "trajectories"}}},
}
wf := h.createWorkflow(project.ID, "HoloBrain Policy Training Draft", nodes, edges, inputs)
// ML engineer runs 1-epoch smoke test with single-node debug on checkpoint-eval
smokePayload := map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"nodeOverrides": map[string]any{
"holobrain-train": map[string]any{
"hyperparams": []map[string]any{
{"key": "epochs", "value": "1"},
{"key": "batch_size", "value": "4"},
},
},
},
"debugTarget": map[string]any{
"targetNodeId": "checkpoint-eval",
"singleNode": true,
},
"inputs": map[string]any{
"trajectories": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"num_episodes": 2}},
},
}
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), smokePayload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
})
t.Run("TC-T4-03: Multi-Vendor Supplier Workforce Stats and Financial Reconciliation Audit", func(t *testing.T) {
vendorAlphaID := "vendor-alpha"
vendorBetaID := "vendor-beta"
eventModes := make(map[string]string)
var events []domain.AnnotationWorklogEventRecord
// Vendor Alpha Production Worklogs: 95 approved, 5 rejected (95% pass rate)
for i := 1; i <= 95; i++ {
id := fmt.Sprintf("evt-alpha-appr-%d", i)
events = append(events, domain.AnnotationWorklogEventRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
VendorID: vendorAlphaID,
EventType: "quality_approved",
DurationSeconds: 120,
OutputDelta: 1,
})
eventModes[id] = runModeProduction
}
for i := 1; i <= 5; i++ {
id := fmt.Sprintf("evt-alpha-rej-%d", i)
events = append(events, domain.AnnotationWorklogEventRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
VendorID: vendorAlphaID,
EventType: "quality_rejected",
DurationSeconds: 120,
OutputDelta: 1,
})
eventModes[id] = runModeProduction
}
// Vendor Beta Production Worklogs: 88 approved, 12 rejected (88% pass rate)
for i := 1; i <= 88; i++ {
id := fmt.Sprintf("evt-beta-appr-%d", i)
events = append(events, domain.AnnotationWorklogEventRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
VendorID: vendorBetaID,
EventType: "quality_approved",
DurationSeconds: 150,
OutputDelta: 1,
})
eventModes[id] = runModeProduction
}
for i := 1; i <= 12; i++ {
id := fmt.Sprintf("evt-beta-rej-%d", i)
events = append(events, domain.AnnotationWorklogEventRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
VendorID: vendorBetaID,
EventType: "quality_rejected",
DurationSeconds: 150,
OutputDelta: 1,
})
eventModes[id] = runModeProduction
}
// 100 Practice Debug Worklogs from trial annotators (would destroy financial audit if leaked)
for i := 1; i <= 100; i++ {
id := fmt.Sprintf("evt-trial-dbg-%d", i)
events = append(events, domain.AnnotationWorklogEventRecord{
ID: id,
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
VendorID: vendorAlphaID,
EventType: "quality_rejected",
DurationSeconds: 600,
OutputDelta: 5,
})
eventModes[id] = runModeDebug
}
// Calculate Isolated Metrics
metrics := calculateWorkforceMetricsIsolated(events, eventModes)
if metrics.TotalEvents != 300 {
t.Fatalf("expected 300 total events, got %d", metrics.TotalEvents)
}
if metrics.ProductionCount != 200 {
t.Fatalf("expected 200 production events, got %d", metrics.ProductionCount)
}
if metrics.DebugExcludedCount != 100 {
t.Fatalf("expected 100 debug events excluded, got %d", metrics.DebugExcludedCount)
}
// Overall Production Pass Rate: (95+88) / 200 = 183 / 200 = 91.5%
if metrics.QualityPassRate == nil || *metrics.QualityPassRate != 91.5 {
t.Fatalf("expected QualityPassRate = 91.5%%, got %v", metrics.QualityPassRate)
}
})
t.Run("TC-T4-04: Urgent Hotfix and Production Failure Regression Debugging Simulation", func(t *testing.T) {
// Production run fails due to bad regex in transformer
failedProdRun := domain.RunRecord{
ID: "run-prod-failed-001",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: project.ID,
WorkflowID: "wf-hotfix-prod",
WorkflowName: "Hotfix Pipeline",
Status: domain.RunStatusFailed,
Error: "Regex parsing error at step 3: invalid datetime token",
}
if err := h.store.UpsertRun(h.ctx, failedProdRun); err != nil {
t.Fatalf("failed to insert failed run: %v", err)
}
// Engineer fixes in draft and verifies via single-node debug
nodes := []domain.WorkflowNode{
{ID: "node-fix", Type: "operator", Data: domain.WorkflowNodeData{Label: "Fixed DateTime Parser", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "node-fix", TargetPortID: "in"}}},
}
draftWf := h.createWorkflow(project.ID, "Hotfix Draft", nodes, nil, inputs)
debugRec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", draftWf.ID), map[string]any{
"trigger": "manual",
"mode": runModeDebug,
"inputs": map[string]any{"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"sample": "2026-08-19"}}},
})
if debugRec.Code != http.StatusAccepted {
t.Fatalf("failed hotfix debug run: %d", debugRec.Code)
}
// Responder publishes Release v2.0.0
relV2 := h.publishWorkflowRelease("tmpl-hotfix", "Hotfix Pipeline Release", 2, nodes, nil, inputs)
if relV2.Version != 2 {
t.Fatalf("expected version 2, got %d", relV2.Version)
}
// Historical failed production run record remains untouched
historical, err := h.store.GetRun(h.ctx, failedProdRun.ID)
if err != nil || historical == nil {
t.Fatalf("failed to retrieve historical run: %v", err)
}
if historical.Status != domain.RunStatusFailed {
t.Fatalf("historical failed run was mutated! status=%s", historical.Status)
}
})
t.Run("TC-T4-05: High-Concurrency Dual-Track Multi-Tenant Load and Data Integrity Verification", func(t *testing.T) {
const numWorkers = 20
var wg sync.WaitGroup
wg.Add(numWorkers)
errChan := make(chan error, numWorkers)
for i := 0; i < numWorkers; i++ {
workerID := i
go func() {
defer wg.Done()
tenantID := fmt.Sprintf("tenant-%d", workerID%5)
isProd := workerID%2 == 0
if isProd {
// Production run record
run := domain.RunRecord{
ID: fmt.Sprintf("concurrent-prod-%d", workerID),
OrganizationID: tenantID,
ProjectID: fmt.Sprintf("proj-%s", tenantID),
WorkflowID: fmt.Sprintf("wf-%s", tenantID),
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := h.store.UpsertRun(h.ctx, run); err != nil {
errChan <- fmt.Errorf("worker %d upsert prod run error: %w", workerID, err)
return
}
} else {
// Debug run record
run := domain.RunRecord{
ID: fmt.Sprintf("concurrent-dbg-%d", workerID),
OrganizationID: tenantID,
ProjectID: fmt.Sprintf("proj-%s", tenantID),
WorkflowID: fmt.Sprintf("wf-%s", tenantID),
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := h.store.UpsertRun(h.ctx, run); err != nil {
errChan <- fmt.Errorf("worker %d upsert debug run error: %w", workerID, err)
return
}
}
}()
}
wg.Wait()
close(errChan)
for err := range errChan {
if err != nil {
t.Fatalf("concurrent execution failure: %v", err)
}
}
})
}