forked from longtao.wu/droboticflow
1579 lines
60 KiB
Go
1579 lines
60 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
// ============================================================================
|
|
// Dual-Track E2E Test Suite Specification & Harness
|
|
// ============================================================================
|
|
// Covers the complete 4-tier testing methodology:
|
|
// - Tier 1: Feature Coverage (F1 - F13 primary behavior)
|
|
// - Tier 2: Boundary & Corner Cases (Security gates, payload limits, division-by-zero)
|
|
// - Tier 3: Cross-Feature Combinations (Full lifecycles, subtree rerun, multi-tenant)
|
|
// - Tier 4: Real-World Application Scenarios (Autonomous driving, HoloBrain AI, Workforce audit)
|
|
|
|
const (
|
|
runModeProduction = "production"
|
|
runModeDebug = "debug"
|
|
runModeAll = "all"
|
|
)
|
|
|
|
type dualTrackTestHarness struct {
|
|
t *testing.T
|
|
store store.Store
|
|
handler http.Handler
|
|
ctx context.Context
|
|
}
|
|
|
|
func newDualTrackTestHarness(t *testing.T) *dualTrackTestHarness {
|
|
t.Helper()
|
|
dataStore := store.NewMemoryStore()
|
|
h := &dualTrackTestHarness{
|
|
t: t,
|
|
store: dataStore,
|
|
handler: NewHandler(dataStore, nil),
|
|
ctx: context.Background(),
|
|
}
|
|
return h
|
|
}
|
|
|
|
func (h *dualTrackTestHarness) createProject(name, description string) domain.ProjectRecord {
|
|
h.t.Helper()
|
|
p, err := h.store.CreateProject(h.ctx, domain.CreateProjectInput{
|
|
Name: name,
|
|
Description: description,
|
|
})
|
|
if err != nil {
|
|
h.t.Fatalf("failed to create test project: %v", err)
|
|
}
|
|
return *p
|
|
}
|
|
|
|
func (h *dualTrackTestHarness) createStorageConnection(id string, provider domain.StorageProvider) domain.StorageConnectionRecord {
|
|
h.t.Helper()
|
|
conn := domain.StorageConnectionRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
Name: id,
|
|
Provider: provider,
|
|
Endpoint: "http://bos.example.local",
|
|
Bucket: "test-bucket",
|
|
AccessKeyID: "ak-test",
|
|
SecretAccessKey: "sk-test",
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := h.store.UpsertStorageConnection(h.ctx, conn); err != nil {
|
|
h.t.Fatalf("failed to upsert storage connection: %v", err)
|
|
}
|
|
return conn
|
|
}
|
|
|
|
func (h *dualTrackTestHarness) createWorkflow(projectID, name string, nodes []domain.WorkflowNode, edges []domain.WorkflowEdge, inputs []domain.WorkflowInputDefinition) domain.WorkflowRecord {
|
|
h.t.Helper()
|
|
wf, err := h.store.CreateWorkflow(h.ctx, domain.CreateWorkflowInput{
|
|
ProjectID: projectID,
|
|
Name: name,
|
|
Description: "Test Workflow for Dual-Track Isolation",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: nodes,
|
|
Edges: edges,
|
|
WorkflowInputs: inputs,
|
|
})
|
|
if err != nil {
|
|
h.t.Fatalf("failed to create test workflow: %v", err)
|
|
}
|
|
return *wf
|
|
}
|
|
|
|
func (h *dualTrackTestHarness) publishWorkflowRelease(templateID, name string, version int, nodes []domain.WorkflowNode, edges []domain.WorkflowEdge, inputs []domain.WorkflowInputDefinition) domain.WorkflowReleaseRecord {
|
|
h.t.Helper()
|
|
now := time.Now().UTC().Format(time.RFC3339)
|
|
release := domain.WorkflowReleaseRecord{
|
|
ID: fmt.Sprintf("workflow-release-%s-v%d", templateID, version),
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
TemplateID: templateID,
|
|
Version: version,
|
|
Name: name,
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindStandard,
|
|
ExecutionScope: domain.WorkflowExecutionScopeProduction,
|
|
Nodes: nodes,
|
|
Edges: edges,
|
|
WorkflowInputs: inputs,
|
|
ContentDigest: fmt.Sprintf("sha256:release-digest-%s-v%d", templateID, version),
|
|
ReleaseNotes: fmt.Sprintf("Release notes for version %d", version),
|
|
PublishedByUserID: "user-pipeline-engineer",
|
|
PublishedAt: now,
|
|
CreatedAt: now,
|
|
}
|
|
if err := h.store.UpsertWorkflowRelease(h.ctx, release); err != nil {
|
|
h.t.Fatalf("failed to upsert workflow release: %v", err)
|
|
}
|
|
return release
|
|
}
|
|
|
|
func (h *dualTrackTestHarness) serveHTTP(method, path string, body any) *httptest.ResponseRecorder {
|
|
h.t.Helper()
|
|
var bodyReader *bytes.Reader
|
|
if body != nil {
|
|
switch v := body.(type) {
|
|
case string:
|
|
bodyReader = bytes.NewReader([]byte(v))
|
|
case []byte:
|
|
bodyReader = bytes.NewReader(v)
|
|
default:
|
|
payloadBytes, err := json.Marshal(body)
|
|
if err != nil {
|
|
h.t.Fatalf("failed to marshal request body: %v", err)
|
|
}
|
|
bodyReader = bytes.NewReader(payloadBytes)
|
|
}
|
|
} else {
|
|
bodyReader = bytes.NewReader([]byte{})
|
|
}
|
|
|
|
req := httptest.NewRequest(method, path, bodyReader)
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
h.handler.ServeHTTP(rec, req)
|
|
return rec
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// Domain & Security Gate Contract Helpers
|
|
// ----------------------------------------------------------------------------
|
|
|
|
type securityGateResult struct {
|
|
Allowed bool `json:"allowed"`
|
|
StatusCode int `json:"statusCode"`
|
|
Code string `json:"code"`
|
|
Message string `json:"message"`
|
|
Issues []string `json:"issues"`
|
|
}
|
|
|
|
func findRootNodeIDs(nodes []domain.WorkflowNode, edges []domain.WorkflowEdge) map[string]bool {
|
|
targeted := make(map[string]bool)
|
|
for _, e := range edges {
|
|
targeted[e.Target] = true
|
|
}
|
|
roots := make(map[string]bool)
|
|
for _, n := range nodes {
|
|
if !targeted[n.ID] {
|
|
roots[n.ID] = true
|
|
}
|
|
}
|
|
return roots
|
|
}
|
|
|
|
func validateProductionRunSecurityGateContract(release domain.WorkflowReleaseRecord, payload map[string]any) securityGateResult {
|
|
if payload == nil {
|
|
return securityGateResult{Allowed: false, StatusCode: http.StatusBadRequest, Code: "invalid_payload", Message: "Payload must be an object"}
|
|
}
|
|
|
|
// Gate 1: Check for forbidden nodeOverrides
|
|
if overrides, exists := payload["nodeOverrides"]; exists && overrides != nil {
|
|
if overrideMap, ok := overrides.(map[string]any); ok && len(overrideMap) > 0 {
|
|
var issues []string
|
|
for nodeID := range overrideMap {
|
|
issues = append(issues, fmt.Sprintf("nodeOverrides.%s: Node parameter overrides are forbidden in production runs", nodeID))
|
|
}
|
|
return securityGateResult{
|
|
Allowed: false,
|
|
StatusCode: http.StatusUnprocessableEntity,
|
|
Code: "production_input_forbidden",
|
|
Message: "Intermediate node hyperparameters and overrides are forbidden in production run",
|
|
Issues: issues,
|
|
}
|
|
}
|
|
}
|
|
|
|
// Gate 2: Check for forbidden debugTarget
|
|
if debugTarget, exists := payload["debugTarget"]; exists && debugTarget != nil {
|
|
return securityGateResult{
|
|
Allowed: false,
|
|
StatusCode: http.StatusUnprocessableEntity,
|
|
Code: "production_input_forbidden",
|
|
Message: "debugTarget is forbidden in production run",
|
|
}
|
|
}
|
|
|
|
// Gate 3: Check that inputs bind only to root nodes and check payload sizes
|
|
roots := findRootNodeIDs(release.Nodes, release.Edges)
|
|
if rawInputs, exists := payload["inputs"]; exists && rawInputs != nil {
|
|
if inputMap, ok := rawInputs.(map[string]any); ok {
|
|
for key, rawVal := range inputMap {
|
|
// Size check
|
|
if valMap, isMap := rawVal.(map[string]any); isMap {
|
|
if mode, _ := valMap["mode"].(string); mode == "inline" {
|
|
if inlineData, hasInline := valMap["inline"]; hasInline {
|
|
dataBytes, _ := json.Marshal(inlineData)
|
|
if len(dataBytes) > maxInlinePayloadBytes {
|
|
return securityGateResult{
|
|
Allowed: false,
|
|
StatusCode: http.StatusBadRequest,
|
|
Code: "workflow_inputs_invalid",
|
|
Message: fmt.Sprintf("inline payload for '%s' exceeds %d bytes; use ref mode", key, maxInlinePayloadBytes),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check root node binding
|
|
parts := strings.Split(key, ".")
|
|
targetNode := parts[0]
|
|
isRootBinding := roots[targetNode]
|
|
for rootID := range roots {
|
|
if strings.Contains(key, rootID) {
|
|
isRootBinding = true
|
|
break
|
|
}
|
|
}
|
|
for _, def := range release.WorkflowInputs {
|
|
if def.ID == key {
|
|
for _, b := range def.Bindings {
|
|
if roots[b.NodeID] {
|
|
isRootBinding = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if !isRootBinding {
|
|
return securityGateResult{
|
|
Allowed: false,
|
|
StatusCode: http.StatusUnprocessableEntity,
|
|
Code: "production_input_forbidden",
|
|
Message: fmt.Sprintf("workflow inputs may only bind root nodes, unauthorized input '%s'", key),
|
|
Issues: []string{fmt.Sprintf("inputs.%s: Workflow inputs may only bind root nodes in production runs.", key)},
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Gate 4: Required inputs check
|
|
for _, inputDef := range release.WorkflowInputs {
|
|
if inputDef.Required {
|
|
inputMap, _ := payload["inputs"].(map[string]any)
|
|
val, present := inputMap[inputDef.ID]
|
|
if !present || val == nil {
|
|
return securityGateResult{
|
|
Allowed: false,
|
|
StatusCode: http.StatusUnprocessableEntity,
|
|
Code: "workflow_inputs_invalid",
|
|
Message: fmt.Sprintf("Required root input '%s' (%s) is missing", inputDef.Name, inputDef.ID),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return securityGateResult{Allowed: true, StatusCode: http.StatusAccepted}
|
|
}
|
|
|
|
type workforceMetricSummary struct {
|
|
TotalEvents int
|
|
ProductionCount int
|
|
DebugExcludedCount int
|
|
QualityApproved int64
|
|
QualityRejected int64
|
|
AcceptanceApproved int64
|
|
AcceptanceRejected int64
|
|
ReworkCount int64
|
|
SubmittedCount int64
|
|
TotalDurationSeconds int64
|
|
TotalOutputCount int64
|
|
QualityPassRate *float64
|
|
AcceptancePassRate *float64
|
|
ReworkRate *float64
|
|
}
|
|
|
|
func calculateWorkforceMetricsIsolated(events []domain.AnnotationWorklogEventRecord, eventModes map[string]string) workforceMetricSummary {
|
|
var summary workforceMetricSummary
|
|
summary.TotalEvents = len(events)
|
|
|
|
for _, event := range events {
|
|
mode := eventModes[event.ID]
|
|
if mode == runModeDebug {
|
|
summary.DebugExcludedCount++
|
|
continue
|
|
}
|
|
summary.ProductionCount++
|
|
summary.TotalDurationSeconds += event.DurationSeconds
|
|
summary.TotalOutputCount += event.OutputDelta
|
|
|
|
switch event.EventType {
|
|
case "annotation_submitted":
|
|
summary.SubmittedCount++
|
|
case "rework_submitted":
|
|
summary.ReworkCount++
|
|
case "quality_approved":
|
|
summary.QualityApproved++
|
|
case "quality_rejected":
|
|
summary.QualityRejected++
|
|
case "acceptance_approved":
|
|
summary.AcceptanceApproved++
|
|
case "acceptance_rejected":
|
|
summary.AcceptanceRejected++
|
|
}
|
|
}
|
|
|
|
qualityTotal := summary.QualityApproved + summary.QualityRejected
|
|
if qualityTotal > 0 {
|
|
rate := float64(summary.QualityApproved) / float64(qualityTotal) * 100.0
|
|
summary.QualityPassRate = &rate
|
|
}
|
|
|
|
acceptanceTotal := summary.AcceptanceApproved + summary.AcceptanceRejected
|
|
if acceptanceTotal > 0 {
|
|
rate := float64(summary.AcceptanceApproved) / float64(acceptanceTotal) * 100.0
|
|
summary.AcceptancePassRate = &rate
|
|
}
|
|
|
|
reworkTotal := summary.SubmittedCount + summary.ReworkCount
|
|
if reworkTotal > 0 {
|
|
rate := float64(summary.ReworkCount) / float64(reworkTotal) * 100.0
|
|
summary.ReworkRate = &rate
|
|
}
|
|
|
|
return summary
|
|
}
|
|
|
|
type decoratedArtifact struct {
|
|
ID string
|
|
Name string
|
|
IsTemporary bool
|
|
BadgeText string
|
|
EligibleForDeliveryPackage bool
|
|
}
|
|
|
|
func decorateArtifactsByMode(artifacts []domain.ExecutionArtifact, runMode string) []decoratedArtifact {
|
|
isDebug := runMode == runModeDebug
|
|
result := make([]decoratedArtifact, len(artifacts))
|
|
for i, a := range artifacts {
|
|
isTemp := isDebug
|
|
badge := "正式交付产物 (Production)"
|
|
if isTemp {
|
|
badge = "临时产物 (Debug)"
|
|
}
|
|
result[i] = decoratedArtifact{
|
|
ID: a.ID,
|
|
Name: a.Name,
|
|
IsTemporary: isTemp,
|
|
BadgeText: badge,
|
|
EligibleForDeliveryPackage: !isTemp && !isDebug,
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// ============================================================================
|
|
// Tier 1: Feature Coverage (Primary Behavior)
|
|
// ============================================================================
|
|
|
|
func TestDualTrack_Tier1_FeatureCoverage(t *testing.T) {
|
|
h := newDualTrackTestHarness(t)
|
|
project := h.createProject("Tier1-Feature-Coverage", "Primary Behavior Tests")
|
|
|
|
t.Run("TC-T1-F1: Debug Run Trigger on Draft Workflow", func(t *testing.T) {
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "node-source", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source Node", Inputs: []domain.PortDefinition{{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true}}}},
|
|
{ID: "node-ai", Type: "operator", Data: domain.WorkflowNodeData{Label: "AI Node", Inputs: []domain.PortDefinition{{ID: "data", Name: "Data", Type: domain.PortTypeJSON}}}},
|
|
}
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "edge-1", Source: "node-source", Target: "node-ai"},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "node-source", TargetPortID: "in"}}},
|
|
}
|
|
wf := h.createWorkflow(project.ID, "Draft Debug Workflow", nodes, edges, inputs)
|
|
|
|
// Assert draft state
|
|
if wf.DefinitionState != domain.WorkflowDefinitionStateDraft {
|
|
t.Fatalf("expected workflow definition state to be draft, got %v", wf.DefinitionState)
|
|
}
|
|
|
|
// Dispatch debug run trigger on draft
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"inputs": map[string]any{
|
|
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"debug_seed": 42}},
|
|
},
|
|
}
|
|
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), payload)
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected HTTP 202 Accepted on draft debug run, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
var createdRun domain.RunRecord
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &createdRun); err != nil {
|
|
t.Fatalf("failed to decode created run: %v", err)
|
|
}
|
|
|
|
if createdRun.WorkflowID != wf.ID {
|
|
t.Fatalf("expected workflow ID %s, got %s", wf.ID, createdRun.WorkflowID)
|
|
}
|
|
// Verify workflow in database remains in draft state
|
|
storedWf, err := h.store.GetWorkflow(h.ctx, wf.ID)
|
|
if err != nil || storedWf == nil {
|
|
t.Fatalf("workflow record missing: %v", err)
|
|
}
|
|
if storedWf.DefinitionState != domain.WorkflowDefinitionStateDraft {
|
|
t.Fatalf("expected workflow definition to remain draft, got %v", storedWf.DefinitionState)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F2: Production Run on Published WorkflowRelease", func(t *testing.T) {
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "root-bos", Type: "operator", Data: domain.WorkflowNodeData{Label: "BOS Reader", Inputs: []domain.PortDefinition{{ID: "dataset", Name: "Dataset", Type: domain.PortTypeFile, Required: true}}}},
|
|
{ID: "node-infer", Type: "operator", Data: domain.WorkflowNodeData{Label: "Model Inference"}},
|
|
}
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "root-bos", Target: "node-infer"},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "dataset", Name: "Dataset", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "root-bos", TargetPortID: "dataset"}}},
|
|
}
|
|
release := h.publishWorkflowRelease("tmpl-prod-01", "Production Release v1", 1, nodes, edges, inputs)
|
|
|
|
// Validate Production Run trigger request against security gate
|
|
prodPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"dataset": map[string]any{
|
|
"mode": "ref",
|
|
"type": "file",
|
|
"ref": map[string]any{
|
|
"provider": "BOS",
|
|
"connectionId": "bos-conn-prod",
|
|
"bucket": "prod-datasets",
|
|
"prefix": "batch-20260819/",
|
|
"kind": "directory",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
gateResult := validateProductionRunSecurityGateContract(release, prodPayload)
|
|
if !gateResult.Allowed {
|
|
t.Fatalf("expected security gate to allow valid production run, got code=%s msg=%s", gateResult.Code, gateResult.Message)
|
|
}
|
|
if gateResult.StatusCode != http.StatusAccepted {
|
|
t.Fatalf("expected status 202 Accepted, got %d", gateResult.StatusCode)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F3: Release Immutable Metadata and Provenance Binding", func(t *testing.T) {
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "root", Type: "operator", Data: domain.WorkflowNodeData{Label: "Root Node"}},
|
|
}
|
|
release := h.publishWorkflowRelease("tmpl-meta-01", "Metadata Binding Release", 2, nodes, nil, nil)
|
|
|
|
// Assert immutable release properties
|
|
if release.Version != 2 {
|
|
t.Fatalf("expected release version 2, got %d", release.Version)
|
|
}
|
|
if !strings.HasPrefix(release.ContentDigest, "sha256:") {
|
|
t.Fatalf("expected sha256 content digest, got %s", release.ContentDigest)
|
|
}
|
|
if release.ExecutionScope != domain.WorkflowExecutionScopeProduction {
|
|
t.Fatalf("expected production execution scope, got %s", release.ExecutionScope)
|
|
}
|
|
|
|
// Verify release record is queryable and intact in store
|
|
storedRel, err := h.store.GetWorkflowRelease(h.ctx, release.ID)
|
|
if err != nil || storedRel == nil {
|
|
t.Fatalf("failed to retrieve release: %v", err)
|
|
}
|
|
if storedRel.ContentDigest != release.ContentDigest {
|
|
t.Fatalf("content digest mismatch: expected %s, got %s", release.ContentDigest, storedRel.ContentDigest)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F4: Transient Node Overrides In Debug Run Without DB Mutation", func(t *testing.T) {
|
|
baselineHyperparams := []domain.HyperParam{
|
|
{Key: "confidence", Value: "0.85", Type: "number"},
|
|
{Key: "batch_size", Value: "32", Type: "number"},
|
|
}
|
|
nodes := []domain.WorkflowNode{
|
|
{
|
|
ID: "ai-model",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "AI Model",
|
|
Hyperparams: baselineHyperparams,
|
|
Inputs: []domain.PortDefinition{{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true}},
|
|
},
|
|
},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "ai-model", TargetPortID: "in"}}},
|
|
}
|
|
wf := h.createWorkflow(project.ID, "Transient Overrides Workflow", nodes, nil, inputs)
|
|
|
|
// Simulate debug run with transient hyperparam override confidence = 0.45
|
|
debugPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"nodeOverrides": map[string]any{
|
|
"ai-model": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "confidence", "value": "0.45"},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"test": true}},
|
|
},
|
|
}
|
|
|
|
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), debugPayload)
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted on debug run, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
// Verify database workflow record was NOT modified (zero-mutation invariant)
|
|
storedWf, err := h.store.GetWorkflow(h.ctx, wf.ID)
|
|
if err != nil || storedWf == nil {
|
|
t.Fatalf("workflow record missing: %v", err)
|
|
}
|
|
for _, hp := range storedWf.Nodes[0].Data.Hyperparams {
|
|
if hp.Key == "confidence" && hp.Value != "0.85" {
|
|
t.Fatalf("DB workflow hyperparameter was corrupted! expected 0.85, got %s", hp.Value)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F5: Single-Node Debug Target Execution", func(t *testing.T) {
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "node-A", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node A", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
|
|
{ID: "node-B", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node B"}},
|
|
{ID: "node-C", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node C"}},
|
|
{ID: "node-D", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node D"}},
|
|
{ID: "node-E", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node E"}},
|
|
}
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "node-A", Target: "node-B"},
|
|
{ID: "e2", Source: "node-B", Target: "node-C"},
|
|
{ID: "e3", Source: "node-C", Target: "node-D"},
|
|
{ID: "e4", Source: "node-D", Target: "node-E"},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "node-A", TargetPortID: "in"}}},
|
|
}
|
|
wf := h.createWorkflow(project.ID, "Single Node Debug DAG", nodes, edges, inputs)
|
|
|
|
// Target node-C only in debug run
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"debugTarget": map[string]any{
|
|
"targetNodeId": "node-C",
|
|
"singleNode": true,
|
|
},
|
|
"inputs": map[string]any{
|
|
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"inject": "payload-A"}},
|
|
},
|
|
}
|
|
|
|
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), payload)
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted on single node debug run, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F6: Monitoring Run Query Mode Filtering (production/debug/all)", func(t *testing.T) {
|
|
// Seed 5 production runs and 5 debug runs in store
|
|
now := time.Now().UTC()
|
|
var seededRuns []domain.RunRecord
|
|
runModes := make(map[string]string)
|
|
|
|
for i := 1; i <= 5; i++ {
|
|
prodRun := domain.RunRecord{
|
|
ID: fmt.Sprintf("run-prod-%02d", i),
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
WorkflowID: "wf-prod",
|
|
WorkflowName: "Production Pipeline",
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
CreatedAt: now.Add(-time.Duration(i) * time.Hour).Format(time.RFC3339),
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, prodRun); err != nil {
|
|
t.Fatalf("failed to seed prod run: %v", err)
|
|
}
|
|
seededRuns = append(seededRuns, prodRun)
|
|
runModes[prodRun.ID] = runModeProduction
|
|
}
|
|
|
|
for i := 1; i <= 5; i++ {
|
|
dbgRun := domain.RunRecord{
|
|
ID: fmt.Sprintf("run-dbg-%02d", i),
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
WorkflowID: "wf-dbg",
|
|
WorkflowName: "Debug Pipeline",
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
CreatedAt: now.Add(-time.Duration(i+10) * time.Hour).Format(time.RFC3339),
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, dbgRun); err != nil {
|
|
t.Fatalf("failed to seed debug run: %v", err)
|
|
}
|
|
seededRuns = append(seededRuns, dbgRun)
|
|
runModes[dbgRun.ID] = runModeDebug
|
|
}
|
|
|
|
// Filter mode=production
|
|
var prodFiltered []domain.RunRecord
|
|
for _, r := range seededRuns {
|
|
if runModes[r.ID] == runModeProduction {
|
|
prodFiltered = append(prodFiltered, r)
|
|
}
|
|
}
|
|
if len(prodFiltered) != 5 {
|
|
t.Fatalf("expected 5 production runs, got %d", len(prodFiltered))
|
|
}
|
|
|
|
// Filter mode=debug
|
|
var dbgFiltered []domain.RunRecord
|
|
for _, r := range seededRuns {
|
|
if runModes[r.ID] == runModeDebug {
|
|
dbgFiltered = append(dbgFiltered, r)
|
|
}
|
|
}
|
|
if len(dbgFiltered) != 5 {
|
|
t.Fatalf("expected 5 debug runs, got %d", len(dbgFiltered))
|
|
}
|
|
|
|
// Mode=all
|
|
if len(seededRuns) != 10 {
|
|
t.Fatalf("expected 10 combined runs, got %d", len(seededRuns))
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F7: Workforce Metric Calculation Isolation (Excluding Debug Events)", func(t *testing.T) {
|
|
eventModes := make(map[string]string)
|
|
var events []domain.AnnotationWorklogEventRecord
|
|
|
|
// 10 Production events: 9 approved, 1 rejected -> 90% Quality Pass Rate
|
|
for i := 1; i <= 9; i++ {
|
|
id := fmt.Sprintf("evt-prod-appr-%d", i)
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
EventType: "quality_approved",
|
|
DurationSeconds: 100,
|
|
OutputDelta: 10,
|
|
})
|
|
eventModes[id] = runModeProduction
|
|
}
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: "evt-prod-rej-1",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
EventType: "quality_rejected",
|
|
DurationSeconds: 100,
|
|
OutputDelta: 10,
|
|
})
|
|
eventModes["evt-prod-rej-1"] = runModeProduction
|
|
|
|
// 50 Debug events: all rejected (would destroy pass rate to ~15% if contaminated)
|
|
for i := 1; i <= 50; i++ {
|
|
id := fmt.Sprintf("evt-dbg-rej-%d", i)
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
EventType: "quality_rejected",
|
|
DurationSeconds: 600,
|
|
OutputDelta: 50,
|
|
})
|
|
eventModes[id] = runModeDebug
|
|
}
|
|
|
|
metrics := calculateWorkforceMetricsIsolated(events, eventModes)
|
|
if metrics.TotalEvents != 60 {
|
|
t.Fatalf("expected 60 total events, got %d", metrics.TotalEvents)
|
|
}
|
|
if metrics.ProductionCount != 10 {
|
|
t.Fatalf("expected 10 production events, got %d", metrics.ProductionCount)
|
|
}
|
|
if metrics.DebugExcludedCount != 50 {
|
|
t.Fatalf("expected 50 debug events excluded, got %d", metrics.DebugExcludedCount)
|
|
}
|
|
if metrics.QualityPassRate == nil || *metrics.QualityPassRate != 90.0 {
|
|
t.Fatalf("expected QualityPassRate = 90.0%%, got %v", metrics.QualityPassRate)
|
|
}
|
|
if metrics.TotalDurationSeconds != 1000 {
|
|
t.Fatalf("expected TotalDurationSeconds = 1000s, got %d", metrics.TotalDurationSeconds)
|
|
}
|
|
if metrics.TotalOutputCount != 100 {
|
|
t.Fatalf("expected TotalOutputCount = 100, got %d", metrics.TotalOutputCount)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F8: Delivery Package Isolation (Production Only)", func(t *testing.T) {
|
|
// Production package
|
|
prodPkg := domain.DeliveryPackageRecord{
|
|
ID: "pkg-prod-001",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
WorkflowID: "wf-prod",
|
|
ParentRunID: "run-prod-01",
|
|
Status: domain.DeliveryPackagePendingAcceptance,
|
|
}
|
|
if err := h.store.UpsertDeliveryPackage(h.ctx, prodPkg); err != nil {
|
|
t.Fatalf("failed to upsert prod delivery package: %v", err)
|
|
}
|
|
|
|
// Query delivery packages via API
|
|
rec := h.serveHTTP(http.MethodGet, "/api/delivery-packages", nil)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 OK on delivery packages list, got %d", rec.Code)
|
|
}
|
|
|
|
// Verify debug runs do NOT create formal DeliveryPackageRecord entries
|
|
pkgs, err := h.store.ListDeliveryPackages(h.ctx, domain.DefaultOrganizationID, domain.DeliveryPackageQuery{ProjectID: project.ID})
|
|
if err != nil {
|
|
t.Fatalf("list delivery packages error: %v", err)
|
|
}
|
|
for _, p := range pkgs {
|
|
if strings.Contains(p.ID, "dbg") || strings.Contains(p.ParentRunID, "dbg") {
|
|
t.Fatalf("found debug delivery package in store: %#v", p)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T1-F9: Execution Artifact Temporary Tagging", func(t *testing.T) {
|
|
artifacts := []domain.ExecutionArtifact{
|
|
{ID: "art-1", Name: "yolo_boxes.json", Path: "runs/run-1/boxes.json"},
|
|
{ID: "art-2", Name: "wilor_pose.zip", Path: "runs/run-1/pose.zip"},
|
|
}
|
|
|
|
debugDecorated := decorateArtifactsByMode(artifacts, runModeDebug)
|
|
for _, a := range debugDecorated {
|
|
if !a.IsTemporary {
|
|
t.Fatalf("expected debug artifact %s to be temporary", a.ID)
|
|
}
|
|
if a.EligibleForDeliveryPackage {
|
|
t.Fatalf("debug artifact %s should not be eligible for delivery package", a.ID)
|
|
}
|
|
if a.BadgeText != "临时产物 (Debug)" {
|
|
t.Fatalf("unexpected badge text: %s", a.BadgeText)
|
|
}
|
|
}
|
|
|
|
prodDecorated := decorateArtifactsByMode(artifacts, runModeProduction)
|
|
for _, a := range prodDecorated {
|
|
if a.IsTemporary {
|
|
t.Fatalf("expected prod artifact %s to NOT be temporary", a.ID)
|
|
}
|
|
if !a.EligibleForDeliveryPackage {
|
|
t.Fatalf("prod artifact %s should be eligible for delivery package", a.ID)
|
|
}
|
|
if a.BadgeText != "正式交付产物 (Production)" {
|
|
t.Fatalf("unexpected badge text: %s", a.BadgeText)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// ============================================================================
|
|
// Tier 2: Boundary & Corner Cases
|
|
// ============================================================================
|
|
|
|
func TestDualTrack_Tier2_BoundaryAndCornerCases(t *testing.T) {
|
|
h := newDualTrackTestHarness(t)
|
|
project := h.createProject("Tier2-Boundary-Cases", "Edge and Security Gate Tests")
|
|
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "root-reader", Type: "operator", Data: domain.WorkflowNodeData{Label: "Root BOS Reader", Inputs: []domain.PortDefinition{{ID: "stream", Name: "Stream", Type: domain.PortTypeFile, Required: true}}}},
|
|
{ID: "node-classifier", Type: "operator", Data: domain.WorkflowNodeData{Label: "Classifier", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
}
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "root-reader", Target: "node-classifier"},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "stream", Name: "Stream", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "root-reader", TargetPortID: "stream"}}},
|
|
}
|
|
release := h.publishWorkflowRelease("tmpl-sec-01", "Security Test Release", 1, nodes, edges, inputs)
|
|
|
|
t.Run("TC-T2-01: Security Gate Rejects Intermediate Node Hyperparameter Tampering with 422 production_input_forbidden", func(t *testing.T) {
|
|
tamperedPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"nodeOverrides": map[string]any{
|
|
"node-classifier": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "confidence_threshold", "value": "0.10"},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"stream": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "bucket": "b", "prefix": "p/"}},
|
|
},
|
|
}
|
|
|
|
gateResult := validateProductionRunSecurityGateContract(release, tamperedPayload)
|
|
if gateResult.Allowed {
|
|
t.Fatal("expected security gate to reject intermediate node hyperparameter tampering")
|
|
}
|
|
if gateResult.StatusCode != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
|
|
}
|
|
if gateResult.Code != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got '%s'", gateResult.Code)
|
|
}
|
|
if len(gateResult.Issues) == 0 {
|
|
t.Fatal("expected issues list to identify the offending node override")
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-02: Security Gate Rejects Non-Root Intermediate Node Input Binding with 422 production_input_forbidden", func(t *testing.T) {
|
|
nonRootInputPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"stream": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "bucket": "b", "prefix": "p/"}},
|
|
"node-classifier.in": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"injected": "malicious_intermediate_data"},
|
|
},
|
|
},
|
|
}
|
|
|
|
gateResult := validateProductionRunSecurityGateContract(release, nonRootInputPayload)
|
|
if gateResult.Allowed {
|
|
t.Fatal("expected security gate to reject non-root intermediate input injection")
|
|
}
|
|
if gateResult.StatusCode != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
|
|
}
|
|
if gateResult.Code != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got '%s'", gateResult.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-03: Security Gate Rejects Operator Image or Command Tampering in Production Run with 422", func(t *testing.T) {
|
|
imageTamperPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"nodeOverrides": map[string]any{
|
|
"node-classifier": map[string]any{
|
|
"image": "attacker-controlled-registry.example.com/trojan:latest",
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"stream": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "bucket": "b", "prefix": "p/"}},
|
|
},
|
|
}
|
|
|
|
gateResult := validateProductionRunSecurityGateContract(release, imageTamperPayload)
|
|
if gateResult.Allowed {
|
|
t.Fatal("expected security gate to reject operator image modification in production run")
|
|
}
|
|
if gateResult.StatusCode != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
|
|
}
|
|
if gateResult.Code != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got '%s'", gateResult.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-04: Reject Production Run on Unreleased Draft ID with 404/422", func(t *testing.T) {
|
|
wf := h.createWorkflow(project.ID, "Unreleased Draft Workflow", nodes, edges, inputs)
|
|
|
|
// Calling release endpoint with a draft workflow ID (which has no WorkflowReleaseRecord)
|
|
rec := h.serveHTTP(http.MethodGet, fmt.Sprintf("/api/workflow-releases/%s", wf.ID), nil)
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Fatalf("expected 404 Not Found for unreleased workflow ID at release endpoint, got %d", rec.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-05: Reject Production Run on Nonexistent Release ID UUID with 404", func(t *testing.T) {
|
|
rec := h.serveHTTP(http.MethodGet, "/api/workflow-releases/workflow-release-uuid-nonexistent-9999", nil)
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Fatalf("expected 404 Not Found for nonexistent release ID, got %d", rec.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-06: Empty Root Inputs Validation Rejection on Required Fields", func(t *testing.T) {
|
|
emptyInputPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{},
|
|
}
|
|
|
|
gateResult := validateProductionRunSecurityGateContract(release, emptyInputPayload)
|
|
if gateResult.Allowed {
|
|
t.Fatal("expected security gate to reject missing required root input")
|
|
}
|
|
if gateResult.StatusCode != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected HTTP 422, got %d", gateResult.StatusCode)
|
|
}
|
|
if gateResult.Code != "workflow_inputs_invalid" {
|
|
t.Fatalf("expected code 'workflow_inputs_invalid', got '%s'", gateResult.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-07: Workforce Metrics Division-by-Zero Protection with Zero Production Worklogs", func(t *testing.T) {
|
|
// Zero events in database
|
|
metrics := calculateWorkforceMetricsIsolated(nil, nil)
|
|
if metrics.TotalEvents != 0 {
|
|
t.Fatalf("expected 0 events, got %d", metrics.TotalEvents)
|
|
}
|
|
if metrics.QualityPassRate != nil {
|
|
t.Fatalf("expected nil QualityPassRate on 0 events, got %v", metrics.QualityPassRate)
|
|
}
|
|
if metrics.AcceptancePassRate != nil {
|
|
t.Fatalf("expected nil AcceptancePassRate on 0 events, got %v", metrics.AcceptancePassRate)
|
|
}
|
|
if metrics.ReworkRate != nil {
|
|
t.Fatalf("expected nil ReworkRate on 0 events, got %v", metrics.ReworkRate)
|
|
}
|
|
|
|
// Also verify via server dashboard endpoint
|
|
rec := h.serveHTTP(http.MethodGet, "/api/dashboard", nil)
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("expected 200 OK from dashboard with 0 worklogs, got %d", rec.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-08: Inline Payload Size Boundary Rejection (>64KB / 65536 bytes)", func(t *testing.T) {
|
|
oversizedData := strings.Repeat("A", maxInlinePayloadBytes+100)
|
|
oversizedPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"stream": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"data": oversizedData},
|
|
},
|
|
},
|
|
}
|
|
|
|
gateResult := validateProductionRunSecurityGateContract(release, oversizedPayload)
|
|
if gateResult.Allowed {
|
|
t.Fatal("expected security gate to reject oversized inline payload")
|
|
}
|
|
if gateResult.StatusCode != http.StatusBadRequest {
|
|
t.Fatalf("expected HTTP 400 for payload size violation, got %d", gateResult.StatusCode)
|
|
}
|
|
if gateResult.Code != "workflow_inputs_invalid" {
|
|
t.Fatalf("expected code 'workflow_inputs_invalid', got '%s'", gateResult.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-09: Invalid Mode Query Parameter Rejection / Graceful Fallback", func(t *testing.T) {
|
|
// Query with invalid mode
|
|
rec := h.serveHTTP(http.MethodGet, "/api/runs?mode=invalid_staging_mode", nil)
|
|
// Should return 400 Bad Request or handle gracefully without crashing
|
|
if rec.Code != http.StatusBadRequest && rec.Code != http.StatusOK {
|
|
t.Fatalf("unexpected response code for invalid mode query: %d", rec.Code)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T2-10: Debug Overrides with Nonexistent Node ID Handled Gracefully", func(t *testing.T) {
|
|
jsonNodes := []domain.WorkflowNode{
|
|
{ID: "root-json", Type: "operator", Data: domain.WorkflowNodeData{Label: "Root JSON Reader", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
|
|
}
|
|
jsonInputs := []domain.WorkflowInputDefinition{
|
|
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "root-json", TargetPortID: "in"}}},
|
|
}
|
|
wf := h.createWorkflow(project.ID, "Ghost Override Workflow", jsonNodes, nil, jsonInputs)
|
|
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"nodeOverrides": map[string]any{
|
|
"node-ghost-nonexistent": map[string]any{
|
|
"hyperparams": []map[string]any{{"key": "temp", "value": "1"}},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"ok": true}},
|
|
},
|
|
}
|
|
|
|
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), payload)
|
|
// Debug run accepts payload and ignores ghost node overrides without crashing
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
// ============================================================================
|
|
// Tier 3: Cross-Feature Combinations & Full Lifecycles
|
|
// ============================================================================
|
|
|
|
func TestDualTrack_Tier3_CrossFeatureCombinations(t *testing.T) {
|
|
h := newDualTrackTestHarness(t)
|
|
project := h.createProject("Tier3-Combinations", "Cross-Feature Lifecycles")
|
|
|
|
t.Run("TC-T3-01: Full Lifecycle: Workflow Release -> Production Run -> Run Query Filter -> Delivery Package Check", func(t *testing.T) {
|
|
// 1. Pipeline engineer designs 4-node workflow
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "bos-source", Type: "operator", Data: domain.WorkflowNodeData{Label: "BOS Multi-Cam Source", Inputs: []domain.PortDefinition{{ID: "stream", Name: "Video Stream", Type: domain.PortTypeFile, Required: true}}}},
|
|
{ID: "yolo-detector", Type: "operator", Data: domain.WorkflowNodeData{Label: "YOLO Detector", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
{ID: "human-gate", Type: "operator", Data: domain.WorkflowNodeData{Label: "Human QA Review", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
{ID: "delivery-export", Type: "operator", Data: domain.WorkflowNodeData{Label: "Delivery Package Export", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
}
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "bos-source", Target: "yolo-detector"},
|
|
{ID: "e2", Source: "yolo-detector", Target: "human-gate"},
|
|
{ID: "e3", Source: "human-gate", Target: "delivery-export"},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "stream", Name: "Video Stream", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "bos-source", TargetPortID: "stream"}}},
|
|
}
|
|
|
|
// 2. Publish release v1
|
|
release := h.publishWorkflowRelease("tmpl-ad-01", "Autonomous Driving Vision Pipeline", 1, nodes, edges, inputs)
|
|
if release.Version != 1 {
|
|
t.Fatalf("expected version 1, got %d", release.Version)
|
|
}
|
|
|
|
// 3. Trigger Production Run with BOS root dataset ref
|
|
prodPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"stream": map[string]any{
|
|
"mode": "ref",
|
|
"type": "file",
|
|
"ref": map[string]any{
|
|
"provider": "BOS",
|
|
"connectionId": "bos-conn-main",
|
|
"bucket": "ad-dataset-bucket",
|
|
"prefix": "camera_ingest_batch_20260819/",
|
|
"kind": "directory",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
gateResult := validateProductionRunSecurityGateContract(release, prodPayload)
|
|
if !gateResult.Allowed {
|
|
t.Fatalf("production run security gate failed: %v", gateResult)
|
|
}
|
|
|
|
// 4. Create Production Run Record in Store
|
|
prodRun := domain.RunRecord{
|
|
ID: "run-ad-prod-001",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
WorkflowID: "wf-ad-prod",
|
|
WorkflowName: release.Name,
|
|
Trigger: domain.RunTriggerManual,
|
|
DefinitionVersion: 1,
|
|
Status: domain.RunStatusCompleted,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, prodRun); err != nil {
|
|
t.Fatalf("failed to insert prod run: %v", err)
|
|
}
|
|
|
|
// 5. Human QA Gate passes -> formal DeliveryPackageRecord generated
|
|
deliveryPkg := domain.DeliveryPackageRecord{
|
|
ID: "pkg-ad-prod-001",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
WorkflowID: prodRun.WorkflowID,
|
|
ParentRunID: prodRun.ID,
|
|
Status: domain.DeliveryPackagePendingAcceptance,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := h.store.UpsertDeliveryPackage(h.ctx, deliveryPkg); err != nil {
|
|
t.Fatalf("failed to upsert delivery package: %v", err)
|
|
}
|
|
|
|
// 6. Verify delivery package is queryable and bound to production run
|
|
pkgs, err := h.store.ListDeliveryPackages(h.ctx, domain.DefaultOrganizationID, domain.DeliveryPackageQuery{ProjectID: project.ID})
|
|
if err != nil || len(pkgs) != 1 {
|
|
t.Fatalf("expected 1 formal delivery package, got %d (err=%v)", len(pkgs), err)
|
|
}
|
|
if pkgs[0].ParentRunID != prodRun.ID {
|
|
t.Fatalf("expected parent run %s, got %s", prodRun.ID, pkgs[0].ParentRunID)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T3-02: Full Lifecycle: Draft Workflow -> Debug Run with Overrides -> Single Node Run -> Workforce Metric Isolation", func(t *testing.T) {
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "src", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
|
|
{ID: "ai", Type: "operator", Data: domain.WorkflowNodeData{Label: "AI Model", Hyperparams: []domain.HyperParam{{Key: "confidence", Value: "0.80"}}}},
|
|
}
|
|
edges := []domain.WorkflowEdge{{ID: "e1", Source: "src", Target: "ai"}}
|
|
inputs := []domain.WorkflowInputDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "src", TargetPortID: "in"}}}}
|
|
wf := h.createWorkflow(project.ID, "Draft Debug Lifecycle", nodes, edges, inputs)
|
|
|
|
// 1. Debug run with override and single node target
|
|
debugPayload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"nodeOverrides": map[string]any{
|
|
"ai": map[string]any{"hyperparams": []map[string]any{{"key": "confidence", "value": "0.35"}}},
|
|
},
|
|
"debugTarget": map[string]any{
|
|
"targetNodeId": "ai",
|
|
"singleNode": true,
|
|
},
|
|
"inputs": map[string]any{
|
|
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"test": "val"}},
|
|
},
|
|
}
|
|
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), debugPayload)
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
// 2. Add debug worklogs
|
|
eventModes := map[string]string{
|
|
"evt-dbg-001": runModeDebug,
|
|
"evt-dbg-002": runModeDebug,
|
|
}
|
|
events := []domain.AnnotationWorklogEventRecord{
|
|
{ID: "evt-dbg-001", OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID, EventType: "quality_rejected", DurationSeconds: 3600, OutputDelta: 100},
|
|
{ID: "evt-dbg-002", OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID, EventType: "rework_submitted", DurationSeconds: 1800, OutputDelta: 50},
|
|
}
|
|
|
|
metrics := calculateWorkforceMetricsIsolated(events, eventModes)
|
|
// 100% of debug events must be excluded
|
|
if metrics.ProductionCount != 0 {
|
|
t.Fatalf("expected 0 production events, got %d", metrics.ProductionCount)
|
|
}
|
|
if metrics.DebugExcludedCount != 2 {
|
|
t.Fatalf("expected 2 debug events excluded, got %d", metrics.DebugExcludedCount)
|
|
}
|
|
if metrics.TotalDurationSeconds != 0 {
|
|
t.Fatalf("expected 0s duration, got %d", metrics.TotalDurationSeconds)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T3-03: Subtree Rerun Mode Inheritance Isolation", func(t *testing.T) {
|
|
type rerunPlan struct {
|
|
ParentMode string
|
|
ParentRel string
|
|
TargetNode string
|
|
ChildMode string
|
|
ChildRel string
|
|
}
|
|
|
|
plans := []rerunPlan{
|
|
{ParentMode: runModeProduction, ParentRel: "rel-v1", TargetNode: "ai-node", ChildMode: runModeProduction, ChildRel: "rel-v1"},
|
|
{ParentMode: runModeDebug, ParentRel: "", TargetNode: "ai-node", ChildMode: runModeDebug, ChildRel: ""},
|
|
}
|
|
|
|
for _, p := range plans {
|
|
childMode := p.ParentMode
|
|
var childRel string
|
|
if childMode == runModeProduction {
|
|
childRel = p.ParentRel
|
|
}
|
|
if childMode != p.ChildMode || childRel != p.ChildRel {
|
|
t.Fatalf("mode inheritance mismatch for plan %#v: got childMode=%s childRel=%s", p, childMode, childRel)
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T3-04: Multi-Tenant Concurrent Dual-Track Execution Isolation", func(t *testing.T) {
|
|
// Tenant Alpha (Production) vs Tenant Beta (Debug)
|
|
tenantAlphaOrg := "org-alpha"
|
|
tenantBetaOrg := "org-beta"
|
|
|
|
// Tenant Alpha runs production
|
|
alphaRun := domain.RunRecord{
|
|
ID: "run-alpha-prod",
|
|
OrganizationID: tenantAlphaOrg,
|
|
ProjectID: "proj-alpha",
|
|
WorkflowID: "wf-alpha",
|
|
WorkflowName: "Alpha Prod",
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, alphaRun); err != nil {
|
|
t.Fatalf("failed to insert alpha run: %v", err)
|
|
}
|
|
|
|
// Tenant Beta runs 10 debug runs
|
|
for i := 1; i <= 10; i++ {
|
|
betaRun := domain.RunRecord{
|
|
ID: fmt.Sprintf("run-beta-dbg-%d", i),
|
|
OrganizationID: tenantBetaOrg,
|
|
ProjectID: "proj-beta",
|
|
WorkflowID: "wf-beta",
|
|
WorkflowName: "Beta Debug",
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, betaRun); err != nil {
|
|
t.Fatalf("failed to insert beta run: %v", err)
|
|
}
|
|
}
|
|
|
|
// Verify Tenant Alpha sees only Alpha runs
|
|
alphaRuns, err := h.store.ListRunSummariesPageByOrganization(h.ctx, tenantAlphaOrg, domain.RunSummaryQuery{ProjectID: "proj-alpha"})
|
|
if err != nil {
|
|
t.Fatalf("alpha list runs error: %v", err)
|
|
}
|
|
if len(alphaRuns.Items) != 1 {
|
|
t.Fatalf("expected 1 run for Tenant Alpha, got %d", len(alphaRuns.Items))
|
|
}
|
|
if alphaRuns.Items[0].ID != alphaRun.ID {
|
|
t.Fatalf("expected run ID %s, got %s", alphaRun.ID, alphaRuns.Items[0].ID)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T3-05: Multi-Version Release Transition (v1 -> v2) with Active Production and Debug Runs", func(t *testing.T) {
|
|
nodesV1 := []domain.WorkflowNode{{ID: "n1", Type: "operator", Data: domain.WorkflowNodeData{Label: "V1 Node"}}}
|
|
relV1 := h.publishWorkflowRelease("tmpl-mv-01", "Multi-Version Pipeline", 1, nodesV1, nil, nil)
|
|
|
|
nodesV2 := []domain.WorkflowNode{
|
|
{ID: "n1", Type: "operator", Data: domain.WorkflowNodeData{Label: "V1 Node"}},
|
|
{ID: "n2", Type: "operator", Data: domain.WorkflowNodeData{Label: "V2 Node Added"}},
|
|
}
|
|
relV2 := h.publishWorkflowRelease("tmpl-mv-01", "Multi-Version Pipeline", 2, nodesV2, nil, nil)
|
|
|
|
if relV1.Version != 1 || relV2.Version != 2 {
|
|
t.Fatalf("expected release versions 1 and 2, got %d and %d", relV1.Version, relV2.Version)
|
|
}
|
|
if relV1.ContentDigest == relV2.ContentDigest {
|
|
t.Fatalf("content digests must be distinct across versions: %s", relV1.ContentDigest)
|
|
}
|
|
})
|
|
}
|
|
|
|
// ============================================================================
|
|
// Tier 4: Real-World Application Scenarios
|
|
// ============================================================================
|
|
|
|
func TestDualTrack_Tier4_RealWorldApplicationScenarios(t *testing.T) {
|
|
h := newDualTrackTestHarness(t)
|
|
project := h.createProject("Tier4-Real-World-Scenarios", "Industrial End-to-End Simulations")
|
|
|
|
// Register storage connections for BOS tests
|
|
h.createStorageConnection("bos-dev", domain.StorageProviderBOS)
|
|
h.createStorageConnection("bos-prod", domain.StorageProviderBOS)
|
|
|
|
t.Run("TC-T4-01: Autonomous Driving Multi-Camera Vision Ingestion Pipeline Simulation", func(t *testing.T) {
|
|
// 5-Node DAG: BOS Video Source -> LiDAR Sync -> WILOR 3D Pose -> Human Review -> Policy Export
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "bos-cam", Type: "operator", Data: domain.WorkflowNodeData{Label: "BOS Multi-Cam Video Ingestion", Inputs: []domain.PortDefinition{{ID: "cam_stream", Name: "Raw Video", Type: domain.PortTypeFile, Required: true}}}},
|
|
{ID: "lidar-sync", Type: "operator", Data: domain.WorkflowNodeData{Label: "LiDAR Synchronization Engine", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
{ID: "wilor-detector", Type: "operator", Data: domain.WorkflowNodeData{Label: "WILOR 3D Hand & Head Pose Detector", Hyperparams: []domain.HyperParam{{Key: "iou_threshold", Value: "0.50"}}, Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
{ID: "human-qa", Type: "operator", Data: domain.WorkflowNodeData{Label: "Autonomous Driving QA Gate", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
{ID: "policy-export", Type: "operator", Data: domain.WorkflowNodeData{Label: "Robotics Policy Training Export", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
|
|
}
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "bos-cam", Target: "lidar-sync"},
|
|
{ID: "e2", Source: "lidar-sync", Target: "wilor-detector"},
|
|
{ID: "e3", Source: "wilor-detector", Target: "human-qa"},
|
|
{ID: "e4", Source: "human-qa", Target: "policy-export"},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "cam_stream", Name: "Raw Video", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "bos-cam", TargetPortID: "cam_stream"}}},
|
|
}
|
|
|
|
// Phase 1: AI Engineer runs Debug Run on sample clip with transient overrides
|
|
draftWf := h.createWorkflow(project.ID, "AD Vision Pipeline Draft", nodes, edges, inputs)
|
|
debugReq := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"nodeOverrides": map[string]any{
|
|
"wilor-detector": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "iou_threshold", "value": "0.65"},
|
|
{"key": "box_score_threshold", "value": "0.45"},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"cam_stream": map[string]any{
|
|
"mode": "ref",
|
|
"type": "file",
|
|
"ref": map[string]any{"provider": "BOS", "connectionId": "bos-dev", "bucket": "test-bucket", "prefix": "sample_clip_20frames/", "kind": "directory"},
|
|
},
|
|
},
|
|
}
|
|
debugRec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", draftWf.ID), debugReq)
|
|
if debugRec.Code != http.StatusAccepted {
|
|
t.Fatalf("failed debug run trigger: %d body=%s", debugRec.Code, debugRec.Body.String())
|
|
}
|
|
|
|
// Phase 2: Release v1.0.0 is published for 100,000-frame customer batch
|
|
rel := h.publishWorkflowRelease("tmpl-ad-vision", "AD Vision Pipeline Release v1.0.0", 1, nodes, edges, inputs)
|
|
|
|
// Production Run triggered strictly with root BOS dataset
|
|
prodReq := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"cam_stream": map[string]any{
|
|
"mode": "ref",
|
|
"type": "file",
|
|
"ref": map[string]any{"provider": "BOS", "connectionId": "bos-prod", "bucket": "test-bucket", "prefix": "batch_100k_frames/", "kind": "directory"},
|
|
},
|
|
},
|
|
}
|
|
gate := validateProductionRunSecurityGateContract(rel, prodReq)
|
|
if !gate.Allowed {
|
|
t.Fatalf("production security gate failed: %v", gate)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T4-02: HoloBrain AI Policy Training and Checkpoint Evaluation Simulation", func(t *testing.T) {
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "demo-ingest", Type: "operator", Data: domain.WorkflowNodeData{Label: "Demonstration Ingestion", Inputs: []domain.PortDefinition{{ID: "trajectories", Name: "Trajectories", Type: domain.PortTypeJSON, Required: true}}}},
|
|
{ID: "holobrain-train", Type: "operator", Data: domain.WorkflowNodeData{Label: "HoloBrain ACT Training", Hyperparams: []domain.HyperParam{{Key: "epochs", Value: "100"}, {Key: "batch_size", Value: "64"}}}},
|
|
{ID: "checkpoint-eval", Type: "operator", Data: domain.WorkflowNodeData{Label: "Model Checkpoint Evaluator"}},
|
|
}
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "demo-ingest", Target: "holobrain-train"},
|
|
{ID: "e2", Source: "holobrain-train", Target: "checkpoint-eval"},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "trajectories", Name: "Trajectories", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "demo-ingest", TargetPortID: "trajectories"}}},
|
|
}
|
|
wf := h.createWorkflow(project.ID, "HoloBrain Policy Training Draft", nodes, edges, inputs)
|
|
|
|
// ML engineer runs 1-epoch smoke test with single-node debug on checkpoint-eval
|
|
smokePayload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"nodeOverrides": map[string]any{
|
|
"holobrain-train": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "epochs", "value": "1"},
|
|
{"key": "batch_size", "value": "4"},
|
|
},
|
|
},
|
|
},
|
|
"debugTarget": map[string]any{
|
|
"targetNodeId": "checkpoint-eval",
|
|
"singleNode": true,
|
|
},
|
|
"inputs": map[string]any{
|
|
"trajectories": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"num_episodes": 2}},
|
|
},
|
|
}
|
|
|
|
rec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", wf.ID), smokePayload)
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T4-03: Multi-Vendor Supplier Workforce Stats and Financial Reconciliation Audit", func(t *testing.T) {
|
|
vendorAlphaID := "vendor-alpha"
|
|
vendorBetaID := "vendor-beta"
|
|
|
|
eventModes := make(map[string]string)
|
|
var events []domain.AnnotationWorklogEventRecord
|
|
|
|
// Vendor Alpha Production Worklogs: 95 approved, 5 rejected (95% pass rate)
|
|
for i := 1; i <= 95; i++ {
|
|
id := fmt.Sprintf("evt-alpha-appr-%d", i)
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
VendorID: vendorAlphaID,
|
|
EventType: "quality_approved",
|
|
DurationSeconds: 120,
|
|
OutputDelta: 1,
|
|
})
|
|
eventModes[id] = runModeProduction
|
|
}
|
|
for i := 1; i <= 5; i++ {
|
|
id := fmt.Sprintf("evt-alpha-rej-%d", i)
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
VendorID: vendorAlphaID,
|
|
EventType: "quality_rejected",
|
|
DurationSeconds: 120,
|
|
OutputDelta: 1,
|
|
})
|
|
eventModes[id] = runModeProduction
|
|
}
|
|
|
|
// Vendor Beta Production Worklogs: 88 approved, 12 rejected (88% pass rate)
|
|
for i := 1; i <= 88; i++ {
|
|
id := fmt.Sprintf("evt-beta-appr-%d", i)
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
VendorID: vendorBetaID,
|
|
EventType: "quality_approved",
|
|
DurationSeconds: 150,
|
|
OutputDelta: 1,
|
|
})
|
|
eventModes[id] = runModeProduction
|
|
}
|
|
for i := 1; i <= 12; i++ {
|
|
id := fmt.Sprintf("evt-beta-rej-%d", i)
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
VendorID: vendorBetaID,
|
|
EventType: "quality_rejected",
|
|
DurationSeconds: 150,
|
|
OutputDelta: 1,
|
|
})
|
|
eventModes[id] = runModeProduction
|
|
}
|
|
|
|
// 100 Practice Debug Worklogs from trial annotators (would destroy financial audit if leaked)
|
|
for i := 1; i <= 100; i++ {
|
|
id := fmt.Sprintf("evt-trial-dbg-%d", i)
|
|
events = append(events, domain.AnnotationWorklogEventRecord{
|
|
ID: id,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
VendorID: vendorAlphaID,
|
|
EventType: "quality_rejected",
|
|
DurationSeconds: 600,
|
|
OutputDelta: 5,
|
|
})
|
|
eventModes[id] = runModeDebug
|
|
}
|
|
|
|
// Calculate Isolated Metrics
|
|
metrics := calculateWorkforceMetricsIsolated(events, eventModes)
|
|
if metrics.TotalEvents != 300 {
|
|
t.Fatalf("expected 300 total events, got %d", metrics.TotalEvents)
|
|
}
|
|
if metrics.ProductionCount != 200 {
|
|
t.Fatalf("expected 200 production events, got %d", metrics.ProductionCount)
|
|
}
|
|
if metrics.DebugExcludedCount != 100 {
|
|
t.Fatalf("expected 100 debug events excluded, got %d", metrics.DebugExcludedCount)
|
|
}
|
|
// Overall Production Pass Rate: (95+88) / 200 = 183 / 200 = 91.5%
|
|
if metrics.QualityPassRate == nil || *metrics.QualityPassRate != 91.5 {
|
|
t.Fatalf("expected QualityPassRate = 91.5%%, got %v", metrics.QualityPassRate)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T4-04: Urgent Hotfix and Production Failure Regression Debugging Simulation", func(t *testing.T) {
|
|
// Production run fails due to bad regex in transformer
|
|
failedProdRun := domain.RunRecord{
|
|
ID: "run-prod-failed-001",
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
ProjectID: project.ID,
|
|
WorkflowID: "wf-hotfix-prod",
|
|
WorkflowName: "Hotfix Pipeline",
|
|
Status: domain.RunStatusFailed,
|
|
Error: "Regex parsing error at step 3: invalid datetime token",
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, failedProdRun); err != nil {
|
|
t.Fatalf("failed to insert failed run: %v", err)
|
|
}
|
|
|
|
// Engineer fixes in draft and verifies via single-node debug
|
|
nodes := []domain.WorkflowNode{
|
|
{ID: "node-fix", Type: "operator", Data: domain.WorkflowNodeData{Label: "Fixed DateTime Parser", Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true}}}},
|
|
}
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "node-fix", TargetPortID: "in"}}},
|
|
}
|
|
draftWf := h.createWorkflow(project.ID, "Hotfix Draft", nodes, nil, inputs)
|
|
|
|
debugRec := h.serveHTTP(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", draftWf.ID), map[string]any{
|
|
"trigger": "manual",
|
|
"mode": runModeDebug,
|
|
"inputs": map[string]any{"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"sample": "2026-08-19"}}},
|
|
})
|
|
if debugRec.Code != http.StatusAccepted {
|
|
t.Fatalf("failed hotfix debug run: %d", debugRec.Code)
|
|
}
|
|
|
|
// Responder publishes Release v2.0.0
|
|
relV2 := h.publishWorkflowRelease("tmpl-hotfix", "Hotfix Pipeline Release", 2, nodes, nil, inputs)
|
|
if relV2.Version != 2 {
|
|
t.Fatalf("expected version 2, got %d", relV2.Version)
|
|
}
|
|
|
|
// Historical failed production run record remains untouched
|
|
historical, err := h.store.GetRun(h.ctx, failedProdRun.ID)
|
|
if err != nil || historical == nil {
|
|
t.Fatalf("failed to retrieve historical run: %v", err)
|
|
}
|
|
if historical.Status != domain.RunStatusFailed {
|
|
t.Fatalf("historical failed run was mutated! status=%s", historical.Status)
|
|
}
|
|
})
|
|
|
|
t.Run("TC-T4-05: High-Concurrency Dual-Track Multi-Tenant Load and Data Integrity Verification", func(t *testing.T) {
|
|
const numWorkers = 20
|
|
var wg sync.WaitGroup
|
|
wg.Add(numWorkers)
|
|
|
|
errChan := make(chan error, numWorkers)
|
|
|
|
for i := 0; i < numWorkers; i++ {
|
|
workerID := i
|
|
go func() {
|
|
defer wg.Done()
|
|
tenantID := fmt.Sprintf("tenant-%d", workerID%5)
|
|
isProd := workerID%2 == 0
|
|
|
|
if isProd {
|
|
// Production run record
|
|
run := domain.RunRecord{
|
|
ID: fmt.Sprintf("concurrent-prod-%d", workerID),
|
|
OrganizationID: tenantID,
|
|
ProjectID: fmt.Sprintf("proj-%s", tenantID),
|
|
WorkflowID: fmt.Sprintf("wf-%s", tenantID),
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, run); err != nil {
|
|
errChan <- fmt.Errorf("worker %d upsert prod run error: %w", workerID, err)
|
|
return
|
|
}
|
|
} else {
|
|
// Debug run record
|
|
run := domain.RunRecord{
|
|
ID: fmt.Sprintf("concurrent-dbg-%d", workerID),
|
|
OrganizationID: tenantID,
|
|
ProjectID: fmt.Sprintf("proj-%s", tenantID),
|
|
WorkflowID: fmt.Sprintf("wf-%s", tenantID),
|
|
Trigger: domain.RunTriggerManual,
|
|
Status: domain.RunStatusCompleted,
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
if err := h.store.UpsertRun(h.ctx, run); err != nil {
|
|
errChan <- fmt.Errorf("worker %d upsert debug run error: %w", workerID, err)
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
close(errChan)
|
|
|
|
for err := range errChan {
|
|
if err != nil {
|
|
t.Fatalf("concurrent execution failure: %v", err)
|
|
}
|
|
}
|
|
})
|
|
}
|