droboticflow/internal/api/m2_adversarial_challenge_test.go

907 lines
31 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// ============================================================================
// Milestone M2 Empirical Challenger Stress & Adversarial Test Suite
// ============================================================================
func createAdversarialTestDAG() ([]domain.WorkflowNode, []domain.WorkflowEdge, []domain.WorkflowInputDefinition) {
// Topology:
// root-source (root) -> mid-process (intermediate) -> sink-writer (leaf)
nodes := []domain.WorkflowNode{
{
ID: "root-source",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Root Source",
Inputs: []domain.PortDefinition{
{ID: "root_in", Name: "Root In", Type: domain.PortTypeJSON, Required: true},
},
Outputs: []domain.PortDefinition{
{ID: "out", Name: "Out", Type: domain.PortTypeJSON},
},
Hyperparams: []domain.HyperParam{
{Key: "batch_size", Value: "32"},
},
Image: "registry.example.local/operators/source:v1.0.0",
},
},
{
ID: "mid-process",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Middle Processing",
Inputs: []domain.PortDefinition{
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true},
{ID: "aux_in", Name: "Aux In", Type: domain.PortTypeJSON, Required: false},
},
Outputs: []domain.PortDefinition{
{ID: "out", Name: "Out", Type: domain.PortTypeJSON},
},
Hyperparams: []domain.HyperParam{
{Key: "threshold", Value: "0.85"},
{Key: "learning_rate", Value: "0.001"},
},
Image: "registry.example.local/operators/model:v2.1.0",
},
},
{
ID: "sink-writer",
Type: "operator",
Data: domain.WorkflowNodeData{
Label: "Sink Writer",
Inputs: []domain.PortDefinition{
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true},
},
Hyperparams: []domain.HyperParam{
{Key: "destination", Value: "s3://output-bucket/results"},
},
Image: "registry.example.local/operators/sink:v1.0.0",
},
},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "root-source", SourceHandle: "out", Target: "mid-process", TargetHandle: "in"},
{ID: "e2", Source: "mid-process", SourceHandle: "out", Target: "sink-writer", TargetHandle: "in"},
}
inputs := []domain.WorkflowInputDefinition{
{
ID: "source_data",
Name: "Source Dataset",
Type: domain.PortTypeJSON,
Required: true,
Bindings: []domain.WorkflowInputTarget{
{NodeID: "root-source", TargetPortID: "root_in"},
},
},
}
return nodes, edges, inputs
}
func setupAdversarialRelease(ctx context.Context, dataStore store.Store) domain.WorkflowReleaseRecord {
nodes, edges, inputs := createAdversarialTestDAG()
now := time.Now().UTC().Format(time.RFC3339)
release := domain.WorkflowReleaseRecord{
ID: fmt.Sprintf("rel-adv-%d", time.Now().UnixNano()),
OrganizationID: domain.DefaultOrganizationID,
TemplateID: "tmpl-adv-001",
Version: 3,
Name: "Production Pipeline Release v3",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
ExecutionScope: domain.WorkflowExecutionScopeProduction,
Nodes: nodes,
Edges: edges,
WorkflowInputs: inputs,
ContentDigest: "sha256:adv-digest-9876543210abcdef",
ReleaseNotes: "Milestone M2 Adversarial Verification Release",
PublishedByUserID: "challenger-agent",
PublishedAt: now,
CreatedAt: now,
}
_ = dataStore.UpsertWorkflowRelease(ctx, release)
return release
}
func setupAdversarialDraftWorkflow(ctx context.Context, dataStore store.Store, name string) domain.WorkflowRecord {
nodes, edges, inputs := createAdversarialTestDAG()
draftWorkflow, _ := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
Name: name,
Type: domain.WorkflowTypeBatch,
Nodes: nodes,
Edges: edges,
WorkflowInputs: inputs,
})
return *draftWorkflow
}
// ----------------------------------------------------------------------------
// Attack 1: Attempting to pass intermediate node hyperparameters in release runs
// ----------------------------------------------------------------------------
func TestAdversarial_ReleaseRun_IntermediateNodeHyperparams(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
release := setupAdversarialRelease(ctx, dataStore)
sendReleaseRun := func(t *testing.T, payload any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflow-releases/%s/runs", release.ID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
var resp map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
return rec, resp
}
t.Run("Attack 1.1: Single nodeOverrides for intermediate node must fail with 422", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"nodeOverrides": map[string]any{
"mid-process": map[string]any{
"hyperparams": []map[string]any{
{"key": "threshold", "value": "0.01"},
},
},
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"data": [1]int{1}},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422 Unprocessable Entity, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
})
t.Run("Attack 1.2: Multiple nodeOverrides across intermediate and leaf nodes must fail with 422", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"nodeOverrides": map[string]any{
"mid-process": map[string]any{
"image": "evil-registry.local/backdoor:v1",
},
"sink-writer": map[string]any{
"hyperparams": []map[string]any{
{"key": "destination", "value": "s3://hacked-bucket"},
},
},
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"data": [1]int{1}},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
issues, ok := resp["issues"].([]any)
if !ok || len(issues) < 2 {
t.Fatalf("expected at least 2 validation issues reported, got %#v", resp["issues"])
}
})
t.Run("Attack 1.3: Injection of debugTarget in release run must fail with 422", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"debugTarget": map[string]any{
"targetNodeId": "mid-process",
"singleNode": true,
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"data": [1]int{1}},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
})
t.Run("Attack 1.4: Direct topology tampering in release run must fail with 422", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"nodes": []map[string]any{
{"id": "hacked-node", "type": "operator"},
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"data": [1]int{1}},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
})
}
// ----------------------------------------------------------------------------
// Attack 2: Attempting non-root input bindings in release runs
// ----------------------------------------------------------------------------
func TestAdversarial_ReleaseRun_NonRootInputBindings(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
release := setupAdversarialRelease(ctx, dataStore)
sendReleaseRun := func(t *testing.T, payload any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflow-releases/%s/runs", release.ID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
var resp map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
return rec, resp
}
t.Run("Attack 2.1: Direct intermediate node input binding (mid-process.in) rejected", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"ok": true},
},
"mid-process.in": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"tampered": true},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
})
t.Run("Attack 2.2: Leaf node input binding (sink-writer.in) rejected", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"ok": true},
},
"sink-writer.in": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"tampered": true},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
})
t.Run("Attack 2.3: Non-existent node input binding (ghost-node.port) rejected", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"ok": true},
},
"ghost-node.port": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"ghost": true},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
})
t.Run("Attack 2.4: Path traversal attempt in input key rejected", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"ok": true},
},
"../../root-source.root_in": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"traversal": true},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
}
})
t.Run("Attack 2.5: Inline payload exceeding 64KB rejected with 400 workflow_inputs_invalid", func(t *testing.T) {
hugePayload := strings.Repeat("A", 70*1024)
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"huge": hugePayload},
},
},
}
rec, resp := sendReleaseRun(t, payload)
if rec.Code != http.StatusBadRequest {
t.Fatalf("expected 400 Bad Request for >64KB inline payload, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["code"] != "workflow_inputs_invalid" {
t.Fatalf("expected code 'workflow_inputs_invalid', got %v", resp["code"])
}
})
t.Run("Valid Root Input Execution Passes and Stamped with Immutable Provenance", func(t *testing.T) {
freshRelease := setupAdversarialRelease(ctx, dataStore)
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"batch_id": "b-2026-08"},
},
},
}
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflow-releases/%s/runs", freshRelease.ID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
var resp map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d, body: %s", rec.Code, rec.Body.String())
}
if resp["mode"] != "production" {
t.Fatalf("expected mode 'production', got %v", resp["mode"])
}
if resp["releaseId"] != freshRelease.ID {
t.Fatalf("expected releaseId %s, got %v", freshRelease.ID, resp["releaseId"])
}
if resp["workflowVersion"] != float64(freshRelease.Version) {
t.Fatalf("expected workflowVersion %d, got %v", freshRelease.Version, resp["workflowVersion"])
}
if resp["gitCommitSha"] != freshRelease.ContentDigest {
t.Fatalf("expected gitCommitSha %s, got %v", freshRelease.ContentDigest, resp["gitCommitSha"])
}
if resp["imageDigest"] != freshRelease.ContentDigest {
t.Fatalf("expected imageDigest %s, got %v", freshRelease.ContentDigest, resp["imageDigest"])
}
if resp["nodeOverrides"] != nil {
t.Fatalf("expected nodeOverrides to be nil, got %#v", resp["nodeOverrides"])
}
if resp["debugTarget"] != nil {
t.Fatalf("expected debugTarget to be nil, got %#v", resp["debugTarget"])
}
})
}
// ----------------------------------------------------------------------------
// Attack 3: Attempting ghost and malformed overrides in debug runs
// ----------------------------------------------------------------------------
func TestAdversarial_DebugRun_GhostAndMalformedOverrides(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
sendDebugRun := func(t *testing.T, workflowID string, payload any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", workflowID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
var resp map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
return rec, resp
}
t.Run("Attack 3.1: Ghost node in nodeOverrides is gracefully ignored and stripped", func(t *testing.T) {
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.1")
payload := map[string]any{
"trigger": "manual",
"mode": "debug",
"nodeOverrides": map[string]any{
"ghost-phantom-node-999": map[string]any{
"hyperparams": []map[string]any{
{"key": "ghost_key", "value": "ghost_val"},
},
},
"mid-process": map[string]any{
"hyperparams": []map[string]any{
{"key": "threshold", "value": "0.12"},
},
},
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"debug": true},
},
},
}
rec, resp := sendDebugRun(t, wf.ID, payload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
if resp["mode"] != "debug" {
t.Fatalf("expected mode 'debug', got %v", resp["mode"])
}
overrides, ok := resp["nodeOverrides"].(map[string]any)
if !ok {
t.Fatalf("expected nodeOverrides map in response, got %#v", resp["nodeOverrides"])
}
if _, hasGhost := overrides["ghost-phantom-node-999"]; hasGhost {
t.Fatalf("ghost node should have been stripped from nodeOverrides, found: %#v", overrides)
}
if _, hasMid := overrides["mid-process"]; !hasMid {
t.Fatalf("valid override for mid-process should be preserved, got %#v", overrides)
}
})
t.Run("Attack 3.2: Malformed input payload in nodeOverrides rejected with 400 validation error", func(t *testing.T) {
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.2")
payload := map[string]any{
"trigger": "manual",
"mode": "debug",
"nodeOverrides": map[string]any{
"mid-process": map[string]any{
"inputs": map[string]any{
"aux_in": map[string]any{
"mode": "ref",
"type": "json",
// missing required "ref" object for ref mode
},
},
},
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"debug": true},
},
},
}
rec, resp := sendDebugRun(t, wf.ID, payload)
if rec.Code != http.StatusBadRequest {
t.Fatalf("expected 400 for malformed nodeOverride input payload, got %d body=%s", rec.Code, rec.Body.String())
}
issues, ok := resp["issues"].([]any)
if !ok || len(issues) == 0 {
t.Fatalf("expected validation issues in response, got %#v", resp)
}
firstIssue := issues[0].(map[string]any)
if firstIssue["code"] != "workflow_inputs_invalid" {
t.Fatalf("expected issue code 'workflow_inputs_invalid', got %v", firstIssue["code"])
}
})
t.Run("Attack 3.3: debugTarget targeting nonexistent ghost node rejected with 400 debug_target_node_missing", func(t *testing.T) {
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.3")
payload := map[string]any{
"trigger": "manual",
"mode": "debug",
"debugTarget": map[string]any{
"targetNodeId": "ghost-nonexistent-node-XYZ",
"singleNode": true,
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"debug": true},
},
},
}
rec, resp := sendDebugRun(t, wf.ID, payload)
if rec.Code != http.StatusBadRequest {
t.Fatalf("expected 400 for ghost debugTarget node, got %d body=%s", rec.Code, rec.Body.String())
}
issues, ok := resp["issues"].([]any)
if !ok || len(issues) == 0 {
t.Fatalf("expected validation issues in response, got %#v", resp)
}
firstIssue := issues[0].(map[string]any)
if firstIssue["code"] != "debug_target_node_missing" {
t.Fatalf("expected issue code 'debug_target_node_missing', got %v", firstIssue["code"])
}
})
t.Run("Attack 3.4: Explicit production mode on draft workflow rejected with 400 production_run_draft_forbidden", func(t *testing.T) {
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.4")
payload := map[string]any{
"trigger": "manual",
"mode": "production", // forbidden on draft workflow
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"data": 1},
},
},
}
rec, resp := sendDebugRun(t, wf.ID, payload)
if rec.Code != http.StatusBadRequest {
t.Fatalf("expected 400 for production mode on draft workflow, got %d body=%s", rec.Code, rec.Body.String())
}
issues, ok := resp["issues"].([]any)
if !ok || len(issues) == 0 {
t.Fatalf("expected validation issues in response, got %#v", resp)
}
firstIssue := issues[0].(map[string]any)
if firstIssue["code"] != "production_run_draft_forbidden" {
t.Fatalf("expected issue code 'production_run_draft_forbidden', got %v", firstIssue["code"])
}
})
}
// ----------------------------------------------------------------------------
// Attack 4: Validating the DB zero-mutation invariant under concurrent debug runs
// ----------------------------------------------------------------------------
func TestAdversarial_ConcurrentDebugRuns_DBZeroMutationInvariant(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
// Create 50 distinct draft workflows sharing identical initial DAG definitions
const concurrencyLevel = 50
draftWorkflows := make([]domain.WorkflowRecord, concurrencyLevel)
for i := 0; i < concurrencyLevel; i++ {
wf := setupAdversarialDraftWorkflow(ctx, dataStore, fmt.Sprintf("Concurrent Draft %d", i))
draftWorkflows[i] = wf
}
var wg sync.WaitGroup
wg.Add(concurrencyLevel)
errChan := make(chan error, concurrencyLevel)
for i := 0; i < concurrencyLevel; i++ {
go func(idx int) {
defer wg.Done()
targetWF := draftWorkflows[idx]
// Each goroutine prepares a distinct, conflicting override payload
hackedThreshold := fmt.Sprintf("0.%02d", idx)
hackedImage := fmt.Sprintf("hacked-repo.local/runner:worker-%d", idx)
hackedCmd := []string{"bash", "-c", fmt.Sprintf("echo worker-%d", idx)}
payload := map[string]any{
"trigger": "manual",
"mode": "debug",
"nodeOverrides": map[string]any{
"mid-process": map[string]any{
"hyperparams": []map[string]any{
{"key": "threshold", "value": hackedThreshold},
{"key": "learning_rate", "value": "0.999"},
{"key": fmt.Sprintf("unique_key_%d", idx), "value": "injected"},
},
"image": hackedImage,
"containerCommand": hackedCmd,
},
"sink-writer": map[string]any{
"hyperparams": []map[string]any{
{"key": "destination", "value": fmt.Sprintf("s3://worker-%d/dest", idx)},
},
},
},
"inputs": map[string]any{
"source_data": map[string]any{
"mode": "inline",
"type": "json",
"inline": map[string]any{"worker_idx": idx},
},
},
}
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", targetWF.ID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusAccepted {
errChan <- fmt.Errorf("worker %d got status %d: %s", idx, rec.Code, rec.Body.String())
return
}
var resp map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
errChan <- fmt.Errorf("worker %d unmarshal error: %v", idx, err)
return
}
// Validate created run record has its own independent overrides
if resp["mode"] != "debug" {
errChan <- fmt.Errorf("worker %d run mode mismatch: %v", idx, resp["mode"])
return
}
}(i)
}
wg.Wait()
close(errChan)
for err := range errChan {
t.Fatalf("concurrent debug run failed: %v", err)
}
// ------------------------------------------------------------------------
// Verification of DB Zero-Mutation Invariant across all 50 workflows
// ------------------------------------------------------------------------
for i := 0; i < concurrencyLevel; i++ {
finalWorkflow, err := dataStore.GetWorkflow(ctx, draftWorkflows[i].ID)
if err != nil || finalWorkflow == nil {
t.Fatalf("failed to fetch final workflow %d from store: %v", i, err)
}
// Deep semantic equality check on DAG definitions
if !reflect.DeepEqual(draftWorkflows[i].Nodes, finalWorkflow.Nodes) {
t.Fatalf("workflow.Nodes was mutated in DB for workflow %d!", i)
}
if !reflect.DeepEqual(draftWorkflows[i].Edges, finalWorkflow.Edges) {
t.Fatalf("workflow.Edges was mutated in DB for workflow %d!", i)
}
if !reflect.DeepEqual(draftWorkflows[i].Hyperparams, finalWorkflow.Hyperparams) {
t.Fatalf("workflow.Hyperparams was mutated in DB for workflow %d!", i)
}
if !reflect.DeepEqual(draftWorkflows[i].WorkflowInputs, finalWorkflow.WorkflowInputs) {
t.Fatalf("workflow.WorkflowInputs was mutated in DB for workflow %d!", i)
}
// Specifically verify node hyperparameters and images remained pristine
for _, node := range finalWorkflow.Nodes {
if node.ID == "mid-process" {
if node.Data.Image != "registry.example.local/operators/model:v2.1.0" {
t.Fatalf("mid-process image corrupted in DB for workflow %d: %s", i, node.Data.Image)
}
for _, hp := range node.Data.Hyperparams {
if hp.Key == "threshold" && hp.Value != "0.85" {
t.Fatalf("mid-process threshold hyperparam corrupted in DB: %s", hp.Value)
}
if strings.HasPrefix(hp.Key, "unique_key_") {
t.Fatalf("injected hyperparam leaked into DB workflow: %s", hp.Key)
}
}
}
}
}
}
// ----------------------------------------------------------------------------
// Attack 5: Subtree Rerun Provenance and Mode Inheritance
// ----------------------------------------------------------------------------
func TestAdversarial_SubtreeRerun_SecurityAndProvenancePreservation(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
release := setupAdversarialRelease(ctx, dataStore)
// Ensure backing workflow record exists for the release
wfSnapshot := workflowSnapshotFromRelease(release)
_ = dataStore.UpsertWorkflow(ctx, wfSnapshot)
validInputs := map[string]domain.ArtifactPayload{
"source_data": {
Mode: domain.ArtifactTransferModeInline,
Type: domain.PortTypeJSON,
Inline: map[string]any{"data": 123},
},
}
t.Run("Production Subtree Rerun Inherits Release Provenance and Mode", func(t *testing.T) {
// Create a completed root execution, failed mid-process execution
parentRun := domain.RunRecord{
ID: "run-prod-failed-001",
WorkflowID: release.ID,
OrganizationID: domain.DefaultOrganizationID,
Trigger: domain.RunTriggerManual,
Mode: domain.RunModeProduction,
ReleaseID: release.ID,
WorkflowVersion: release.Version,
GitCommitSha: release.ContentDigest,
ImageDigest: release.ContentDigest,
Status: domain.RunStatusFailed,
Inputs: validInputs,
Attempt: 1,
NodeExecutions: []domain.NodeExecutionRecord{
{
NodeID: "root-source",
Status: domain.NodeExecutionStatusCompleted,
},
{
NodeID: "mid-process",
Status: domain.NodeExecutionStatusFailed,
},
},
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
_ = dataStore.UpsertRun(ctx, parentRun)
// Rerun subtree from mid-process
payload := map[string]any{
"nodeId": "mid-process",
}
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/runs/%s/rerun-subtree", parentRun.ID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
var childRun domain.RunRecord
_ = json.Unmarshal(rec.Body.Bytes(), &childRun)
if childRun.ParentRunID != parentRun.ID {
t.Fatalf("expected parentRunId %s, got %s", parentRun.ID, childRun.ParentRunID)
}
if childRun.Mode != domain.RunModeProduction {
t.Fatalf("expected mode 'production', got %s", childRun.Mode)
}
if childRun.ReleaseID != release.ID {
t.Fatalf("expected releaseId %s, got %s", release.ID, childRun.ReleaseID)
}
if childRun.WorkflowVersion != release.Version {
t.Fatalf("expected workflowVersion %d, got %d", release.Version, childRun.WorkflowVersion)
}
if childRun.GitCommitSha != release.ContentDigest {
t.Fatalf("expected gitCommitSha %s, got %s", release.ContentDigest, childRun.GitCommitSha)
}
if childRun.Attempt != 2 {
t.Fatalf("expected attempt 2, got %d", childRun.Attempt)
}
})
t.Run("Debug Subtree Rerun Inherits Debug Mode and NodeOverrides", func(t *testing.T) {
debugRelease := setupAdversarialRelease(ctx, dataStore)
wfDebugSnapshot := workflowSnapshotFromRelease(debugRelease)
_ = dataStore.UpsertWorkflow(ctx, wfDebugSnapshot)
parentDebugRun := domain.RunRecord{
ID: "run-debug-failed-002",
WorkflowID: debugRelease.ID,
OrganizationID: domain.DefaultOrganizationID,
Trigger: domain.RunTriggerManual,
Mode: domain.RunModeDebug,
Inputs: validInputs,
NodeOverrides: map[string]domain.NodeOverride{
"mid-process": {
Hyperparams: []domain.HyperParam{
{Key: "threshold", Value: "0.15"},
},
},
},
Status: domain.RunStatusFailed,
Attempt: 1,
NodeExecutions: []domain.NodeExecutionRecord{
{
NodeID: "root-source",
Status: domain.NodeExecutionStatusCompleted,
},
{
NodeID: "mid-process",
Status: domain.NodeExecutionStatusFailed,
},
},
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
_ = dataStore.UpsertRun(ctx, parentDebugRun)
payload := map[string]any{
"nodeId": "mid-process",
}
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/runs/%s/rerun-subtree", parentDebugRun.ID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
var childRun domain.RunRecord
_ = json.Unmarshal(rec.Body.Bytes(), &childRun)
if childRun.Mode != domain.RunModeDebug {
t.Fatalf("expected mode 'debug', got %s", childRun.Mode)
}
if childRun.NodeOverrides == nil || len(childRun.NodeOverrides["mid-process"].Hyperparams) == 0 {
t.Fatalf("expected nodeOverrides preserved on child debug run, got %#v", childRun.NodeOverrides)
}
if childRun.NodeOverrides["mid-process"].Hyperparams[0].Value != "0.15" {
t.Fatalf("expected threshold override '0.15', got %s", childRun.NodeOverrides["mid-process"].Hyperparams[0].Value)
}
})
}