forked from longtao.wu/droboticflow
907 lines
31 KiB
Go
907 lines
31 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"reflect"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
// ============================================================================
|
|
// Milestone M2 Empirical Challenger Stress & Adversarial Test Suite
|
|
// ============================================================================
|
|
|
|
func createAdversarialTestDAG() ([]domain.WorkflowNode, []domain.WorkflowEdge, []domain.WorkflowInputDefinition) {
|
|
// Topology:
|
|
// root-source (root) -> mid-process (intermediate) -> sink-writer (leaf)
|
|
nodes := []domain.WorkflowNode{
|
|
{
|
|
ID: "root-source",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Root Source",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "root_in", Name: "Root In", Type: domain.PortTypeJSON, Required: true},
|
|
},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out", Name: "Out", Type: domain.PortTypeJSON},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "batch_size", Value: "32"},
|
|
},
|
|
Image: "registry.example.local/operators/source:v1.0.0",
|
|
},
|
|
},
|
|
{
|
|
ID: "mid-process",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Middle Processing",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true},
|
|
{ID: "aux_in", Name: "Aux In", Type: domain.PortTypeJSON, Required: false},
|
|
},
|
|
Outputs: []domain.PortDefinition{
|
|
{ID: "out", Name: "Out", Type: domain.PortTypeJSON},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "threshold", Value: "0.85"},
|
|
{Key: "learning_rate", Value: "0.001"},
|
|
},
|
|
Image: "registry.example.local/operators/model:v2.1.0",
|
|
},
|
|
},
|
|
{
|
|
ID: "sink-writer",
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: "Sink Writer",
|
|
Inputs: []domain.PortDefinition{
|
|
{ID: "in", Name: "In", Type: domain.PortTypeJSON, Required: true},
|
|
},
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "destination", Value: "s3://output-bucket/results"},
|
|
},
|
|
Image: "registry.example.local/operators/sink:v1.0.0",
|
|
},
|
|
},
|
|
}
|
|
|
|
edges := []domain.WorkflowEdge{
|
|
{ID: "e1", Source: "root-source", SourceHandle: "out", Target: "mid-process", TargetHandle: "in"},
|
|
{ID: "e2", Source: "mid-process", SourceHandle: "out", Target: "sink-writer", TargetHandle: "in"},
|
|
}
|
|
|
|
inputs := []domain.WorkflowInputDefinition{
|
|
{
|
|
ID: "source_data",
|
|
Name: "Source Dataset",
|
|
Type: domain.PortTypeJSON,
|
|
Required: true,
|
|
Bindings: []domain.WorkflowInputTarget{
|
|
{NodeID: "root-source", TargetPortID: "root_in"},
|
|
},
|
|
},
|
|
}
|
|
|
|
return nodes, edges, inputs
|
|
}
|
|
|
|
func setupAdversarialRelease(ctx context.Context, dataStore store.Store) domain.WorkflowReleaseRecord {
|
|
nodes, edges, inputs := createAdversarialTestDAG()
|
|
now := time.Now().UTC().Format(time.RFC3339)
|
|
release := domain.WorkflowReleaseRecord{
|
|
ID: fmt.Sprintf("rel-adv-%d", time.Now().UnixNano()),
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
TemplateID: "tmpl-adv-001",
|
|
Version: 3,
|
|
Name: "Production Pipeline Release v3",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindStandard,
|
|
ExecutionScope: domain.WorkflowExecutionScopeProduction,
|
|
Nodes: nodes,
|
|
Edges: edges,
|
|
WorkflowInputs: inputs,
|
|
ContentDigest: "sha256:adv-digest-9876543210abcdef",
|
|
ReleaseNotes: "Milestone M2 Adversarial Verification Release",
|
|
PublishedByUserID: "challenger-agent",
|
|
PublishedAt: now,
|
|
CreatedAt: now,
|
|
}
|
|
_ = dataStore.UpsertWorkflowRelease(ctx, release)
|
|
return release
|
|
}
|
|
|
|
func setupAdversarialDraftWorkflow(ctx context.Context, dataStore store.Store, name string) domain.WorkflowRecord {
|
|
nodes, edges, inputs := createAdversarialTestDAG()
|
|
draftWorkflow, _ := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
|
Name: name,
|
|
Type: domain.WorkflowTypeBatch,
|
|
Nodes: nodes,
|
|
Edges: edges,
|
|
WorkflowInputs: inputs,
|
|
})
|
|
return *draftWorkflow
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// Attack 1: Attempting to pass intermediate node hyperparameters in release runs
|
|
// ----------------------------------------------------------------------------
|
|
|
|
func TestAdversarial_ReleaseRun_IntermediateNodeHyperparams(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
handler := NewHandler(dataStore, nil)
|
|
release := setupAdversarialRelease(ctx, dataStore)
|
|
|
|
sendReleaseRun := func(t *testing.T, payload any) (*httptest.ResponseRecorder, map[string]any) {
|
|
t.Helper()
|
|
bodyBytes, _ := json.Marshal(payload)
|
|
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflow-releases/%s/runs", release.ID), bytes.NewReader(bodyBytes))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
var resp map[string]any
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
|
|
return rec, resp
|
|
}
|
|
|
|
t.Run("Attack 1.1: Single nodeOverrides for intermediate node must fail with 422", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"nodeOverrides": map[string]any{
|
|
"mid-process": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "threshold", "value": "0.01"},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"data": [1]int{1}},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422 Unprocessable Entity, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 1.2: Multiple nodeOverrides across intermediate and leaf nodes must fail with 422", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"nodeOverrides": map[string]any{
|
|
"mid-process": map[string]any{
|
|
"image": "evil-registry.local/backdoor:v1",
|
|
},
|
|
"sink-writer": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "destination", "value": "s3://hacked-bucket"},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"data": [1]int{1}},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
issues, ok := resp["issues"].([]any)
|
|
if !ok || len(issues) < 2 {
|
|
t.Fatalf("expected at least 2 validation issues reported, got %#v", resp["issues"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 1.3: Injection of debugTarget in release run must fail with 422", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"debugTarget": map[string]any{
|
|
"targetNodeId": "mid-process",
|
|
"singleNode": true,
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"data": [1]int{1}},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 1.4: Direct topology tampering in release run must fail with 422", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"nodes": []map[string]any{
|
|
{"id": "hacked-node", "type": "operator"},
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"data": [1]int{1}},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
})
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// Attack 2: Attempting non-root input bindings in release runs
|
|
// ----------------------------------------------------------------------------
|
|
|
|
func TestAdversarial_ReleaseRun_NonRootInputBindings(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
handler := NewHandler(dataStore, nil)
|
|
release := setupAdversarialRelease(ctx, dataStore)
|
|
|
|
sendReleaseRun := func(t *testing.T, payload any) (*httptest.ResponseRecorder, map[string]any) {
|
|
t.Helper()
|
|
bodyBytes, _ := json.Marshal(payload)
|
|
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflow-releases/%s/runs", release.ID), bytes.NewReader(bodyBytes))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
var resp map[string]any
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
|
|
return rec, resp
|
|
}
|
|
|
|
t.Run("Attack 2.1: Direct intermediate node input binding (mid-process.in) rejected", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"ok": true},
|
|
},
|
|
"mid-process.in": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"tampered": true},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 2.2: Leaf node input binding (sink-writer.in) rejected", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"ok": true},
|
|
},
|
|
"sink-writer.in": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"tampered": true},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 2.3: Non-existent node input binding (ghost-node.port) rejected", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"ok": true},
|
|
},
|
|
"ghost-node.port": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"ghost": true},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 2.4: Path traversal attempt in input key rejected", func(t *testing.T) {
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"ok": true},
|
|
},
|
|
"../../root-source.root_in": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"traversal": true},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusUnprocessableEntity {
|
|
t.Fatalf("expected 422, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "production_input_forbidden" {
|
|
t.Fatalf("expected code 'production_input_forbidden', got %v", resp["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 2.5: Inline payload exceeding 64KB rejected with 400 workflow_inputs_invalid", func(t *testing.T) {
|
|
hugePayload := strings.Repeat("A", 70*1024)
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"huge": hugePayload},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendReleaseRun(t, payload)
|
|
if rec.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 Bad Request for >64KB inline payload, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["code"] != "workflow_inputs_invalid" {
|
|
t.Fatalf("expected code 'workflow_inputs_invalid', got %v", resp["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Valid Root Input Execution Passes and Stamped with Immutable Provenance", func(t *testing.T) {
|
|
freshRelease := setupAdversarialRelease(ctx, dataStore)
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"batch_id": "b-2026-08"},
|
|
},
|
|
},
|
|
}
|
|
bodyBytes, _ := json.Marshal(payload)
|
|
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflow-releases/%s/runs", freshRelease.ID), bytes.NewReader(bodyBytes))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
var resp map[string]any
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
|
|
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted, got %d, body: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["mode"] != "production" {
|
|
t.Fatalf("expected mode 'production', got %v", resp["mode"])
|
|
}
|
|
if resp["releaseId"] != freshRelease.ID {
|
|
t.Fatalf("expected releaseId %s, got %v", freshRelease.ID, resp["releaseId"])
|
|
}
|
|
if resp["workflowVersion"] != float64(freshRelease.Version) {
|
|
t.Fatalf("expected workflowVersion %d, got %v", freshRelease.Version, resp["workflowVersion"])
|
|
}
|
|
if resp["gitCommitSha"] != freshRelease.ContentDigest {
|
|
t.Fatalf("expected gitCommitSha %s, got %v", freshRelease.ContentDigest, resp["gitCommitSha"])
|
|
}
|
|
if resp["imageDigest"] != freshRelease.ContentDigest {
|
|
t.Fatalf("expected imageDigest %s, got %v", freshRelease.ContentDigest, resp["imageDigest"])
|
|
}
|
|
if resp["nodeOverrides"] != nil {
|
|
t.Fatalf("expected nodeOverrides to be nil, got %#v", resp["nodeOverrides"])
|
|
}
|
|
if resp["debugTarget"] != nil {
|
|
t.Fatalf("expected debugTarget to be nil, got %#v", resp["debugTarget"])
|
|
}
|
|
})
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// Attack 3: Attempting ghost and malformed overrides in debug runs
|
|
// ----------------------------------------------------------------------------
|
|
|
|
func TestAdversarial_DebugRun_GhostAndMalformedOverrides(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
sendDebugRun := func(t *testing.T, workflowID string, payload any) (*httptest.ResponseRecorder, map[string]any) {
|
|
t.Helper()
|
|
bodyBytes, _ := json.Marshal(payload)
|
|
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", workflowID), bytes.NewReader(bodyBytes))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
var resp map[string]any
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
|
|
return rec, resp
|
|
}
|
|
|
|
t.Run("Attack 3.1: Ghost node in nodeOverrides is gracefully ignored and stripped", func(t *testing.T) {
|
|
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.1")
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": "debug",
|
|
"nodeOverrides": map[string]any{
|
|
"ghost-phantom-node-999": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "ghost_key", "value": "ghost_val"},
|
|
},
|
|
},
|
|
"mid-process": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "threshold", "value": "0.12"},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"debug": true},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendDebugRun(t, wf.ID, payload)
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
if resp["mode"] != "debug" {
|
|
t.Fatalf("expected mode 'debug', got %v", resp["mode"])
|
|
}
|
|
overrides, ok := resp["nodeOverrides"].(map[string]any)
|
|
if !ok {
|
|
t.Fatalf("expected nodeOverrides map in response, got %#v", resp["nodeOverrides"])
|
|
}
|
|
if _, hasGhost := overrides["ghost-phantom-node-999"]; hasGhost {
|
|
t.Fatalf("ghost node should have been stripped from nodeOverrides, found: %#v", overrides)
|
|
}
|
|
if _, hasMid := overrides["mid-process"]; !hasMid {
|
|
t.Fatalf("valid override for mid-process should be preserved, got %#v", overrides)
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 3.2: Malformed input payload in nodeOverrides rejected with 400 validation error", func(t *testing.T) {
|
|
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.2")
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": "debug",
|
|
"nodeOverrides": map[string]any{
|
|
"mid-process": map[string]any{
|
|
"inputs": map[string]any{
|
|
"aux_in": map[string]any{
|
|
"mode": "ref",
|
|
"type": "json",
|
|
// missing required "ref" object for ref mode
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"debug": true},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendDebugRun(t, wf.ID, payload)
|
|
if rec.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 for malformed nodeOverride input payload, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
issues, ok := resp["issues"].([]any)
|
|
if !ok || len(issues) == 0 {
|
|
t.Fatalf("expected validation issues in response, got %#v", resp)
|
|
}
|
|
firstIssue := issues[0].(map[string]any)
|
|
if firstIssue["code"] != "workflow_inputs_invalid" {
|
|
t.Fatalf("expected issue code 'workflow_inputs_invalid', got %v", firstIssue["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 3.3: debugTarget targeting nonexistent ghost node rejected with 400 debug_target_node_missing", func(t *testing.T) {
|
|
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.3")
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": "debug",
|
|
"debugTarget": map[string]any{
|
|
"targetNodeId": "ghost-nonexistent-node-XYZ",
|
|
"singleNode": true,
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"debug": true},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendDebugRun(t, wf.ID, payload)
|
|
if rec.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 for ghost debugTarget node, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
issues, ok := resp["issues"].([]any)
|
|
if !ok || len(issues) == 0 {
|
|
t.Fatalf("expected validation issues in response, got %#v", resp)
|
|
}
|
|
firstIssue := issues[0].(map[string]any)
|
|
if firstIssue["code"] != "debug_target_node_missing" {
|
|
t.Fatalf("expected issue code 'debug_target_node_missing', got %v", firstIssue["code"])
|
|
}
|
|
})
|
|
|
|
t.Run("Attack 3.4: Explicit production mode on draft workflow rejected with 400 production_run_draft_forbidden", func(t *testing.T) {
|
|
wf := setupAdversarialDraftWorkflow(ctx, dataStore, "Draft 3.4")
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": "production", // forbidden on draft workflow
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"data": 1},
|
|
},
|
|
},
|
|
}
|
|
rec, resp := sendDebugRun(t, wf.ID, payload)
|
|
if rec.Code != http.StatusBadRequest {
|
|
t.Fatalf("expected 400 for production mode on draft workflow, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
issues, ok := resp["issues"].([]any)
|
|
if !ok || len(issues) == 0 {
|
|
t.Fatalf("expected validation issues in response, got %#v", resp)
|
|
}
|
|
firstIssue := issues[0].(map[string]any)
|
|
if firstIssue["code"] != "production_run_draft_forbidden" {
|
|
t.Fatalf("expected issue code 'production_run_draft_forbidden', got %v", firstIssue["code"])
|
|
}
|
|
})
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// Attack 4: Validating the DB zero-mutation invariant under concurrent debug runs
|
|
// ----------------------------------------------------------------------------
|
|
|
|
func TestAdversarial_ConcurrentDebugRuns_DBZeroMutationInvariant(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
handler := NewHandler(dataStore, nil)
|
|
|
|
// Create 50 distinct draft workflows sharing identical initial DAG definitions
|
|
const concurrencyLevel = 50
|
|
draftWorkflows := make([]domain.WorkflowRecord, concurrencyLevel)
|
|
|
|
for i := 0; i < concurrencyLevel; i++ {
|
|
wf := setupAdversarialDraftWorkflow(ctx, dataStore, fmt.Sprintf("Concurrent Draft %d", i))
|
|
draftWorkflows[i] = wf
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(concurrencyLevel)
|
|
errChan := make(chan error, concurrencyLevel)
|
|
|
|
for i := 0; i < concurrencyLevel; i++ {
|
|
go func(idx int) {
|
|
defer wg.Done()
|
|
|
|
targetWF := draftWorkflows[idx]
|
|
|
|
// Each goroutine prepares a distinct, conflicting override payload
|
|
hackedThreshold := fmt.Sprintf("0.%02d", idx)
|
|
hackedImage := fmt.Sprintf("hacked-repo.local/runner:worker-%d", idx)
|
|
hackedCmd := []string{"bash", "-c", fmt.Sprintf("echo worker-%d", idx)}
|
|
|
|
payload := map[string]any{
|
|
"trigger": "manual",
|
|
"mode": "debug",
|
|
"nodeOverrides": map[string]any{
|
|
"mid-process": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "threshold", "value": hackedThreshold},
|
|
{"key": "learning_rate", "value": "0.999"},
|
|
{"key": fmt.Sprintf("unique_key_%d", idx), "value": "injected"},
|
|
},
|
|
"image": hackedImage,
|
|
"containerCommand": hackedCmd,
|
|
},
|
|
"sink-writer": map[string]any{
|
|
"hyperparams": []map[string]any{
|
|
{"key": "destination", "value": fmt.Sprintf("s3://worker-%d/dest", idx)},
|
|
},
|
|
},
|
|
},
|
|
"inputs": map[string]any{
|
|
"source_data": map[string]any{
|
|
"mode": "inline",
|
|
"type": "json",
|
|
"inline": map[string]any{"worker_idx": idx},
|
|
},
|
|
},
|
|
}
|
|
|
|
bodyBytes, _ := json.Marshal(payload)
|
|
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", targetWF.ID), bytes.NewReader(bodyBytes))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
|
|
if rec.Code != http.StatusAccepted {
|
|
errChan <- fmt.Errorf("worker %d got status %d: %s", idx, rec.Code, rec.Body.String())
|
|
return
|
|
}
|
|
|
|
var resp map[string]any
|
|
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
|
|
errChan <- fmt.Errorf("worker %d unmarshal error: %v", idx, err)
|
|
return
|
|
}
|
|
|
|
// Validate created run record has its own independent overrides
|
|
if resp["mode"] != "debug" {
|
|
errChan <- fmt.Errorf("worker %d run mode mismatch: %v", idx, resp["mode"])
|
|
return
|
|
}
|
|
}(i)
|
|
}
|
|
|
|
wg.Wait()
|
|
close(errChan)
|
|
|
|
for err := range errChan {
|
|
t.Fatalf("concurrent debug run failed: %v", err)
|
|
}
|
|
|
|
// ------------------------------------------------------------------------
|
|
// Verification of DB Zero-Mutation Invariant across all 50 workflows
|
|
// ------------------------------------------------------------------------
|
|
for i := 0; i < concurrencyLevel; i++ {
|
|
finalWorkflow, err := dataStore.GetWorkflow(ctx, draftWorkflows[i].ID)
|
|
if err != nil || finalWorkflow == nil {
|
|
t.Fatalf("failed to fetch final workflow %d from store: %v", i, err)
|
|
}
|
|
|
|
// Deep semantic equality check on DAG definitions
|
|
if !reflect.DeepEqual(draftWorkflows[i].Nodes, finalWorkflow.Nodes) {
|
|
t.Fatalf("workflow.Nodes was mutated in DB for workflow %d!", i)
|
|
}
|
|
if !reflect.DeepEqual(draftWorkflows[i].Edges, finalWorkflow.Edges) {
|
|
t.Fatalf("workflow.Edges was mutated in DB for workflow %d!", i)
|
|
}
|
|
if !reflect.DeepEqual(draftWorkflows[i].Hyperparams, finalWorkflow.Hyperparams) {
|
|
t.Fatalf("workflow.Hyperparams was mutated in DB for workflow %d!", i)
|
|
}
|
|
if !reflect.DeepEqual(draftWorkflows[i].WorkflowInputs, finalWorkflow.WorkflowInputs) {
|
|
t.Fatalf("workflow.WorkflowInputs was mutated in DB for workflow %d!", i)
|
|
}
|
|
|
|
// Specifically verify node hyperparameters and images remained pristine
|
|
for _, node := range finalWorkflow.Nodes {
|
|
if node.ID == "mid-process" {
|
|
if node.Data.Image != "registry.example.local/operators/model:v2.1.0" {
|
|
t.Fatalf("mid-process image corrupted in DB for workflow %d: %s", i, node.Data.Image)
|
|
}
|
|
for _, hp := range node.Data.Hyperparams {
|
|
if hp.Key == "threshold" && hp.Value != "0.85" {
|
|
t.Fatalf("mid-process threshold hyperparam corrupted in DB: %s", hp.Value)
|
|
}
|
|
if strings.HasPrefix(hp.Key, "unique_key_") {
|
|
t.Fatalf("injected hyperparam leaked into DB workflow: %s", hp.Key)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// Attack 5: Subtree Rerun Provenance and Mode Inheritance
|
|
// ----------------------------------------------------------------------------
|
|
|
|
func TestAdversarial_SubtreeRerun_SecurityAndProvenancePreservation(t *testing.T) {
|
|
ctx := context.Background()
|
|
dataStore := store.NewMemoryStore()
|
|
handler := NewHandler(dataStore, nil)
|
|
release := setupAdversarialRelease(ctx, dataStore)
|
|
|
|
// Ensure backing workflow record exists for the release
|
|
wfSnapshot := workflowSnapshotFromRelease(release)
|
|
_ = dataStore.UpsertWorkflow(ctx, wfSnapshot)
|
|
|
|
validInputs := map[string]domain.ArtifactPayload{
|
|
"source_data": {
|
|
Mode: domain.ArtifactTransferModeInline,
|
|
Type: domain.PortTypeJSON,
|
|
Inline: map[string]any{"data": 123},
|
|
},
|
|
}
|
|
|
|
t.Run("Production Subtree Rerun Inherits Release Provenance and Mode", func(t *testing.T) {
|
|
// Create a completed root execution, failed mid-process execution
|
|
parentRun := domain.RunRecord{
|
|
ID: "run-prod-failed-001",
|
|
WorkflowID: release.ID,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
Trigger: domain.RunTriggerManual,
|
|
Mode: domain.RunModeProduction,
|
|
ReleaseID: release.ID,
|
|
WorkflowVersion: release.Version,
|
|
GitCommitSha: release.ContentDigest,
|
|
ImageDigest: release.ContentDigest,
|
|
Status: domain.RunStatusFailed,
|
|
Inputs: validInputs,
|
|
Attempt: 1,
|
|
NodeExecutions: []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "root-source",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
},
|
|
{
|
|
NodeID: "mid-process",
|
|
Status: domain.NodeExecutionStatusFailed,
|
|
},
|
|
},
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
_ = dataStore.UpsertRun(ctx, parentRun)
|
|
|
|
// Rerun subtree from mid-process
|
|
payload := map[string]any{
|
|
"nodeId": "mid-process",
|
|
}
|
|
bodyBytes, _ := json.Marshal(payload)
|
|
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/runs/%s/rerun-subtree", parentRun.ID), bytes.NewReader(bodyBytes))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
var childRun domain.RunRecord
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &childRun)
|
|
|
|
if childRun.ParentRunID != parentRun.ID {
|
|
t.Fatalf("expected parentRunId %s, got %s", parentRun.ID, childRun.ParentRunID)
|
|
}
|
|
if childRun.Mode != domain.RunModeProduction {
|
|
t.Fatalf("expected mode 'production', got %s", childRun.Mode)
|
|
}
|
|
if childRun.ReleaseID != release.ID {
|
|
t.Fatalf("expected releaseId %s, got %s", release.ID, childRun.ReleaseID)
|
|
}
|
|
if childRun.WorkflowVersion != release.Version {
|
|
t.Fatalf("expected workflowVersion %d, got %d", release.Version, childRun.WorkflowVersion)
|
|
}
|
|
if childRun.GitCommitSha != release.ContentDigest {
|
|
t.Fatalf("expected gitCommitSha %s, got %s", release.ContentDigest, childRun.GitCommitSha)
|
|
}
|
|
if childRun.Attempt != 2 {
|
|
t.Fatalf("expected attempt 2, got %d", childRun.Attempt)
|
|
}
|
|
})
|
|
|
|
t.Run("Debug Subtree Rerun Inherits Debug Mode and NodeOverrides", func(t *testing.T) {
|
|
debugRelease := setupAdversarialRelease(ctx, dataStore)
|
|
wfDebugSnapshot := workflowSnapshotFromRelease(debugRelease)
|
|
_ = dataStore.UpsertWorkflow(ctx, wfDebugSnapshot)
|
|
|
|
parentDebugRun := domain.RunRecord{
|
|
ID: "run-debug-failed-002",
|
|
WorkflowID: debugRelease.ID,
|
|
OrganizationID: domain.DefaultOrganizationID,
|
|
Trigger: domain.RunTriggerManual,
|
|
Mode: domain.RunModeDebug,
|
|
Inputs: validInputs,
|
|
NodeOverrides: map[string]domain.NodeOverride{
|
|
"mid-process": {
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "threshold", Value: "0.15"},
|
|
},
|
|
},
|
|
},
|
|
Status: domain.RunStatusFailed,
|
|
Attempt: 1,
|
|
NodeExecutions: []domain.NodeExecutionRecord{
|
|
{
|
|
NodeID: "root-source",
|
|
Status: domain.NodeExecutionStatusCompleted,
|
|
},
|
|
{
|
|
NodeID: "mid-process",
|
|
Status: domain.NodeExecutionStatusFailed,
|
|
},
|
|
},
|
|
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
|
}
|
|
_ = dataStore.UpsertRun(ctx, parentDebugRun)
|
|
|
|
payload := map[string]any{
|
|
"nodeId": "mid-process",
|
|
}
|
|
bodyBytes, _ := json.Marshal(payload)
|
|
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/runs/%s/rerun-subtree", parentDebugRun.ID), bytes.NewReader(bodyBytes))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
rec := httptest.NewRecorder()
|
|
handler.ServeHTTP(rec, req)
|
|
|
|
if rec.Code != http.StatusAccepted {
|
|
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
|
|
var childRun domain.RunRecord
|
|
_ = json.Unmarshal(rec.Body.Bytes(), &childRun)
|
|
|
|
if childRun.Mode != domain.RunModeDebug {
|
|
t.Fatalf("expected mode 'debug', got %s", childRun.Mode)
|
|
}
|
|
if childRun.NodeOverrides == nil || len(childRun.NodeOverrides["mid-process"].Hyperparams) == 0 {
|
|
t.Fatalf("expected nodeOverrides preserved on child debug run, got %#v", childRun.NodeOverrides)
|
|
}
|
|
if childRun.NodeOverrides["mid-process"].Hyperparams[0].Value != "0.15" {
|
|
t.Fatalf("expected threshold override '0.15', got %s", childRun.NodeOverrides["mid-process"].Hyperparams[0].Value)
|
|
}
|
|
})
|
|
}
|