droboticflow/internal/api/workflow_releases_test.go
2026-08-31 13:27:18 +08:00

520 lines
21 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func TestProjectWorkflowInstancePersistsReleasedProjectWorkflow(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project := domain.ProjectRecord{ID: "project-release-instance", OrganizationID: domain.DefaultOrganizationID, Name: "Project"}
if err := dataStore.UpsertProject(ctx, project); err != nil {
t.Fatal(err)
}
release := domain.WorkflowReleaseRecord{
ID: "release-project-instance", OrganizationID: domain.DefaultOrganizationID, TemplateID: "template-project-instance", Version: 4,
Name: "Released workflow", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindStandard,
ExecutionScope: domain.WorkflowExecutionScopeProduction, Nodes: []domain.WorkflowNode{{ID: "root", Type: "operator", Data: domain.WorkflowNodeData{Label: "root"}}},
CreatedAt: nowISO(), PublishedAt: nowISO(),
}
if err := dataStore.UpsertWorkflowRelease(ctx, release); err != nil {
t.Fatal(err)
}
handler := NewHandler(dataStore, nil)
body, _ := json.Marshal(map[string]string{"releaseId": release.ID})
request := httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflow-instances", bytes.NewReader(body))
request.Header.Set("Content-Type", "application/json")
request.AddCookie(&http.Cookie{Name: "droboticflow_csrf", Value: "project-release-csrf"})
request.Header.Set("X-Droboticflow-CSRF-Token", "project-release-csrf")
request = request.WithContext(context.WithValue(request.Context(), authContextKey{}, &authenticatedPrincipal{
User: domain.UserRecord{ID: "admin-project-release", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin},
Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive},
}))
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusCreated {
t.Fatalf("create project workflow instance status=%d body=%s", recorder.Code, recorder.Body.String())
}
var response struct {
Workflow domain.WorkflowRecord `json:"workflow"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
stored, err := dataStore.GetWorkflow(ctx, response.Workflow.ID)
if err != nil || stored == nil {
t.Fatalf("stored project workflow=%#v err=%v", stored, err)
}
if stored.ProjectID != project.ID || stored.DefinitionState != domain.WorkflowDefinitionStateReleased || stored.DefinitionVersion != release.Version || stored.DefinitionFamilyID != release.TemplateID {
t.Fatalf("project release binding was persisted as a draft: %#v", stored)
}
}
func TestProjectWorkflowInstanceBindsValidatedSupplier(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project := domain.ProjectRecord{ID: "project-supplier-binding", OrganizationID: domain.DefaultOrganizationID, Name: "Project"}
if err := dataStore.UpsertProject(ctx, project); err != nil {
t.Fatal(err)
}
supplier := domain.SupplierRecord{ID: "supplier-production", OrganizationID: domain.DefaultOrganizationID, Name: "Production supplier", Status: "active", CreatedAt: nowISO(), UpdatedAt: nowISO()}
if err := dataStore.UpsertSupplier(ctx, supplier); err != nil {
t.Fatal(err)
}
for index, role := range []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator, domain.AnnotationRoleQualityReviewer, domain.AnnotationRoleAcceptanceReviewer} {
if err := dataStore.UpsertProjectMember(ctx, domain.ProjectMemberRecord{
ID: fmt.Sprintf("member-%d", index), OrganizationID: domain.DefaultOrganizationID, ProjectID: project.ID,
UserID: fmt.Sprintf("user-%d", index), SupplierID: supplier.ID, Roles: []domain.AnnotationAssignmentRole{role}, CreatedAt: nowISO(), UpdatedAt: nowISO(),
}); err != nil {
t.Fatal(err)
}
}
release := domain.WorkflowReleaseRecord{
ID: "release-supplier-binding", OrganizationID: domain.DefaultOrganizationID, TemplateID: "template-supplier-binding", Version: 1,
Name: "Released human workflow", Type: domain.WorkflowTypeBatch, Kind: domain.WorkflowKindStandard,
ExecutionScope: domain.WorkflowExecutionScopeProduction,
Nodes: []domain.WorkflowNode{{ID: "human", Type: "operator", Data: domain.WorkflowNodeData{
Label: "Human gate", OperatorID: "operator-builtin-annotation-gate",
Hyperparams: []domain.HyperParam{{Key: "ANNOTATION_SUPPLIER_ID", Value: "release-must-not-own-supplier"}},
}}},
CreatedAt: nowISO(), PublishedAt: nowISO(),
}
if err := dataStore.UpsertWorkflowRelease(ctx, release); err != nil {
t.Fatal(err)
}
handler := NewHandler(dataStore, nil)
requestBody, _ := json.Marshal(map[string]string{"releaseId": release.ID, "supplierId": supplier.ID})
request := httptest.NewRequest(http.MethodPost, "/api/projects/"+project.ID+"/workflow-instances", bytes.NewReader(requestBody))
request.Header.Set("Content-Type", "application/json")
request.AddCookie(&http.Cookie{Name: "droboticflow_csrf", Value: "supplier-binding-csrf"})
request.Header.Set("X-Droboticflow-CSRF-Token", "supplier-binding-csrf")
request = request.WithContext(context.WithValue(request.Context(), authContextKey{}, &authenticatedPrincipal{
User: domain.UserRecord{ID: "admin-supplier-binding", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAdmin},
Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive},
}))
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusCreated {
t.Fatalf("create bound workflow instance status=%d body=%s", recorder.Code, recorder.Body.String())
}
var response struct {
Instance domain.ProjectWorkflowInstanceRecord `json:"instance"`
Workflow domain.WorkflowRecord `json:"workflow"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if response.Instance.SupplierID != supplier.ID {
t.Fatalf("instance supplier=%q want=%q", response.Instance.SupplierID, supplier.ID)
}
params := response.Workflow.Nodes[0].Data.Hyperparams
if len(params) != 1 || params[0].Key != "ANNOTATION_SUPPLIER_ID" || params[0].Value != supplier.ID {
t.Fatalf("workflow supplier binding=%#v", params)
}
if release.Nodes[0].Data.Hyperparams[0].Value != "release-must-not-own-supplier" {
t.Fatalf("release snapshot was mutated: %#v", release.Nodes[0].Data.Hyperparams)
}
}
func TestWorkflowReleaseRunEndpointSecurityGate(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
conn := domain.StorageConnectionRecord{
ID: "bos-conn-test",
OrganizationID: domain.DefaultOrganizationID,
Name: "Test BOS Connection",
Provider: domain.StorageProviderBOS,
Endpoint: "http://bos.example.local",
Bucket: "prod-bucket",
AccessKeyID: "ak-test",
SecretAccessKey: "sk-test",
CreatedAt: time.Now().UTC().Format(time.RFC3339),
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := dataStore.UpsertStorageConnection(ctx, conn); err != nil {
t.Fatalf("upsert storage connection: %v", err)
}
nodes := []domain.WorkflowNode{
{ID: "root-bos", Type: "operator", Data: domain.WorkflowNodeData{Label: "BOS Reader", Inputs: []domain.PortDefinition{{ID: "in_file", Name: "Input File", Type: domain.PortTypeFile, Required: true}}}},
{ID: "mid-infer", Type: "operator", Data: domain.WorkflowNodeData{Label: "Model Inference", Hyperparams: []domain.HyperParam{{Key: "thresh", Value: "0.8"}}, Inputs: []domain.PortDefinition{{ID: "in", Name: "In", Type: domain.PortTypeFile}}}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "root-bos", Target: "mid-infer"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "in_file", Name: "Input File", Type: domain.PortTypeFile, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "root-bos", TargetPortID: "in_file"}}},
}
release := domain.WorkflowReleaseRecord{
ID: "rel-sec-gate-001",
OrganizationID: domain.DefaultOrganizationID,
TemplateID: "tmpl-sec-01",
Version: 1,
Name: "Production Release",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
ExecutionScope: domain.WorkflowExecutionScopeProduction,
Nodes: nodes,
Edges: edges,
WorkflowInputs: inputs,
ContentDigest: "sha256:abcd1234efgh5678",
PublishedAt: time.Now().UTC().Format(time.RFC3339),
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := dataStore.UpsertWorkflowRelease(ctx, release); err != nil {
t.Fatalf("upsert release: %v", err)
}
sendReleaseRun := func(t *testing.T, releaseID string, payload any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflow-releases/%s/runs", releaseID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
var resp map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
return rec, resp
}
t.Run("Valid Production Run Returns 202 and Stamps Provenance", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"in_file": map[string]any{
"mode": "ref",
"type": "file",
"ref": map[string]any{
"provider": "BOS",
"connectionId": "bos-conn-test",
"bucket": "prod-bucket",
"prefix": "batch-1/",
"kind": "directory",
},
},
},
}
rec, resp := sendReleaseRun(t, release.ID, payload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
if resp["mode"] != "production" {
t.Fatalf("expected mode production, got %v", resp["mode"])
}
if resp["releaseId"] != release.ID {
t.Fatalf("expected releaseId %s, got %v", release.ID, resp["releaseId"])
}
if resp["gitCommitSha"] != release.ContentDigest {
t.Fatalf("expected gitCommitSha %s, got %v", release.ContentDigest, resp["gitCommitSha"])
}
if resp["imageDigest"] != release.ContentDigest {
t.Fatalf("expected imageDigest %s, got %v", release.ContentDigest, resp["imageDigest"])
}
})
t.Run("Rejects Node Parameter Overrides with 422 production_input_forbidden", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"nodeOverrides": map[string]any{
"mid-infer": map[string]any{
"hyperparams": []map[string]any{{"key": "thresh", "value": "0.1"}},
},
},
"inputs": map[string]any{
"in_file": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "connectionId": "bos-conn-test", "bucket": "b", "prefix": "p/"}},
},
}
rec, resp := sendReleaseRun(t, release.ID, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422 Unprocessable Entity, got %d body=%s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code production_input_forbidden, got %v", resp["code"])
}
})
t.Run("Rejects DebugTarget with 422 production_input_forbidden", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"debugTarget": map[string]any{
"targetNodeId": "mid-infer",
"singleNode": true,
},
"inputs": map[string]any{
"in_file": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "connectionId": "bos-conn-test", "bucket": "b", "prefix": "p/"}},
},
}
rec, resp := sendReleaseRun(t, release.ID, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422 Unprocessable Entity, got %d body=%s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code production_input_forbidden, got %v", resp["code"])
}
})
t.Run("Rejects Non-Root Intermediate Input Binding with 422 production_input_forbidden", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"in_file": map[string]any{"mode": "ref", "type": "file", "ref": map[string]any{"provider": "BOS", "connectionId": "bos-conn-test", "bucket": "b", "prefix": "p/"}},
"mid-infer.in": map[string]any{"mode": "inline", "type": "file", "inline": map[string]any{"fake": true}},
},
}
rec, resp := sendReleaseRun(t, release.ID, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d body=%s", rec.Code, rec.Body.String())
}
if resp["code"] != "production_input_forbidden" {
t.Fatalf("expected code production_input_forbidden, got %v", resp["code"])
}
})
t.Run("Rejects Missing Required Root Input with 422 workflow_inputs_invalid", func(t *testing.T) {
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{},
}
rec, resp := sendReleaseRun(t, release.ID, payload)
if rec.Code != http.StatusUnprocessableEntity {
t.Fatalf("expected 422, got %d body=%s", rec.Code, rec.Body.String())
}
if resp["code"] != "workflow_inputs_invalid" {
t.Fatalf("expected code workflow_inputs_invalid, got %v", resp["code"])
}
})
t.Run("Rejects Nonexistent Release ID with 404", func(t *testing.T) {
rec, _ := sendReleaseRun(t, "nonexistent-rel-id", map[string]any{"trigger": "manual"})
if rec.Code != http.StatusNotFound {
t.Fatalf("expected 404 Not Found, got %d body=%s", rec.Code, rec.Body.String())
}
})
}
func TestWorkflowDraftDebugRunAndOverrides(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
createTestWorkflow := func(name string) *domain.WorkflowRecord {
nodes := []domain.WorkflowNode{
{ID: "src-node", Type: "operator", Data: domain.WorkflowNodeData{Label: "Source", Inputs: []domain.PortDefinition{{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true}}}},
{ID: "model-node", Type: "operator", Data: domain.WorkflowNodeData{Label: "Model", Hyperparams: []domain.HyperParam{{Key: "confidence", Value: "0.85"}}}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "src-node", Target: "model-node"},
}
inputs := []domain.WorkflowInputDefinition{
{ID: "in", Name: "Input", Type: domain.PortTypeJSON, Required: true, Bindings: []domain.WorkflowInputTarget{{NodeID: "src-node", TargetPortID: "in"}}},
}
wf, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
Name: name,
Type: domain.WorkflowTypeBatch,
Nodes: nodes,
Edges: edges,
WorkflowInputs: inputs,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
return wf
}
sendWorkflowRun := func(t *testing.T, workflowID string, payload any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
bodyBytes, _ := json.Marshal(payload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/workflows/%s/run", workflowID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
var resp map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &resp)
return rec, resp
}
t.Run("Draft Workflow Automatically Defaults to Debug Run", func(t *testing.T) {
wf := createTestWorkflow("Draft Auto Debug")
payload := map[string]any{
"trigger": "manual",
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"test": 123}},
},
}
rec, resp := sendWorkflowRun(t, wf.ID, payload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
if resp["mode"] != "debug" {
t.Fatalf("expected mode debug, got %v", resp["mode"])
}
})
t.Run("Production Run on Draft Workflow is Rejected with 400", func(t *testing.T) {
wf := createTestWorkflow("Draft Prod Reject")
payload := map[string]any{
"trigger": "manual",
"mode": "production",
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"test": 123}},
},
}
rec, resp := sendWorkflowRun(t, wf.ID, payload)
if rec.Code != http.StatusBadRequest {
t.Fatalf("expected 400 Bad Request on draft production run, got %d body=%s", rec.Code, rec.Body.String())
}
issues, ok := resp["issues"].([]any)
if !ok || len(issues) == 0 {
t.Fatalf("expected validation issues, got %v", resp)
}
})
t.Run("Transient Node Overrides Attached to RunRecord Without DB Mutation", func(t *testing.T) {
wf := createTestWorkflow("Draft Node Overrides")
payload := map[string]any{
"trigger": "manual",
"mode": "debug",
"nodeOverrides": map[string]any{
"model-node": map[string]any{
"hyperparams": []map[string]any{{"key": "confidence", "value": "0.35"}},
},
"ghost-node-nonexistent": map[string]any{
"hyperparams": []map[string]any{{"key": "foo", "value": "bar"}},
},
},
"inputs": map[string]any{
"in": map[string]any{"mode": "inline", "type": "json", "inline": map[string]any{"ok": true}},
},
}
rec, resp := sendWorkflowRun(t, wf.ID, payload)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted, got %d body=%s", rec.Code, rec.Body.String())
}
overrides, ok := resp["nodeOverrides"].(map[string]any)
if !ok {
t.Fatalf("expected nodeOverrides on run, got %v", resp["nodeOverrides"])
}
if _, exists := overrides["model-node"]; !exists {
t.Fatalf("expected model-node override present")
}
if _, exists := overrides["ghost-node-nonexistent"]; exists {
t.Fatalf("expected ghost node override to be sanitized/ignored")
}
// Verify zero-mutation invariant on database workflow
storedWf, err := dataStore.GetWorkflow(ctx, wf.ID)
if err != nil || storedWf == nil {
t.Fatalf("get workflow: %v", err)
}
for _, hp := range storedWf.Nodes[1].Data.Hyperparams {
if hp.Key == "confidence" && hp.Value != "0.85" {
t.Fatalf("DB workflow hyperparams mutated! expected 0.85, got %s", hp.Value)
}
}
})
}
func TestSubtreeRerunInheritsModeAndProvenance(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
nodes := []domain.WorkflowNode{
{ID: "node-A", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node A"}},
{ID: "node-B", Type: "operator", Data: domain.WorkflowNodeData{Label: "Node B"}},
}
edges := []domain.WorkflowEdge{
{ID: "e1", Source: "node-A", Target: "node-B"},
}
wf, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
Name: "Subtree Rerun Workflow",
Type: domain.WorkflowTypeBatch,
Nodes: nodes,
Edges: edges,
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
sourceRun := domain.RunRecord{
ID: "parent-dbg-run-001",
OrganizationID: domain.DefaultOrganizationID,
WorkflowID: wf.ID,
WorkflowName: wf.Name,
Trigger: domain.RunTriggerManual,
Mode: domain.RunModeDebug,
ReleaseID: "rel-dbg-001",
WorkflowVersion: 2,
GitCommitSha: "sha256:source-commit-digest",
ImageDigest: "sha256:source-image-digest",
NodeOverrides: map[string]domain.NodeOverride{
"node-B": {
Hyperparams: []domain.HyperParam{{Key: "learning_rate", Value: "0.01"}},
},
},
Status: domain.RunStatusFailed,
NodeExecutions: []domain.NodeExecutionRecord{
{NodeID: "node-A", Status: domain.NodeExecutionStatusCompleted},
{NodeID: "node-B", Status: domain.NodeExecutionStatusFailed},
},
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := dataStore.UpsertRun(ctx, sourceRun); err != nil {
t.Fatalf("upsert source run: %v", err)
}
// Update workflow to reflect failed run
wf.Status = domain.WorkflowStatusFailed
wf.LastRunID = sourceRun.ID
if err := dataStore.UpdateWorkflowRunState(ctx, wf.ID, wf.Status, wf.LastRunID, time.Now().UTC().Format(time.RFC3339)); err != nil {
t.Fatalf("update workflow state: %v", err)
}
rerunPayload := map[string]any{
"nodeId": "node-B",
}
bodyBytes, _ := json.Marshal(rerunPayload)
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/api/runs/%s/rerun-subtree", sourceRun.ID), bytes.NewReader(bodyBytes))
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusAccepted {
t.Fatalf("expected 202 Accepted on rerun subtree, got %d body=%s", rec.Code, rec.Body.String())
}
var childResp map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &childResp); err != nil {
t.Fatalf("unmarshal child run resp: %v", err)
}
if childResp["mode"] != "debug" {
t.Fatalf("expected mode debug inherited from parent, got %v", childResp["mode"])
}
if childResp["releaseId"] != sourceRun.ReleaseID {
t.Fatalf("expected releaseId %s, got %v", sourceRun.ReleaseID, childResp["releaseId"])
}
if childResp["gitCommitSha"] != sourceRun.GitCommitSha {
t.Fatalf("expected gitCommitSha %s, got %v", sourceRun.GitCommitSha, childResp["gitCommitSha"])
}
if childResp["imageDigest"] != sourceRun.ImageDigest {
t.Fatalf("expected imageDigest %s, got %v", sourceRun.ImageDigest, childResp["imageDigest"])
}
if childResp["parentRunId"] != sourceRun.ID {
t.Fatalf("expected parentRunId %s, got %v", sourceRun.ID, childResp["parentRunId"])
}
}