droboticflow/internal/api/challenger_m3_stress_test.go

798 lines
33 KiB
Go

package api
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"sync"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
// ============================================================================
// Challenger M3 Adversarial Stress & Empirical Verification Test Suite
// ============================================================================
func TestChallengerM3_MassiveDebugWorklogInjectionStress(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
s := &server{store: dataStore}
now := time.Now().UTC()
from := now.Add(-15 * 24 * time.Hour)
to := now.Add(15 * 24 * time.Hour)
adminPrincipal := &authenticatedPrincipal{
User: domain.UserRecord{
ID: "admin-1",
OrganizationID: domain.DefaultOrganizationID,
Role: domain.UserRoleAdmin,
Name: "Admin User",
},
Organization: domain.OrganizationRecord{
ID: domain.DefaultOrganizationID,
Status: domain.OrganizationStatusActive,
},
}
_ = dataStore.UpsertUser(ctx, adminPrincipal.User)
// 1. Seed baseline production users, vendors, and workgroups
users := []domain.UserRecord{
{ID: "worker-alice", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAnnotator, Name: "Alice Worker"},
{ID: "worker-bob", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAnnotator, Name: "Bob Worker"},
{ID: "worker-charlie", OrganizationID: domain.DefaultOrganizationID, Role: domain.UserRoleAnnotator, Name: "Charlie Worker"},
}
for _, u := range users {
if err := dataStore.UpsertUser(ctx, u); err != nil {
t.Fatalf("failed to seed user: %v", err)
}
}
// 2. Seed baseline production worklog events
prodEvents := []domain.AnnotationWorklogEventRecord{
{ID: "pe-1", OrganizationID: domain.DefaultOrganizationID, ProjectID: "proj-1", TaskID: "t-1", UserID: "worker-alice", WorkgroupID: "wg-1", VendorID: "vendor-a", EventType: "annotation_submitted", Mode: domain.RunModeProduction, DurationSeconds: 300, OutputDelta: 10, CreatedAt: now.Add(-2 * time.Hour).Format(time.RFC3339Nano)},
{ID: "pe-2", OrganizationID: domain.DefaultOrganizationID, ProjectID: "proj-1", TaskID: "t-1", UserID: "worker-alice", WorkgroupID: "wg-1", VendorID: "vendor-a", EventType: "quality_approved", Mode: domain.RunModeProduction, DurationSeconds: 60, OutputDelta: 0, CreatedAt: now.Add(-1 * time.Hour).Format(time.RFC3339Nano)},
{ID: "pe-3", OrganizationID: domain.DefaultOrganizationID, ProjectID: "proj-1", TaskID: "t-2", UserID: "worker-bob", WorkgroupID: "wg-2", VendorID: "vendor-b", EventType: "annotation_submitted", Mode: domain.RunModeProduction, DurationSeconds: 450, OutputDelta: 15, CreatedAt: now.Add(-3 * time.Hour).Format(time.RFC3339Nano)},
{ID: "pe-4", OrganizationID: domain.DefaultOrganizationID, ProjectID: "proj-1", TaskID: "t-2", UserID: "worker-bob", WorkgroupID: "wg-2", VendorID: "vendor-b", EventType: "rework_submitted", Mode: domain.RunModeProduction, DurationSeconds: 150, OutputDelta: 5, CreatedAt: now.Add(-2 * time.Hour).Format(time.RFC3339Nano)},
{ID: "pe-5", OrganizationID: domain.DefaultOrganizationID, ProjectID: "proj-1", TaskID: "t-2", UserID: "worker-bob", WorkgroupID: "wg-2", VendorID: "vendor-b", EventType: "quality_rejected", Mode: domain.RunModeProduction, DurationSeconds: 40, OutputDelta: 0, CreatedAt: now.Add(-1 * time.Hour).Format(time.RFC3339Nano)},
{ID: "pe-6", OrganizationID: domain.DefaultOrganizationID, ProjectID: "proj-1", TaskID: "t-3", UserID: "worker-charlie", WorkgroupID: "wg-1", VendorID: "vendor-a", EventType: "acceptance_approved", Mode: domain.RunModeProduction, DurationSeconds: 100, OutputDelta: 20, CreatedAt: now.Add(-30 * time.Minute).Format(time.RFC3339Nano)},
}
for _, pe := range prodEvents {
if err := dataStore.UpsertAnnotationWorklogEvent(ctx, pe); err != nil {
t.Fatalf("failed to insert prod event: %v", err)
}
}
// 3. Seed baseline production runs and delivery packages
prodRun := domain.RunRecord{
ID: "run-prod-1",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-1",
Status: domain.RunStatusCompleted,
Mode: domain.RunModeProduction,
DurationMS: 12000,
CreatedAt: now.Add(-4 * time.Hour).Format(time.RFC3339Nano),
}
_ = dataStore.UpsertRun(ctx, prodRun)
prodPkg := domain.DeliveryPackageRecord{
ID: "pkg-prod-1",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-1",
Status: domain.DeliveryPackagePendingAcceptance,
Mode: domain.RunModeProduction,
Temporary: false,
CreatedAt: now.Add(-4 * time.Hour).Format(time.RFC3339Nano),
}
_ = dataStore.UpsertDeliveryPackage(ctx, prodPkg)
// 4. Capture baseline workforce summary across all scopes
baselineSummaryPlatform := &domain.DashboardSummary{}
if err := s.populateDashboardWorkforce(ctx, domain.DefaultOrganizationID, adminPrincipal, from, to, baselineSummaryPlatform); err != nil {
t.Fatalf("failed to populate baseline platform workforce: %v", err)
}
alicePrincipal := &authenticatedPrincipal{User: users[0], Organization: adminPrincipal.Organization}
baselineSummaryAlice := &domain.DashboardSummary{}
if err := s.populateDashboardWorkforce(ctx, domain.DefaultOrganizationID, alicePrincipal, from, to, baselineSummaryAlice); err != nil {
t.Fatalf("failed to populate baseline alice workforce: %v", err)
}
// Verify baseline platform numbers
if baselineSummaryPlatform.Workforce.Totals.SubmittedCount != 2 {
t.Fatalf("expected baseline SubmittedCount == 2, got %d", baselineSummaryPlatform.Workforce.Totals.SubmittedCount)
}
if baselineSummaryPlatform.Workforce.Totals.ReworkCount != 1 {
t.Fatalf("expected baseline ReworkCount == 1, got %d", baselineSummaryPlatform.Workforce.Totals.ReworkCount)
}
if baselineSummaryPlatform.Workforce.Totals.QualityApproved != 1 {
t.Fatalf("expected baseline QualityApproved == 1, got %d", baselineSummaryPlatform.Workforce.Totals.QualityApproved)
}
if baselineSummaryPlatform.Workforce.Totals.QualityRejected != 1 {
t.Fatalf("expected baseline QualityRejected == 1, got %d", baselineSummaryPlatform.Workforce.Totals.QualityRejected)
}
if baselineSummaryPlatform.Workforce.Totals.AcceptanceApproved != 1 {
t.Fatalf("expected baseline AcceptanceApproved == 1, got %d", baselineSummaryPlatform.Workforce.Totals.AcceptanceApproved)
}
if baselineSummaryPlatform.Workforce.Totals.DurationSeconds != 1100 {
t.Fatalf("expected baseline DurationSeconds == 1100, got %d", baselineSummaryPlatform.Workforce.Totals.DurationSeconds)
}
if baselineSummaryPlatform.Workforce.Totals.OutputCount != 50 {
t.Fatalf("expected baseline OutputCount == 50, got %d", baselineSummaryPlatform.Workforce.Totals.OutputCount)
}
// Snapshot JSON representation of baseline
baselinePlatformJSON, _ := json.Marshal(baselineSummaryPlatform.Workforce)
baselineAliceJSON, _ := json.Marshal(baselineSummaryAlice.Workforce)
// 5. INJECT 1,200 ADVERSARIAL DEBUG WORKLOG EVENTS
debugEventTypes := []string{"annotation_submitted", "rework_submitted", "quality_rejected", "acceptance_rejected"}
for i := 0; i < 1200; i++ {
eventType := debugEventTypes[i%len(debugEventTypes)]
userID := users[i%len(users)].ID
if i%3 == 0 {
userID = fmt.Sprintf("rogue-debug-user-%d", i)
}
debugEvent := domain.AnnotationWorklogEventRecord{
ID: fmt.Sprintf("adversarial-debug-event-%04d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-1",
TaskID: fmt.Sprintf("task-dbg-%04d", i),
UserID: userID,
WorkgroupID: fmt.Sprintf("wg-%d", i%5),
VendorID: fmt.Sprintf("vendor-%d", i%3),
EventType: eventType,
Mode: domain.RunModeDebug,
Temporary: true,
DurationSeconds: int64(100000 + i*1000), // extreme durations
OutputDelta: int64(500 + i), // extreme outputs
CreatedAt: now.Add(time.Duration(i-600) * time.Minute).Format(time.RFC3339Nano),
}
if err := dataStore.UpsertAnnotationWorklogEvent(ctx, debugEvent); err != nil {
t.Fatalf("failed to insert debug event %d: %v", i, err)
}
}
// 6. INJECT 300 ADVERSARIAL DEBUG RUNS (Failed, huge durations)
for i := 0; i < 300; i++ {
debugRun := domain.RunRecord{
ID: fmt.Sprintf("run-dbg-stress-%04d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-1",
Status: domain.RunStatusFailed,
Mode: domain.RunModeDebug,
DurationMS: 999999999,
CreatedAt: now.Add(time.Duration(i-150) * time.Minute).Format(time.RFC3339Nano),
}
_ = dataStore.UpsertRun(ctx, debugRun)
}
// 7. INJECT 200 ADVERSARIAL DEBUG DELIVERY PACKAGES
for i := 0; i < 200; i++ {
debugPkg := domain.DeliveryPackageRecord{
ID: fmt.Sprintf("pkg-dbg-stress-%04d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-1",
Status: domain.DeliveryPackagePendingAcceptance,
Mode: domain.RunModeDebug,
Temporary: true,
CreatedAt: now.Add(time.Duration(i-100) * time.Minute).Format(time.RFC3339Nano),
}
_ = dataStore.UpsertDeliveryPackage(ctx, debugPkg)
}
// 8. Re-evaluate populateDashboardWorkforce across all scopes
stressedSummaryPlatform := &domain.DashboardSummary{}
if err := s.populateDashboardWorkforce(ctx, domain.DefaultOrganizationID, adminPrincipal, from, to, stressedSummaryPlatform); err != nil {
t.Fatalf("failed to populate stressed platform workforce: %v", err)
}
stressedSummaryAlice := &domain.DashboardSummary{}
if err := s.populateDashboardWorkforce(ctx, domain.DefaultOrganizationID, alicePrincipal, from, to, stressedSummaryAlice); err != nil {
t.Fatalf("failed to populate stressed alice workforce: %v", err)
}
stressedPlatformJSON, _ := json.Marshal(stressedSummaryPlatform.Workforce)
stressedAliceJSON, _ := json.Marshal(stressedSummaryAlice.Workforce)
// 9. ASSERT STRICT 100% EQUALITY (ZERO LEAKAGE)
if !bytes.Equal(baselinePlatformJSON, stressedPlatformJSON) {
t.Fatalf("CRITICAL WORKFORCE METRIC LEAKAGE DETECTED (Platform Scope)!\nBaseline:\n%s\nStressed:\n%s", string(baselinePlatformJSON), string(stressedPlatformJSON))
}
if !bytes.Equal(baselineAliceJSON, stressedAliceJSON) {
t.Fatalf("CRITICAL WORKFORCE METRIC LEAKAGE DETECTED (Self Scope)!\nBaseline:\n%s\nStressed:\n%s", string(baselineAliceJSON), string(stressedAliceJSON))
}
if !reflect.DeepEqual(baselineSummaryPlatform.Workforce, stressedSummaryPlatform.Workforce) {
t.Fatalf("Workforce summary struct mismatch after 1200 debug events injection!")
}
}
func TestChallengerM3_DashboardHTTPEndpointFullIsolation(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
now := time.Now().UTC()
adminUser := domain.UserRecord{
ID: "admin-http",
OrganizationID: domain.DefaultOrganizationID,
Role: domain.UserRoleAdmin,
Name: "Admin HTTP",
}
_ = dataStore.UpsertUser(ctx, adminUser)
adminPrincipal := &authenticatedPrincipal{
User: adminUser,
Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive},
}
// 1. Seed initial production state
_ = dataStore.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{
ID: "prod-wl-1",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-http",
TaskID: "task-1",
UserID: adminUser.ID,
WorkgroupID: "wg-1",
VendorID: "vendor-1",
EventType: "annotation_submitted",
Mode: domain.RunModeProduction,
DurationSeconds: 200,
OutputDelta: 5,
CreatedAt: now.Format(time.RFC3339Nano),
})
_ = dataStore.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{
ID: "pkg-prod-http",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-http",
Status: domain.DeliveryPackagePendingAcceptance,
Mode: domain.RunModeProduction,
CreatedAt: now.Format(time.RFC3339Nano),
})
_ = dataStore.UpsertRun(ctx, domain.RunRecord{
ID: "run-prod-http",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-http",
Status: domain.RunStatusCompleted,
Mode: domain.RunModeProduction,
DurationMS: 5000,
CreatedAt: now.Format(time.RFC3339Nano),
})
// Perform Baseline HTTP Request
reqBaseline := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil)
reqBaseline = reqBaseline.WithContext(context.WithValue(reqBaseline.Context(), authContextKey{}, adminPrincipal))
recBaseline := httptest.NewRecorder()
handler.ServeHTTP(recBaseline, reqBaseline)
if recBaseline.Code != http.StatusOK {
t.Fatalf("expected 200 OK on baseline dashboard, got %d body=%s", recBaseline.Code, recBaseline.Body.String())
}
var baselineDashboard domain.DashboardSummary
if err := json.Unmarshal(recBaseline.Body.Bytes(), &baselineDashboard); err != nil {
t.Fatalf("failed to decode baseline dashboard: %v", err)
}
// 2. Inject 500 Debug Events, 100 Debug Runs, 100 Debug Packages
for i := 0; i < 500; i++ {
_ = dataStore.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{
ID: fmt.Sprintf("dbg-wl-%d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-http",
TaskID: fmt.Sprintf("task-d-%d", i),
UserID: adminUser.ID,
WorkgroupID: "wg-1",
VendorID: "vendor-1",
EventType: "rework_submitted",
Mode: domain.RunModeDebug,
Temporary: true,
DurationSeconds: 999999,
OutputDelta: 9999,
CreatedAt: now.Format(time.RFC3339Nano),
})
}
for i := 0; i < 100; i++ {
_ = dataStore.UpsertRun(ctx, domain.RunRecord{
ID: fmt.Sprintf("dbg-run-%d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-http",
Status: domain.RunStatusFailed,
Mode: domain.RunModeDebug,
DurationMS: 9999999,
CreatedAt: now.Format(time.RFC3339Nano),
})
_ = dataStore.UpsertDeliveryPackage(ctx, domain.DeliveryPackageRecord{
ID: fmt.Sprintf("dbg-pkg-%d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-http",
Status: domain.DeliveryPackagePendingAcceptance,
Mode: domain.RunModeDebug,
Temporary: true,
CreatedAt: now.Format(time.RFC3339Nano),
})
}
// Perform Stressed HTTP Request
reqStressed := httptest.NewRequest(http.MethodGet, "/api/dashboard", nil)
reqStressed = reqStressed.WithContext(context.WithValue(reqStressed.Context(), authContextKey{}, adminPrincipal))
recStressed := httptest.NewRecorder()
handler.ServeHTTP(recStressed, reqStressed)
if recStressed.Code != http.StatusOK {
t.Fatalf("expected 200 OK on stressed dashboard, got %d body=%s", recStressed.Code, recStressed.Body.String())
}
var stressedDashboard domain.DashboardSummary
if err := json.Unmarshal(recStressed.Body.Bytes(), &stressedDashboard); err != nil {
t.Fatalf("failed to decode stressed dashboard: %v", err)
}
// 3. Verify complete immunity
if stressedDashboard.Workforce.Totals.SubmittedCount != baselineDashboard.Workforce.Totals.SubmittedCount {
t.Fatalf("Dashboard Workforce Totals.SubmittedCount leaked! baseline=%d, stressed=%d",
baselineDashboard.Workforce.Totals.SubmittedCount, stressedDashboard.Workforce.Totals.SubmittedCount)
}
if stressedDashboard.Workforce.Totals.ReworkCount != baselineDashboard.Workforce.Totals.ReworkCount {
t.Fatalf("Dashboard Workforce Totals.ReworkCount leaked! baseline=%d, stressed=%d",
baselineDashboard.Workforce.Totals.ReworkCount, stressedDashboard.Workforce.Totals.ReworkCount)
}
if stressedDashboard.Workforce.Totals.DurationSeconds != baselineDashboard.Workforce.Totals.DurationSeconds {
t.Fatalf("Dashboard Workforce Totals.DurationSeconds leaked! baseline=%d, stressed=%d",
baselineDashboard.Workforce.Totals.DurationSeconds, stressedDashboard.Workforce.Totals.DurationSeconds)
}
if stressedDashboard.Data.PackageCount != baselineDashboard.Data.PackageCount {
t.Fatalf("Dashboard Data.PackageCount leaked! baseline=%d, stressed=%d",
baselineDashboard.Data.PackageCount, stressedDashboard.Data.PackageCount)
}
if stressedDashboard.Tasks.Failed != baselineDashboard.Tasks.Failed {
t.Fatalf("Dashboard Tasks.Failed leaked! baseline=%d, stressed=%d",
baselineDashboard.Tasks.Failed, stressedDashboard.Tasks.Failed)
}
if stressedDashboard.Jobs.SuccessRate != baselineDashboard.Jobs.SuccessRate {
t.Fatalf("Dashboard Jobs.SuccessRate leaked! baseline=%v, stressed=%v",
baselineDashboard.Jobs.SuccessRate, stressedDashboard.Jobs.SuccessRate)
}
}
func TestChallengerM3_WorkloadSummaryEndpointIsolation(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
now := time.Now().UTC()
adminUser := domain.UserRecord{
ID: "admin-wl",
OrganizationID: domain.DefaultOrganizationID,
Role: domain.UserRoleAdmin,
Name: "Admin WL",
}
_ = dataStore.UpsertUser(ctx, adminUser)
adminPrincipal := &authenticatedPrincipal{
User: adminUser,
Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive},
}
// Seed 50 production events and 150 debug events
for i := 0; i < 50; i++ {
event := domain.AnnotationWorklogEventRecord{
ID: fmt.Sprintf("prod-wl-%03d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-stress",
TaskID: fmt.Sprintf("task-p-%03d", i),
UserID: "worker-p",
WorkgroupID: "wg-prod",
VendorID: "vendor-prod",
EventType: "annotation_submitted",
Mode: domain.RunModeProduction,
DurationSeconds: 100,
OutputDelta: 2,
CreatedAt: now.Format(time.RFC3339Nano),
}
_ = dataStore.UpsertAnnotationWorklogEvent(ctx, event)
}
for i := 0; i < 150; i++ {
event := domain.AnnotationWorklogEventRecord{
ID: fmt.Sprintf("debug-wl-%03d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-stress",
TaskID: fmt.Sprintf("task-d-%03d", i),
UserID: "worker-d",
WorkgroupID: "wg-debug",
VendorID: "vendor-debug",
EventType: "rework_submitted",
Mode: domain.RunModeDebug,
Temporary: true,
DurationSeconds: 9999,
OutputDelta: 500,
CreatedAt: now.Format(time.RFC3339Nano),
}
_ = dataStore.UpsertAnnotationWorklogEvent(ctx, event)
}
// 1. Query with default mode (must only return 50 prod events)
reqDefault := httptest.NewRequest(http.MethodGet, "/api/annotation-workload-summary?projectId=proj-stress", nil)
reqDefault = reqDefault.WithContext(context.WithValue(reqDefault.Context(), authContextKey{}, adminPrincipal))
recDefault := httptest.NewRecorder()
handler.ServeHTTP(recDefault, reqDefault)
if recDefault.Code != http.StatusOK {
t.Fatalf("expected 200 OK, got %d", recDefault.Code)
}
var resDefault domain.AnnotationWorkloadSummary
if err := json.Unmarshal(recDefault.Body.Bytes(), &resDefault); err != nil {
t.Fatalf("failed to decode response: %v", err)
}
if resDefault.EventCount != 50 {
t.Fatalf("expected default EventCount == 50, got %d", resDefault.EventCount)
}
if len(resDefault.Items) != 1 || resDefault.Items[0].SubmittedCount != 50 || resDefault.Items[0].ReworkCount != 0 {
t.Fatalf("expected 50 submitted and 0 rework in prod aggregate, got %+v", resDefault.Items)
}
// 2. Query with mode=production
reqProd := httptest.NewRequest(http.MethodGet, "/api/annotation-workload-summary?projectId=proj-stress&mode=production", nil)
reqProd = reqProd.WithContext(context.WithValue(reqProd.Context(), authContextKey{}, adminPrincipal))
recProd := httptest.NewRecorder()
handler.ServeHTTP(recProd, reqProd)
if recProd.Code != http.StatusOK {
t.Fatalf("expected 200 OK, got %d", recProd.Code)
}
var resProd domain.AnnotationWorkloadSummary
_ = json.Unmarshal(recProd.Body.Bytes(), &resProd)
if resProd.EventCount != 50 {
t.Fatalf("expected mode=production EventCount == 50, got %d", resProd.EventCount)
}
// 3. Query with mode=debug
reqDebug := httptest.NewRequest(http.MethodGet, "/api/annotation-workload-summary?projectId=proj-stress&mode=debug", nil)
reqDebug = reqDebug.WithContext(context.WithValue(reqDebug.Context(), authContextKey{}, adminPrincipal))
recDebug := httptest.NewRecorder()
handler.ServeHTTP(recDebug, reqDebug)
if recDebug.Code != http.StatusOK {
t.Fatalf("expected 200 OK, got %d", recDebug.Code)
}
var resDebug domain.AnnotationWorkloadSummary
_ = json.Unmarshal(recDebug.Body.Bytes(), &resDebug)
if resDebug.EventCount != 150 {
t.Fatalf("expected mode=debug EventCount == 150, got %d", resDebug.EventCount)
}
if len(resDebug.Items) != 1 || resDebug.Items[0].ReworkCount != 150 || resDebug.Items[0].SubmittedCount != 0 {
t.Fatalf("expected 150 rework in debug aggregate, got %+v", resDebug.Items)
}
// 4. Query with mode=all
reqAll := httptest.NewRequest(http.MethodGet, "/api/annotation-workload-summary?projectId=proj-stress&mode=all", nil)
reqAll = reqAll.WithContext(context.WithValue(reqAll.Context(), authContextKey{}, adminPrincipal))
recAll := httptest.NewRecorder()
handler.ServeHTTP(recAll, reqAll)
if recAll.Code != http.StatusOK {
t.Fatalf("expected 200 OK, got %d", recAll.Code)
}
var resAll domain.AnnotationWorkloadSummary
_ = json.Unmarshal(recAll.Body.Bytes(), &resAll)
if resAll.EventCount != 200 {
t.Fatalf("expected mode=all EventCount == 200, got %d", resAll.EventCount)
}
if len(resAll.Items) != 2 {
t.Fatalf("expected 2 aggregate items in mode=all, got %d", len(resAll.Items))
}
}
func TestChallengerM3_DeliveryPackageEndpointIsolation(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
now := time.Now().UTC().Format(time.RFC3339)
adminUser := domain.UserRecord{
ID: "admin-pkg",
OrganizationID: domain.DefaultOrganizationID,
Role: domain.UserRoleAdmin,
Name: "Admin PKG",
}
_ = dataStore.UpsertUser(ctx, adminUser)
adminPrincipal := &authenticatedPrincipal{
User: adminUser,
Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive},
}
// Seed 40 production packages and 60 debug packages
for i := 0; i < 40; i++ {
pkg := domain.DeliveryPackageRecord{
ID: fmt.Sprintf("pkg-p-%03d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-pkg",
Status: domain.DeliveryPackageBuilding,
Mode: domain.RunModeProduction,
Temporary: false,
CreatedAt: now,
}
_ = dataStore.UpsertDeliveryPackage(ctx, pkg)
}
for i := 0; i < 60; i++ {
pkg := domain.DeliveryPackageRecord{
ID: fmt.Sprintf("pkg-d-%03d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-pkg",
Status: domain.DeliveryPackageBuilding,
Mode: domain.RunModeDebug,
Temporary: true,
CreatedAt: now,
}
_ = dataStore.UpsertDeliveryPackage(ctx, pkg)
}
type listResp struct {
Items []domain.DeliveryPackageRecord `json:"items"`
}
// 1. List with mode=production
reqProd := httptest.NewRequest(http.MethodGet, "/api/delivery-packages?projectId=proj-pkg&mode=production", nil)
reqProd = reqProd.WithContext(context.WithValue(reqProd.Context(), authContextKey{}, adminPrincipal))
recProd := httptest.NewRecorder()
handler.ServeHTTP(recProd, reqProd)
if recProd.Code != http.StatusOK {
t.Fatalf("expected 200 OK, got %d", recProd.Code)
}
var resProd listResp
if err := json.Unmarshal(recProd.Body.Bytes(), &resProd); err != nil {
t.Fatalf("failed to decode packages: %v", err)
}
if len(resProd.Items) != 40 {
t.Fatalf("expected 40 packages for mode=production, got %d", len(resProd.Items))
}
for _, p := range resProd.Items {
if p.Mode == domain.RunModeDebug || p.Temporary {
t.Fatalf("leak in production delivery packages listing: %+v", p)
}
}
// 2. List with mode=debug
reqDebug := httptest.NewRequest(http.MethodGet, "/api/delivery-packages?projectId=proj-pkg&mode=debug", nil)
reqDebug = reqDebug.WithContext(context.WithValue(reqDebug.Context(), authContextKey{}, adminPrincipal))
recDebug := httptest.NewRecorder()
handler.ServeHTTP(recDebug, reqDebug)
if recDebug.Code != http.StatusOK {
t.Fatalf("expected 200 OK, got %d", recDebug.Code)
}
var resDebug listResp
_ = json.Unmarshal(recDebug.Body.Bytes(), &resDebug)
if len(resDebug.Items) != 60 {
t.Fatalf("expected 60 debug packages, got %d", len(resDebug.Items))
}
for _, p := range resDebug.Items {
if p.Mode != domain.RunModeDebug || !p.Temporary {
t.Fatalf("expected debug package with temporary=true, got %+v", p)
}
}
// 3. List with mode=all
reqAll := httptest.NewRequest(http.MethodGet, "/api/delivery-packages?projectId=proj-pkg&mode=all", nil)
reqAll = reqAll.WithContext(context.WithValue(reqAll.Context(), authContextKey{}, adminPrincipal))
recAll := httptest.NewRecorder()
handler.ServeHTTP(recAll, reqAll)
if recAll.Code != http.StatusOK {
t.Fatalf("expected 200 OK, got %d", recAll.Code)
}
var resAll listResp
_ = json.Unmarshal(recAll.Body.Bytes(), &resAll)
if len(resAll.Items) != 100 {
t.Fatalf("expected 100 total packages for mode=all, got %d", len(resAll.Items))
}
}
func TestChallengerM3_DeliveryPackageDecisionWorklogInheritance(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
handler := NewHandler(dataStore, nil)
now := time.Now().UTC().Format(time.RFC3339)
adminUser := domain.UserRecord{
ID: "admin-decision",
OrganizationID: domain.DefaultOrganizationID,
Role: domain.UserRoleAdmin,
Name: "Admin Decision",
}
_ = dataStore.UpsertUser(ctx, adminUser)
adminPrincipal := &authenticatedPrincipal{
User: adminUser,
Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive},
}
// 1. Create a debug human item and delivery package
debugItem := domain.HumanItemRecord{
ID: "human-item-dbg",
OrganizationID: domain.DefaultOrganizationID,
BatchID: "batch-dbg",
ProjectID: "proj-1",
Status: domain.HumanItemClaimed,
CurrentStageID: "acceptance",
AssignedUserID: adminUser.ID,
Version: 1,
CreatedAt: now,
UpdatedAt: now,
}
_ = dataStore.UpsertHumanItem(ctx, debugItem)
debugPkg := domain.DeliveryPackageRecord{
ID: "pkg-dbg-decision",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-1",
Status: domain.DeliveryPackagePendingAcceptance,
Mode: domain.RunModeDebug,
Temporary: true,
AssignedUserID: adminUser.ID,
AcceptanceHumanItemID: debugItem.ID,
ReleasePartitionID: "part-dbg",
Version: 1,
CreatedAt: now,
UpdatedAt: now,
}
_ = dataStore.UpsertDeliveryPackage(ctx, debugPkg)
// Make decision on debug package
decisionPayload := `{"decision":"rejected","reasonCodes":["MISSING_ANNOTATION"],"comment":"test reject","version":1}`
req := httptest.NewRequest(http.MethodPost, "/api/delivery-packages/pkg-dbg-decision/decisions", bytes.NewReader([]byte(decisionPayload)))
req.Header.Set("Content-Type", "application/json")
csrfToken := "test-csrf-token"
req.AddCookie(&http.Cookie{Name: csrfCookieName, Value: csrfToken})
req.Header.Set(csrfHeaderName, csrfToken)
req = req.WithContext(context.WithValue(req.Context(), authContextKey{}, adminPrincipal))
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusCreated {
t.Fatalf("expected 201 Created on delivery package decision, got %d body=%s", rec.Code, rec.Body.String())
}
// Verify the emitted worklog event
events, err := dataStore.ListAnnotationWorklogEvents(ctx, domain.DefaultOrganizationID, "proj-1")
if err != nil {
t.Fatalf("failed to list worklog events: %v", err)
}
var foundDebugDecisionEvent bool
for _, ev := range events {
if ev.TaskID == "pkg-dbg-decision" {
foundDebugDecisionEvent = true
if ev.Mode != domain.RunModeDebug {
t.Fatalf("expected emitted decision event to have Mode=debug, got %s", ev.Mode)
}
if !ev.Temporary {
t.Fatalf("expected emitted decision event to have Temporary=true")
}
if ev.EventType != "acceptance_rejected" {
t.Fatalf("expected EventType=acceptance_rejected, got %s", ev.EventType)
}
}
}
if !foundDebugDecisionEvent {
t.Fatalf("decision event for pkg-dbg-decision was not emitted")
}
// Verify workforce metrics exclude this rejected acceptance event
summary := &domain.DashboardSummary{}
s := &server{store: dataStore}
from := time.Now().Add(-24 * time.Hour)
to := time.Now().Add(24 * time.Hour)
_ = s.populateDashboardWorkforce(ctx, domain.DefaultOrganizationID, adminPrincipal, from, to, summary)
if summary.Workforce.Totals.AcceptanceRejected != 0 {
t.Fatalf("expected Totals.AcceptanceRejected == 0, got %d", summary.Workforce.Totals.AcceptanceRejected)
}
}
func TestChallengerM3_ConcurrentIngestionAndQueryStress(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
s := &server{store: dataStore}
now := time.Now().UTC()
from := now.Add(-24 * time.Hour)
to := now.Add(24 * time.Hour)
adminPrincipal := &authenticatedPrincipal{
User: domain.UserRecord{
ID: "admin-conc",
OrganizationID: domain.DefaultOrganizationID,
Role: domain.UserRoleAdmin,
Name: "Admin Conc",
},
Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive},
}
_ = dataStore.UpsertUser(ctx, adminPrincipal.User)
// Seed 10 baseline prod events
for i := 0; i < 10; i++ {
_ = dataStore.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{
ID: fmt.Sprintf("conc-prod-%d", i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-conc",
TaskID: fmt.Sprintf("task-cp-%d", i),
UserID: "worker-conc",
WorkgroupID: "wg-conc",
VendorID: "vendor-conc",
EventType: "annotation_submitted",
Mode: domain.RunModeProduction,
DurationSeconds: 100,
OutputDelta: 1,
CreatedAt: now.Format(time.RFC3339Nano),
})
}
const goroutines = 20
const operationsPerGoroutine = 50
var wg sync.WaitGroup
wg.Add(goroutines * 2)
// Ingestion goroutines (constantly inserting debug events and runs)
for g := 0; g < goroutines; g++ {
go func(routineID int) {
defer wg.Done()
for i := 0; i < operationsPerGoroutine; i++ {
_ = dataStore.UpsertAnnotationWorklogEvent(ctx, domain.AnnotationWorklogEventRecord{
ID: fmt.Sprintf("conc-dbg-%d-%d", routineID, i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-conc",
TaskID: fmt.Sprintf("task-cd-%d-%d", routineID, i),
UserID: fmt.Sprintf("user-cd-%d", routineID),
WorkgroupID: "wg-conc",
VendorID: "vendor-conc",
EventType: "rework_submitted",
Mode: domain.RunModeDebug,
Temporary: true,
DurationSeconds: 99999,
OutputDelta: 500,
CreatedAt: now.Format(time.RFC3339Nano),
})
_ = dataStore.UpsertRun(ctx, domain.RunRecord{
ID: fmt.Sprintf("conc-run-%d-%d", routineID, i),
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "proj-conc",
Status: domain.RunStatusFailed,
Mode: domain.RunModeDebug,
DurationMS: 999999,
CreatedAt: now.Format(time.RFC3339Nano),
})
}
}(g)
}
// Query goroutines (constantly verifying workforce isolation during ingestion)
for g := 0; g < goroutines; g++ {
go func(routineID int) {
defer wg.Done()
for i := 0; i < operationsPerGoroutine; i++ {
summary := &domain.DashboardSummary{}
if err := s.populateDashboardWorkforce(ctx, domain.DefaultOrganizationID, adminPrincipal, from, to, summary); err != nil {
t.Errorf("routine %d step %d error: %v", routineID, i, err)
return
}
// Totals MUST ALWAYS be 10 submitted count, 0 rework count, 1000 duration
if summary.Workforce.Totals.SubmittedCount != 10 {
t.Errorf("CONCURRENCY ISOLATION VIOLATION: SubmittedCount = %d, expected 10", summary.Workforce.Totals.SubmittedCount)
return
}
if summary.Workforce.Totals.ReworkCount != 0 {
t.Errorf("CONCURRENCY ISOLATION VIOLATION: ReworkCount = %d, expected 0", summary.Workforce.Totals.ReworkCount)
return
}
if summary.Workforce.Totals.DurationSeconds != 1000 {
t.Errorf("CONCURRENCY ISOLATION VIOLATION: DurationSeconds = %d, expected 1000", summary.Workforce.Totals.DurationSeconds)
return
}
}
}(g)
}
wg.Wait()
}