253 lines
10 KiB
Go

package worker
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/store"
)
func TestWorkerMetricsEndpoint(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime := &Runtime{store: dataStore}
runtime.metrics.RecordProcessAvailable(25, nil)
runtime.metrics.RecordDispatch(3, 1, 2)
runtime.metrics.RecordNodeExecutionStarted()
runtime.metrics.RecordNodeExecutionCompleted(42)
handler := NewHTTPHandler(dataStore, runtime)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/worker-api/metrics", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
body := recorder.Body.String()
for _, expected := range []string{
"droboticflow_worker_process_available_total 1",
`droboticflow_worker_process_available_duration_ms_bucket{le="100"} 1`,
"droboticflow_worker_process_available_duration_ms_sum 25",
"droboticflow_worker_process_available_duration_ms_count 1",
"droboticflow_worker_dispatched_runs_total 2",
"droboticflow_worker_node_executions_started_total 1",
"droboticflow_worker_node_executions_completed_total 1",
`droboticflow_worker_node_execution_duration_ms_bucket{le="100"} 1`,
"droboticflow_worker_node_execution_duration_ms_sum 42",
"droboticflow_worker_node_execution_duration_ms_count 1",
"droboticflow_worker_last_dispatch_queue_depth 3",
} {
if !strings.Contains(body, expected) {
t.Fatalf("expected metrics body to contain %q, got:\n%s", expected, body)
}
}
}
func TestWorkerReadyReportsBuildAndLiveSchedulerState(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime := &Runtime{store: dataStore, launcher: testLauncher{}, operatorRunnerImage: "registry/runner:abc123"}
runtime.SetBuildInfo(BuildInfo{BuildCommit: "abc123", BuildTime: "2026-07-10T08:00:00Z", Image: "registry/worker:abc123"})
runtime.SetRunnerBuildInfo(BuildInfo{BuildCommit: "runner123", BuildTime: "2026-07-10T07:59:00Z", Image: "registry/runner:release"})
runtime.SetConfiguredPollInterval(2 * time.Second)
runtime.effectivePollMS.Store(750)
runtime.metrics.RecordProcessAvailable(42, nil)
handler := NewHTTPHandler(dataStore, runtime)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/worker-api/ready", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("expected ready 200, got %d body=%s", recorder.Code, recorder.Body.String())
}
var payload ReadinessSnapshot
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode readiness: %v", err)
}
if !payload.Ready || payload.BuildCommit != "abc123" || payload.Image != "registry/worker:abc123" || payload.RunnerImage != "registry/runner:release" {
t.Fatalf("unexpected readiness payload: %#v", payload)
}
if payload.RunnerBuildCommit != "runner123" || payload.RunnerBuildTime != "2026-07-10T07:59:00Z" {
t.Fatalf("expected explicit runner build metadata, got %#v", payload)
}
if payload.ConfiguredPollIntervalMS != 2000 || payload.EffectivePollIntervalMS != 750 || payload.CycleDurationMS != 42 {
t.Fatalf("expected scheduler details, got %#v", payload)
}
}
func TestWorkerReadyFailsBeforeFirstSuccessfulCycleWhileHealthStaysLive(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime := &Runtime{store: dataStore, launcher: testLauncher{}}
handler := NewHTTPHandler(dataStore, runtime)
readyRecorder := httptest.NewRecorder()
handler.ServeHTTP(readyRecorder, httptest.NewRequest(http.MethodGet, "/worker-api/ready", nil))
if readyRecorder.Code != http.StatusServiceUnavailable || !strings.Contains(readyRecorder.Body.String(), "has not completed") {
t.Fatalf("expected not-ready response before first cycle, got %d %s", readyRecorder.Code, readyRecorder.Body.String())
}
healthRecorder := httptest.NewRecorder()
handler.ServeHTTP(healthRecorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
if healthRecorder.Code != http.StatusOK {
t.Fatalf("liveness must remain healthy, got %d %s", healthRecorder.Code, healthRecorder.Body.String())
}
}
func TestWorkerHealthReportsExplicitRunnerBuildMetadata(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime := &Runtime{store: dataStore, launcher: testLauncher{}, operatorRunnerImage: "registry/runner:tag-is-not-a-commit"}
runtime.SetRunnerBuildInfo(BuildInfo{
BuildCommit: "explicit-runner-commit",
BuildTime: "2026-07-10T07:58:00Z",
Image: "registry/runner:tag-is-not-a-commit",
})
handler := NewHTTPHandler(dataStore, runtime)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("expected health 200, got %d body=%s", recorder.Code, recorder.Body.String())
}
var payload struct {
RunnerImage string `json:"runnerImage"`
RunnerBuildCommit string `json:"runnerBuildCommit"`
RunnerBuildTime string `json:"runnerBuildTime"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode health payload: %v", err)
}
if payload.RunnerImage != "registry/runner:tag-is-not-a-commit" || payload.RunnerBuildCommit != "explicit-runner-commit" || payload.RunnerBuildTime != "2026-07-10T07:58:00Z" {
t.Fatalf("unexpected runner health metadata: %#v", payload)
}
}
func TestWorkerStatsCanReturnOrganizationScopedCounts(t *testing.T) {
dataStore := store.NewMemoryStore()
for _, run := range []domain.RunRecord{
{ID: "run-org-a-audit", OrganizationID: "org-a", Status: domain.RunStatusWaitingForAudit},
{ID: "run-org-a-running", OrganizationID: "org-a", Status: domain.RunStatusRunning},
{ID: "run-org-b-audit", OrganizationID: "org-b", Status: domain.RunStatusWaitingForAudit},
{ID: "run-org-b-audit-2", OrganizationID: "org-b", Status: domain.RunStatusWaitingForAudit},
} {
if err := dataStore.UpsertRun(context.Background(), run); err != nil {
t.Fatalf("upsert run: %v", err)
}
}
runtime := &Runtime{store: dataStore}
runtime.SetRunnerBuildInfo(BuildInfo{BuildCommit: "runner-stats-commit", BuildTime: "2026-07-10T07:57:00Z", Image: "registry/runner:stats"})
handler := NewHTTPHandler(dataStore, runtime)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/worker-api/stats?organizationId=org-a", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String())
}
var payload struct {
Scope string `json:"scope"`
OrganizationID string `json:"organizationId"`
Running int `json:"running"`
WaitingForAudit int `json:"waitingForAudit"`
Global map[string]int `json:"global"`
RunnerBuildCommit string `json:"runnerBuildCommit"`
RunnerBuildTime string `json:"runnerBuildTime"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode stats payload: %v", err)
}
if payload.Scope != "organization" || payload.OrganizationID != "org-a" {
t.Fatalf("expected organization scoped stats, got %#v", payload)
}
if payload.Running != 1 || payload.WaitingForAudit != 1 {
t.Fatalf("expected org-a counts running=1 waiting=1, got %#v", payload)
}
if payload.Global["waitingForAudit"] != 3 {
t.Fatalf("expected global waitingForAudit=3, got %#v", payload.Global)
}
if payload.RunnerBuildCommit != "runner-stats-commit" || payload.RunnerBuildTime != "2026-07-10T07:57:00Z" {
t.Fatalf("expected runner metadata in stats, got %#v", payload)
}
}
func TestWorkerCapacityEndpointReturnsGPUCapacity(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime := &Runtime{
store: dataStore,
operatorRunnerImage: "registry/runner:release",
launcher: testLauncher{
gpuCapacity: func(_ context.Context) (*k8s.GPUCapacitySnapshot, error) {
return &k8s.GPUCapacitySnapshot{
ResourceName: "nvidia.com/gpu",
SelectorSummary: "<none>",
Nodes: []k8s.GPUCapacityNode{{Name: "cpu-node", Allocatable: 0, SelectorMatched: true}},
TotalAllocatable: 0,
SchedulableAllocatable: 0,
MaxAllocatable: 0,
SchedulableNodeCount: 0,
Available: false,
CheckedAt: "2026-06-26T01:00:00Z",
}, nil
},
},
}
handler := NewHTTPHandler(dataStore, runtime)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/worker-api/capacity", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String())
}
var payload struct {
GPU k8s.GPUCapacitySnapshot `json:"gpu"`
RunnerImage string `json:"runnerImage"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("decode capacity payload: %v", err)
}
if payload.GPU.Available || len(payload.GPU.Nodes) != 1 || payload.GPU.Nodes[0].Name != "cpu-node" || payload.RunnerImage != "registry/runner:release" {
t.Fatalf("unexpected gpu capacity payload: %#v", payload.GPU)
}
}
func TestWorkerReadinessUsesBoundedSuccessfulCycleDuration(t *testing.T) {
for _, test := range []struct {
name string
duration uint64
age time.Duration
failedCycle bool
ready bool
}{
{"long healthy cycle", 82000, 70 * time.Second, false, true},
{"long stalled cycle", 82000, 190 * time.Second, false, false},
{"short cycle remains strict", 10, 40 * time.Second, false, false},
{"duration allowance capped", 3600000, 6 * time.Minute, false, false},
{"failed cycle cannot extend deadline", 10, 40 * time.Second, true, false},
} {
t.Run(test.name, func(t *testing.T) {
runtime := &Runtime{store: store.NewMemoryStore(), launcher: testLauncher{}}
runtime.SetConfiguredPollInterval(2 * time.Second)
runtime.effectivePollMS.Store(2000)
runtime.metrics.RecordProcessAvailable(test.duration, nil)
runtime.metrics.lastSuccessfulCycleUnixNano.Store(time.Now().Add(-test.age).UnixNano())
if test.failedCycle {
runtime.metrics.RecordProcessAvailable(3600000, errors.New("failed cycle"))
}
result := runtime.Readiness(t.Context())
if result.Ready != test.ready {
t.Fatalf("ready=%v reason=%s", result.Ready, result.Reason)
}
if result.ReadinessWindowMS > 300000 {
t.Fatal("unbounded adaptive readiness window")
}
})
}
}