forked from longtao.wu/droboticflow
253 lines
10 KiB
Go
253 lines
10 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/k8s"
|
|
"droboticflow/internal/store"
|
|
)
|
|
|
|
func TestWorkerMetricsEndpoint(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
runtime := &Runtime{store: dataStore}
|
|
runtime.metrics.RecordProcessAvailable(25, nil)
|
|
runtime.metrics.RecordDispatch(3, 1, 2)
|
|
runtime.metrics.RecordNodeExecutionStarted()
|
|
runtime.metrics.RecordNodeExecutionCompleted(42)
|
|
|
|
handler := NewHTTPHandler(dataStore, runtime)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(http.MethodGet, "/worker-api/metrics", nil)
|
|
handler.ServeHTTP(recorder, request)
|
|
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d", recorder.Code)
|
|
}
|
|
body := recorder.Body.String()
|
|
for _, expected := range []string{
|
|
"droboticflow_worker_process_available_total 1",
|
|
`droboticflow_worker_process_available_duration_ms_bucket{le="100"} 1`,
|
|
"droboticflow_worker_process_available_duration_ms_sum 25",
|
|
"droboticflow_worker_process_available_duration_ms_count 1",
|
|
"droboticflow_worker_dispatched_runs_total 2",
|
|
"droboticflow_worker_node_executions_started_total 1",
|
|
"droboticflow_worker_node_executions_completed_total 1",
|
|
`droboticflow_worker_node_execution_duration_ms_bucket{le="100"} 1`,
|
|
"droboticflow_worker_node_execution_duration_ms_sum 42",
|
|
"droboticflow_worker_node_execution_duration_ms_count 1",
|
|
"droboticflow_worker_last_dispatch_queue_depth 3",
|
|
} {
|
|
if !strings.Contains(body, expected) {
|
|
t.Fatalf("expected metrics body to contain %q, got:\n%s", expected, body)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestWorkerReadyReportsBuildAndLiveSchedulerState(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
runtime := &Runtime{store: dataStore, launcher: testLauncher{}, operatorRunnerImage: "registry/runner:abc123"}
|
|
runtime.SetBuildInfo(BuildInfo{BuildCommit: "abc123", BuildTime: "2026-07-10T08:00:00Z", Image: "registry/worker:abc123"})
|
|
runtime.SetRunnerBuildInfo(BuildInfo{BuildCommit: "runner123", BuildTime: "2026-07-10T07:59:00Z", Image: "registry/runner:release"})
|
|
runtime.SetConfiguredPollInterval(2 * time.Second)
|
|
runtime.effectivePollMS.Store(750)
|
|
runtime.metrics.RecordProcessAvailable(42, nil)
|
|
|
|
handler := NewHTTPHandler(dataStore, runtime)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(http.MethodGet, "/worker-api/ready", nil)
|
|
handler.ServeHTTP(recorder, request)
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("expected ready 200, got %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var payload ReadinessSnapshot
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
|
t.Fatalf("decode readiness: %v", err)
|
|
}
|
|
if !payload.Ready || payload.BuildCommit != "abc123" || payload.Image != "registry/worker:abc123" || payload.RunnerImage != "registry/runner:release" {
|
|
t.Fatalf("unexpected readiness payload: %#v", payload)
|
|
}
|
|
if payload.RunnerBuildCommit != "runner123" || payload.RunnerBuildTime != "2026-07-10T07:59:00Z" {
|
|
t.Fatalf("expected explicit runner build metadata, got %#v", payload)
|
|
}
|
|
if payload.ConfiguredPollIntervalMS != 2000 || payload.EffectivePollIntervalMS != 750 || payload.CycleDurationMS != 42 {
|
|
t.Fatalf("expected scheduler details, got %#v", payload)
|
|
}
|
|
}
|
|
|
|
func TestWorkerReadyFailsBeforeFirstSuccessfulCycleWhileHealthStaysLive(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
runtime := &Runtime{store: dataStore, launcher: testLauncher{}}
|
|
handler := NewHTTPHandler(dataStore, runtime)
|
|
|
|
readyRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(readyRecorder, httptest.NewRequest(http.MethodGet, "/worker-api/ready", nil))
|
|
if readyRecorder.Code != http.StatusServiceUnavailable || !strings.Contains(readyRecorder.Body.String(), "has not completed") {
|
|
t.Fatalf("expected not-ready response before first cycle, got %d %s", readyRecorder.Code, readyRecorder.Body.String())
|
|
}
|
|
|
|
healthRecorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(healthRecorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
|
|
if healthRecorder.Code != http.StatusOK {
|
|
t.Fatalf("liveness must remain healthy, got %d %s", healthRecorder.Code, healthRecorder.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestWorkerHealthReportsExplicitRunnerBuildMetadata(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
runtime := &Runtime{store: dataStore, launcher: testLauncher{}, operatorRunnerImage: "registry/runner:tag-is-not-a-commit"}
|
|
runtime.SetRunnerBuildInfo(BuildInfo{
|
|
BuildCommit: "explicit-runner-commit",
|
|
BuildTime: "2026-07-10T07:58:00Z",
|
|
Image: "registry/runner:tag-is-not-a-commit",
|
|
})
|
|
handler := NewHTTPHandler(dataStore, runtime)
|
|
recorder := httptest.NewRecorder()
|
|
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
|
|
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("expected health 200, got %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var payload struct {
|
|
RunnerImage string `json:"runnerImage"`
|
|
RunnerBuildCommit string `json:"runnerBuildCommit"`
|
|
RunnerBuildTime string `json:"runnerBuildTime"`
|
|
}
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
|
t.Fatalf("decode health payload: %v", err)
|
|
}
|
|
if payload.RunnerImage != "registry/runner:tag-is-not-a-commit" || payload.RunnerBuildCommit != "explicit-runner-commit" || payload.RunnerBuildTime != "2026-07-10T07:58:00Z" {
|
|
t.Fatalf("unexpected runner health metadata: %#v", payload)
|
|
}
|
|
}
|
|
|
|
func TestWorkerStatsCanReturnOrganizationScopedCounts(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
for _, run := range []domain.RunRecord{
|
|
{ID: "run-org-a-audit", OrganizationID: "org-a", Status: domain.RunStatusWaitingForAudit},
|
|
{ID: "run-org-a-running", OrganizationID: "org-a", Status: domain.RunStatusRunning},
|
|
{ID: "run-org-b-audit", OrganizationID: "org-b", Status: domain.RunStatusWaitingForAudit},
|
|
{ID: "run-org-b-audit-2", OrganizationID: "org-b", Status: domain.RunStatusWaitingForAudit},
|
|
} {
|
|
if err := dataStore.UpsertRun(context.Background(), run); err != nil {
|
|
t.Fatalf("upsert run: %v", err)
|
|
}
|
|
}
|
|
runtime := &Runtime{store: dataStore}
|
|
runtime.SetRunnerBuildInfo(BuildInfo{BuildCommit: "runner-stats-commit", BuildTime: "2026-07-10T07:57:00Z", Image: "registry/runner:stats"})
|
|
handler := NewHTTPHandler(dataStore, runtime)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(http.MethodGet, "/worker-api/stats?organizationId=org-a", nil)
|
|
handler.ServeHTTP(recorder, request)
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var payload struct {
|
|
Scope string `json:"scope"`
|
|
OrganizationID string `json:"organizationId"`
|
|
Running int `json:"running"`
|
|
WaitingForAudit int `json:"waitingForAudit"`
|
|
Global map[string]int `json:"global"`
|
|
RunnerBuildCommit string `json:"runnerBuildCommit"`
|
|
RunnerBuildTime string `json:"runnerBuildTime"`
|
|
}
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
|
t.Fatalf("decode stats payload: %v", err)
|
|
}
|
|
if payload.Scope != "organization" || payload.OrganizationID != "org-a" {
|
|
t.Fatalf("expected organization scoped stats, got %#v", payload)
|
|
}
|
|
if payload.Running != 1 || payload.WaitingForAudit != 1 {
|
|
t.Fatalf("expected org-a counts running=1 waiting=1, got %#v", payload)
|
|
}
|
|
if payload.Global["waitingForAudit"] != 3 {
|
|
t.Fatalf("expected global waitingForAudit=3, got %#v", payload.Global)
|
|
}
|
|
if payload.RunnerBuildCommit != "runner-stats-commit" || payload.RunnerBuildTime != "2026-07-10T07:57:00Z" {
|
|
t.Fatalf("expected runner metadata in stats, got %#v", payload)
|
|
}
|
|
}
|
|
|
|
func TestWorkerCapacityEndpointReturnsGPUCapacity(t *testing.T) {
|
|
dataStore := store.NewMemoryStore()
|
|
runtime := &Runtime{
|
|
store: dataStore,
|
|
operatorRunnerImage: "registry/runner:release",
|
|
launcher: testLauncher{
|
|
gpuCapacity: func(_ context.Context) (*k8s.GPUCapacitySnapshot, error) {
|
|
return &k8s.GPUCapacitySnapshot{
|
|
ResourceName: "nvidia.com/gpu",
|
|
SelectorSummary: "<none>",
|
|
Nodes: []k8s.GPUCapacityNode{{Name: "cpu-node", Allocatable: 0, SelectorMatched: true}},
|
|
TotalAllocatable: 0,
|
|
SchedulableAllocatable: 0,
|
|
MaxAllocatable: 0,
|
|
SchedulableNodeCount: 0,
|
|
Available: false,
|
|
CheckedAt: "2026-06-26T01:00:00Z",
|
|
}, nil
|
|
},
|
|
},
|
|
}
|
|
|
|
handler := NewHTTPHandler(dataStore, runtime)
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(http.MethodGet, "/worker-api/capacity", nil)
|
|
handler.ServeHTTP(recorder, request)
|
|
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var payload struct {
|
|
GPU k8s.GPUCapacitySnapshot `json:"gpu"`
|
|
RunnerImage string `json:"runnerImage"`
|
|
}
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
|
t.Fatalf("decode capacity payload: %v", err)
|
|
}
|
|
if payload.GPU.Available || len(payload.GPU.Nodes) != 1 || payload.GPU.Nodes[0].Name != "cpu-node" || payload.RunnerImage != "registry/runner:release" {
|
|
t.Fatalf("unexpected gpu capacity payload: %#v", payload.GPU)
|
|
}
|
|
}
|
|
|
|
func TestWorkerReadinessUsesBoundedSuccessfulCycleDuration(t *testing.T) {
|
|
for _, test := range []struct {
|
|
name string
|
|
duration uint64
|
|
age time.Duration
|
|
failedCycle bool
|
|
ready bool
|
|
}{
|
|
{"long healthy cycle", 82000, 70 * time.Second, false, true},
|
|
{"long stalled cycle", 82000, 190 * time.Second, false, false},
|
|
{"short cycle remains strict", 10, 40 * time.Second, false, false},
|
|
{"duration allowance capped", 3600000, 6 * time.Minute, false, false},
|
|
{"failed cycle cannot extend deadline", 10, 40 * time.Second, true, false},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
runtime := &Runtime{store: store.NewMemoryStore(), launcher: testLauncher{}}
|
|
runtime.SetConfiguredPollInterval(2 * time.Second)
|
|
runtime.effectivePollMS.Store(2000)
|
|
runtime.metrics.RecordProcessAvailable(test.duration, nil)
|
|
runtime.metrics.lastSuccessfulCycleUnixNano.Store(time.Now().Add(-test.age).UnixNano())
|
|
if test.failedCycle {
|
|
runtime.metrics.RecordProcessAvailable(3600000, errors.New("failed cycle"))
|
|
}
|
|
result := runtime.Readiness(t.Context())
|
|
if result.Ready != test.ready {
|
|
t.Fatalf("ready=%v reason=%s", result.Ready, result.Reason)
|
|
}
|
|
if result.ReadinessWindowMS > 300000 {
|
|
t.Fatal("unbounded adaptive readiness window")
|
|
}
|
|
})
|
|
}
|
|
}
|