d-robotics dc81aaf5fe 🐛 Preserve deployed worker retry protections before scheduler rollout
(cherry picked from commit 6f8b6a7f01bfdbb55f501c60e121ede64d52c5a4)
2026-09-29 08:54:35 -07:00

352 lines
15 KiB
Go

package worker
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"github.com/go-chi/chi/v5"
)
type HTTPServer struct {
store store.Store
runtime *Runtime
}
func NewHTTPHandler(dataStore store.Store, runtime *Runtime) http.Handler {
server := &HTTPServer{store: dataStore, runtime: runtime}
router := chi.NewRouter()
router.Get("/worker-api/health", server.handleHealth)
router.Get("/worker-api/ready", server.handleReady)
router.Get("/worker-api/stats", server.handleStats)
router.Get("/worker-api/metrics", server.handleMetrics)
router.Get("/worker-api/capacity", server.handleCapacity)
router.Post("/worker-api/process-once", server.handleProcessOnce)
router.Post("/worker-api/runs/{id}/retry", server.handleRetry)
return router
}
func (server *HTTPServer) handleHealth(writer http.ResponseWriter, _ *http.Request) {
build := BuildInfo{}
runnerBuild := BuildInfo{}
if server.runtime != nil {
build = server.runtime.BuildInfo()
runnerBuild = server.runtime.RunnerBuildInfo()
if runnerBuild.Image == "" {
runnerBuild.Image = server.runtime.operatorRunnerImage
}
}
renderWorkerJSON(writer, http.StatusOK, map[string]any{
"status": "ok",
"service": "droboticflow-worker",
"buildCommit": build.BuildCommit,
"buildTime": build.BuildTime,
"image": build.Image,
"runnerImage": runnerBuild.Image,
"runnerBuildCommit": runnerBuild.BuildCommit,
"runnerBuildTime": runnerBuild.BuildTime,
})
}
func (server *HTTPServer) handleReady(writer http.ResponseWriter, request *http.Request) {
if server.runtime == nil {
renderWorkerJSON(writer, http.StatusServiceUnavailable, ReadinessSnapshot{Status: "not_ready", Service: "droboticflow-worker", Reason: "worker runtime is not initialized"})
return
}
snapshot := server.runtime.Readiness(request.Context())
status := http.StatusOK
if !snapshot.Ready {
status = http.StatusServiceUnavailable
}
renderWorkerJSON(writer, status, snapshot)
}
func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http.Request) {
counts, err := server.store.CountRunsByStatus(request.Context())
if err != nil {
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
organizationID := request.URL.Query().Get("organizationId")
tenantCounts := map[domain.RunStatus]int{}
if organizationID != "" {
runs, listErr := server.store.ListRunsByOrganization(request.Context(), organizationID, "")
if listErr != nil {
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": listErr.Error()})
return
}
for _, run := range runs {
tenantCounts[run.Status]++
}
}
response := map[string]any{
"queued": counts[domain.RunStatusQueued],
"running": counts[domain.RunStatusRunning],
"waitingForAudit": counts[domain.RunStatusWaitingForAudit],
"scope": "global",
"metrics": server.runtime.MetricsSnapshot(),
}
readiness := server.runtime.statusSnapshot()
response["buildCommit"] = readiness.BuildCommit
response["buildTime"] = readiness.BuildTime
response["image"] = readiness.Image
response["runnerImage"] = readiness.RunnerImage
response["runnerBuildCommit"] = readiness.RunnerBuildCommit
response["runnerBuildTime"] = readiness.RunnerBuildTime
response["lastCycleAt"] = readiness.LastCycleAt
response["lastSuccessfulCycleAt"] = readiness.LastSuccessfulCycleAt
response["cycleDurationMs"] = readiness.CycleDurationMS
response["configuredPollIntervalMs"] = readiness.ConfiguredPollIntervalMS
response["effectivePollIntervalMs"] = readiness.EffectivePollIntervalMS
response["stuckRunCount"] = readiness.StuckRunCount
response["orphanRuntimeCount"] = readiness.OrphanRuntimeCount
response["orphanRuntimeCleanedCount"] = readiness.OrphanRuntimeCleanedCount
response["orphanRuntimeCleanupErrorCount"] = readiness.OrphanRuntimeCleanupErrorCount
if organizationID != "" {
response["organizationId"] = organizationID
response["global"] = map[string]int{
"queued": counts[domain.RunStatusQueued],
"running": counts[domain.RunStatusRunning],
"waitingForAudit": counts[domain.RunStatusWaitingForAudit],
}
response["queued"] = tenantCounts[domain.RunStatusQueued]
response["running"] = tenantCounts[domain.RunStatusRunning]
response["waitingForAudit"] = tenantCounts[domain.RunStatusWaitingForAudit]
response["scope"] = "organization"
}
renderWorkerJSON(writer, http.StatusOK, response)
}
func (server *HTTPServer) handleCapacity(writer http.ResponseWriter, request *http.Request) {
if server.runtime == nil {
renderWorkerJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "worker runtime is not configured"})
return
}
gpu, err := server.runtime.GPUCapacity(request.Context())
if err != nil {
renderWorkerJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error()})
return
}
renderWorkerJSON(writer, http.StatusOK, map[string]any{"gpu": gpu, "runnerImage": server.runtime.operatorRunnerImage})
}
func (server *HTTPServer) handleMetrics(writer http.ResponseWriter, request *http.Request) {
counts, err := server.store.CountRunsByStatus(request.Context())
if err != nil {
http.Error(writer, err.Error(), http.StatusInternalServerError)
return
}
metrics := server.runtime.MetricsSnapshot()
writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
fmt.Fprintf(writer, "# TYPE droboticflow_worker_runs gauge\n")
for _, status := range []domain.RunStatus{domain.RunStatusQueued, domain.RunStatusRunning, domain.RunStatusWaitingForAudit, domain.RunStatusCompleted, domain.RunStatusFailed, domain.RunStatusCancelled} {
fmt.Fprintf(writer, "droboticflow_worker_runs{status=%q} %d\n", status, counts[status])
}
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_process_available_total %d\n", metrics.ProcessAvailableTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_errors_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_process_available_errors_total %d\n", metrics.ProcessAvailableErrors)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_duration_ms_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_process_available_duration_ms_total %d\n", metrics.ProcessAvailableDurationMS)
writeDurationHistogram(
writer,
"droboticflow_worker_process_available_duration_ms",
metrics.ProcessAvailableDurationMS,
metrics.ProcessAvailableTotal,
metrics.ProcessAvailableDurationBuckets,
)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_dispatched_runs_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_dispatched_runs_total %d\n", metrics.DispatchedRunsTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_started_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_executions_started_total %d\n", metrics.NodeExecutionsStartedTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_completed_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_executions_completed_total %d\n", metrics.NodeExecutionsCompletedTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_failed_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_executions_failed_total %d\n", metrics.NodeExecutionsFailedTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_execution_duration_ms_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_execution_duration_ms_total %d\n", metrics.NodeExecutionDurationMS)
writeDurationHistogram(
writer,
"droboticflow_worker_node_execution_duration_ms",
metrics.NodeExecutionDurationMS,
metrics.NodeExecutionsCompletedTotal+metrics.NodeExecutionsFailedTotal,
metrics.NodeExecutionDurationBuckets,
)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_queue_depth gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_queue_depth %d\n", metrics.LastDispatchQueueDepth)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_running_count gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_running_count %d\n", metrics.LastDispatchRunningCount)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_selected_count gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_selected_count %d\n", metrics.LastDispatchSelectedCount)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_cycle_duration_ms gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_cycle_duration_ms %d\n", metrics.CycleDurationMS)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_stuck_runs gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_stuck_runs %d\n", metrics.StuckRunCount)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_orphan_runtimes gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_orphan_runtimes %d\n", metrics.OrphanRuntimeCount)
if server.runtime != nil {
readiness := server.runtime.statusSnapshot()
fmt.Fprintf(writer, "# TYPE droboticflow_worker_configured_poll_interval_ms gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_configured_poll_interval_ms %d\n", readiness.ConfiguredPollIntervalMS)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_effective_poll_interval_ms gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_effective_poll_interval_ms %d\n", readiness.EffectivePollIntervalMS)
}
}
func writeDurationHistogram(writer http.ResponseWriter, name string, sum uint64, count uint64, buckets []DurationBucketSnapshot) {
fmt.Fprintf(writer, "# TYPE %s histogram\n", name)
for _, bucket := range buckets {
fmt.Fprintf(writer, "%s_bucket{le=%q} %d\n", name, fmt.Sprintf("%d", bucket.UpperBoundMS), bucket.Count)
}
fmt.Fprintf(writer, "%s_bucket{le=%q} %d\n", name, "+Inf", count)
fmt.Fprintf(writer, "%s_sum %d\n", name, sum)
fmt.Fprintf(writer, "%s_count %d\n", name, count)
}
func (server *HTTPServer) handleProcessOnce(writer http.ResponseWriter, request *http.Request) {
result, err := server.runtime.ProcessOnce(context.WithoutCancel(request.Context()))
if err != nil {
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
renderWorkerJSON(writer, http.StatusOK, result)
}
func (server *HTTPServer) handleRetry(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
if run == nil {
renderWorkerJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
if run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled {
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Only failed or cancelled runs can be retried"})
return
}
// A distinct owner serializes retry requests with each other and the scheduler.
owner := domain.NewID("retry")
ctx, cancel := context.WithTimeout(context.WithoutCancel(request.Context()), 20*time.Second)
defer cancel()
run, acquired, err := server.store.AcquireRunLease(ctx, run.ID, owner, time.Minute)
if err != nil {
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
if !acquired || run == nil {
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Run is currently leased"})
return
}
defer func() {
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer releaseCancel()
_ = server.store.ReleaseRunLease(releaseCtx, run.ID, owner)
}()
if run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled {
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Run is no longer retryable"})
return
}
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
if workflow == nil {
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Workflow not found"})
return
}
baseline, err := retryBaseline(*workflow, run)
if err != nil {
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": err.Error()})
return
}
run.RetryHistory = append(run.RetryHistory, domain.RunRetryAttempt{Attempt: run.Attempt, RetriedAt: nowISO(), Status: run.Status, Error: run.Error, NodeExecutions: run.NodeExecutions, K8sRefs: run.K8sRefs})
// Legacy runs and recovered nodes can have a higher attempt than the run.
attempt := runExecutionAttempt(run)
for _, execution := range run.NodeExecutions {
if execution.Attempt > attempt {
attempt = execution.Attempt
}
}
for _, ref := range run.K8sRefs {
if ref.Attempt > attempt {
attempt = ref.Attempt
}
}
run.Attempt = attempt + 1
run.Status = domain.RunStatusQueued
run.StartedAt = ""
run.EndedAt = ""
run.DurationMS = 0
run.CurrentNodeID = ""
run.StopMode = ""
run.StopRequestedAt = ""
run.NodeExecutions = baseline
run.ResourceSeries = nil
run.Artifacts = nil
run.K8sRefs = nil
run.Error = ""
run.Failure = nil
run.Progress = nil
run.AvailableAt = nowISO()
run.UpdatedAt = nowISO()
appendLog(run, logEntry("INFO", "Explicit retry queued; previous attempt retained", ""))
if err := server.store.UpsertRun(ctx, *run); err != nil {
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
renderWorkerJSON(writer, http.StatusAccepted, run)
}
func renderWorkerJSON(writer http.ResponseWriter, status int, payload any) {
writer.Header().Set("Content-Type", "application/json")
writer.WriteHeader(status)
_ = json.NewEncoder(writer).Encode(payload)
}
// Scoped runs must never restart without their frozen upstream executions.
func retryBaseline(workflow domain.WorkflowRecord, run *domain.RunRecord) ([]domain.NodeExecutionRecord, error) {
allowed := rerunAllowedNodeSet(workflow, run)
if allowed == nil {
if run.Trigger == domain.RunTriggerHumanRelease || run.Trigger == domain.RunTriggerRerunSubtree {
return nil, fmt.Errorf("Scoped retry requires a restart node")
}
return nil, nil
}
for id := range allowed {
if domain.GetNodeByID(workflow.Nodes, id) == nil {
return nil, fmt.Errorf("Retry node %s is missing from workflow", id)
}
}
baseline := []domain.NodeExecutionRecord{}
completed := map[string]domain.NodeExecutionRecord{}
for _, execution := range run.NodeExecutions {
if _, reset := allowed[execution.NodeID]; !reset && execution.Status == domain.NodeExecutionStatusCompleted {
baseline = append(baseline, execution)
completed[execution.NodeID] = execution
}
}
for _, edge := range workflow.Edges {
if _, target := allowed[edge.Target]; !target {
continue
}
if _, source := allowed[edge.Source]; source {
continue
}
execution, ok := completed[edge.Source]
if !ok || (run.Trigger == domain.RunTriggerHumanRelease && len(execution.Artifacts) == 0) {
return nil, fmt.Errorf("Retry requires preserved upstream input from %s", edge.Source)
}
}
return baseline, nil
}