forked from longtao.wu/droboticflow
352 lines
15 KiB
Go
352 lines
15 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/store"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
)
|
|
|
|
type HTTPServer struct {
|
|
store store.Store
|
|
runtime *Runtime
|
|
}
|
|
|
|
func NewHTTPHandler(dataStore store.Store, runtime *Runtime) http.Handler {
|
|
server := &HTTPServer{store: dataStore, runtime: runtime}
|
|
router := chi.NewRouter()
|
|
|
|
router.Get("/worker-api/health", server.handleHealth)
|
|
router.Get("/worker-api/ready", server.handleReady)
|
|
router.Get("/worker-api/stats", server.handleStats)
|
|
router.Get("/worker-api/metrics", server.handleMetrics)
|
|
router.Get("/worker-api/capacity", server.handleCapacity)
|
|
router.Post("/worker-api/process-once", server.handleProcessOnce)
|
|
router.Post("/worker-api/runs/{id}/retry", server.handleRetry)
|
|
|
|
return router
|
|
}
|
|
|
|
func (server *HTTPServer) handleHealth(writer http.ResponseWriter, _ *http.Request) {
|
|
build := BuildInfo{}
|
|
runnerBuild := BuildInfo{}
|
|
if server.runtime != nil {
|
|
build = server.runtime.BuildInfo()
|
|
runnerBuild = server.runtime.RunnerBuildInfo()
|
|
if runnerBuild.Image == "" {
|
|
runnerBuild.Image = server.runtime.operatorRunnerImage
|
|
}
|
|
}
|
|
renderWorkerJSON(writer, http.StatusOK, map[string]any{
|
|
"status": "ok",
|
|
"service": "droboticflow-worker",
|
|
"buildCommit": build.BuildCommit,
|
|
"buildTime": build.BuildTime,
|
|
"image": build.Image,
|
|
"runnerImage": runnerBuild.Image,
|
|
"runnerBuildCommit": runnerBuild.BuildCommit,
|
|
"runnerBuildTime": runnerBuild.BuildTime,
|
|
})
|
|
}
|
|
|
|
func (server *HTTPServer) handleReady(writer http.ResponseWriter, request *http.Request) {
|
|
if server.runtime == nil {
|
|
renderWorkerJSON(writer, http.StatusServiceUnavailable, ReadinessSnapshot{Status: "not_ready", Service: "droboticflow-worker", Reason: "worker runtime is not initialized"})
|
|
return
|
|
}
|
|
snapshot := server.runtime.Readiness(request.Context())
|
|
status := http.StatusOK
|
|
if !snapshot.Ready {
|
|
status = http.StatusServiceUnavailable
|
|
}
|
|
renderWorkerJSON(writer, status, snapshot)
|
|
}
|
|
|
|
func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http.Request) {
|
|
counts, err := server.store.CountRunsByStatus(request.Context())
|
|
if err != nil {
|
|
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
organizationID := request.URL.Query().Get("organizationId")
|
|
tenantCounts := map[domain.RunStatus]int{}
|
|
if organizationID != "" {
|
|
runs, listErr := server.store.ListRunsByOrganization(request.Context(), organizationID, "")
|
|
if listErr != nil {
|
|
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": listErr.Error()})
|
|
return
|
|
}
|
|
for _, run := range runs {
|
|
tenantCounts[run.Status]++
|
|
}
|
|
}
|
|
response := map[string]any{
|
|
"queued": counts[domain.RunStatusQueued],
|
|
"running": counts[domain.RunStatusRunning],
|
|
"waitingForAudit": counts[domain.RunStatusWaitingForAudit],
|
|
"scope": "global",
|
|
"metrics": server.runtime.MetricsSnapshot(),
|
|
}
|
|
readiness := server.runtime.statusSnapshot()
|
|
response["buildCommit"] = readiness.BuildCommit
|
|
response["buildTime"] = readiness.BuildTime
|
|
response["image"] = readiness.Image
|
|
response["runnerImage"] = readiness.RunnerImage
|
|
response["runnerBuildCommit"] = readiness.RunnerBuildCommit
|
|
response["runnerBuildTime"] = readiness.RunnerBuildTime
|
|
response["lastCycleAt"] = readiness.LastCycleAt
|
|
response["lastSuccessfulCycleAt"] = readiness.LastSuccessfulCycleAt
|
|
response["cycleDurationMs"] = readiness.CycleDurationMS
|
|
response["configuredPollIntervalMs"] = readiness.ConfiguredPollIntervalMS
|
|
response["effectivePollIntervalMs"] = readiness.EffectivePollIntervalMS
|
|
response["stuckRunCount"] = readiness.StuckRunCount
|
|
response["orphanRuntimeCount"] = readiness.OrphanRuntimeCount
|
|
response["orphanRuntimeCleanedCount"] = readiness.OrphanRuntimeCleanedCount
|
|
response["orphanRuntimeCleanupErrorCount"] = readiness.OrphanRuntimeCleanupErrorCount
|
|
if organizationID != "" {
|
|
response["organizationId"] = organizationID
|
|
response["global"] = map[string]int{
|
|
"queued": counts[domain.RunStatusQueued],
|
|
"running": counts[domain.RunStatusRunning],
|
|
"waitingForAudit": counts[domain.RunStatusWaitingForAudit],
|
|
}
|
|
response["queued"] = tenantCounts[domain.RunStatusQueued]
|
|
response["running"] = tenantCounts[domain.RunStatusRunning]
|
|
response["waitingForAudit"] = tenantCounts[domain.RunStatusWaitingForAudit]
|
|
response["scope"] = "organization"
|
|
}
|
|
renderWorkerJSON(writer, http.StatusOK, response)
|
|
}
|
|
|
|
func (server *HTTPServer) handleCapacity(writer http.ResponseWriter, request *http.Request) {
|
|
if server.runtime == nil {
|
|
renderWorkerJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "worker runtime is not configured"})
|
|
return
|
|
}
|
|
gpu, err := server.runtime.GPUCapacity(request.Context())
|
|
if err != nil {
|
|
renderWorkerJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
renderWorkerJSON(writer, http.StatusOK, map[string]any{"gpu": gpu, "runnerImage": server.runtime.operatorRunnerImage})
|
|
}
|
|
|
|
func (server *HTTPServer) handleMetrics(writer http.ResponseWriter, request *http.Request) {
|
|
counts, err := server.store.CountRunsByStatus(request.Context())
|
|
if err != nil {
|
|
http.Error(writer, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
metrics := server.runtime.MetricsSnapshot()
|
|
writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_runs gauge\n")
|
|
for _, status := range []domain.RunStatus{domain.RunStatusQueued, domain.RunStatusRunning, domain.RunStatusWaitingForAudit, domain.RunStatusCompleted, domain.RunStatusFailed, domain.RunStatusCancelled} {
|
|
fmt.Fprintf(writer, "droboticflow_worker_runs{status=%q} %d\n", status, counts[status])
|
|
}
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_process_available_total %d\n", metrics.ProcessAvailableTotal)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_errors_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_process_available_errors_total %d\n", metrics.ProcessAvailableErrors)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_duration_ms_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_process_available_duration_ms_total %d\n", metrics.ProcessAvailableDurationMS)
|
|
writeDurationHistogram(
|
|
writer,
|
|
"droboticflow_worker_process_available_duration_ms",
|
|
metrics.ProcessAvailableDurationMS,
|
|
metrics.ProcessAvailableTotal,
|
|
metrics.ProcessAvailableDurationBuckets,
|
|
)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_dispatched_runs_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_dispatched_runs_total %d\n", metrics.DispatchedRunsTotal)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_started_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_node_executions_started_total %d\n", metrics.NodeExecutionsStartedTotal)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_completed_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_node_executions_completed_total %d\n", metrics.NodeExecutionsCompletedTotal)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_failed_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_node_executions_failed_total %d\n", metrics.NodeExecutionsFailedTotal)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_execution_duration_ms_total counter\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_node_execution_duration_ms_total %d\n", metrics.NodeExecutionDurationMS)
|
|
writeDurationHistogram(
|
|
writer,
|
|
"droboticflow_worker_node_execution_duration_ms",
|
|
metrics.NodeExecutionDurationMS,
|
|
metrics.NodeExecutionsCompletedTotal+metrics.NodeExecutionsFailedTotal,
|
|
metrics.NodeExecutionDurationBuckets,
|
|
)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_queue_depth gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_queue_depth %d\n", metrics.LastDispatchQueueDepth)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_running_count gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_running_count %d\n", metrics.LastDispatchRunningCount)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_selected_count gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_selected_count %d\n", metrics.LastDispatchSelectedCount)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_cycle_duration_ms gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_cycle_duration_ms %d\n", metrics.CycleDurationMS)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_stuck_runs gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_stuck_runs %d\n", metrics.StuckRunCount)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_orphan_runtimes gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_orphan_runtimes %d\n", metrics.OrphanRuntimeCount)
|
|
if server.runtime != nil {
|
|
readiness := server.runtime.statusSnapshot()
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_configured_poll_interval_ms gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_configured_poll_interval_ms %d\n", readiness.ConfiguredPollIntervalMS)
|
|
fmt.Fprintf(writer, "# TYPE droboticflow_worker_effective_poll_interval_ms gauge\n")
|
|
fmt.Fprintf(writer, "droboticflow_worker_effective_poll_interval_ms %d\n", readiness.EffectivePollIntervalMS)
|
|
}
|
|
}
|
|
|
|
func writeDurationHistogram(writer http.ResponseWriter, name string, sum uint64, count uint64, buckets []DurationBucketSnapshot) {
|
|
fmt.Fprintf(writer, "# TYPE %s histogram\n", name)
|
|
for _, bucket := range buckets {
|
|
fmt.Fprintf(writer, "%s_bucket{le=%q} %d\n", name, fmt.Sprintf("%d", bucket.UpperBoundMS), bucket.Count)
|
|
}
|
|
fmt.Fprintf(writer, "%s_bucket{le=%q} %d\n", name, "+Inf", count)
|
|
fmt.Fprintf(writer, "%s_sum %d\n", name, sum)
|
|
fmt.Fprintf(writer, "%s_count %d\n", name, count)
|
|
}
|
|
|
|
func (server *HTTPServer) handleProcessOnce(writer http.ResponseWriter, request *http.Request) {
|
|
result, err := server.runtime.ProcessOnce(context.WithoutCancel(request.Context()))
|
|
if err != nil {
|
|
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
renderWorkerJSON(writer, http.StatusOK, result)
|
|
}
|
|
|
|
func (server *HTTPServer) handleRetry(writer http.ResponseWriter, request *http.Request) {
|
|
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "id"))
|
|
if err != nil {
|
|
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
if run == nil {
|
|
renderWorkerJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
|
|
return
|
|
}
|
|
if run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled {
|
|
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Only failed or cancelled runs can be retried"})
|
|
return
|
|
}
|
|
// A distinct owner serializes retry requests with each other and the scheduler.
|
|
owner := domain.NewID("retry")
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(request.Context()), 20*time.Second)
|
|
defer cancel()
|
|
run, acquired, err := server.store.AcquireRunLease(ctx, run.ID, owner, time.Minute)
|
|
if err != nil {
|
|
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
if !acquired || run == nil {
|
|
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Run is currently leased"})
|
|
return
|
|
}
|
|
defer func() {
|
|
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer releaseCancel()
|
|
_ = server.store.ReleaseRunLease(releaseCtx, run.ID, owner)
|
|
}()
|
|
if run.Status != domain.RunStatusFailed && run.Status != domain.RunStatusCancelled {
|
|
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Run is no longer retryable"})
|
|
return
|
|
}
|
|
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
|
|
if err != nil {
|
|
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
if workflow == nil {
|
|
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": "Workflow not found"})
|
|
return
|
|
}
|
|
baseline, err := retryBaseline(*workflow, run)
|
|
if err != nil {
|
|
renderWorkerJSON(writer, http.StatusConflict, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
run.RetryHistory = append(run.RetryHistory, domain.RunRetryAttempt{Attempt: run.Attempt, RetriedAt: nowISO(), Status: run.Status, Error: run.Error, NodeExecutions: run.NodeExecutions, K8sRefs: run.K8sRefs})
|
|
// Legacy runs and recovered nodes can have a higher attempt than the run.
|
|
attempt := runExecutionAttempt(run)
|
|
for _, execution := range run.NodeExecutions {
|
|
if execution.Attempt > attempt {
|
|
attempt = execution.Attempt
|
|
}
|
|
}
|
|
for _, ref := range run.K8sRefs {
|
|
if ref.Attempt > attempt {
|
|
attempt = ref.Attempt
|
|
}
|
|
}
|
|
run.Attempt = attempt + 1
|
|
run.Status = domain.RunStatusQueued
|
|
run.StartedAt = ""
|
|
run.EndedAt = ""
|
|
run.DurationMS = 0
|
|
run.CurrentNodeID = ""
|
|
run.StopMode = ""
|
|
run.StopRequestedAt = ""
|
|
run.NodeExecutions = baseline
|
|
run.ResourceSeries = nil
|
|
run.Artifacts = nil
|
|
run.K8sRefs = nil
|
|
run.Error = ""
|
|
run.Failure = nil
|
|
run.Progress = nil
|
|
run.AvailableAt = nowISO()
|
|
run.UpdatedAt = nowISO()
|
|
appendLog(run, logEntry("INFO", "Explicit retry queued; previous attempt retained", ""))
|
|
if err := server.store.UpsertRun(ctx, *run); err != nil {
|
|
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
renderWorkerJSON(writer, http.StatusAccepted, run)
|
|
}
|
|
|
|
func renderWorkerJSON(writer http.ResponseWriter, status int, payload any) {
|
|
writer.Header().Set("Content-Type", "application/json")
|
|
writer.WriteHeader(status)
|
|
_ = json.NewEncoder(writer).Encode(payload)
|
|
}
|
|
|
|
// Scoped runs must never restart without their frozen upstream executions.
|
|
func retryBaseline(workflow domain.WorkflowRecord, run *domain.RunRecord) ([]domain.NodeExecutionRecord, error) {
|
|
allowed := rerunAllowedNodeSet(workflow, run)
|
|
if allowed == nil {
|
|
if run.Trigger == domain.RunTriggerHumanRelease || run.Trigger == domain.RunTriggerRerunSubtree {
|
|
return nil, fmt.Errorf("Scoped retry requires a restart node")
|
|
}
|
|
return nil, nil
|
|
}
|
|
for id := range allowed {
|
|
if domain.GetNodeByID(workflow.Nodes, id) == nil {
|
|
return nil, fmt.Errorf("Retry node %s is missing from workflow", id)
|
|
}
|
|
}
|
|
baseline := []domain.NodeExecutionRecord{}
|
|
completed := map[string]domain.NodeExecutionRecord{}
|
|
for _, execution := range run.NodeExecutions {
|
|
if _, reset := allowed[execution.NodeID]; !reset && execution.Status == domain.NodeExecutionStatusCompleted {
|
|
baseline = append(baseline, execution)
|
|
completed[execution.NodeID] = execution
|
|
}
|
|
}
|
|
for _, edge := range workflow.Edges {
|
|
if _, target := allowed[edge.Target]; !target {
|
|
continue
|
|
}
|
|
if _, source := allowed[edge.Source]; source {
|
|
continue
|
|
}
|
|
execution, ok := completed[edge.Source]
|
|
if !ok || (run.Trigger == domain.RunTriggerHumanRelease && len(execution.Artifacts) == 0) {
|
|
return nil, fmt.Errorf("Retry requires preserved upstream input from %s", edge.Source)
|
|
}
|
|
}
|
|
return baseline, nil
|
|
}
|