📈 Add full verification and worker metrics

This commit is contained in:
eust-w 2026-06-15 17:42:58 +08:00
parent 452c94e52b
commit 5253de26a1
13 changed files with 306 additions and 32 deletions

View File

@ -62,7 +62,7 @@ Drobodata is a workflow orchestration platform for embodied AI and vision pipeli
- `npm run lint`: type-check frontend and verify Go formatting
- `npm test`: Go unit and integration tests
- `npm run build`: build frontend assets and Go binaries
- `npm run verify`: lint + test + build
- `npm run verify`: lint, Go tests, Python operator-runner tests, frontend node tests, build, dependency audit, and Kubernetes manifest dry-run validation
- `npm run smoke`: local smoke test against the running API and worker
- `npm run reconcile:data -- <api-base-url>`: remove legacy demo data, reset settings, and bootstrap built-in operators through the real API
- `python3 scripts/wjzh_operator_regression.py --preflight-only`: preflight the production-style WJZH real-data operator regression against a live API
@ -96,10 +96,12 @@ Key environment variables:
The frontend never reads the AI key directly. Workflow generation and operator generation are proxied through the Go API so secrets stay on the server side.
The settings page can update K8s access details, the worker defaults, the global `timezone`, `provider`, `baseURL`, `model`, and the server-side AI key. `GET /api/settings` only returns whether secrets exist via `k8s.hasKubeconfig` / `ai.hasAPIKey`; it does not echo plaintext kubeconfig or API keys back to the browser. Timestamps in the UI are rendered in the configured IANA timezone with compact minute-level formatting for cards, lists, and summaries, while logs keep finer granularity when needed. OSS-related bucket and path settings belong on OSS operators in workflows, not in the global system settings page. Scan workflows can now reference reusable BOS storage connections via `scanConfig.connectionId`, so endpoint and AK/SK are stored once in `/api/storage/connections` and resolved by the API/worker at preview and execution time instead of being duplicated into each workflow.
The Kubernetes runtime now assumes a shared workspace PVC. Worker writes per-node `input-artifacts.json` files into the shared workspace, operator-runner Jobs read those files through `INPUT_ARTIFACTS_PATH`, then persist `result.json` and `stage-manifest.json` back into the same PVC. Large artifact lists no longer need to travel through giant env vars or stdout payloads. Node CPU and memory limits are applied to Kubernetes Job container resources; GPU requests use `nvidia.com/gpu` when the node or default resource config sets a positive GPU count. GPU scheduling requires a cluster with GPU nodes and the NVIDIA device plugin/runtime installed. Optional GPU node selector, tolerations, and runtime class settings are only injected into Jobs that request a positive GPU count.
The Kubernetes runtime now assumes a shared workspace PVC. Worker writes per-node `input-artifacts.json` files into the shared workspace, operator-runner Jobs read those files through `INPUT_ARTIFACTS_PATH`, then persist `result.json` and `stage-manifest.json` back into the same PVC. Large artifact lists no longer need to travel through giant env vars or stdout payloads. Node CPU and memory limits are applied to Kubernetes Job container resources; GPU requests use `nvidia.com/gpu` when the node or default resource config sets a positive GPU count. GPU scheduling requires a cluster with GPU nodes and the NVIDIA device plugin/runtime installed. Optional GPU node selector, tolerations, and runtime class settings are only injected into Jobs that request a positive GPU count. Invalid GPU selector or tolerations configuration fails worker startup instead of being ignored.
Kubernetes Job failures include operator logs plus Job conditions, Pod phase, container waiting or termination reasons, and Pod events in the run error path. This makes image pull, scheduling, PVC mount, timeout, and GPU allocation failures visible without separately describing the Pod.
The worker exposes lightweight runtime metrics at `/worker-api/metrics` in Prometheus text format. The endpoint reports run status gauges, dispatch counters, node execution counters, cumulative duration counters, and the latest dispatch queue depth.
Workflow execution concurrency now has two levels:
- system default: `settings.worker.maxConcurrentRuns`

View File

@ -508,6 +508,7 @@ Base path: `/worker-api`
- `GET /worker-api/health`: worker health check
- `GET /worker-api/stats`: current queue/running/audit counts
- `GET /worker-api/metrics`: Prometheus text metrics for queue depth, dispatch counts, node execution counts, and cumulative durations
- `POST /worker-api/process-once`: process one queued run immediately
- `POST /worker-api/runs/:id/retry`: reset a run to `queued` and clear prior execution state

View File

@ -184,9 +184,10 @@ http://<node-ip>:30080
- The worker runs with Kubernetes RBAC that allows creating and watching Jobs in the namespace.
- The operator-runner Job template inherits the private registry pull secret name from `K8S_IMAGE_PULL_SECRET_NAME`.
- Operator-runner Jobs apply CPU and memory requests/limits from system defaults and node overrides. A positive GPU value is emitted as `nvidia.com/gpu` in both requests and limits; the target cluster must have GPU nodes and the NVIDIA device plugin/runtime configured.
- GPU-only scheduling settings are optional: `OPERATOR_GPU_NODE_SELECTOR` accepts comma-separated `key=value` pairs, `OPERATOR_GPU_TOLERATIONS_JSON` accepts a Kubernetes tolerations JSON array, and `OPERATOR_GPU_RUNTIME_CLASS` sets `runtimeClassName`. These settings are only injected into Jobs that request a positive GPU count.
- GPU-only scheduling settings are optional: `OPERATOR_GPU_NODE_SELECTOR` accepts comma-separated `key=value` pairs, `OPERATOR_GPU_TOLERATIONS_JSON` accepts a Kubernetes tolerations JSON array, and `OPERATOR_GPU_RUNTIME_CLASS` sets `runtimeClassName`. These settings are only injected into Jobs that request a positive GPU count. Invalid selector or tolerations values fail worker startup.
- The operator-runner is not a standalone Deployment. It is launched on demand as a Python Job container.
- Failed operator-runner Jobs report Job conditions, Pod phase, container waiting or termination reasons, Pod events, and available logs back through the run error path.
- Worker runtime metrics are available from `GET /worker-api/metrics` in Prometheus text format.
- AI generation runs through the API Deployment. `AI_API_KEY` is mounted through the `droboticflow-ai-secret` Secret and must not be baked into frontend env vars.
- `DROBOTICFLOW_API_TOKEN` is mounted through `droboticflow-auth-secret`. When enabled, build the frontend with `VITE_API_TOKEN` or call APIs with `Authorization: Bearer <token>`. Public LoadBalancer API exposure requires this token.
- If you use DashScope-compatible models, prefer a backend-allowed API key. Do not deploy tool-only Coding Plan keys into the cluster.

View File

@ -60,6 +60,12 @@ Run the full verification loop:
npm run verify
```
This includes TypeScript/Go linting, Go tests, Python operator-runner tests, frontend node tests, production build, `npm audit --audit-level=moderate`, and Kubernetes manifest dry-run validation.
## Runtime Metrics
The worker exposes Prometheus text metrics at `/worker-api/metrics`. The endpoint includes run status gauges, ProcessAvailable counters, dispatch counters, node execution counters, cumulative node execution duration, and the most recent dispatch queue depth.
Start the stack locally:
```bash

View File

@ -734,6 +734,7 @@ func NewHandler(dataStore store.Store, aiService ai.Service) http.Handler {
router.Get("/api/health", instance.handleHealth)
router.Get("/worker-api/health", instance.handleWorkerAPIHealth)
router.Get("/worker-api/stats", instance.handleWorkerAPIStats)
router.Get("/worker-api/metrics", instance.handleWorkerAPIMetrics)
router.Post("/worker-api/process-once", instance.handleWorkerAPIProcessOnce)
router.Post("/worker-api/runs/{id}/retry", instance.handleWorkerAPIRetry)
router.Get("/api/ai/status", instance.handleAIStatus)
@ -920,6 +921,10 @@ func (server *server) handleWorkerAPIStats(writer http.ResponseWriter, request *
proxyWorkerAPI(writer, request, "/worker-api/stats")
}
func (server *server) handleWorkerAPIMetrics(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/metrics")
}
func (server *server) handleWorkerAPIProcessOnce(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/process-once")
}

View File

@ -179,6 +179,38 @@ func TestWorkerAPIProcessOnceIsProxiedThroughAPI(t *testing.T) {
}
}
func TestWorkerAPIMetricsIsProxiedThroughAPI(t *testing.T) {
worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.Method != http.MethodGet {
t.Fatalf("unexpected upstream method: %s", request.Method)
}
if request.URL.Path != "/worker-api/metrics" {
t.Fatalf("unexpected upstream path: %s", request.URL.Path)
}
writer.Header().Set("Content-Type", "text/plain; version=0.0.4")
writer.WriteHeader(http.StatusOK)
_, _ = writer.Write([]byte("droboticflow_worker_runs{status=\"queued\"} 1\n"))
}))
defer worker.Close()
originalResolver := workerAPIBaseURLResolver
workerAPIBaseURLResolver = func() string { return worker.URL }
t.Cleanup(func() {
workerAPIBaseURLResolver = originalResolver
})
handler := NewHandler(store.NewMemoryStore(), nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/metrics", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String())
}
if !strings.Contains(recorder.Body.String(), "droboticflow_worker_runs") {
t.Fatalf("unexpected worker metrics payload: %s", recorder.Body.String())
}
}
func TestAPIHandlerSupportsDashboardCRUDAndRunQueueing(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()

View File

@ -140,42 +140,47 @@ func resourceRequirements(input LaunchInput) corev1.ResourceRequirements {
return requirements
}
func parseNodeSelector(value string) map[string]string {
func parseNodeSelector(value string) (map[string]string, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil
return nil, nil
}
result := map[string]string{}
for _, item := range strings.Split(trimmed, ",") {
parts := strings.SplitN(strings.TrimSpace(item), "=", 2)
if len(parts) != 2 {
trimmedItem := strings.TrimSpace(item)
if trimmedItem == "" {
continue
}
parts := strings.SplitN(trimmedItem, "=", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("invalid gpu node selector %q: expected key=value", trimmedItem)
}
key := strings.TrimSpace(parts[0])
val := strings.TrimSpace(parts[1])
if key != "" && val != "" {
result[key] = val
if key == "" || val == "" {
return nil, fmt.Errorf("invalid gpu node selector %q: key and value are required", trimmedItem)
}
result[key] = val
}
if len(result) == 0 {
return nil
return nil, nil
}
return result
return result, nil
}
func parseTolerations(value string) []corev1.Toleration {
func parseTolerations(value string) ([]corev1.Toleration, error) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return nil
return nil, nil
}
var items []corev1.Toleration
if err := json.Unmarshal([]byte(trimmed), &items); err != nil {
return nil
return nil, fmt.Errorf("invalid gpu tolerations json: %w", err)
}
if len(items) == 0 {
return nil
return nil, nil
}
return items
return items, nil
}
func cloneStringMap(items map[string]string) map[string]string {
@ -322,13 +327,21 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
if !strings.HasPrefix(trimmedMountPath, "/") {
return nil, fmt.Errorf("workspace mount path must be absolute")
}
parsedGPUNodeSelector, err := parseNodeSelector(gpuNodeSelector)
if err != nil {
return nil, err
}
parsedGPUTolerations, err := parseTolerations(gpuTolerationsJSON)
if err != nil {
return nil, err
}
launcher := &Launcher{
namespace: namespace,
imagePullSecretName: imagePullSecretName,
workspacePVCName: trimmedPVCName,
workspaceMountPath: trimmedMountPath,
gpuNodeSelector: parseNodeSelector(gpuNodeSelector),
gpuTolerations: parseTolerations(gpuTolerationsJSON),
gpuNodeSelector: parsedGPUNodeSelector,
gpuTolerations: parsedGPUTolerations,
gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass),
}
config, err := loadRestConfig()

View File

@ -113,6 +113,15 @@ func TestNewLauncherRequiresWorkspacePVCName(t *testing.T) {
}
}
func TestNewLauncherRejectsInvalidGPUSchedulingConfig(t *testing.T) {
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "accelerator", "", ""); err == nil || !strings.Contains(err.Error(), "invalid gpu node selector") {
t.Fatalf("expected invalid gpu node selector error, got %v", err)
}
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "", "not-json", ""); err == nil || !strings.Contains(err.Error(), "invalid gpu tolerations json") {
t.Fatalf("expected invalid gpu tolerations error, got %v", err)
}
}
func TestWaitDeadlineForLaunchUsesInputTimeoutWhenContextHasNoDeadline(t *testing.T) {
now := time.Date(2026, time.April, 12, 0, 0, 0, 0, time.UTC)
deadline := waitDeadlineForLaunch(context.Background(), LaunchInput{TimeoutSeconds: 600}, now)

View File

@ -2,6 +2,7 @@ package worker
import (
"encoding/json"
"fmt"
"net/http"
"droboticflow/internal/domain"
@ -21,6 +22,7 @@ func NewHTTPHandler(dataStore store.Store, runtime *Runtime) http.Handler {
router.Get("/worker-api/health", server.handleHealth)
router.Get("/worker-api/stats", server.handleStats)
router.Get("/worker-api/metrics", server.handleMetrics)
router.Post("/worker-api/process-once", server.handleProcessOnce)
router.Post("/worker-api/runs/{id}/retry", server.handleRetry)
@ -37,20 +39,68 @@ func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http.
renderWorkerJSON(writer, http.StatusInternalServerError, map[string]string{"error": err.Error()})
return
}
response := map[string]int{"queued": 0, "running": 0, "waitingForAudit": 0}
queued := 0
running := 0
waitingForAudit := 0
for _, run := range runs {
switch run.Status {
case domain.RunStatusQueued:
response["queued"]++
queued++
case domain.RunStatusRunning:
response["running"]++
running++
case domain.RunStatusWaitingForAudit:
response["waitingForAudit"]++
waitingForAudit++
}
}
response := map[string]any{
"queued": queued,
"running": running,
"waitingForAudit": waitingForAudit,
"metrics": server.runtime.MetricsSnapshot(),
}
renderWorkerJSON(writer, http.StatusOK, response)
}
func (server *HTTPServer) handleMetrics(writer http.ResponseWriter, request *http.Request) {
runs, err := server.store.ListRuns(request.Context(), "")
if err != nil {
http.Error(writer, err.Error(), http.StatusInternalServerError)
return
}
counts := map[domain.RunStatus]int{}
for _, run := range runs {
counts[run.Status]++
}
metrics := server.runtime.MetricsSnapshot()
writer.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
fmt.Fprintf(writer, "# TYPE droboticflow_worker_runs gauge\n")
for _, status := range []domain.RunStatus{domain.RunStatusQueued, domain.RunStatusRunning, domain.RunStatusWaitingForAudit, domain.RunStatusCompleted, domain.RunStatusFailed, domain.RunStatusCancelled} {
fmt.Fprintf(writer, "droboticflow_worker_runs{status=%q} %d\n", status, counts[status])
}
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_process_available_total %d\n", metrics.ProcessAvailableTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_errors_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_process_available_errors_total %d\n", metrics.ProcessAvailableErrors)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_process_available_duration_ms_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_process_available_duration_ms_total %d\n", metrics.ProcessAvailableDurationMS)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_dispatched_runs_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_dispatched_runs_total %d\n", metrics.DispatchedRunsTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_started_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_executions_started_total %d\n", metrics.NodeExecutionsStartedTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_completed_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_executions_completed_total %d\n", metrics.NodeExecutionsCompletedTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_executions_failed_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_executions_failed_total %d\n", metrics.NodeExecutionsFailedTotal)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_node_execution_duration_ms_total counter\n")
fmt.Fprintf(writer, "droboticflow_worker_node_execution_duration_ms_total %d\n", metrics.NodeExecutionDurationMS)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_queue_depth gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_queue_depth %d\n", metrics.LastDispatchQueueDepth)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_running_count gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_running_count %d\n", metrics.LastDispatchRunningCount)
fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_selected_count gauge\n")
fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_selected_count %d\n", metrics.LastDispatchSelectedCount)
}
func (server *HTTPServer) handleProcessOnce(writer http.ResponseWriter, request *http.Request) {
run, err := server.runtime.ProcessOnce(request.Context())
if err != nil {

View File

@ -0,0 +1,40 @@
package worker
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"droboticflow/internal/store"
)
func TestWorkerMetricsEndpoint(t *testing.T) {
dataStore := store.NewMemoryStore()
runtime := &Runtime{store: dataStore}
runtime.metrics.RecordProcessAvailable(25, nil)
runtime.metrics.RecordDispatch(3, 1, 2)
runtime.metrics.RecordNodeExecutionStarted()
runtime.metrics.RecordNodeExecutionCompleted(42)
handler := NewHTTPHandler(dataStore, runtime)
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, "/worker-api/metrics", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
body := recorder.Body.String()
for _, expected := range []string{
"droboticflow_worker_process_available_total 1",
"droboticflow_worker_dispatched_runs_total 2",
"droboticflow_worker_node_executions_started_total 1",
"droboticflow_worker_node_executions_completed_total 1",
"droboticflow_worker_last_dispatch_queue_depth 3",
} {
if !strings.Contains(body, expected) {
t.Fatalf("expected metrics body to contain %q, got:\n%s", expected, body)
}
}
}

View File

@ -0,0 +1,78 @@
package worker
import "sync/atomic"
type MetricsSnapshot struct {
ProcessAvailableTotal uint64
ProcessAvailableErrors uint64
ProcessAvailableDurationMS uint64
DispatchedRunsTotal uint64
NodeExecutionsStartedTotal uint64
NodeExecutionsCompletedTotal uint64
NodeExecutionsFailedTotal uint64
NodeExecutionDurationMS uint64
LastDispatchQueueDepth uint64
LastDispatchRunningCount uint64
LastDispatchSelectedCount uint64
}
type WorkerMetrics struct {
processAvailableTotal atomic.Uint64
processAvailableErrors atomic.Uint64
processAvailableDurationMS atomic.Uint64
dispatchedRunsTotal atomic.Uint64
nodeExecutionsStartedTotal atomic.Uint64
nodeExecutionsCompletedTotal atomic.Uint64
nodeExecutionsFailedTotal atomic.Uint64
nodeExecutionDurationMS atomic.Uint64
lastDispatchQueueDepth atomic.Uint64
lastDispatchRunningCount atomic.Uint64
lastDispatchSelectedCount atomic.Uint64
}
func (metrics *WorkerMetrics) RecordProcessAvailable(durationMS uint64, err error) {
metrics.processAvailableTotal.Add(1)
metrics.processAvailableDurationMS.Add(durationMS)
if err != nil {
metrics.processAvailableErrors.Add(1)
}
}
func (metrics *WorkerMetrics) RecordDispatch(queueDepth int, runningCount int, selectedCount int) {
metrics.lastDispatchQueueDepth.Store(uint64(max(queueDepth, 0)))
metrics.lastDispatchRunningCount.Store(uint64(max(runningCount, 0)))
metrics.lastDispatchSelectedCount.Store(uint64(max(selectedCount, 0)))
if selectedCount > 0 {
metrics.dispatchedRunsTotal.Add(uint64(selectedCount))
}
}
func (metrics *WorkerMetrics) RecordNodeExecutionStarted() {
metrics.nodeExecutionsStartedTotal.Add(1)
}
func (metrics *WorkerMetrics) RecordNodeExecutionCompleted(durationMS uint64) {
metrics.nodeExecutionsCompletedTotal.Add(1)
metrics.nodeExecutionDurationMS.Add(durationMS)
}
func (metrics *WorkerMetrics) RecordNodeExecutionFailed(durationMS uint64) {
metrics.nodeExecutionsFailedTotal.Add(1)
metrics.nodeExecutionDurationMS.Add(durationMS)
}
func (metrics *WorkerMetrics) Snapshot() MetricsSnapshot {
return MetricsSnapshot{
ProcessAvailableTotal: metrics.processAvailableTotal.Load(),
ProcessAvailableErrors: metrics.processAvailableErrors.Load(),
ProcessAvailableDurationMS: metrics.processAvailableDurationMS.Load(),
DispatchedRunsTotal: metrics.dispatchedRunsTotal.Load(),
NodeExecutionsStartedTotal: metrics.nodeExecutionsStartedTotal.Load(),
NodeExecutionsCompletedTotal: metrics.nodeExecutionsCompletedTotal.Load(),
NodeExecutionsFailedTotal: metrics.nodeExecutionsFailedTotal.Load(),
NodeExecutionDurationMS: metrics.nodeExecutionDurationMS.Load(),
LastDispatchQueueDepth: metrics.lastDispatchQueueDepth.Load(),
LastDispatchRunningCount: metrics.lastDispatchRunningCount.Load(),
LastDispatchSelectedCount: metrics.lastDispatchSelectedCount.Load(),
}
}

View File

@ -27,6 +27,7 @@ type Runtime struct {
operatorRunnerImage string
workspaceRoot string
launcher jobLauncher
metrics WorkerMetrics
}
var activeDispatchStatuses = map[domain.RunStatus]struct{}{
@ -59,6 +60,10 @@ func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName str
}, nil
}
func (runtime *Runtime) MetricsSnapshot() MetricsSnapshot {
return runtime.metrics.Snapshot()
}
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}
@ -235,6 +240,11 @@ func shouldEnqueueScheduledRun(workflow domain.WorkflowRecord, now time.Time) bo
}
func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) []string {
selected, _, _ := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
return selected
}
func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) {
globalLimit := globalConcurrencyLimit(settings)
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
@ -261,7 +271,7 @@ func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.Wor
available := globalLimit - activeTotal
if available <= 0 {
return nil
return nil, len(queued), activeTotal
}
sort.Slice(queued, func(left int, right int) bool {
@ -287,7 +297,7 @@ func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.Wor
activeByWorkflow[run.WorkflowID]++
available--
}
return selected
return selected, len(queued), activeTotal
}
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
@ -343,28 +353,41 @@ func (runtime *Runtime) processClaimedRun(claimed *domain.RunRecord) {
}
func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
startedAt := time.Now()
var processErr error
defer func() {
runtime.metrics.RecordProcessAvailable(uint64(time.Since(startedAt).Milliseconds()), processErr)
}()
if err := runtime.reconcileActiveRuns(ctx); err != nil {
return err
processErr = err
return processErr
}
if err := runtime.enqueueStreamingWorkflows(ctx); err != nil {
return err
processErr = err
return processErr
}
settings, err := runtime.store.GetSettings(ctx)
if err != nil {
return err
processErr = err
return processErr
}
workflows, err := runtime.store.ListAllWorkflows(ctx)
if err != nil {
return err
processErr = err
return processErr
}
runs, err := runtime.store.ListRuns(ctx, "")
if err != nil {
return err
processErr = err
return processErr
}
for _, runID := range selectQueuedRunsForDispatch(runs, workflows, settings) {
selectedRunIDs, queueDepth, runningCount := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
runtime.metrics.RecordDispatch(queueDepth, runningCount, len(selectedRunIDs))
for _, runID := range selectedRunIDs {
claimed, err := runtime.store.ClaimQueuedRun(ctx, runID)
if err != nil {
return err
processErr = err
return processErr
}
if claimed == nil {
continue
@ -1293,6 +1316,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
defer cancelExecution()
results := make(chan nodeExecutionOutcome, nodeConcurrencyLimit)
runningNodes := make(map[string]struct{})
nodeStartedAt := make(map[string]time.Time)
var persistedControl *domain.RunRecord
gracefulStopRequested := false
forceStopRequested := false
@ -1367,6 +1391,8 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
runningNodes[node.ID] = struct{}{}
nodeStartedAt[node.ID] = time.Now()
runtime.metrics.RecordNodeExecutionStarted()
launched = true
runSnapshot := *currentRun
@ -1473,6 +1499,12 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
outcome := <-results
delete(runningNodes, outcome.nodeID)
startedAt := nodeStartedAt[outcome.nodeID]
delete(nodeStartedAt, outcome.nodeID)
nodeDurationMS := uint64(0)
if !startedAt.IsZero() {
nodeDurationMS = uint64(time.Since(startedAt).Milliseconds())
}
persistedControl, err := runtime.loadRunControlState(ctx, currentRun.ID)
if err != nil {
return nil, err
@ -1504,6 +1536,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
}
continue
}
runtime.metrics.RecordNodeExecutionFailed(nodeDurationMS)
cancelExecution()
execution.Status = domain.NodeExecutionStatusFailed
execution.EndedAt = nowISO()
@ -1540,6 +1573,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
return currentRun, nil
}
runtime.metrics.RecordNodeExecutionCompleted(nodeDurationMS)
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, outcome.result.Logs...)

View File

@ -22,7 +22,10 @@
"lint:web": "tsc --noEmit",
"lint:go": "bash -lc 'test -z \"$(gofmt -l cmd internal)\"'",
"test": "go test ./...",
"verify": "npm run lint && npm test && npm run build",
"test:python": "python3 -m pytest backend/operator-runner tests/backend",
"test:frontend": "node --test tests/frontend/*.test.mjs",
"audit:moderate": "npm audit --audit-level=moderate",
"verify": "npm run lint && npm test && npm run test:python && npm run test:frontend && npm run build && npm run audit:moderate && npm run k8s:validate",
"docker:build:api": "docker build -f Dockerfile.api -t droboticflow-api:local .",
"docker:build:worker": "docker build -f Dockerfile.worker -t droboticflow-worker:local .",
"docker:build:operator-runner": "docker build -f Dockerfile.operator-runner -t droboticflow-operator-runner:local .",