diff --git a/Dockerfile.api b/Dockerfile.api index a8e4b3ed..615c057b 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -1,3 +1,6 @@ +ARG DROBOTICFLOW_BUILD_COMMIT=unknown +ARG DROBOTICFLOW_BUILD_TIME=unknown + FROM node:22-alpine AS web-build WORKDIR /app @@ -21,9 +24,15 @@ COPY internal ./internal RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-api ./cmd/api FROM debian:bookworm-slim AS runtime +ARG DROBOTICFLOW_BUILD_COMMIT +ARG DROBOTICFLOW_BUILD_TIME WORKDIR /app ENV PORT=3001 ENV DIST_DIR=/app/dist +ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT} +ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME} +LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT} +LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME} RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow diff --git a/Dockerfile.operator-runner b/Dockerfile.operator-runner index 8bf6a2b3..13635208 100644 --- a/Dockerfile.operator-runner +++ b/Dockerfile.operator-runner @@ -1,7 +1,16 @@ +ARG DROBOTICFLOW_BUILD_COMMIT=unknown +ARG DROBOTICFLOW_BUILD_TIME=unknown + FROM python:3.11-slim AS runtime +ARG DROBOTICFLOW_BUILD_COMMIT +ARG DROBOTICFLOW_BUILD_TIME WORKDIR /app ENV PYTHONUNBUFFERED=1 ENV PYTHONPATH=/app +ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT} +ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME} +LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT} +LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME} RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir numpy opencv-python-headless bce-python-sdk pymongo h5py pyarrow mcap PyYAML zarr nvidia-ml-py diff --git a/Dockerfile.worker b/Dockerfile.worker index a7a01a50..22caccc1 100644 --- a/Dockerfile.worker +++ b/Dockerfile.worker @@ -1,3 +1,6 @@ +ARG DROBOTICFLOW_BUILD_COMMIT=unknown +ARG DROBOTICFLOW_BUILD_TIME=unknown + FROM golang:1.24-bookworm AS build WORKDIR /src ARG GOPROXY=https://proxy.golang.org,direct @@ -10,8 +13,14 @@ COPY internal ./internal RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-worker ./cmd/worker FROM debian:bookworm-slim AS runtime +ARG DROBOTICFLOW_BUILD_COMMIT +ARG DROBOTICFLOW_BUILD_TIME WORKDIR /app ENV WORKER_PORT=3002 +ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT} +ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME} +LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT} +LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME} RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow diff --git a/internal/api/server.go b/internal/api/server.go index f1ecbe32..cd477da7 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -6048,11 +6048,12 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec if run.Status != domain.RunStatusRunning && run.Status != domain.RunStatusQueued && run.Status != domain.RunStatusStopping { return } - nodeID, nodeLabel := "", "" + nodeID, nodeLabel, nodeStartedAt := "", "", "" for _, execution := range run.NodeExecutions { if execution.Status == domain.NodeExecutionStatusRunning { nodeID = execution.NodeID nodeLabel = execution.NodeLabel + nodeStartedAt = execution.StartedAt break } } @@ -6069,7 +6070,7 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec } tailCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - tail, err := launcher.TailNodePodLogs(tailCtx, run.ID, nodeID, 300) + tail, err := launcher.TailNodePodLogsWithTimestamps(tailCtx, run.ID, nodeID, 300) if err != nil || strings.TrimSpace(tail) == "" { return } @@ -6077,18 +6078,36 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec if strings.TrimSpace(label) == "" { label = nodeID } + fallbackTimestamp := firstNonEmptyText(nodeStartedAt, run.StartedAt, run.CreatedAt, "1970-01-01T00:00:00Z") + run.Logs = append(run.Logs, livePodLogEntries(tail, label, fallbackTimestamp)...) +} + +func livePodLogEntries(tail string, label string, fallbackTimestamp string) []domain.LogEntryRecord { + entries := make([]domain.LogEntryRecord, 0) for _, line := range strings.Split(tail, "\n") { trimmed := strings.TrimSpace(line) if trimmed == "" { continue } - run.Logs = append(run.Logs, domain.LogEntryRecord{ - Timestamp: nowISO(), + timestamp := fallbackTimestamp + message := trimmed + if candidate, remainder, found := strings.Cut(trimmed, " "); found { + if _, err := time.Parse(time.RFC3339Nano, candidate); err == nil { + timestamp = candidate + message = strings.TrimSpace(remainder) + } + } + if message == "" { + continue + } + entries = append(entries, domain.LogEntryRecord{ + Timestamp: timestamp, Level: "INFO", - Message: trimmed, + Message: message, Operator: label, }) } + return entries } func (server *server) handleRunsResources(writer http.ResponseWriter, request *http.Request) { diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 7a9676ad..86e4ce42 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -191,6 +191,20 @@ func TestJSONResponsesUseGzipWhenTheClientSupportsIt(t *testing.T) { } } +func TestLivePodLogEntriesKeepKubernetesTimestampsStable(t *testing.T) { + tail := "2026-07-10T10:20:30.123456789Z first line\nsecond line\n" + entries := livePodLogEntries(tail, "Train", "2026-07-10T10:00:00Z") + if len(entries) != 2 { + t.Fatalf("entries = %#v", entries) + } + if entries[0].Timestamp != "2026-07-10T10:20:30.123456789Z" || entries[0].Message != "first line" { + t.Fatalf("timestamped entry = %#v", entries[0]) + } + if entries[1].Timestamp != "2026-07-10T10:00:00Z" || entries[1].Message != "second line" { + t.Fatalf("fallback entry = %#v", entries[1]) + } +} + func TestNewHandlerReportsInvalidConfiguration(t *testing.T) { t.Setenv("PORT", "invalid") diff --git a/internal/config/config.go b/internal/config/config.go index 50fd1fef..a67ae2de 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -145,12 +145,21 @@ func LoadValidated() (AppConfig, error) { CustomOperatorImageImporterImage: getenvTrimmed("CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE", "quay.io/skopeo/stable:v1.16.1"), CustomOperatorImageImportTimeoutSeconds: customOperatorImageImportTimeoutSeconds, CustomOperatorImageArchiveMaxMiB: customOperatorImageArchiveMaxMiB, - BuildCommit: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_COMMIT")), - BuildTime: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_TIME")), + BuildCommit: firstNonEmptyTrimmed(os.Getenv("DROBOTICFLOW_IMAGE_BUILD_COMMIT"), os.Getenv("DROBOTICFLOW_BUILD_COMMIT")), + BuildTime: firstNonEmptyTrimmed(os.Getenv("DROBOTICFLOW_IMAGE_BUILD_TIME"), os.Getenv("DROBOTICFLOW_BUILD_TIME")), Image: strings.TrimSpace(os.Getenv("DROBOTICFLOW_IMAGE")), }, nil } +func firstNonEmptyTrimmed(values ...string) string { + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + } + return "" +} + func parseMongoURIEnv(key string) (string, error) { value := strings.TrimSpace(os.Getenv(key)) if value == "" { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 04111c5f..44c51175 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -38,6 +38,8 @@ var configEnvKeys = []string{ "CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB", "DROBOTICFLOW_BUILD_COMMIT", "DROBOTICFLOW_BUILD_TIME", + "DROBOTICFLOW_IMAGE_BUILD_COMMIT", + "DROBOTICFLOW_IMAGE_BUILD_TIME", "DROBOTICFLOW_IMAGE", } @@ -148,6 +150,8 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) { t.Setenv("CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB", "1024") t.Setenv("DROBOTICFLOW_BUILD_COMMIT", " 3cda38c ") t.Setenv("DROBOTICFLOW_BUILD_TIME", " 2026-07-10T12:00:00Z ") + t.Setenv("DROBOTICFLOW_IMAGE_BUILD_COMMIT", " image-commit ") + t.Setenv("DROBOTICFLOW_IMAGE_BUILD_TIME", " 2026-07-10T11:59:00Z ") t.Setenv("DROBOTICFLOW_IMAGE", " ccr.example/droboticflow-api:3cda38c ") cfg, err := LoadValidated() @@ -193,7 +197,7 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) { if !cfg.RequireSessionAuth || !cfg.SessionCookieSecure { t.Fatalf("expected explicit boolean envs to enable auth and secure cookie") } - if cfg.BuildCommit != "3cda38c" || cfg.BuildTime != "2026-07-10T12:00:00Z" || cfg.Image != "ccr.example/droboticflow-api:3cda38c" { + if cfg.BuildCommit != "image-commit" || cfg.BuildTime != "2026-07-10T11:59:00Z" || cfg.Image != "ccr.example/droboticflow-api:3cda38c" { t.Fatalf("build metadata = commit %q time %q image %q", cfg.BuildCommit, cfg.BuildTime, cfg.Image) } } diff --git a/internal/k8s/launcher.go b/internal/k8s/launcher.go index 79f0d54b..7cf1c125 100644 --- a/internal/k8s/launcher.go +++ b/internal/k8s/launcher.go @@ -1040,6 +1040,16 @@ func (launcher *Launcher) SamplePodGPUForRun(ctx context.Context, runID string, // result.json (which the worker persists only at node completion). Returns "" when // no Running pod exists, so the caller falls back to the stored run logs. func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) { + return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, false) +} + +// TailNodePodLogsWithTimestamps keeps the Kubernetes source timestamp on each +// line so repeated API polls can deduplicate the same live log entry. +func (launcher *Launcher) TailNodePodLogsWithTimestamps(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) { + return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, true) +} + +func (launcher *Launcher) tailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64, timestamps bool) (string, error) { if tailLines <= 0 { tailLines = 300 } @@ -1051,10 +1061,14 @@ func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nod return "", nil } tail := tailLines - stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, &corev1.PodLogOptions{ + options := &corev1.PodLogOptions{ Container: "operator-runner", TailLines: &tail, - }).Stream(ctx) + } + if timestamps { + options.Timestamps = true + } + stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, options).Stream(ctx) if err != nil { return "", err } diff --git a/internal/store/memory.go b/internal/store/memory.go index 55fbd2e9..54eb4c3f 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -569,6 +569,31 @@ func (store *MemoryStore) ListAllWorkflowsByOrganization(_ context.Context, orga return items, nil } +func (store *MemoryStore) ListWorkerWorkflows(_ context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + wanted := make(map[string]struct{}, len(workflowIDs)) + for _, workflowID := range workflowIDs { + if workflowID != "" { + wanted[workflowID] = struct{}{} + } + } + items := make([]domain.WorkflowRecord, 0) + for _, workflow := range store.workflows { + _, explicitlyRequested := wanted[workflow.ID] + if !explicitlyRequested && (workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule)) { + continue + } + items = append(items, clone(workflow)) + } + sortWorkflows(items) + limit = normalizedWorkerQueryLimit(limit, 500, 1000) + if len(items) > limit { + items = items[:limit] + } + return items, nil +} + func (store *MemoryStore) GetWorkflow(_ context.Context, id string) (*domain.WorkflowRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -970,6 +995,80 @@ func (store *MemoryStore) ListRunSummariesPageByOrganization(_ context.Context, return store.listRunSummariesPageLocked(organizationID, query), nil } +func normalizedWorkerQueryLimit(limit int, defaultLimit int, maxLimit int) int { + if limit <= 0 { + return defaultLimit + } + if limit > maxLimit { + return maxLimit + } + return limit +} + +func isReconcileRunSummary(item domain.RunSummary) bool { + return item.Status == domain.RunStatusRunning || + item.Status == domain.RunStatusStopping || + (item.Status == domain.RunStatusQueued && item.CurrentNodeID != "") +} + +func (store *MemoryStore) ListReconcileRunSummaries(_ context.Context, limit int) ([]domain.RunSummary, error) { + store.mu.Lock() + defer store.mu.Unlock() + items := make([]domain.RunSummary, 0) + for _, run := range store.runs { + summary := summarizeRun(run) + if isReconcileRunSummary(summary) { + items = append(items, compactRunSummary(summary)) + } + } + sort.Slice(items, func(left int, right int) bool { + if items[left].UpdatedAt == items[right].UpdatedAt { + return items[left].ID < items[right].ID + } + return items[left].UpdatedAt < items[right].UpdatedAt + }) + limit = normalizedWorkerQueryLimit(limit, 500, 1000) + if len(items) > limit { + items = items[:limit] + } + return items, nil +} + +func newWorkerRunCounts() WorkerRunCounts { + return WorkerRunCounts{ + ActiveByWorkflow: map[string]int{}, + InflightByWorkflow: map[string]int{}, + } +} + +func addWorkerRunCount(counts *WorkerRunCounts, workflowID string, status domain.RunStatus, count int) { + if count <= 0 { + return + } + switch status { + case domain.RunStatusQueued: + counts.QueuedTotal += count + counts.InflightByWorkflow[workflowID] += count + case domain.RunStatusRunning, domain.RunStatusStopping: + counts.ActiveTotal += count + counts.ActiveByWorkflow[workflowID] += count + counts.InflightByWorkflow[workflowID] += count + case domain.RunStatusWaitingForAudit: + counts.ActiveByWorkflow[workflowID] += count + counts.InflightByWorkflow[workflowID] += count + } +} + +func (store *MemoryStore) CountWorkerRuns(_ context.Context) (WorkerRunCounts, error) { + store.mu.Lock() + defer store.mu.Unlock() + counts := newWorkerRunCounts() + for _, run := range store.runs { + addWorkerRunCount(&counts, run.WorkflowID, run.Status, 1) + } + return counts, nil +} + func (store *MemoryStore) hasActiveRunLocked(workflowID string) bool { for _, run := range store.runs { if run.WorkflowID == workflowID && isActiveRunStatus(run.Status) { diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go index 9491cd15..9ec8ac88 100644 --- a/internal/store/memory_test.go +++ b/internal/store/memory_test.go @@ -789,6 +789,66 @@ func TestListRunSummariesPageFiltersCountsAndAdvancesCursor(t *testing.T) { } } +func TestWorkerQueriesReturnOnlyDispatchAndReconcileState(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + scheduled := domain.WorkflowRecord{ + ID: "workflow-scheduled", + Type: domain.WorkflowTypeStreaming, + Schedule: &domain.WorkflowSchedule{Mode: domain.WorkflowScheduleModeIntervalSeconds, Interval: 60}, + CreatedAt: "2026-07-10T01:00:00Z", + UpdatedAt: "2026-07-10T01:00:00Z", + } + requested := domain.WorkflowRecord{ID: "workflow-requested", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"} + unrelated := domain.WorkflowRecord{ID: "workflow-unrelated", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"} + for _, workflow := range []domain.WorkflowRecord{scheduled, requested, unrelated} { + if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil { + t.Fatalf("upsert workflow %s: %v", workflow.ID, err) + } + } + workflows, err := dataStore.ListWorkerWorkflows(ctx, []string{requested.ID}, 20) + if err != nil { + t.Fatalf("list worker workflows: %v", err) + } + workflowIDs := map[string]bool{} + for _, workflow := range workflows { + workflowIDs[workflow.ID] = true + } + if !workflowIDs[scheduled.ID] || !workflowIDs[requested.ID] || workflowIDs[unrelated.ID] { + t.Fatalf("unexpected worker workflows: %#v", workflowIDs) + } + + runs := []domain.RunRecord{ + {ID: "run-queued", WorkflowID: requested.ID, Status: domain.RunStatusQueued, UpdatedAt: "2026-07-10T01:00:00Z"}, + {ID: "run-claimed", WorkflowID: requested.ID, Status: domain.RunStatusQueued, CurrentNodeID: "node-a", UpdatedAt: "2026-07-10T02:00:00Z"}, + {ID: "run-running", WorkflowID: scheduled.ID, Status: domain.RunStatusRunning, UpdatedAt: "2026-07-10T03:00:00Z"}, + {ID: "run-audit", WorkflowID: scheduled.ID, Status: domain.RunStatusWaitingForAudit, UpdatedAt: "2026-07-10T04:00:00Z"}, + {ID: "run-completed", WorkflowID: scheduled.ID, Status: domain.RunStatusCompleted, UpdatedAt: "2026-07-10T05:00:00Z"}, + } + for _, run := range runs { + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run %s: %v", run.ID, err) + } + } + reconcile, err := dataStore.ListReconcileRunSummaries(ctx, 20) + if err != nil { + t.Fatalf("list reconcile runs: %v", err) + } + if len(reconcile) != 2 || reconcile[0].ID != "run-claimed" || reconcile[1].ID != "run-running" { + t.Fatalf("unexpected reconcile runs: %#v", reconcile) + } + counts, err := dataStore.CountWorkerRuns(ctx) + if err != nil { + t.Fatalf("count worker runs: %v", err) + } + if counts.QueuedTotal != 2 || counts.ActiveTotal != 1 { + t.Fatalf("unexpected worker totals: %#v", counts) + } + if counts.ActiveByWorkflow[scheduled.ID] != 2 || counts.InflightByWorkflow[requested.ID] != 2 { + t.Fatalf("unexpected worker workflow counts: %#v", counts) + } +} + func TestUpdateRunProgressAndFailureUseStructuredFields(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() diff --git a/internal/store/mongo.go b/internal/store/mongo.go index a83ddd5d..2302043a 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -409,6 +409,26 @@ func (store *MongoStore) ListAllWorkflowsByOrganization(ctx context.Context, org return decodeMany[domain.WorkflowRecord](ctx, cursor) } +func (store *MongoStore) ListWorkerWorkflows(ctx context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) { + filters := bson.A{bson.M{ + "type": domain.WorkflowTypeStreaming, + "schedule": bson.M{"$exists": true, "$ne": nil}, + }} + if len(workflowIDs) > 0 { + filters = append(filters, bson.M{"id": bson.M{"$in": workflowIDs}}) + } + limit = normalizedWorkerQueryLimit(limit, 500, 1000) + cursor, err := store.workflows.Find( + ctx, + bson.M{"$or": filters}, + options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)), + ) + if err != nil { + return nil, err + } + return decodeMany[domain.WorkflowRecord](ctx, cursor) +} + func (store *MongoStore) GetWorkflow(ctx context.Context, id string) (*domain.WorkflowRecord, error) { var item domain.WorkflowRecord err := store.workflows.FindOne(ctx, bson.M{"id": id}).Decode(&item) @@ -910,6 +930,78 @@ func (store *MongoStore) ListRunSummariesPageByOrganization(ctx context.Context, return store.listRunSummariesPage(ctx, organizationID, query) } +func (store *MongoStore) ListReconcileRunSummaries(ctx context.Context, limit int) ([]domain.RunSummary, error) { + limit = normalizedWorkerQueryLimit(limit, 500, 1000) + filter := bson.M{"$or": bson.A{ + bson.M{"status": bson.M{"$in": bson.A{domain.RunStatusRunning, domain.RunStatusStopping}}}, + bson.M{ + "status": domain.RunStatusQueued, + "currentNodeId": bson.M{"$exists": true, "$nin": bson.A{"", nil}}, + }, + }} + cursor, err := store.runs.Find( + ctx, + filter, + options.Find(). + SetProjection(runSummaryPageProjection()). + SetSort(bson.D{{Key: "updatedAt", Value: 1}, {Key: "id", Value: 1}}). + SetLimit(int64(limit)), + ) + if err != nil { + return nil, err + } + items, err := decodeMany[domain.RunSummary](ctx, cursor) + if err != nil { + return nil, err + } + for index := range items { + items[index] = compactRunSummary(items[index]) + } + return items, nil +} + +func (store *MongoStore) CountWorkerRuns(ctx context.Context) (WorkerRunCounts, error) { + statuses := bson.A{ + domain.RunStatusQueued, + domain.RunStatusRunning, + domain.RunStatusStopping, + domain.RunStatusWaitingForAudit, + } + pipeline := mongo.Pipeline{ + bson.D{{Key: "$match", Value: bson.M{"status": bson.M{"$in": statuses}}}}, + bson.D{{Key: "$group", Value: bson.M{ + "_id": bson.M{ + "workflowId": "$workflowId", + "status": "$status", + }, + "count": bson.M{"$sum": 1}, + }}}, + } + cursor, err := store.runs.Aggregate(ctx, pipeline) + if err != nil { + return WorkerRunCounts{}, err + } + defer cursor.Close(ctx) + counts := newWorkerRunCounts() + for cursor.Next(ctx) { + var item struct { + Key struct { + WorkflowID string `bson:"workflowId"` + Status domain.RunStatus `bson:"status"` + } `bson:"_id"` + Count int `bson:"count"` + } + if err := cursor.Decode(&item); err != nil { + return WorkerRunCounts{}, err + } + addWorkerRunCount(&counts, item.Key.WorkflowID, item.Key.Status, item.Count) + } + if err := cursor.Err(); err != nil { + return WorkerRunCounts{}, err + } + return counts, nil +} + func (store *MongoStore) HasActiveRun(ctx context.Context, workflowID string) (bool, error) { count, err := store.runs.CountDocuments(ctx, bson.M{ "workflowId": workflowID, diff --git a/internal/store/store.go b/internal/store/store.go index a82cbb3e..92b95acd 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -13,6 +13,13 @@ var ( ErrDuplicateUserEmail = errors.New("user email must be unique") ) +type WorkerRunCounts struct { + ActiveByWorkflow map[string]int + InflightByWorkflow map[string]int + ActiveTotal int + QueuedTotal int +} + type Store interface { Ready(context.Context) error EnsureDefaultOrganization(context.Context) (*domain.OrganizationRecord, error) @@ -40,6 +47,7 @@ type Store interface { ListWorkflowsByProject(context.Context, string) ([]domain.WorkflowRecord, error) ListAllWorkflows(context.Context) ([]domain.WorkflowRecord, error) ListAllWorkflowsByOrganization(context.Context, string) ([]domain.WorkflowRecord, error) + ListWorkerWorkflows(context.Context, []string, int) ([]domain.WorkflowRecord, error) GetWorkflow(context.Context, string) (*domain.WorkflowRecord, error) CreateWorkflow(context.Context, domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) DeleteWorkflow(context.Context, string) (bool, error) @@ -68,6 +76,8 @@ type Store interface { ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error) ListRunSummariesPage(context.Context, domain.RunSummaryQuery) (domain.RunSummaryPage, error) ListRunSummariesPageByOrganization(context.Context, string, domain.RunSummaryQuery) (domain.RunSummaryPage, error) + ListReconcileRunSummaries(context.Context, int) ([]domain.RunSummary, error) + CountWorkerRuns(context.Context) (WorkerRunCounts, error) HasActiveRun(context.Context, string) (bool, error) GetRun(context.Context, string) (*domain.RunRecord, error) CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error) diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 1f380d5e..cf570099 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -161,12 +161,11 @@ var workflowActiveDispatchStatuses = map[domain.RunStatus]struct{}{ domain.RunStatusWaitingForAudit: {}, } -var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{ - domain.RunStatusQueued: {}, - domain.RunStatusRunning: {}, - domain.RunStatusStopping: {}, - domain.RunStatusWaitingForAudit: {}, -} +const ( + workerQueuedRunLimit = 100 + workerReconcileRunLimit = 500 + workerWorkflowQueryLimit = 500 +) func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string) (*Runtime, error) { launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds) @@ -702,13 +701,6 @@ func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.Wor } func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) { - globalLimit := globalConcurrencyLimit(settings) - workflowByID := make(map[string]domain.WorkflowRecord, len(workflows)) - for _, workflow := range workflows { - workflowByID[workflow.ID] = workflow - } - now := time.Now().UTC() - activeByWorkflow := map[string]int{} activeTotal := 0 queued := make([]domain.RunRecord, 0) @@ -721,29 +713,49 @@ func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []d continue } if run.Status == domain.RunStatusQueued { - if !runDispatchReady(run, now) { - continue - } queued = append(queued, run) } } + selected := selectQueuedRunsForDispatchSnapshot(queued, workflows, settings, activeByWorkflow, activeTotal) + return selected, len(queued), activeTotal +} + +func selectQueuedRunsForDispatchSnapshot( + queued []domain.RunRecord, + workflows []domain.WorkflowRecord, + settings *domain.SettingsRecord, + activeByWorkflow map[string]int, + activeTotal int, +) []string { + globalLimit := globalConcurrencyLimit(settings) + workflowByID := make(map[string]domain.WorkflowRecord, len(workflows)) + for _, workflow := range workflows { + workflowByID[workflow.ID] = workflow + } + now := time.Now().UTC() + ready := make([]domain.RunRecord, 0, len(queued)) + for _, run := range queued { + if run.Status == domain.RunStatusQueued && runDispatchReady(run, now) { + ready = append(ready, run) + } + } available := globalLimit - activeTotal if available <= 0 { - return nil, len(queued), activeTotal + return nil } - sort.Slice(queued, func(left int, right int) bool { - leftAvailableAt := runAvailableAt(queued[left]) - rightAvailableAt := runAvailableAt(queued[right]) + sort.Slice(ready, func(left int, right int) bool { + leftAvailableAt := runAvailableAt(ready[left]) + rightAvailableAt := runAvailableAt(ready[right]) if leftAvailableAt.Equal(rightAvailableAt) { - return queued[left].CreatedAt < queued[right].CreatedAt + return ready[left].CreatedAt < ready[right].CreatedAt } return leftAvailableAt.Before(rightAvailableAt) }) selected := make([]string, 0, available) - for _, run := range queued { + for _, run := range ready { if available == 0 { break } @@ -756,15 +768,15 @@ func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []d activeByWorkflow[run.WorkflowID]++ available-- } - return selected, len(queued), activeTotal + return selected } func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { - workflows, err := runtime.store.ListAllWorkflows(ctx) + workflows, err := runtime.store.ListWorkerWorkflows(ctx, nil, workerWorkflowQueryLimit) if err != nil { return err } - runs, err := runtime.store.ListRunSummaries(ctx, "") + counts, err := runtime.store.CountWorkerRuns(ctx) if err != nil { return err } @@ -772,10 +784,32 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { if err != nil { return err } - return runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, runs, settings) + return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings) } func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Context, workflows []domain.WorkflowRecord, runs []domain.RunSummary, settings *domain.SettingsRecord) error { + counts := store.WorkerRunCounts{ + ActiveByWorkflow: map[string]int{}, + InflightByWorkflow: map[string]int{}, + } + for _, run := range runs { + switch run.Status { + case domain.RunStatusQueued: + counts.QueuedTotal++ + counts.InflightByWorkflow[run.WorkflowID]++ + case domain.RunStatusRunning, domain.RunStatusStopping: + counts.ActiveTotal++ + counts.ActiveByWorkflow[run.WorkflowID]++ + counts.InflightByWorkflow[run.WorkflowID]++ + case domain.RunStatusWaitingForAudit: + counts.ActiveByWorkflow[run.WorkflowID]++ + counts.InflightByWorkflow[run.WorkflowID]++ + } + } + return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings) +} + +func (runtime *Runtime) enqueueStreamingWorkflowsWithCounts(ctx context.Context, workflows []domain.WorkflowRecord, counts *store.WorkerRunCounts, settings *domain.SettingsRecord) error { now := time.Now().UTC() for _, workflow := range workflows { if !shouldEnqueueScheduledRun(workflow, now) { @@ -791,14 +825,7 @@ func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Conte continue } } - inflightCount := 0 - for _, run := range runs { - if run.WorkflowID == workflow.ID { - if _, ok := queuedOrActiveStatuses[run.Status]; ok { - inflightCount++ - } - } - } + inflightCount := counts.InflightByWorkflow[workflow.ID] if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) { continue } @@ -817,6 +844,8 @@ func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Conte return err } } + counts.QueuedTotal++ + counts.InflightByWorkflow[workflow.ID]++ } return nil } @@ -948,7 +977,7 @@ func (runtime *Runtime) orphanMinimumRuntimeAge() time.Duration { return runtime.orphanMinimumAge } -func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context, summaries []domain.RunSummary) { +func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) { now := time.Now().UTC() interval := runtime.orphanCheckInterval if interval <= 0 { @@ -966,6 +995,11 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context, summaries runtime.metrics.RecordOrphanRuntimes(0) return } + summaries, err := runtime.store.ListRunSummaries(ctx, "") + if err != nil { + fmt.Printf("[worker] list run ids for orphan reconciliation failed: %v\n", err) + return + } knownRunIDs := make(map[string]struct{}, len(summaries)) for _, summary := range summaries { knownRunIDs[summary.ID] = struct{}{} @@ -990,28 +1024,56 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { return processErr } runtime.updateEffectivePollInterval(settings) - workflows, err := runtime.store.ListAllWorkflows(ctx) + reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit) if err != nil { processErr = err return processErr } - summaries, err := runtime.store.ListRunSummaries(ctx, "") + if err := runtime.reconcileActiveRunsWithSummaries(ctx, reconcileRuns); err != nil { + processErr = err + return processErr + } + queuedPage, err := runtime.store.ListRunSummariesPage(ctx, domain.RunSummaryQuery{ + Status: domain.RunStatusQueued, + Limit: workerQueuedRunLimit, + }) if err != nil { processErr = err return processErr } - if err := runtime.reconcileActiveRunsWithSummaries(ctx, summaries); err != nil { + counts, err := runtime.store.CountWorkerRuns(ctx) + if err != nil { processErr = err return processErr } - if err := runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, summaries, settings); err != nil { + workflowIDs := make([]string, 0, len(queuedPage.Items)) + workflowSet := make(map[string]struct{}, len(queuedPage.Items)) + for _, run := range queuedPage.Items { + if _, exists := workflowSet[run.WorkflowID]; exists { + continue + } + workflowSet[run.WorkflowID] = struct{}{} + workflowIDs = append(workflowIDs, run.WorkflowID) + } + workflows, err := runtime.store.ListWorkerWorkflows(ctx, workflowIDs, workerWorkflowQueryLimit) + if err != nil { processErr = err return processErr } - runtime.reconcileOrphanedRuntimes(ctx, summaries) - runs := runRecordsFromSummaries(summaries) - selectedRunIDs, queueDepth, runningCount := selectQueuedRunsForDispatchWithStats(runs, workflows, settings) - runtime.metrics.RecordDispatch(queueDepth, runningCount, len(selectedRunIDs)) + if err := runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings); err != nil { + processErr = err + return processErr + } + runtime.reconcileOrphanedRuntimes(ctx) + queuedRuns := runRecordsFromSummaries(queuedPage.Items) + selectedRunIDs := selectQueuedRunsForDispatchSnapshot( + queuedRuns, + workflows, + settings, + counts.ActiveByWorkflow, + counts.ActiveTotal, + ) + runtime.metrics.RecordDispatch(counts.QueuedTotal, counts.ActiveTotal, len(selectedRunIDs)) for _, runID := range selectedRunIDs { if !runtime.tryMarkRunActive(runID) { continue @@ -1366,7 +1428,7 @@ func (runtime *Runtime) reconcileActiveRunsWithSummaries(ctx context.Context, ru } func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error { - runs, err := runtime.store.ListRunSummaries(ctx, "") + runs, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit) if err != nil { return err } diff --git a/scripts/build-poc-images-kaniko.sh b/scripts/build-poc-images-kaniko.sh index eb33a6d4..1ebb9a7a 100755 --- a/scripts/build-poc-images-kaniko.sh +++ b/scripts/build-poc-images-kaniko.sh @@ -43,8 +43,10 @@ POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}" POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}" if [[ -z "${IMAGE_TAG:-}" ]]; then - IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)" + IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)" fi +POC_KANIKO_SOURCE_COMMIT="${POC_KANIKO_SOURCE_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}" +POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" # shellcheck source=scripts/lib/image-config.sh source "$ROOT_DIR/scripts/lib/image-config.sh" @@ -176,6 +178,8 @@ YAML printf ' - --build-arg=OPERATOR_GPU_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" printf ' - --build-arg=OPERATOR_HOLOBRAIN_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" printf ' - --build-arg=OPERATOR_YOLO_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" + printf ' - --build-arg=DROBOTICFLOW_BUILD_COMMIT=%s\n' "$POC_KANIKO_SOURCE_COMMIT" + printf ' - --build-arg=DROBOTICFLOW_BUILD_TIME=%s\n' "$POC_KANIKO_BUILD_TIME" IFS=',' read -r -a mirrors <<<"$POC_KANIKO_REGISTRY_MIRRORS" for mirror in "${mirrors[@]}"; do mirror="$(printf '%s' "$mirror" | xargs)" diff --git a/src/pages/WorkflowEditor.tsx b/src/pages/WorkflowEditor.tsx index 193c8502..fce0deb8 100644 --- a/src/pages/WorkflowEditor.tsx +++ b/src/pages/WorkflowEditor.tsx @@ -1817,6 +1817,9 @@ export default function WorkflowEditor() { let controller: AbortController | null = null; const refreshLogs = async () => { + if (cancelled || document.visibilityState !== "visible") { + return; + } controller?.abort(); controller = new AbortController(); setIsLiveRunLogsLoading(true); @@ -1837,18 +1840,31 @@ export default function WorkflowEditor() { } finally { if (!cancelled) { setIsLiveRunLogsLoading(false); - if (activeRunInFlight) { + if (activeRunInFlight && document.visibilityState === "visible") { timer = window.setTimeout(refreshLogs, 5000); } } } }; - void refreshLogs(); + const handleVisibilityChange = () => { + window.clearTimeout(timer); + if (document.visibilityState === "hidden") { + controller?.abort(); + return; + } + void refreshLogs(); + }; + + if (document.visibilityState === "visible") { + void refreshLogs(); + } + document.addEventListener("visibilitychange", handleVisibilityChange); return () => { cancelled = true; window.clearTimeout(timer); controller?.abort(); + document.removeEventListener("visibilitychange", handleVisibilityChange); }; }, [activeRun?.id, activeRunInFlight, liveRunLogsRefreshToken, nodeConfigPanelTab, selectedNode?.id]); diff --git a/tests/backend/build_poc_images_kaniko_script_test.py b/tests/backend/build_poc_images_kaniko_script_test.py index 43fa24de..5e426b45 100644 --- a/tests/backend/build_poc_images_kaniko_script_test.py +++ b/tests/backend/build_poc_images_kaniko_script_test.py @@ -111,11 +111,13 @@ def run_script( "POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS", "POC_KANIKO_CONTEXT_SOURCE", "POC_KANIKO_BUILD_OPERATORS", + "POC_KANIKO_BUILD_TIME", "POC_KANIKO_OPERATOR_MANIFEST", "POC_KANIKO_OPERATOR_CONCURRENCY", "POC_KANIKO_OPERATOR_RETRIES", "POC_KANIKO_IMAGE", "POC_KANIKO_NODE_NAME", + "POC_KANIKO_SOURCE_COMMIT", "POC_KANIKO_GOPROXY", "POC_KANIKO_REGISTRY_MIRRORS", "POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK", @@ -161,6 +163,8 @@ def test_poc_kaniko_build_creates_three_jobs_with_registry_mirror(tmp_path): "POC_KANIKO_CLEAN_CONTEXT": "false", "POC_KANIKO_GOPROXY": "https://goproxy.cn,direct", "POC_KANIKO_NODE_NAME": "worker-cpu-01", + "POC_KANIKO_SOURCE_COMMIT": "0123456789abcdef", + "POC_KANIKO_BUILD_TIME": "2026-07-10T10:30:00Z", }, ) @@ -177,6 +181,8 @@ def test_poc_kaniko_build_creates_three_jobs_with_registry_mirror(tmp_path): manifests = "\n---\n".join(path.read_text(encoding="utf-8") for path in sorted(applied_dir.glob("*.yaml"))) assert manifests.count("kind: Job") == 3 assert "--build-arg=GOPROXY=https://goproxy.cn,direct" in manifests + assert "--build-arg=DROBOTICFLOW_BUILD_COMMIT=0123456789abcdef" in manifests + assert "--build-arg=DROBOTICFLOW_BUILD_TIME=2026-07-10T10:30:00Z" in manifests assert "--registry-mirror=mirror.gcr.io" in manifests assert "--skip-default-registry-fallback" in manifests assert "Dockerfile.api" in manifests