⚡ tighten worker scheduling and runtime visibility

This commit is contained in:
eust-w 2026-07-11 00:10:34 +08:00
parent d097d32aa5
commit 66a15d4cf4
16 changed files with 493 additions and 57 deletions

View File

@ -1,3 +1,6 @@
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
ARG DROBOTICFLOW_BUILD_TIME=unknown
FROM node:22-alpine AS web-build
WORKDIR /app
@ -21,9 +24,15 @@ COPY internal ./internal
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-api ./cmd/api
FROM debian:bookworm-slim AS runtime
ARG DROBOTICFLOW_BUILD_COMMIT
ARG DROBOTICFLOW_BUILD_TIME
WORKDIR /app
ENV PORT=3001
ENV DIST_DIR=/app/dist
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow

View File

@ -1,7 +1,16 @@
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
ARG DROBOTICFLOW_BUILD_TIME=unknown
FROM python:3.11-slim AS runtime
ARG DROBOTICFLOW_BUILD_COMMIT
ARG DROBOTICFLOW_BUILD_TIME
WORKDIR /app
ENV PYTHONUNBUFFERED=1
ENV PYTHONPATH=/app
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir numpy opencv-python-headless bce-python-sdk pymongo h5py pyarrow mcap PyYAML zarr nvidia-ml-py

View File

@ -1,3 +1,6 @@
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
ARG DROBOTICFLOW_BUILD_TIME=unknown
FROM golang:1.24-bookworm AS build
WORKDIR /src
ARG GOPROXY=https://proxy.golang.org,direct
@ -10,8 +13,14 @@ COPY internal ./internal
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-worker ./cmd/worker
FROM debian:bookworm-slim AS runtime
ARG DROBOTICFLOW_BUILD_COMMIT
ARG DROBOTICFLOW_BUILD_TIME
WORKDIR /app
ENV WORKER_PORT=3002
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow

View File

@ -6048,11 +6048,12 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec
if run.Status != domain.RunStatusRunning && run.Status != domain.RunStatusQueued && run.Status != domain.RunStatusStopping {
return
}
nodeID, nodeLabel := "", ""
nodeID, nodeLabel, nodeStartedAt := "", "", ""
for _, execution := range run.NodeExecutions {
if execution.Status == domain.NodeExecutionStatusRunning {
nodeID = execution.NodeID
nodeLabel = execution.NodeLabel
nodeStartedAt = execution.StartedAt
break
}
}
@ -6069,7 +6070,7 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec
}
tailCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
tail, err := launcher.TailNodePodLogs(tailCtx, run.ID, nodeID, 300)
tail, err := launcher.TailNodePodLogsWithTimestamps(tailCtx, run.ID, nodeID, 300)
if err != nil || strings.TrimSpace(tail) == "" {
return
}
@ -6077,18 +6078,36 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec
if strings.TrimSpace(label) == "" {
label = nodeID
}
fallbackTimestamp := firstNonEmptyText(nodeStartedAt, run.StartedAt, run.CreatedAt, "1970-01-01T00:00:00Z")
run.Logs = append(run.Logs, livePodLogEntries(tail, label, fallbackTimestamp)...)
}
func livePodLogEntries(tail string, label string, fallbackTimestamp string) []domain.LogEntryRecord {
entries := make([]domain.LogEntryRecord, 0)
for _, line := range strings.Split(tail, "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "" {
continue
}
run.Logs = append(run.Logs, domain.LogEntryRecord{
Timestamp: nowISO(),
timestamp := fallbackTimestamp
message := trimmed
if candidate, remainder, found := strings.Cut(trimmed, " "); found {
if _, err := time.Parse(time.RFC3339Nano, candidate); err == nil {
timestamp = candidate
message = strings.TrimSpace(remainder)
}
}
if message == "" {
continue
}
entries = append(entries, domain.LogEntryRecord{
Timestamp: timestamp,
Level: "INFO",
Message: trimmed,
Message: message,
Operator: label,
})
}
return entries
}
func (server *server) handleRunsResources(writer http.ResponseWriter, request *http.Request) {

View File

@ -191,6 +191,20 @@ func TestJSONResponsesUseGzipWhenTheClientSupportsIt(t *testing.T) {
}
}
func TestLivePodLogEntriesKeepKubernetesTimestampsStable(t *testing.T) {
tail := "2026-07-10T10:20:30.123456789Z first line\nsecond line\n"
entries := livePodLogEntries(tail, "Train", "2026-07-10T10:00:00Z")
if len(entries) != 2 {
t.Fatalf("entries = %#v", entries)
}
if entries[0].Timestamp != "2026-07-10T10:20:30.123456789Z" || entries[0].Message != "first line" {
t.Fatalf("timestamped entry = %#v", entries[0])
}
if entries[1].Timestamp != "2026-07-10T10:00:00Z" || entries[1].Message != "second line" {
t.Fatalf("fallback entry = %#v", entries[1])
}
}
func TestNewHandlerReportsInvalidConfiguration(t *testing.T) {
t.Setenv("PORT", "invalid")

View File

@ -145,12 +145,21 @@ func LoadValidated() (AppConfig, error) {
CustomOperatorImageImporterImage: getenvTrimmed("CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE", "quay.io/skopeo/stable:v1.16.1"),
CustomOperatorImageImportTimeoutSeconds: customOperatorImageImportTimeoutSeconds,
CustomOperatorImageArchiveMaxMiB: customOperatorImageArchiveMaxMiB,
BuildCommit: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_COMMIT")),
BuildTime: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_TIME")),
BuildCommit: firstNonEmptyTrimmed(os.Getenv("DROBOTICFLOW_IMAGE_BUILD_COMMIT"), os.Getenv("DROBOTICFLOW_BUILD_COMMIT")),
BuildTime: firstNonEmptyTrimmed(os.Getenv("DROBOTICFLOW_IMAGE_BUILD_TIME"), os.Getenv("DROBOTICFLOW_BUILD_TIME")),
Image: strings.TrimSpace(os.Getenv("DROBOTICFLOW_IMAGE")),
}, nil
}
func firstNonEmptyTrimmed(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func parseMongoURIEnv(key string) (string, error) {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {

View File

@ -38,6 +38,8 @@ var configEnvKeys = []string{
"CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB",
"DROBOTICFLOW_BUILD_COMMIT",
"DROBOTICFLOW_BUILD_TIME",
"DROBOTICFLOW_IMAGE_BUILD_COMMIT",
"DROBOTICFLOW_IMAGE_BUILD_TIME",
"DROBOTICFLOW_IMAGE",
}
@ -148,6 +150,8 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) {
t.Setenv("CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB", "1024")
t.Setenv("DROBOTICFLOW_BUILD_COMMIT", " 3cda38c ")
t.Setenv("DROBOTICFLOW_BUILD_TIME", " 2026-07-10T12:00:00Z ")
t.Setenv("DROBOTICFLOW_IMAGE_BUILD_COMMIT", " image-commit ")
t.Setenv("DROBOTICFLOW_IMAGE_BUILD_TIME", " 2026-07-10T11:59:00Z ")
t.Setenv("DROBOTICFLOW_IMAGE", " ccr.example/droboticflow-api:3cda38c ")
cfg, err := LoadValidated()
@ -193,7 +197,7 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) {
if !cfg.RequireSessionAuth || !cfg.SessionCookieSecure {
t.Fatalf("expected explicit boolean envs to enable auth and secure cookie")
}
if cfg.BuildCommit != "3cda38c" || cfg.BuildTime != "2026-07-10T12:00:00Z" || cfg.Image != "ccr.example/droboticflow-api:3cda38c" {
if cfg.BuildCommit != "image-commit" || cfg.BuildTime != "2026-07-10T11:59:00Z" || cfg.Image != "ccr.example/droboticflow-api:3cda38c" {
t.Fatalf("build metadata = commit %q time %q image %q", cfg.BuildCommit, cfg.BuildTime, cfg.Image)
}
}

View File

@ -1040,6 +1040,16 @@ func (launcher *Launcher) SamplePodGPUForRun(ctx context.Context, runID string,
// result.json (which the worker persists only at node completion). Returns "" when
// no Running pod exists, so the caller falls back to the stored run logs.
func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, false)
}
// TailNodePodLogsWithTimestamps keeps the Kubernetes source timestamp on each
// line so repeated API polls can deduplicate the same live log entry.
func (launcher *Launcher) TailNodePodLogsWithTimestamps(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, true)
}
func (launcher *Launcher) tailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64, timestamps bool) (string, error) {
if tailLines <= 0 {
tailLines = 300
}
@ -1051,10 +1061,14 @@ func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nod
return "", nil
}
tail := tailLines
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, &corev1.PodLogOptions{
options := &corev1.PodLogOptions{
Container: "operator-runner",
TailLines: &tail,
}).Stream(ctx)
}
if timestamps {
options.Timestamps = true
}
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, options).Stream(ctx)
if err != nil {
return "", err
}

View File

@ -569,6 +569,31 @@ func (store *MemoryStore) ListAllWorkflowsByOrganization(_ context.Context, orga
return items, nil
}
func (store *MemoryStore) ListWorkerWorkflows(_ context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
wanted := make(map[string]struct{}, len(workflowIDs))
for _, workflowID := range workflowIDs {
if workflowID != "" {
wanted[workflowID] = struct{}{}
}
}
items := make([]domain.WorkflowRecord, 0)
for _, workflow := range store.workflows {
_, explicitlyRequested := wanted[workflow.ID]
if !explicitlyRequested && (workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule)) {
continue
}
items = append(items, clone(workflow))
}
sortWorkflows(items)
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
if len(items) > limit {
items = items[:limit]
}
return items, nil
}
func (store *MemoryStore) GetWorkflow(_ context.Context, id string) (*domain.WorkflowRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
@ -970,6 +995,80 @@ func (store *MemoryStore) ListRunSummariesPageByOrganization(_ context.Context,
return store.listRunSummariesPageLocked(organizationID, query), nil
}
func normalizedWorkerQueryLimit(limit int, defaultLimit int, maxLimit int) int {
if limit <= 0 {
return defaultLimit
}
if limit > maxLimit {
return maxLimit
}
return limit
}
func isReconcileRunSummary(item domain.RunSummary) bool {
return item.Status == domain.RunStatusRunning ||
item.Status == domain.RunStatusStopping ||
(item.Status == domain.RunStatusQueued && item.CurrentNodeID != "")
}
func (store *MemoryStore) ListReconcileRunSummaries(_ context.Context, limit int) ([]domain.RunSummary, error) {
store.mu.Lock()
defer store.mu.Unlock()
items := make([]domain.RunSummary, 0)
for _, run := range store.runs {
summary := summarizeRun(run)
if isReconcileRunSummary(summary) {
items = append(items, compactRunSummary(summary))
}
}
sort.Slice(items, func(left int, right int) bool {
if items[left].UpdatedAt == items[right].UpdatedAt {
return items[left].ID < items[right].ID
}
return items[left].UpdatedAt < items[right].UpdatedAt
})
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
if len(items) > limit {
items = items[:limit]
}
return items, nil
}
func newWorkerRunCounts() WorkerRunCounts {
return WorkerRunCounts{
ActiveByWorkflow: map[string]int{},
InflightByWorkflow: map[string]int{},
}
}
func addWorkerRunCount(counts *WorkerRunCounts, workflowID string, status domain.RunStatus, count int) {
if count <= 0 {
return
}
switch status {
case domain.RunStatusQueued:
counts.QueuedTotal += count
counts.InflightByWorkflow[workflowID] += count
case domain.RunStatusRunning, domain.RunStatusStopping:
counts.ActiveTotal += count
counts.ActiveByWorkflow[workflowID] += count
counts.InflightByWorkflow[workflowID] += count
case domain.RunStatusWaitingForAudit:
counts.ActiveByWorkflow[workflowID] += count
counts.InflightByWorkflow[workflowID] += count
}
}
func (store *MemoryStore) CountWorkerRuns(_ context.Context) (WorkerRunCounts, error) {
store.mu.Lock()
defer store.mu.Unlock()
counts := newWorkerRunCounts()
for _, run := range store.runs {
addWorkerRunCount(&counts, run.WorkflowID, run.Status, 1)
}
return counts, nil
}
func (store *MemoryStore) hasActiveRunLocked(workflowID string) bool {
for _, run := range store.runs {
if run.WorkflowID == workflowID && isActiveRunStatus(run.Status) {

View File

@ -789,6 +789,66 @@ func TestListRunSummariesPageFiltersCountsAndAdvancesCursor(t *testing.T) {
}
}
func TestWorkerQueriesReturnOnlyDispatchAndReconcileState(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
scheduled := domain.WorkflowRecord{
ID: "workflow-scheduled",
Type: domain.WorkflowTypeStreaming,
Schedule: &domain.WorkflowSchedule{Mode: domain.WorkflowScheduleModeIntervalSeconds, Interval: 60},
CreatedAt: "2026-07-10T01:00:00Z",
UpdatedAt: "2026-07-10T01:00:00Z",
}
requested := domain.WorkflowRecord{ID: "workflow-requested", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"}
unrelated := domain.WorkflowRecord{ID: "workflow-unrelated", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"}
for _, workflow := range []domain.WorkflowRecord{scheduled, requested, unrelated} {
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
t.Fatalf("upsert workflow %s: %v", workflow.ID, err)
}
}
workflows, err := dataStore.ListWorkerWorkflows(ctx, []string{requested.ID}, 20)
if err != nil {
t.Fatalf("list worker workflows: %v", err)
}
workflowIDs := map[string]bool{}
for _, workflow := range workflows {
workflowIDs[workflow.ID] = true
}
if !workflowIDs[scheduled.ID] || !workflowIDs[requested.ID] || workflowIDs[unrelated.ID] {
t.Fatalf("unexpected worker workflows: %#v", workflowIDs)
}
runs := []domain.RunRecord{
{ID: "run-queued", WorkflowID: requested.ID, Status: domain.RunStatusQueued, UpdatedAt: "2026-07-10T01:00:00Z"},
{ID: "run-claimed", WorkflowID: requested.ID, Status: domain.RunStatusQueued, CurrentNodeID: "node-a", UpdatedAt: "2026-07-10T02:00:00Z"},
{ID: "run-running", WorkflowID: scheduled.ID, Status: domain.RunStatusRunning, UpdatedAt: "2026-07-10T03:00:00Z"},
{ID: "run-audit", WorkflowID: scheduled.ID, Status: domain.RunStatusWaitingForAudit, UpdatedAt: "2026-07-10T04:00:00Z"},
{ID: "run-completed", WorkflowID: scheduled.ID, Status: domain.RunStatusCompleted, UpdatedAt: "2026-07-10T05:00:00Z"},
}
for _, run := range runs {
if err := dataStore.UpsertRun(ctx, run); err != nil {
t.Fatalf("upsert run %s: %v", run.ID, err)
}
}
reconcile, err := dataStore.ListReconcileRunSummaries(ctx, 20)
if err != nil {
t.Fatalf("list reconcile runs: %v", err)
}
if len(reconcile) != 2 || reconcile[0].ID != "run-claimed" || reconcile[1].ID != "run-running" {
t.Fatalf("unexpected reconcile runs: %#v", reconcile)
}
counts, err := dataStore.CountWorkerRuns(ctx)
if err != nil {
t.Fatalf("count worker runs: %v", err)
}
if counts.QueuedTotal != 2 || counts.ActiveTotal != 1 {
t.Fatalf("unexpected worker totals: %#v", counts)
}
if counts.ActiveByWorkflow[scheduled.ID] != 2 || counts.InflightByWorkflow[requested.ID] != 2 {
t.Fatalf("unexpected worker workflow counts: %#v", counts)
}
}
func TestUpdateRunProgressAndFailureUseStructuredFields(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()

View File

@ -409,6 +409,26 @@ func (store *MongoStore) ListAllWorkflowsByOrganization(ctx context.Context, org
return decodeMany[domain.WorkflowRecord](ctx, cursor)
}
func (store *MongoStore) ListWorkerWorkflows(ctx context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) {
filters := bson.A{bson.M{
"type": domain.WorkflowTypeStreaming,
"schedule": bson.M{"$exists": true, "$ne": nil},
}}
if len(workflowIDs) > 0 {
filters = append(filters, bson.M{"id": bson.M{"$in": workflowIDs}})
}
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
cursor, err := store.workflows.Find(
ctx,
bson.M{"$or": filters},
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
return decodeMany[domain.WorkflowRecord](ctx, cursor)
}
func (store *MongoStore) GetWorkflow(ctx context.Context, id string) (*domain.WorkflowRecord, error) {
var item domain.WorkflowRecord
err := store.workflows.FindOne(ctx, bson.M{"id": id}).Decode(&item)
@ -910,6 +930,78 @@ func (store *MongoStore) ListRunSummariesPageByOrganization(ctx context.Context,
return store.listRunSummariesPage(ctx, organizationID, query)
}
func (store *MongoStore) ListReconcileRunSummaries(ctx context.Context, limit int) ([]domain.RunSummary, error) {
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
filter := bson.M{"$or": bson.A{
bson.M{"status": bson.M{"$in": bson.A{domain.RunStatusRunning, domain.RunStatusStopping}}},
bson.M{
"status": domain.RunStatusQueued,
"currentNodeId": bson.M{"$exists": true, "$nin": bson.A{"", nil}},
},
}}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runSummaryPageProjection()).
SetSort(bson.D{{Key: "updatedAt", Value: 1}, {Key: "id", Value: 1}}).
SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
items, err := decodeMany[domain.RunSummary](ctx, cursor)
if err != nil {
return nil, err
}
for index := range items {
items[index] = compactRunSummary(items[index])
}
return items, nil
}
func (store *MongoStore) CountWorkerRuns(ctx context.Context) (WorkerRunCounts, error) {
statuses := bson.A{
domain.RunStatusQueued,
domain.RunStatusRunning,
domain.RunStatusStopping,
domain.RunStatusWaitingForAudit,
}
pipeline := mongo.Pipeline{
bson.D{{Key: "$match", Value: bson.M{"status": bson.M{"$in": statuses}}}},
bson.D{{Key: "$group", Value: bson.M{
"_id": bson.M{
"workflowId": "$workflowId",
"status": "$status",
},
"count": bson.M{"$sum": 1},
}}},
}
cursor, err := store.runs.Aggregate(ctx, pipeline)
if err != nil {
return WorkerRunCounts{}, err
}
defer cursor.Close(ctx)
counts := newWorkerRunCounts()
for cursor.Next(ctx) {
var item struct {
Key struct {
WorkflowID string `bson:"workflowId"`
Status domain.RunStatus `bson:"status"`
} `bson:"_id"`
Count int `bson:"count"`
}
if err := cursor.Decode(&item); err != nil {
return WorkerRunCounts{}, err
}
addWorkerRunCount(&counts, item.Key.WorkflowID, item.Key.Status, item.Count)
}
if err := cursor.Err(); err != nil {
return WorkerRunCounts{}, err
}
return counts, nil
}
func (store *MongoStore) HasActiveRun(ctx context.Context, workflowID string) (bool, error) {
count, err := store.runs.CountDocuments(ctx, bson.M{
"workflowId": workflowID,

View File

@ -13,6 +13,13 @@ var (
ErrDuplicateUserEmail = errors.New("user email must be unique")
)
type WorkerRunCounts struct {
ActiveByWorkflow map[string]int
InflightByWorkflow map[string]int
ActiveTotal int
QueuedTotal int
}
type Store interface {
Ready(context.Context) error
EnsureDefaultOrganization(context.Context) (*domain.OrganizationRecord, error)
@ -40,6 +47,7 @@ type Store interface {
ListWorkflowsByProject(context.Context, string) ([]domain.WorkflowRecord, error)
ListAllWorkflows(context.Context) ([]domain.WorkflowRecord, error)
ListAllWorkflowsByOrganization(context.Context, string) ([]domain.WorkflowRecord, error)
ListWorkerWorkflows(context.Context, []string, int) ([]domain.WorkflowRecord, error)
GetWorkflow(context.Context, string) (*domain.WorkflowRecord, error)
CreateWorkflow(context.Context, domain.CreateWorkflowInput) (*domain.WorkflowRecord, error)
DeleteWorkflow(context.Context, string) (bool, error)
@ -68,6 +76,8 @@ type Store interface {
ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error)
ListRunSummariesPage(context.Context, domain.RunSummaryQuery) (domain.RunSummaryPage, error)
ListRunSummariesPageByOrganization(context.Context, string, domain.RunSummaryQuery) (domain.RunSummaryPage, error)
ListReconcileRunSummaries(context.Context, int) ([]domain.RunSummary, error)
CountWorkerRuns(context.Context) (WorkerRunCounts, error)
HasActiveRun(context.Context, string) (bool, error)
GetRun(context.Context, string) (*domain.RunRecord, error)
CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error)

View File

@ -161,12 +161,11 @@ var workflowActiveDispatchStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusWaitingForAudit: {},
}
var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusQueued: {},
domain.RunStatusRunning: {},
domain.RunStatusStopping: {},
domain.RunStatusWaitingForAudit: {},
}
const (
workerQueuedRunLimit = 100
workerReconcileRunLimit = 500
workerWorkflowQueryLimit = 500
)
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string) (*Runtime, error) {
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
@ -702,13 +701,6 @@ func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.Wor
}
func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) {
globalLimit := globalConcurrencyLimit(settings)
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
now := time.Now().UTC()
activeByWorkflow := map[string]int{}
activeTotal := 0
queued := make([]domain.RunRecord, 0)
@ -721,29 +713,49 @@ func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []d
continue
}
if run.Status == domain.RunStatusQueued {
if !runDispatchReady(run, now) {
continue
}
queued = append(queued, run)
}
}
selected := selectQueuedRunsForDispatchSnapshot(queued, workflows, settings, activeByWorkflow, activeTotal)
return selected, len(queued), activeTotal
}
func selectQueuedRunsForDispatchSnapshot(
queued []domain.RunRecord,
workflows []domain.WorkflowRecord,
settings *domain.SettingsRecord,
activeByWorkflow map[string]int,
activeTotal int,
) []string {
globalLimit := globalConcurrencyLimit(settings)
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
now := time.Now().UTC()
ready := make([]domain.RunRecord, 0, len(queued))
for _, run := range queued {
if run.Status == domain.RunStatusQueued && runDispatchReady(run, now) {
ready = append(ready, run)
}
}
available := globalLimit - activeTotal
if available <= 0 {
return nil, len(queued), activeTotal
return nil
}
sort.Slice(queued, func(left int, right int) bool {
leftAvailableAt := runAvailableAt(queued[left])
rightAvailableAt := runAvailableAt(queued[right])
sort.Slice(ready, func(left int, right int) bool {
leftAvailableAt := runAvailableAt(ready[left])
rightAvailableAt := runAvailableAt(ready[right])
if leftAvailableAt.Equal(rightAvailableAt) {
return queued[left].CreatedAt < queued[right].CreatedAt
return ready[left].CreatedAt < ready[right].CreatedAt
}
return leftAvailableAt.Before(rightAvailableAt)
})
selected := make([]string, 0, available)
for _, run := range queued {
for _, run := range ready {
if available == 0 {
break
}
@ -756,15 +768,15 @@ func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []d
activeByWorkflow[run.WorkflowID]++
available--
}
return selected, len(queued), activeTotal
return selected
}
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
workflows, err := runtime.store.ListAllWorkflows(ctx)
workflows, err := runtime.store.ListWorkerWorkflows(ctx, nil, workerWorkflowQueryLimit)
if err != nil {
return err
}
runs, err := runtime.store.ListRunSummaries(ctx, "")
counts, err := runtime.store.CountWorkerRuns(ctx)
if err != nil {
return err
}
@ -772,10 +784,32 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
if err != nil {
return err
}
return runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, runs, settings)
return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings)
}
func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Context, workflows []domain.WorkflowRecord, runs []domain.RunSummary, settings *domain.SettingsRecord) error {
counts := store.WorkerRunCounts{
ActiveByWorkflow: map[string]int{},
InflightByWorkflow: map[string]int{},
}
for _, run := range runs {
switch run.Status {
case domain.RunStatusQueued:
counts.QueuedTotal++
counts.InflightByWorkflow[run.WorkflowID]++
case domain.RunStatusRunning, domain.RunStatusStopping:
counts.ActiveTotal++
counts.ActiveByWorkflow[run.WorkflowID]++
counts.InflightByWorkflow[run.WorkflowID]++
case domain.RunStatusWaitingForAudit:
counts.ActiveByWorkflow[run.WorkflowID]++
counts.InflightByWorkflow[run.WorkflowID]++
}
}
return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings)
}
func (runtime *Runtime) enqueueStreamingWorkflowsWithCounts(ctx context.Context, workflows []domain.WorkflowRecord, counts *store.WorkerRunCounts, settings *domain.SettingsRecord) error {
now := time.Now().UTC()
for _, workflow := range workflows {
if !shouldEnqueueScheduledRun(workflow, now) {
@ -791,14 +825,7 @@ func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Conte
continue
}
}
inflightCount := 0
for _, run := range runs {
if run.WorkflowID == workflow.ID {
if _, ok := queuedOrActiveStatuses[run.Status]; ok {
inflightCount++
}
}
}
inflightCount := counts.InflightByWorkflow[workflow.ID]
if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) {
continue
}
@ -817,6 +844,8 @@ func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Conte
return err
}
}
counts.QueuedTotal++
counts.InflightByWorkflow[workflow.ID]++
}
return nil
}
@ -948,7 +977,7 @@ func (runtime *Runtime) orphanMinimumRuntimeAge() time.Duration {
return runtime.orphanMinimumAge
}
func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context, summaries []domain.RunSummary) {
func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
now := time.Now().UTC()
interval := runtime.orphanCheckInterval
if interval <= 0 {
@ -966,6 +995,11 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context, summaries
runtime.metrics.RecordOrphanRuntimes(0)
return
}
summaries, err := runtime.store.ListRunSummaries(ctx, "")
if err != nil {
fmt.Printf("[worker] list run ids for orphan reconciliation failed: %v\n", err)
return
}
knownRunIDs := make(map[string]struct{}, len(summaries))
for _, summary := range summaries {
knownRunIDs[summary.ID] = struct{}{}
@ -990,28 +1024,56 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
return processErr
}
runtime.updateEffectivePollInterval(settings)
workflows, err := runtime.store.ListAllWorkflows(ctx)
reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
if err != nil {
processErr = err
return processErr
}
summaries, err := runtime.store.ListRunSummaries(ctx, "")
if err := runtime.reconcileActiveRunsWithSummaries(ctx, reconcileRuns); err != nil {
processErr = err
return processErr
}
queuedPage, err := runtime.store.ListRunSummariesPage(ctx, domain.RunSummaryQuery{
Status: domain.RunStatusQueued,
Limit: workerQueuedRunLimit,
})
if err != nil {
processErr = err
return processErr
}
if err := runtime.reconcileActiveRunsWithSummaries(ctx, summaries); err != nil {
counts, err := runtime.store.CountWorkerRuns(ctx)
if err != nil {
processErr = err
return processErr
}
if err := runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, summaries, settings); err != nil {
workflowIDs := make([]string, 0, len(queuedPage.Items))
workflowSet := make(map[string]struct{}, len(queuedPage.Items))
for _, run := range queuedPage.Items {
if _, exists := workflowSet[run.WorkflowID]; exists {
continue
}
workflowSet[run.WorkflowID] = struct{}{}
workflowIDs = append(workflowIDs, run.WorkflowID)
}
workflows, err := runtime.store.ListWorkerWorkflows(ctx, workflowIDs, workerWorkflowQueryLimit)
if err != nil {
processErr = err
return processErr
}
runtime.reconcileOrphanedRuntimes(ctx, summaries)
runs := runRecordsFromSummaries(summaries)
selectedRunIDs, queueDepth, runningCount := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
runtime.metrics.RecordDispatch(queueDepth, runningCount, len(selectedRunIDs))
if err := runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings); err != nil {
processErr = err
return processErr
}
runtime.reconcileOrphanedRuntimes(ctx)
queuedRuns := runRecordsFromSummaries(queuedPage.Items)
selectedRunIDs := selectQueuedRunsForDispatchSnapshot(
queuedRuns,
workflows,
settings,
counts.ActiveByWorkflow,
counts.ActiveTotal,
)
runtime.metrics.RecordDispatch(counts.QueuedTotal, counts.ActiveTotal, len(selectedRunIDs))
for _, runID := range selectedRunIDs {
if !runtime.tryMarkRunActive(runID) {
continue
@ -1366,7 +1428,7 @@ func (runtime *Runtime) reconcileActiveRunsWithSummaries(ctx context.Context, ru
}
func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error {
runs, err := runtime.store.ListRunSummaries(ctx, "")
runs, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
if err != nil {
return err
}

View File

@ -43,8 +43,10 @@ POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}"
POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}"
if [[ -z "${IMAGE_TAG:-}" ]]; then
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)"
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)"
fi
POC_KANIKO_SOURCE_COMMIT="${POC_KANIKO_SOURCE_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
# shellcheck source=scripts/lib/image-config.sh
source "$ROOT_DIR/scripts/lib/image-config.sh"
@ -176,6 +178,8 @@ YAML
printf ' - --build-arg=OPERATOR_GPU_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
printf ' - --build-arg=OPERATOR_HOLOBRAIN_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
printf ' - --build-arg=OPERATOR_YOLO_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
printf ' - --build-arg=DROBOTICFLOW_BUILD_COMMIT=%s\n' "$POC_KANIKO_SOURCE_COMMIT"
printf ' - --build-arg=DROBOTICFLOW_BUILD_TIME=%s\n' "$POC_KANIKO_BUILD_TIME"
IFS=',' read -r -a mirrors <<<"$POC_KANIKO_REGISTRY_MIRRORS"
for mirror in "${mirrors[@]}"; do
mirror="$(printf '%s' "$mirror" | xargs)"

View File

@ -1817,6 +1817,9 @@ export default function WorkflowEditor() {
let controller: AbortController | null = null;
const refreshLogs = async () => {
if (cancelled || document.visibilityState !== "visible") {
return;
}
controller?.abort();
controller = new AbortController();
setIsLiveRunLogsLoading(true);
@ -1837,18 +1840,31 @@ export default function WorkflowEditor() {
} finally {
if (!cancelled) {
setIsLiveRunLogsLoading(false);
if (activeRunInFlight) {
if (activeRunInFlight && document.visibilityState === "visible") {
timer = window.setTimeout(refreshLogs, 5000);
}
}
}
};
void refreshLogs();
const handleVisibilityChange = () => {
window.clearTimeout(timer);
if (document.visibilityState === "hidden") {
controller?.abort();
return;
}
void refreshLogs();
};
if (document.visibilityState === "visible") {
void refreshLogs();
}
document.addEventListener("visibilitychange", handleVisibilityChange);
return () => {
cancelled = true;
window.clearTimeout(timer);
controller?.abort();
document.removeEventListener("visibilitychange", handleVisibilityChange);
};
}, [activeRun?.id, activeRunInFlight, liveRunLogsRefreshToken, nodeConfigPanelTab, selectedNode?.id]);

View File

@ -111,11 +111,13 @@ def run_script(
"POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS",
"POC_KANIKO_CONTEXT_SOURCE",
"POC_KANIKO_BUILD_OPERATORS",
"POC_KANIKO_BUILD_TIME",
"POC_KANIKO_OPERATOR_MANIFEST",
"POC_KANIKO_OPERATOR_CONCURRENCY",
"POC_KANIKO_OPERATOR_RETRIES",
"POC_KANIKO_IMAGE",
"POC_KANIKO_NODE_NAME",
"POC_KANIKO_SOURCE_COMMIT",
"POC_KANIKO_GOPROXY",
"POC_KANIKO_REGISTRY_MIRRORS",
"POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK",
@ -161,6 +163,8 @@ def test_poc_kaniko_build_creates_three_jobs_with_registry_mirror(tmp_path):
"POC_KANIKO_CLEAN_CONTEXT": "false",
"POC_KANIKO_GOPROXY": "https://goproxy.cn,direct",
"POC_KANIKO_NODE_NAME": "worker-cpu-01",
"POC_KANIKO_SOURCE_COMMIT": "0123456789abcdef",
"POC_KANIKO_BUILD_TIME": "2026-07-10T10:30:00Z",
},
)
@ -177,6 +181,8 @@ def test_poc_kaniko_build_creates_three_jobs_with_registry_mirror(tmp_path):
manifests = "\n---\n".join(path.read_text(encoding="utf-8") for path in sorted(applied_dir.glob("*.yaml")))
assert manifests.count("kind: Job") == 3
assert "--build-arg=GOPROXY=https://goproxy.cn,direct" in manifests
assert "--build-arg=DROBOTICFLOW_BUILD_COMMIT=0123456789abcdef" in manifests
assert "--build-arg=DROBOTICFLOW_BUILD_TIME=2026-07-10T10:30:00Z" in manifests
assert "--registry-mirror=mirror.gcr.io" in manifests
assert "--skip-default-registry-fallback" in manifests
assert "Dockerfile.api" in manifests