⚡ tighten worker scheduling and runtime visibility
This commit is contained in:
parent
d097d32aa5
commit
66a15d4cf4
@ -1,3 +1,6 @@
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
|
||||
ARG DROBOTICFLOW_BUILD_TIME=unknown
|
||||
|
||||
FROM node:22-alpine AS web-build
|
||||
WORKDIR /app
|
||||
|
||||
@ -21,9 +24,15 @@ COPY internal ./internal
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-api ./cmd/api
|
||||
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT
|
||||
ARG DROBOTICFLOW_BUILD_TIME
|
||||
WORKDIR /app
|
||||
ENV PORT=3001
|
||||
ENV DIST_DIR=/app/dist
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
|
||||
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||
RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow
|
||||
|
||||
@ -1,7 +1,16 @@
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
|
||||
ARG DROBOTICFLOW_BUILD_TIME=unknown
|
||||
|
||||
FROM python:3.11-slim AS runtime
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT
|
||||
ARG DROBOTICFLOW_BUILD_TIME
|
||||
WORKDIR /app
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PYTHONPATH=/app
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
|
||||
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||
RUN pip install --no-cache-dir numpy opencv-python-headless bce-python-sdk pymongo h5py pyarrow mcap PyYAML zarr nvidia-ml-py
|
||||
|
||||
@ -1,3 +1,6 @@
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
|
||||
ARG DROBOTICFLOW_BUILD_TIME=unknown
|
||||
|
||||
FROM golang:1.24-bookworm AS build
|
||||
WORKDIR /src
|
||||
ARG GOPROXY=https://proxy.golang.org,direct
|
||||
@ -10,8 +13,14 @@ COPY internal ./internal
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-worker ./cmd/worker
|
||||
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT
|
||||
ARG DROBOTICFLOW_BUILD_TIME
|
||||
WORKDIR /app
|
||||
ENV WORKER_PORT=3002
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
|
||||
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
|
||||
RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow
|
||||
|
||||
@ -6048,11 +6048,12 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec
|
||||
if run.Status != domain.RunStatusRunning && run.Status != domain.RunStatusQueued && run.Status != domain.RunStatusStopping {
|
||||
return
|
||||
}
|
||||
nodeID, nodeLabel := "", ""
|
||||
nodeID, nodeLabel, nodeStartedAt := "", "", ""
|
||||
for _, execution := range run.NodeExecutions {
|
||||
if execution.Status == domain.NodeExecutionStatusRunning {
|
||||
nodeID = execution.NodeID
|
||||
nodeLabel = execution.NodeLabel
|
||||
nodeStartedAt = execution.StartedAt
|
||||
break
|
||||
}
|
||||
}
|
||||
@ -6069,7 +6070,7 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec
|
||||
}
|
||||
tailCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
tail, err := launcher.TailNodePodLogs(tailCtx, run.ID, nodeID, 300)
|
||||
tail, err := launcher.TailNodePodLogsWithTimestamps(tailCtx, run.ID, nodeID, 300)
|
||||
if err != nil || strings.TrimSpace(tail) == "" {
|
||||
return
|
||||
}
|
||||
@ -6077,18 +6078,36 @@ func (server *server) appendLiveNodeLogs(ctx context.Context, run *domain.RunRec
|
||||
if strings.TrimSpace(label) == "" {
|
||||
label = nodeID
|
||||
}
|
||||
fallbackTimestamp := firstNonEmptyText(nodeStartedAt, run.StartedAt, run.CreatedAt, "1970-01-01T00:00:00Z")
|
||||
run.Logs = append(run.Logs, livePodLogEntries(tail, label, fallbackTimestamp)...)
|
||||
}
|
||||
|
||||
func livePodLogEntries(tail string, label string, fallbackTimestamp string) []domain.LogEntryRecord {
|
||||
entries := make([]domain.LogEntryRecord, 0)
|
||||
for _, line := range strings.Split(tail, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
run.Logs = append(run.Logs, domain.LogEntryRecord{
|
||||
Timestamp: nowISO(),
|
||||
timestamp := fallbackTimestamp
|
||||
message := trimmed
|
||||
if candidate, remainder, found := strings.Cut(trimmed, " "); found {
|
||||
if _, err := time.Parse(time.RFC3339Nano, candidate); err == nil {
|
||||
timestamp = candidate
|
||||
message = strings.TrimSpace(remainder)
|
||||
}
|
||||
}
|
||||
if message == "" {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, domain.LogEntryRecord{
|
||||
Timestamp: timestamp,
|
||||
Level: "INFO",
|
||||
Message: trimmed,
|
||||
Message: message,
|
||||
Operator: label,
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func (server *server) handleRunsResources(writer http.ResponseWriter, request *http.Request) {
|
||||
|
||||
@ -191,6 +191,20 @@ func TestJSONResponsesUseGzipWhenTheClientSupportsIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLivePodLogEntriesKeepKubernetesTimestampsStable(t *testing.T) {
|
||||
tail := "2026-07-10T10:20:30.123456789Z first line\nsecond line\n"
|
||||
entries := livePodLogEntries(tail, "Train", "2026-07-10T10:00:00Z")
|
||||
if len(entries) != 2 {
|
||||
t.Fatalf("entries = %#v", entries)
|
||||
}
|
||||
if entries[0].Timestamp != "2026-07-10T10:20:30.123456789Z" || entries[0].Message != "first line" {
|
||||
t.Fatalf("timestamped entry = %#v", entries[0])
|
||||
}
|
||||
if entries[1].Timestamp != "2026-07-10T10:00:00Z" || entries[1].Message != "second line" {
|
||||
t.Fatalf("fallback entry = %#v", entries[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewHandlerReportsInvalidConfiguration(t *testing.T) {
|
||||
t.Setenv("PORT", "invalid")
|
||||
|
||||
|
||||
@ -145,12 +145,21 @@ func LoadValidated() (AppConfig, error) {
|
||||
CustomOperatorImageImporterImage: getenvTrimmed("CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE", "quay.io/skopeo/stable:v1.16.1"),
|
||||
CustomOperatorImageImportTimeoutSeconds: customOperatorImageImportTimeoutSeconds,
|
||||
CustomOperatorImageArchiveMaxMiB: customOperatorImageArchiveMaxMiB,
|
||||
BuildCommit: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_COMMIT")),
|
||||
BuildTime: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_TIME")),
|
||||
BuildCommit: firstNonEmptyTrimmed(os.Getenv("DROBOTICFLOW_IMAGE_BUILD_COMMIT"), os.Getenv("DROBOTICFLOW_BUILD_COMMIT")),
|
||||
BuildTime: firstNonEmptyTrimmed(os.Getenv("DROBOTICFLOW_IMAGE_BUILD_TIME"), os.Getenv("DROBOTICFLOW_BUILD_TIME")),
|
||||
Image: strings.TrimSpace(os.Getenv("DROBOTICFLOW_IMAGE")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func firstNonEmptyTrimmed(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseMongoURIEnv(key string) (string, error) {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
|
||||
@ -38,6 +38,8 @@ var configEnvKeys = []string{
|
||||
"CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB",
|
||||
"DROBOTICFLOW_BUILD_COMMIT",
|
||||
"DROBOTICFLOW_BUILD_TIME",
|
||||
"DROBOTICFLOW_IMAGE_BUILD_COMMIT",
|
||||
"DROBOTICFLOW_IMAGE_BUILD_TIME",
|
||||
"DROBOTICFLOW_IMAGE",
|
||||
}
|
||||
|
||||
@ -148,6 +150,8 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) {
|
||||
t.Setenv("CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB", "1024")
|
||||
t.Setenv("DROBOTICFLOW_BUILD_COMMIT", " 3cda38c ")
|
||||
t.Setenv("DROBOTICFLOW_BUILD_TIME", " 2026-07-10T12:00:00Z ")
|
||||
t.Setenv("DROBOTICFLOW_IMAGE_BUILD_COMMIT", " image-commit ")
|
||||
t.Setenv("DROBOTICFLOW_IMAGE_BUILD_TIME", " 2026-07-10T11:59:00Z ")
|
||||
t.Setenv("DROBOTICFLOW_IMAGE", " ccr.example/droboticflow-api:3cda38c ")
|
||||
|
||||
cfg, err := LoadValidated()
|
||||
@ -193,7 +197,7 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) {
|
||||
if !cfg.RequireSessionAuth || !cfg.SessionCookieSecure {
|
||||
t.Fatalf("expected explicit boolean envs to enable auth and secure cookie")
|
||||
}
|
||||
if cfg.BuildCommit != "3cda38c" || cfg.BuildTime != "2026-07-10T12:00:00Z" || cfg.Image != "ccr.example/droboticflow-api:3cda38c" {
|
||||
if cfg.BuildCommit != "image-commit" || cfg.BuildTime != "2026-07-10T11:59:00Z" || cfg.Image != "ccr.example/droboticflow-api:3cda38c" {
|
||||
t.Fatalf("build metadata = commit %q time %q image %q", cfg.BuildCommit, cfg.BuildTime, cfg.Image)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1040,6 +1040,16 @@ func (launcher *Launcher) SamplePodGPUForRun(ctx context.Context, runID string,
|
||||
// result.json (which the worker persists only at node completion). Returns "" when
|
||||
// no Running pod exists, so the caller falls back to the stored run logs.
|
||||
func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
|
||||
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, false)
|
||||
}
|
||||
|
||||
// TailNodePodLogsWithTimestamps keeps the Kubernetes source timestamp on each
|
||||
// line so repeated API polls can deduplicate the same live log entry.
|
||||
func (launcher *Launcher) TailNodePodLogsWithTimestamps(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) {
|
||||
return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines, true)
|
||||
}
|
||||
|
||||
func (launcher *Launcher) tailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64, timestamps bool) (string, error) {
|
||||
if tailLines <= 0 {
|
||||
tailLines = 300
|
||||
}
|
||||
@ -1051,10 +1061,14 @@ func (launcher *Launcher) TailNodePodLogs(ctx context.Context, runID string, nod
|
||||
return "", nil
|
||||
}
|
||||
tail := tailLines
|
||||
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, &corev1.PodLogOptions{
|
||||
options := &corev1.PodLogOptions{
|
||||
Container: "operator-runner",
|
||||
TailLines: &tail,
|
||||
}).Stream(ctx)
|
||||
}
|
||||
if timestamps {
|
||||
options.Timestamps = true
|
||||
}
|
||||
stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(podName, options).Stream(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@ -569,6 +569,31 @@ func (store *MemoryStore) ListAllWorkflowsByOrganization(_ context.Context, orga
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListWorkerWorkflows(_ context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
wanted := make(map[string]struct{}, len(workflowIDs))
|
||||
for _, workflowID := range workflowIDs {
|
||||
if workflowID != "" {
|
||||
wanted[workflowID] = struct{}{}
|
||||
}
|
||||
}
|
||||
items := make([]domain.WorkflowRecord, 0)
|
||||
for _, workflow := range store.workflows {
|
||||
_, explicitlyRequested := wanted[workflow.ID]
|
||||
if !explicitlyRequested && (workflow.Type != domain.WorkflowTypeStreaming || !domain.WorkflowScheduleEnabled(workflow.Schedule)) {
|
||||
continue
|
||||
}
|
||||
items = append(items, clone(workflow))
|
||||
}
|
||||
sortWorkflows(items)
|
||||
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) GetWorkflow(_ context.Context, id string) (*domain.WorkflowRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
@ -970,6 +995,80 @@ func (store *MemoryStore) ListRunSummariesPageByOrganization(_ context.Context,
|
||||
return store.listRunSummariesPageLocked(organizationID, query), nil
|
||||
}
|
||||
|
||||
func normalizedWorkerQueryLimit(limit int, defaultLimit int, maxLimit int) int {
|
||||
if limit <= 0 {
|
||||
return defaultLimit
|
||||
}
|
||||
if limit > maxLimit {
|
||||
return maxLimit
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func isReconcileRunSummary(item domain.RunSummary) bool {
|
||||
return item.Status == domain.RunStatusRunning ||
|
||||
item.Status == domain.RunStatusStopping ||
|
||||
(item.Status == domain.RunStatusQueued && item.CurrentNodeID != "")
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListReconcileRunSummaries(_ context.Context, limit int) ([]domain.RunSummary, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
items := make([]domain.RunSummary, 0)
|
||||
for _, run := range store.runs {
|
||||
summary := summarizeRun(run)
|
||||
if isReconcileRunSummary(summary) {
|
||||
items = append(items, compactRunSummary(summary))
|
||||
}
|
||||
}
|
||||
sort.Slice(items, func(left int, right int) bool {
|
||||
if items[left].UpdatedAt == items[right].UpdatedAt {
|
||||
return items[left].ID < items[right].ID
|
||||
}
|
||||
return items[left].UpdatedAt < items[right].UpdatedAt
|
||||
})
|
||||
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func newWorkerRunCounts() WorkerRunCounts {
|
||||
return WorkerRunCounts{
|
||||
ActiveByWorkflow: map[string]int{},
|
||||
InflightByWorkflow: map[string]int{},
|
||||
}
|
||||
}
|
||||
|
||||
func addWorkerRunCount(counts *WorkerRunCounts, workflowID string, status domain.RunStatus, count int) {
|
||||
if count <= 0 {
|
||||
return
|
||||
}
|
||||
switch status {
|
||||
case domain.RunStatusQueued:
|
||||
counts.QueuedTotal += count
|
||||
counts.InflightByWorkflow[workflowID] += count
|
||||
case domain.RunStatusRunning, domain.RunStatusStopping:
|
||||
counts.ActiveTotal += count
|
||||
counts.ActiveByWorkflow[workflowID] += count
|
||||
counts.InflightByWorkflow[workflowID] += count
|
||||
case domain.RunStatusWaitingForAudit:
|
||||
counts.ActiveByWorkflow[workflowID] += count
|
||||
counts.InflightByWorkflow[workflowID] += count
|
||||
}
|
||||
}
|
||||
|
||||
func (store *MemoryStore) CountWorkerRuns(_ context.Context) (WorkerRunCounts, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
counts := newWorkerRunCounts()
|
||||
for _, run := range store.runs {
|
||||
addWorkerRunCount(&counts, run.WorkflowID, run.Status, 1)
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) hasActiveRunLocked(workflowID string) bool {
|
||||
for _, run := range store.runs {
|
||||
if run.WorkflowID == workflowID && isActiveRunStatus(run.Status) {
|
||||
|
||||
@ -789,6 +789,66 @@ func TestListRunSummariesPageFiltersCountsAndAdvancesCursor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerQueriesReturnOnlyDispatchAndReconcileState(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
scheduled := domain.WorkflowRecord{
|
||||
ID: "workflow-scheduled",
|
||||
Type: domain.WorkflowTypeStreaming,
|
||||
Schedule: &domain.WorkflowSchedule{Mode: domain.WorkflowScheduleModeIntervalSeconds, Interval: 60},
|
||||
CreatedAt: "2026-07-10T01:00:00Z",
|
||||
UpdatedAt: "2026-07-10T01:00:00Z",
|
||||
}
|
||||
requested := domain.WorkflowRecord{ID: "workflow-requested", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"}
|
||||
unrelated := domain.WorkflowRecord{ID: "workflow-unrelated", Type: domain.WorkflowTypeBatch, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"}
|
||||
for _, workflow := range []domain.WorkflowRecord{scheduled, requested, unrelated} {
|
||||
if err := dataStore.UpsertWorkflow(ctx, workflow); err != nil {
|
||||
t.Fatalf("upsert workflow %s: %v", workflow.ID, err)
|
||||
}
|
||||
}
|
||||
workflows, err := dataStore.ListWorkerWorkflows(ctx, []string{requested.ID}, 20)
|
||||
if err != nil {
|
||||
t.Fatalf("list worker workflows: %v", err)
|
||||
}
|
||||
workflowIDs := map[string]bool{}
|
||||
for _, workflow := range workflows {
|
||||
workflowIDs[workflow.ID] = true
|
||||
}
|
||||
if !workflowIDs[scheduled.ID] || !workflowIDs[requested.ID] || workflowIDs[unrelated.ID] {
|
||||
t.Fatalf("unexpected worker workflows: %#v", workflowIDs)
|
||||
}
|
||||
|
||||
runs := []domain.RunRecord{
|
||||
{ID: "run-queued", WorkflowID: requested.ID, Status: domain.RunStatusQueued, UpdatedAt: "2026-07-10T01:00:00Z"},
|
||||
{ID: "run-claimed", WorkflowID: requested.ID, Status: domain.RunStatusQueued, CurrentNodeID: "node-a", UpdatedAt: "2026-07-10T02:00:00Z"},
|
||||
{ID: "run-running", WorkflowID: scheduled.ID, Status: domain.RunStatusRunning, UpdatedAt: "2026-07-10T03:00:00Z"},
|
||||
{ID: "run-audit", WorkflowID: scheduled.ID, Status: domain.RunStatusWaitingForAudit, UpdatedAt: "2026-07-10T04:00:00Z"},
|
||||
{ID: "run-completed", WorkflowID: scheduled.ID, Status: domain.RunStatusCompleted, UpdatedAt: "2026-07-10T05:00:00Z"},
|
||||
}
|
||||
for _, run := range runs {
|
||||
if err := dataStore.UpsertRun(ctx, run); err != nil {
|
||||
t.Fatalf("upsert run %s: %v", run.ID, err)
|
||||
}
|
||||
}
|
||||
reconcile, err := dataStore.ListReconcileRunSummaries(ctx, 20)
|
||||
if err != nil {
|
||||
t.Fatalf("list reconcile runs: %v", err)
|
||||
}
|
||||
if len(reconcile) != 2 || reconcile[0].ID != "run-claimed" || reconcile[1].ID != "run-running" {
|
||||
t.Fatalf("unexpected reconcile runs: %#v", reconcile)
|
||||
}
|
||||
counts, err := dataStore.CountWorkerRuns(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("count worker runs: %v", err)
|
||||
}
|
||||
if counts.QueuedTotal != 2 || counts.ActiveTotal != 1 {
|
||||
t.Fatalf("unexpected worker totals: %#v", counts)
|
||||
}
|
||||
if counts.ActiveByWorkflow[scheduled.ID] != 2 || counts.InflightByWorkflow[requested.ID] != 2 {
|
||||
t.Fatalf("unexpected worker workflow counts: %#v", counts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateRunProgressAndFailureUseStructuredFields(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
|
||||
@ -409,6 +409,26 @@ func (store *MongoStore) ListAllWorkflowsByOrganization(ctx context.Context, org
|
||||
return decodeMany[domain.WorkflowRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListWorkerWorkflows(ctx context.Context, workflowIDs []string, limit int) ([]domain.WorkflowRecord, error) {
|
||||
filters := bson.A{bson.M{
|
||||
"type": domain.WorkflowTypeStreaming,
|
||||
"schedule": bson.M{"$exists": true, "$ne": nil},
|
||||
}}
|
||||
if len(workflowIDs) > 0 {
|
||||
filters = append(filters, bson.M{"id": bson.M{"$in": workflowIDs}})
|
||||
}
|
||||
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
|
||||
cursor, err := store.workflows.Find(
|
||||
ctx,
|
||||
bson.M{"$or": filters},
|
||||
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.WorkflowRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) GetWorkflow(ctx context.Context, id string) (*domain.WorkflowRecord, error) {
|
||||
var item domain.WorkflowRecord
|
||||
err := store.workflows.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
||||
@ -910,6 +930,78 @@ func (store *MongoStore) ListRunSummariesPageByOrganization(ctx context.Context,
|
||||
return store.listRunSummariesPage(ctx, organizationID, query)
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListReconcileRunSummaries(ctx context.Context, limit int) ([]domain.RunSummary, error) {
|
||||
limit = normalizedWorkerQueryLimit(limit, 500, 1000)
|
||||
filter := bson.M{"$or": bson.A{
|
||||
bson.M{"status": bson.M{"$in": bson.A{domain.RunStatusRunning, domain.RunStatusStopping}}},
|
||||
bson.M{
|
||||
"status": domain.RunStatusQueued,
|
||||
"currentNodeId": bson.M{"$exists": true, "$nin": bson.A{"", nil}},
|
||||
},
|
||||
}}
|
||||
cursor, err := store.runs.Find(
|
||||
ctx,
|
||||
filter,
|
||||
options.Find().
|
||||
SetProjection(runSummaryPageProjection()).
|
||||
SetSort(bson.D{{Key: "updatedAt", Value: 1}, {Key: "id", Value: 1}}).
|
||||
SetLimit(int64(limit)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, err := decodeMany[domain.RunSummary](ctx, cursor)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for index := range items {
|
||||
items[index] = compactRunSummary(items[index])
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) CountWorkerRuns(ctx context.Context) (WorkerRunCounts, error) {
|
||||
statuses := bson.A{
|
||||
domain.RunStatusQueued,
|
||||
domain.RunStatusRunning,
|
||||
domain.RunStatusStopping,
|
||||
domain.RunStatusWaitingForAudit,
|
||||
}
|
||||
pipeline := mongo.Pipeline{
|
||||
bson.D{{Key: "$match", Value: bson.M{"status": bson.M{"$in": statuses}}}},
|
||||
bson.D{{Key: "$group", Value: bson.M{
|
||||
"_id": bson.M{
|
||||
"workflowId": "$workflowId",
|
||||
"status": "$status",
|
||||
},
|
||||
"count": bson.M{"$sum": 1},
|
||||
}}},
|
||||
}
|
||||
cursor, err := store.runs.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return WorkerRunCounts{}, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
counts := newWorkerRunCounts()
|
||||
for cursor.Next(ctx) {
|
||||
var item struct {
|
||||
Key struct {
|
||||
WorkflowID string `bson:"workflowId"`
|
||||
Status domain.RunStatus `bson:"status"`
|
||||
} `bson:"_id"`
|
||||
Count int `bson:"count"`
|
||||
}
|
||||
if err := cursor.Decode(&item); err != nil {
|
||||
return WorkerRunCounts{}, err
|
||||
}
|
||||
addWorkerRunCount(&counts, item.Key.WorkflowID, item.Key.Status, item.Count)
|
||||
}
|
||||
if err := cursor.Err(); err != nil {
|
||||
return WorkerRunCounts{}, err
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) HasActiveRun(ctx context.Context, workflowID string) (bool, error) {
|
||||
count, err := store.runs.CountDocuments(ctx, bson.M{
|
||||
"workflowId": workflowID,
|
||||
|
||||
@ -13,6 +13,13 @@ var (
|
||||
ErrDuplicateUserEmail = errors.New("user email must be unique")
|
||||
)
|
||||
|
||||
type WorkerRunCounts struct {
|
||||
ActiveByWorkflow map[string]int
|
||||
InflightByWorkflow map[string]int
|
||||
ActiveTotal int
|
||||
QueuedTotal int
|
||||
}
|
||||
|
||||
type Store interface {
|
||||
Ready(context.Context) error
|
||||
EnsureDefaultOrganization(context.Context) (*domain.OrganizationRecord, error)
|
||||
@ -40,6 +47,7 @@ type Store interface {
|
||||
ListWorkflowsByProject(context.Context, string) ([]domain.WorkflowRecord, error)
|
||||
ListAllWorkflows(context.Context) ([]domain.WorkflowRecord, error)
|
||||
ListAllWorkflowsByOrganization(context.Context, string) ([]domain.WorkflowRecord, error)
|
||||
ListWorkerWorkflows(context.Context, []string, int) ([]domain.WorkflowRecord, error)
|
||||
GetWorkflow(context.Context, string) (*domain.WorkflowRecord, error)
|
||||
CreateWorkflow(context.Context, domain.CreateWorkflowInput) (*domain.WorkflowRecord, error)
|
||||
DeleteWorkflow(context.Context, string) (bool, error)
|
||||
@ -68,6 +76,8 @@ type Store interface {
|
||||
ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error)
|
||||
ListRunSummariesPage(context.Context, domain.RunSummaryQuery) (domain.RunSummaryPage, error)
|
||||
ListRunSummariesPageByOrganization(context.Context, string, domain.RunSummaryQuery) (domain.RunSummaryPage, error)
|
||||
ListReconcileRunSummaries(context.Context, int) ([]domain.RunSummary, error)
|
||||
CountWorkerRuns(context.Context) (WorkerRunCounts, error)
|
||||
HasActiveRun(context.Context, string) (bool, error)
|
||||
GetRun(context.Context, string) (*domain.RunRecord, error)
|
||||
CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error)
|
||||
|
||||
@ -161,12 +161,11 @@ var workflowActiveDispatchStatuses = map[domain.RunStatus]struct{}{
|
||||
domain.RunStatusWaitingForAudit: {},
|
||||
}
|
||||
|
||||
var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
|
||||
domain.RunStatusQueued: {},
|
||||
domain.RunStatusRunning: {},
|
||||
domain.RunStatusStopping: {},
|
||||
domain.RunStatusWaitingForAudit: {},
|
||||
}
|
||||
const (
|
||||
workerQueuedRunLimit = 100
|
||||
workerReconcileRunLimit = 500
|
||||
workerWorkflowQueryLimit = 500
|
||||
)
|
||||
|
||||
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int, globalBOSEndpoint string) (*Runtime, error) {
|
||||
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
|
||||
@ -702,13 +701,6 @@ func selectQueuedRunsForDispatch(runs []domain.RunRecord, workflows []domain.Wor
|
||||
}
|
||||
|
||||
func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []domain.WorkflowRecord, settings *domain.SettingsRecord) ([]string, int, int) {
|
||||
globalLimit := globalConcurrencyLimit(settings)
|
||||
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
|
||||
for _, workflow := range workflows {
|
||||
workflowByID[workflow.ID] = workflow
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
|
||||
activeByWorkflow := map[string]int{}
|
||||
activeTotal := 0
|
||||
queued := make([]domain.RunRecord, 0)
|
||||
@ -721,29 +713,49 @@ func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []d
|
||||
continue
|
||||
}
|
||||
if run.Status == domain.RunStatusQueued {
|
||||
if !runDispatchReady(run, now) {
|
||||
continue
|
||||
}
|
||||
queued = append(queued, run)
|
||||
}
|
||||
}
|
||||
selected := selectQueuedRunsForDispatchSnapshot(queued, workflows, settings, activeByWorkflow, activeTotal)
|
||||
return selected, len(queued), activeTotal
|
||||
}
|
||||
|
||||
func selectQueuedRunsForDispatchSnapshot(
|
||||
queued []domain.RunRecord,
|
||||
workflows []domain.WorkflowRecord,
|
||||
settings *domain.SettingsRecord,
|
||||
activeByWorkflow map[string]int,
|
||||
activeTotal int,
|
||||
) []string {
|
||||
globalLimit := globalConcurrencyLimit(settings)
|
||||
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
|
||||
for _, workflow := range workflows {
|
||||
workflowByID[workflow.ID] = workflow
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
ready := make([]domain.RunRecord, 0, len(queued))
|
||||
for _, run := range queued {
|
||||
if run.Status == domain.RunStatusQueued && runDispatchReady(run, now) {
|
||||
ready = append(ready, run)
|
||||
}
|
||||
}
|
||||
|
||||
available := globalLimit - activeTotal
|
||||
if available <= 0 {
|
||||
return nil, len(queued), activeTotal
|
||||
return nil
|
||||
}
|
||||
|
||||
sort.Slice(queued, func(left int, right int) bool {
|
||||
leftAvailableAt := runAvailableAt(queued[left])
|
||||
rightAvailableAt := runAvailableAt(queued[right])
|
||||
sort.Slice(ready, func(left int, right int) bool {
|
||||
leftAvailableAt := runAvailableAt(ready[left])
|
||||
rightAvailableAt := runAvailableAt(ready[right])
|
||||
if leftAvailableAt.Equal(rightAvailableAt) {
|
||||
return queued[left].CreatedAt < queued[right].CreatedAt
|
||||
return ready[left].CreatedAt < ready[right].CreatedAt
|
||||
}
|
||||
return leftAvailableAt.Before(rightAvailableAt)
|
||||
})
|
||||
|
||||
selected := make([]string, 0, available)
|
||||
for _, run := range queued {
|
||||
for _, run := range ready {
|
||||
if available == 0 {
|
||||
break
|
||||
}
|
||||
@ -756,15 +768,15 @@ func selectQueuedRunsForDispatchWithStats(runs []domain.RunRecord, workflows []d
|
||||
activeByWorkflow[run.WorkflowID]++
|
||||
available--
|
||||
}
|
||||
return selected, len(queued), activeTotal
|
||||
return selected
|
||||
}
|
||||
|
||||
func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
|
||||
workflows, err := runtime.store.ListAllWorkflows(ctx)
|
||||
workflows, err := runtime.store.ListWorkerWorkflows(ctx, nil, workerWorkflowQueryLimit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runs, err := runtime.store.ListRunSummaries(ctx, "")
|
||||
counts, err := runtime.store.CountWorkerRuns(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -772,10 +784,32 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, runs, settings)
|
||||
return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings)
|
||||
}
|
||||
|
||||
func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Context, workflows []domain.WorkflowRecord, runs []domain.RunSummary, settings *domain.SettingsRecord) error {
|
||||
counts := store.WorkerRunCounts{
|
||||
ActiveByWorkflow: map[string]int{},
|
||||
InflightByWorkflow: map[string]int{},
|
||||
}
|
||||
for _, run := range runs {
|
||||
switch run.Status {
|
||||
case domain.RunStatusQueued:
|
||||
counts.QueuedTotal++
|
||||
counts.InflightByWorkflow[run.WorkflowID]++
|
||||
case domain.RunStatusRunning, domain.RunStatusStopping:
|
||||
counts.ActiveTotal++
|
||||
counts.ActiveByWorkflow[run.WorkflowID]++
|
||||
counts.InflightByWorkflow[run.WorkflowID]++
|
||||
case domain.RunStatusWaitingForAudit:
|
||||
counts.ActiveByWorkflow[run.WorkflowID]++
|
||||
counts.InflightByWorkflow[run.WorkflowID]++
|
||||
}
|
||||
}
|
||||
return runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings)
|
||||
}
|
||||
|
||||
func (runtime *Runtime) enqueueStreamingWorkflowsWithCounts(ctx context.Context, workflows []domain.WorkflowRecord, counts *store.WorkerRunCounts, settings *domain.SettingsRecord) error {
|
||||
now := time.Now().UTC()
|
||||
for _, workflow := range workflows {
|
||||
if !shouldEnqueueScheduledRun(workflow, now) {
|
||||
@ -791,14 +825,7 @@ func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Conte
|
||||
continue
|
||||
}
|
||||
}
|
||||
inflightCount := 0
|
||||
for _, run := range runs {
|
||||
if run.WorkflowID == workflow.ID {
|
||||
if _, ok := queuedOrActiveStatuses[run.Status]; ok {
|
||||
inflightCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
inflightCount := counts.InflightByWorkflow[workflow.ID]
|
||||
if inflightCount >= effectiveWorkflowConcurrencyLimit(&workflow, settings) {
|
||||
continue
|
||||
}
|
||||
@ -817,6 +844,8 @@ func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Conte
|
||||
return err
|
||||
}
|
||||
}
|
||||
counts.QueuedTotal++
|
||||
counts.InflightByWorkflow[workflow.ID]++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -948,7 +977,7 @@ func (runtime *Runtime) orphanMinimumRuntimeAge() time.Duration {
|
||||
return runtime.orphanMinimumAge
|
||||
}
|
||||
|
||||
func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context, summaries []domain.RunSummary) {
|
||||
func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context) {
|
||||
now := time.Now().UTC()
|
||||
interval := runtime.orphanCheckInterval
|
||||
if interval <= 0 {
|
||||
@ -966,6 +995,11 @@ func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context, summaries
|
||||
runtime.metrics.RecordOrphanRuntimes(0)
|
||||
return
|
||||
}
|
||||
summaries, err := runtime.store.ListRunSummaries(ctx, "")
|
||||
if err != nil {
|
||||
fmt.Printf("[worker] list run ids for orphan reconciliation failed: %v\n", err)
|
||||
return
|
||||
}
|
||||
knownRunIDs := make(map[string]struct{}, len(summaries))
|
||||
for _, summary := range summaries {
|
||||
knownRunIDs[summary.ID] = struct{}{}
|
||||
@ -990,28 +1024,56 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
||||
return processErr
|
||||
}
|
||||
runtime.updateEffectivePollInterval(settings)
|
||||
workflows, err := runtime.store.ListAllWorkflows(ctx)
|
||||
reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
|
||||
if err != nil {
|
||||
processErr = err
|
||||
return processErr
|
||||
}
|
||||
summaries, err := runtime.store.ListRunSummaries(ctx, "")
|
||||
if err := runtime.reconcileActiveRunsWithSummaries(ctx, reconcileRuns); err != nil {
|
||||
processErr = err
|
||||
return processErr
|
||||
}
|
||||
queuedPage, err := runtime.store.ListRunSummariesPage(ctx, domain.RunSummaryQuery{
|
||||
Status: domain.RunStatusQueued,
|
||||
Limit: workerQueuedRunLimit,
|
||||
})
|
||||
if err != nil {
|
||||
processErr = err
|
||||
return processErr
|
||||
}
|
||||
if err := runtime.reconcileActiveRunsWithSummaries(ctx, summaries); err != nil {
|
||||
counts, err := runtime.store.CountWorkerRuns(ctx)
|
||||
if err != nil {
|
||||
processErr = err
|
||||
return processErr
|
||||
}
|
||||
if err := runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, summaries, settings); err != nil {
|
||||
workflowIDs := make([]string, 0, len(queuedPage.Items))
|
||||
workflowSet := make(map[string]struct{}, len(queuedPage.Items))
|
||||
for _, run := range queuedPage.Items {
|
||||
if _, exists := workflowSet[run.WorkflowID]; exists {
|
||||
continue
|
||||
}
|
||||
workflowSet[run.WorkflowID] = struct{}{}
|
||||
workflowIDs = append(workflowIDs, run.WorkflowID)
|
||||
}
|
||||
workflows, err := runtime.store.ListWorkerWorkflows(ctx, workflowIDs, workerWorkflowQueryLimit)
|
||||
if err != nil {
|
||||
processErr = err
|
||||
return processErr
|
||||
}
|
||||
runtime.reconcileOrphanedRuntimes(ctx, summaries)
|
||||
runs := runRecordsFromSummaries(summaries)
|
||||
selectedRunIDs, queueDepth, runningCount := selectQueuedRunsForDispatchWithStats(runs, workflows, settings)
|
||||
runtime.metrics.RecordDispatch(queueDepth, runningCount, len(selectedRunIDs))
|
||||
if err := runtime.enqueueStreamingWorkflowsWithCounts(ctx, workflows, &counts, settings); err != nil {
|
||||
processErr = err
|
||||
return processErr
|
||||
}
|
||||
runtime.reconcileOrphanedRuntimes(ctx)
|
||||
queuedRuns := runRecordsFromSummaries(queuedPage.Items)
|
||||
selectedRunIDs := selectQueuedRunsForDispatchSnapshot(
|
||||
queuedRuns,
|
||||
workflows,
|
||||
settings,
|
||||
counts.ActiveByWorkflow,
|
||||
counts.ActiveTotal,
|
||||
)
|
||||
runtime.metrics.RecordDispatch(counts.QueuedTotal, counts.ActiveTotal, len(selectedRunIDs))
|
||||
for _, runID := range selectedRunIDs {
|
||||
if !runtime.tryMarkRunActive(runID) {
|
||||
continue
|
||||
@ -1366,7 +1428,7 @@ func (runtime *Runtime) reconcileActiveRunsWithSummaries(ctx context.Context, ru
|
||||
}
|
||||
|
||||
func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error {
|
||||
runs, err := runtime.store.ListRunSummaries(ctx, "")
|
||||
runs, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -43,8 +43,10 @@ POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}"
|
||||
POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}"
|
||||
|
||||
if [[ -z "${IMAGE_TAG:-}" ]]; then
|
||||
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)"
|
||||
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(date +%Y%m%d%H%M%S)"
|
||||
fi
|
||||
POC_KANIKO_SOURCE_COMMIT="${POC_KANIKO_SOURCE_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
|
||||
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
|
||||
# shellcheck source=scripts/lib/image-config.sh
|
||||
source "$ROOT_DIR/scripts/lib/image-config.sh"
|
||||
@ -176,6 +178,8 @@ YAML
|
||||
printf ' - --build-arg=OPERATOR_GPU_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
|
||||
printf ' - --build-arg=OPERATOR_HOLOBRAIN_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
|
||||
printf ' - --build-arg=OPERATOR_YOLO_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
|
||||
printf ' - --build-arg=DROBOTICFLOW_BUILD_COMMIT=%s\n' "$POC_KANIKO_SOURCE_COMMIT"
|
||||
printf ' - --build-arg=DROBOTICFLOW_BUILD_TIME=%s\n' "$POC_KANIKO_BUILD_TIME"
|
||||
IFS=',' read -r -a mirrors <<<"$POC_KANIKO_REGISTRY_MIRRORS"
|
||||
for mirror in "${mirrors[@]}"; do
|
||||
mirror="$(printf '%s' "$mirror" | xargs)"
|
||||
|
||||
@ -1817,6 +1817,9 @@ export default function WorkflowEditor() {
|
||||
let controller: AbortController | null = null;
|
||||
|
||||
const refreshLogs = async () => {
|
||||
if (cancelled || document.visibilityState !== "visible") {
|
||||
return;
|
||||
}
|
||||
controller?.abort();
|
||||
controller = new AbortController();
|
||||
setIsLiveRunLogsLoading(true);
|
||||
@ -1837,18 +1840,31 @@ export default function WorkflowEditor() {
|
||||
} finally {
|
||||
if (!cancelled) {
|
||||
setIsLiveRunLogsLoading(false);
|
||||
if (activeRunInFlight) {
|
||||
if (activeRunInFlight && document.visibilityState === "visible") {
|
||||
timer = window.setTimeout(refreshLogs, 5000);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void refreshLogs();
|
||||
const handleVisibilityChange = () => {
|
||||
window.clearTimeout(timer);
|
||||
if (document.visibilityState === "hidden") {
|
||||
controller?.abort();
|
||||
return;
|
||||
}
|
||||
void refreshLogs();
|
||||
};
|
||||
|
||||
if (document.visibilityState === "visible") {
|
||||
void refreshLogs();
|
||||
}
|
||||
document.addEventListener("visibilitychange", handleVisibilityChange);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearTimeout(timer);
|
||||
controller?.abort();
|
||||
document.removeEventListener("visibilitychange", handleVisibilityChange);
|
||||
};
|
||||
}, [activeRun?.id, activeRunInFlight, liveRunLogsRefreshToken, nodeConfigPanelTab, selectedNode?.id]);
|
||||
|
||||
|
||||
@ -111,11 +111,13 @@ def run_script(
|
||||
"POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS",
|
||||
"POC_KANIKO_CONTEXT_SOURCE",
|
||||
"POC_KANIKO_BUILD_OPERATORS",
|
||||
"POC_KANIKO_BUILD_TIME",
|
||||
"POC_KANIKO_OPERATOR_MANIFEST",
|
||||
"POC_KANIKO_OPERATOR_CONCURRENCY",
|
||||
"POC_KANIKO_OPERATOR_RETRIES",
|
||||
"POC_KANIKO_IMAGE",
|
||||
"POC_KANIKO_NODE_NAME",
|
||||
"POC_KANIKO_SOURCE_COMMIT",
|
||||
"POC_KANIKO_GOPROXY",
|
||||
"POC_KANIKO_REGISTRY_MIRRORS",
|
||||
"POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK",
|
||||
@ -161,6 +163,8 @@ def test_poc_kaniko_build_creates_three_jobs_with_registry_mirror(tmp_path):
|
||||
"POC_KANIKO_CLEAN_CONTEXT": "false",
|
||||
"POC_KANIKO_GOPROXY": "https://goproxy.cn,direct",
|
||||
"POC_KANIKO_NODE_NAME": "worker-cpu-01",
|
||||
"POC_KANIKO_SOURCE_COMMIT": "0123456789abcdef",
|
||||
"POC_KANIKO_BUILD_TIME": "2026-07-10T10:30:00Z",
|
||||
},
|
||||
)
|
||||
|
||||
@ -177,6 +181,8 @@ def test_poc_kaniko_build_creates_three_jobs_with_registry_mirror(tmp_path):
|
||||
manifests = "\n---\n".join(path.read_text(encoding="utf-8") for path in sorted(applied_dir.glob("*.yaml")))
|
||||
assert manifests.count("kind: Job") == 3
|
||||
assert "--build-arg=GOPROXY=https://goproxy.cn,direct" in manifests
|
||||
assert "--build-arg=DROBOTICFLOW_BUILD_COMMIT=0123456789abcdef" in manifests
|
||||
assert "--build-arg=DROBOTICFLOW_BUILD_TIME=2026-07-10T10:30:00Z" in manifests
|
||||
assert "--registry-mirror=mirror.gcr.io" in manifests
|
||||
assert "--skip-default-registry-fallback" in manifests
|
||||
assert "Dockerfile.api" in manifests
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user