From ddb4cf5b5f627696b100d3887b9f14be301ece26 Mon Sep 17 00:00:00 2001 From: eust-w Date: Mon, 26 Oct 2026 14:54:01 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Stop=20the=20Runner=20reporting?= =?UTF-8?q?=20the=20control=20plane's=20own=20start-up=20as=20errors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local service starts the Runner together with the control plane, so its first polls were refused until the API listened, and every restart left an ECONNREFUSED error line every five seconds in runner.log that looked like a failure. Until the Runner has reached the control plane once, and for two minutes at most, a refused connection is now noted once as waiting for it to start; reaching it is noted too. A control plane that never comes up, and any other failure, are still errors. Co-Authored-By: Claude Opus 5.5 --- apps/runner/src/log.ts | 18 ++++++++++++++++++ apps/runner/src/main.ts | 12 ++++++++++-- tests/runner-log.test.ts | 19 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/runner-log.test.ts diff --git a/apps/runner/src/log.ts b/apps/runner/src/log.ts index 2596e18..fcb2e9c 100644 --- a/apps/runner/src/log.ts +++ b/apps/runner/src/log.ts @@ -5,3 +5,21 @@ export function log(level: 'info' | 'warn' | 'error', message: string, fields: Record = {}) { console.error(JSON.stringify({ time: new Date().toISOString(), level, message, ...fields })); } + +/** How long after start a refused connection still means the control plane is starting with us. */ +export const STARTUP_GRACE_MS = 120_000; + +/** + * The local service starts the Runner together with the control plane, so its first polls are + * refused until the API listens. Until the Runner has reached it once, and for a limited time, + * that is expected rather than a failure worth an error line every few seconds. + */ +export function startingUp(error: unknown, reachedOnce: boolean, startedAt: number, now = Date.now()) { + if (reachedOnce || now - startedAt >= STARTUP_GRACE_MS) return false; + const failure = error as { code?: unknown; cause?: { code?: unknown }; message?: unknown } | undefined; + return ( + failure?.code === 'ECONNREFUSED' || + failure?.cause?.code === 'ECONNREFUSED' || + /\bECONNREFUSED\b/.test(String(failure?.message ?? '')) + ); +} diff --git a/apps/runner/src/main.ts b/apps/runner/src/main.ts index 65fd7a9..35a2e34 100644 --- a/apps/runner/src/main.ts +++ b/apps/runner/src/main.ts @@ -10,7 +10,7 @@ import { OutboundRunner } from './runner.js'; import { createRunnerTransport } from './transport.js'; import { createRunnerAdapters } from './adapters.js'; import { RunnerHealth } from './health.js'; -import { log } from './log.js'; +import { log, startingUp } from './log.js'; async function main() { requireDeploymentMode(process.env); @@ -52,6 +52,9 @@ async function main() { process.once('SIGINT', () => { stopped = true; }); + const startedAt = Date.now(); + let reachedOnce = false; + let waitingLogged = false; try { while (!stopped) { let phase: 'poll' | 'task' = 'poll'; @@ -59,6 +62,8 @@ async function main() { const task = (await request( `/internal/runner/tasks?environmentId=${encodeURIComponent(process.env.WF_ENVIRONMENT_ID!)}`, )) as SignedRunnerTask | null; + if (!reachedOnce) log('info', 'Connected to the control plane'); + reachedOnce = true; await health.update(task ? 'executing' : 'idle', true); if (task) { phase = 'task'; @@ -78,7 +83,10 @@ async function main() { } } catch (error) { await health.update(phase === 'poll' ? 'poll_error' : 'task_error'); - log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase }); + if (phase === 'poll' && startingUp(error, reachedOnce, startedAt)) { + if (!waitingLogged) log('info', 'Waiting for the control plane to start'); + waitingLogged = true; + } else log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase }); } await new Promise(resolve => setTimeout(resolve, 5_000)); } diff --git a/tests/runner-log.test.ts b/tests/runner-log.test.ts new file mode 100644 index 0000000..69c8b42 --- /dev/null +++ b/tests/runner-log.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { STARTUP_GRACE_MS, startingUp } from '../apps/runner/src/log.js'; + +describe('Runner log while the local service starts', () => { + const refused = Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:4000'), { code: 'ECONNREFUSED' }); + const started = 1_000_000; + + it('treats a refused connection as the control plane still starting, only before it first answers', () => { + expect(startingUp(refused, false, started, started + 5_000)).toBe(true); + // fetch wraps the socket error as its cause. + expect(startingUp(new TypeError('fetch failed', { cause: refused }), false, started, started + 5_000)).toBe(true); + expect(startingUp(refused, true, started, started + 5_000)).toBe(false); + }); + it('reports a control plane that never comes up, and any other failure, as errors', () => { + expect(startingUp(refused, false, started, started + STARTUP_GRACE_MS)).toBe(false); + expect(startingUp(new Error('Runner token rejected'), false, started, started + 5_000)).toBe(false); + expect(startingUp(undefined, false, started, started + 5_000)).toBe(false); + }); +});