diff --git a/apps/runner/src/log.ts b/apps/runner/src/log.ts index 2596e18..fcb2e9c 100644 --- a/apps/runner/src/log.ts +++ b/apps/runner/src/log.ts @@ -5,3 +5,21 @@ export function log(level: 'info' | 'warn' | 'error', message: string, fields: Record = {}) { console.error(JSON.stringify({ time: new Date().toISOString(), level, message, ...fields })); } + +/** How long after start a refused connection still means the control plane is starting with us. */ +export const STARTUP_GRACE_MS = 120_000; + +/** + * The local service starts the Runner together with the control plane, so its first polls are + * refused until the API listens. Until the Runner has reached it once, and for a limited time, + * that is expected rather than a failure worth an error line every few seconds. + */ +export function startingUp(error: unknown, reachedOnce: boolean, startedAt: number, now = Date.now()) { + if (reachedOnce || now - startedAt >= STARTUP_GRACE_MS) return false; + const failure = error as { code?: unknown; cause?: { code?: unknown }; message?: unknown } | undefined; + return ( + failure?.code === 'ECONNREFUSED' || + failure?.cause?.code === 'ECONNREFUSED' || + /\bECONNREFUSED\b/.test(String(failure?.message ?? '')) + ); +} diff --git a/apps/runner/src/main.ts b/apps/runner/src/main.ts index 65fd7a9..35a2e34 100644 --- a/apps/runner/src/main.ts +++ b/apps/runner/src/main.ts @@ -10,7 +10,7 @@ import { OutboundRunner } from './runner.js'; import { createRunnerTransport } from './transport.js'; import { createRunnerAdapters } from './adapters.js'; import { RunnerHealth } from './health.js'; -import { log } from './log.js'; +import { log, startingUp } from './log.js'; async function main() { requireDeploymentMode(process.env); @@ -52,6 +52,9 @@ async function main() { process.once('SIGINT', () => { stopped = true; }); + const startedAt = Date.now(); + let reachedOnce = false; + let waitingLogged = false; try { while (!stopped) { let phase: 'poll' | 'task' = 'poll'; @@ -59,6 +62,8 @@ async function main() { const task = (await request( `/internal/runner/tasks?environmentId=${encodeURIComponent(process.env.WF_ENVIRONMENT_ID!)}`, )) as SignedRunnerTask | null; + if (!reachedOnce) log('info', 'Connected to the control plane'); + reachedOnce = true; await health.update(task ? 'executing' : 'idle', true); if (task) { phase = 'task'; @@ -78,7 +83,10 @@ async function main() { } } catch (error) { await health.update(phase === 'poll' ? 'poll_error' : 'task_error'); - log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase }); + if (phase === 'poll' && startingUp(error, reachedOnce, startedAt)) { + if (!waitingLogged) log('info', 'Waiting for the control plane to start'); + waitingLogged = true; + } else log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase }); } await new Promise(resolve => setTimeout(resolve, 5_000)); } diff --git a/tests/runner-log.test.ts b/tests/runner-log.test.ts new file mode 100644 index 0000000..69c8b42 --- /dev/null +++ b/tests/runner-log.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { STARTUP_GRACE_MS, startingUp } from '../apps/runner/src/log.js'; + +describe('Runner log while the local service starts', () => { + const refused = Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:4000'), { code: 'ECONNREFUSED' }); + const started = 1_000_000; + + it('treats a refused connection as the control plane still starting, only before it first answers', () => { + expect(startingUp(refused, false, started, started + 5_000)).toBe(true); + // fetch wraps the socket error as its cause. + expect(startingUp(new TypeError('fetch failed', { cause: refused }), false, started, started + 5_000)).toBe(true); + expect(startingUp(refused, true, started, started + 5_000)).toBe(false); + }); + it('reports a control plane that never comes up, and any other failure, as errors', () => { + expect(startingUp(refused, false, started, started + STARTUP_GRACE_MS)).toBe(false); + expect(startingUp(new Error('Runner token rejected'), false, started, started + 5_000)).toBe(false); + expect(startingUp(undefined, false, started, started + 5_000)).toBe(false); + }); +});