🐛 Stop the Runner reporting the control plane's own start-up as errors

The local service starts the Runner together with the control plane,
so its first polls were refused until the API listened, and every
restart left an ECONNREFUSED error line every five seconds in
runner.log that looked like a failure.

Until the Runner has reached the control plane once, and for two
minutes at most, a refused connection is now noted once as waiting for
it to start; reaching it is noted too. A control plane that never
comes up, and any other failure, are still errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
eust-w 2026-10-26 14:54:01 +08:00
parent 01acc236a0
commit ddb4cf5b5f
3 changed files with 47 additions and 2 deletions

View File

@ -5,3 +5,21 @@
export function log(level: 'info' | 'warn' | 'error', message: string, fields: Record<string, unknown> = {}) {
console.error(JSON.stringify({ time: new Date().toISOString(), level, message, ...fields }));
}
/** How long after start a refused connection still means the control plane is starting with us. */
export const STARTUP_GRACE_MS = 120_000;
/**
* The local service starts the Runner together with the control plane, so its first polls are
* refused until the API listens. Until the Runner has reached it once, and for a limited time,
* that is expected rather than a failure worth an error line every few seconds.
*/
export function startingUp(error: unknown, reachedOnce: boolean, startedAt: number, now = Date.now()) {
if (reachedOnce || now - startedAt >= STARTUP_GRACE_MS) return false;
const failure = error as { code?: unknown; cause?: { code?: unknown }; message?: unknown } | undefined;
return (
failure?.code === 'ECONNREFUSED' ||
failure?.cause?.code === 'ECONNREFUSED' ||
/\bECONNREFUSED\b/.test(String(failure?.message ?? ''))
);
}

View File

@ -10,7 +10,7 @@ import { OutboundRunner } from './runner.js';
import { createRunnerTransport } from './transport.js';
import { createRunnerAdapters } from './adapters.js';
import { RunnerHealth } from './health.js';
import { log } from './log.js';
import { log, startingUp } from './log.js';
async function main() {
requireDeploymentMode(process.env);
@ -52,6 +52,9 @@ async function main() {
process.once('SIGINT', () => {
stopped = true;
});
const startedAt = Date.now();
let reachedOnce = false;
let waitingLogged = false;
try {
while (!stopped) {
let phase: 'poll' | 'task' = 'poll';
@ -59,6 +62,8 @@ async function main() {
const task = (await request(
`/internal/runner/tasks?environmentId=${encodeURIComponent(process.env.WF_ENVIRONMENT_ID!)}`,
)) as SignedRunnerTask | null;
if (!reachedOnce) log('info', 'Connected to the control plane');
reachedOnce = true;
await health.update(task ? 'executing' : 'idle', true);
if (task) {
phase = 'task';
@ -78,7 +83,10 @@ async function main() {
}
} catch (error) {
await health.update(phase === 'poll' ? 'poll_error' : 'task_error');
log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase });
if (phase === 'poll' && startingUp(error, reachedOnce, startedAt)) {
if (!waitingLogged) log('info', 'Waiting for the control plane to start');
waitingLogged = true;
} else log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase });
}
await new Promise(resolve => setTimeout(resolve, 5_000));
}

19
tests/runner-log.test.ts Normal file
View File

@ -0,0 +1,19 @@
import { describe, expect, it } from 'vitest';
import { STARTUP_GRACE_MS, startingUp } from '../apps/runner/src/log.js';
describe('Runner log while the local service starts', () => {
const refused = Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:4000'), { code: 'ECONNREFUSED' });
const started = 1_000_000;
it('treats a refused connection as the control plane still starting, only before it first answers', () => {
expect(startingUp(refused, false, started, started + 5_000)).toBe(true);
// fetch wraps the socket error as its cause.
expect(startingUp(new TypeError('fetch failed', { cause: refused }), false, started, started + 5_000)).toBe(true);
expect(startingUp(refused, true, started, started + 5_000)).toBe(false);
});
it('reports a control plane that never comes up, and any other failure, as errors', () => {
expect(startingUp(refused, false, started, started + STARTUP_GRACE_MS)).toBe(false);
expect(startingUp(new Error('Runner token rejected'), false, started, started + 5_000)).toBe(false);
expect(startingUp(undefined, false, started, started + 5_000)).toBe(false);
});
});