🐛 Stop the Runner reporting the control plane's own start-up as errors
The local service starts the Runner together with the control plane, so its first polls were refused until the API listened, and every restart left an ECONNREFUSED error line every five seconds in runner.log that looked like a failure. Until the Runner has reached the control plane once, and for two minutes at most, a refused connection is now noted once as waiting for it to start; reaching it is noted too. A control plane that never comes up, and any other failure, are still errors. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
01acc236a0
commit
ddb4cf5b5f
@ -5,3 +5,21 @@
|
||||
export function log(level: 'info' | 'warn' | 'error', message: string, fields: Record<string, unknown> = {}) {
|
||||
console.error(JSON.stringify({ time: new Date().toISOString(), level, message, ...fields }));
|
||||
}
|
||||
|
||||
/** How long after start a refused connection still means the control plane is starting with us. */
|
||||
export const STARTUP_GRACE_MS = 120_000;
|
||||
|
||||
/**
|
||||
* The local service starts the Runner together with the control plane, so its first polls are
|
||||
* refused until the API listens. Until the Runner has reached it once, and for a limited time,
|
||||
* that is expected rather than a failure worth an error line every few seconds.
|
||||
*/
|
||||
export function startingUp(error: unknown, reachedOnce: boolean, startedAt: number, now = Date.now()) {
|
||||
if (reachedOnce || now - startedAt >= STARTUP_GRACE_MS) return false;
|
||||
const failure = error as { code?: unknown; cause?: { code?: unknown }; message?: unknown } | undefined;
|
||||
return (
|
||||
failure?.code === 'ECONNREFUSED' ||
|
||||
failure?.cause?.code === 'ECONNREFUSED' ||
|
||||
/\bECONNREFUSED\b/.test(String(failure?.message ?? ''))
|
||||
);
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@ import { OutboundRunner } from './runner.js';
|
||||
import { createRunnerTransport } from './transport.js';
|
||||
import { createRunnerAdapters } from './adapters.js';
|
||||
import { RunnerHealth } from './health.js';
|
||||
import { log } from './log.js';
|
||||
import { log, startingUp } from './log.js';
|
||||
|
||||
async function main() {
|
||||
requireDeploymentMode(process.env);
|
||||
@ -52,6 +52,9 @@ async function main() {
|
||||
process.once('SIGINT', () => {
|
||||
stopped = true;
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
let reachedOnce = false;
|
||||
let waitingLogged = false;
|
||||
try {
|
||||
while (!stopped) {
|
||||
let phase: 'poll' | 'task' = 'poll';
|
||||
@ -59,6 +62,8 @@ async function main() {
|
||||
const task = (await request(
|
||||
`/internal/runner/tasks?environmentId=${encodeURIComponent(process.env.WF_ENVIRONMENT_ID!)}`,
|
||||
)) as SignedRunnerTask | null;
|
||||
if (!reachedOnce) log('info', 'Connected to the control plane');
|
||||
reachedOnce = true;
|
||||
await health.update(task ? 'executing' : 'idle', true);
|
||||
if (task) {
|
||||
phase = 'task';
|
||||
@ -78,7 +83,10 @@ async function main() {
|
||||
}
|
||||
} catch (error) {
|
||||
await health.update(phase === 'poll' ? 'poll_error' : 'task_error');
|
||||
log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase });
|
||||
if (phase === 'poll' && startingUp(error, reachedOnce, startedAt)) {
|
||||
if (!waitingLogged) log('info', 'Waiting for the control plane to start');
|
||||
waitingLogged = true;
|
||||
} else log('error', error instanceof Error ? error.message : 'Runner poll failed', { phase });
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 5_000));
|
||||
}
|
||||
|
||||
19
tests/runner-log.test.ts
Normal file
19
tests/runner-log.test.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { STARTUP_GRACE_MS, startingUp } from '../apps/runner/src/log.js';
|
||||
|
||||
describe('Runner log while the local service starts', () => {
|
||||
const refused = Object.assign(new Error('connect ECONNREFUSED 127.0.0.1:4000'), { code: 'ECONNREFUSED' });
|
||||
const started = 1_000_000;
|
||||
|
||||
it('treats a refused connection as the control plane still starting, only before it first answers', () => {
|
||||
expect(startingUp(refused, false, started, started + 5_000)).toBe(true);
|
||||
// fetch wraps the socket error as its cause.
|
||||
expect(startingUp(new TypeError('fetch failed', { cause: refused }), false, started, started + 5_000)).toBe(true);
|
||||
expect(startingUp(refused, true, started, started + 5_000)).toBe(false);
|
||||
});
|
||||
it('reports a control plane that never comes up, and any other failure, as errors', () => {
|
||||
expect(startingUp(refused, false, started, started + STARTUP_GRACE_MS)).toBe(false);
|
||||
expect(startingUp(new Error('Runner token rejected'), false, started, started + 5_000)).toBe(false);
|
||||
expect(startingUp(undefined, false, started, started + 5_000)).toBe(false);
|
||||
});
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user