🐛 Give a blue-green candidate and a fresh switch time to settle
The Compose and Kubernetes adapters checked a candidate once. A container that had started but did not listen yet made the request throw, and the deployment was reported as of unknown state, to be reconciled by hand; one stray error from the stable entry right after the switch rolled a healthy release back. The candidate checks and the checks after the switch are now repeated, six attempts two to ten seconds apart, as the POC's already were, and the Runner logs each repeat. A candidate that never answers is reported as not switched, with the previous release still serving. A missing address or missing checks are reported at once instead of being asked again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
6722319314
commit
821c059081
@ -29,25 +29,37 @@ export async function createRunnerAdapters(env: Record<string, string | undefine
|
||||
),
|
||||
};
|
||||
}
|
||||
// Candidates that take a moment to answer, and entries still settling after a switch, are told in the log.
|
||||
const patience = { ...DEFAULT_PATIENCE, onRetry: (notice: string) => log('warn', notice) };
|
||||
return {
|
||||
compose: new ComposeDeploymentAdapter({
|
||||
secretBroker: new DeploymentSecretBroker(),
|
||||
dockerConfig: env.WF_DEPLOYMENT_DOCKER_CONFIG,
|
||||
directory: env.WF_COMPOSE_DIRECTORY,
|
||||
project: env.WF_COMPOSE_PROJECT,
|
||||
healthUrl: env.WF_HEALTH_URL,
|
||||
dockerContext: env.WF_DOCKER_CONTEXT,
|
||||
candidateHealthUrlTemplate: env.WF_CANDIDATE_HEALTH_URL_TEMPLATE,
|
||||
proxyAdminUrl: env.WF_CADDY_ADMIN_URL,
|
||||
proxyConfigPath: env.WF_CADDY_PROXY_CONFIG_PATH,
|
||||
proxyDialHost: env.WF_CADDY_DIAL_HOST,
|
||||
}),
|
||||
kubernetes: new KubernetesDeploymentAdapter({
|
||||
secretBroker: new DeploymentSecretBroker(),
|
||||
context: env.WF_KUBE_CONTEXT,
|
||||
namespace: env.WF_KUBE_NAMESPACE,
|
||||
healthUrl: env.WF_HEALTH_URL,
|
||||
candidateHealthUrlTemplate: env.WF_CANDIDATE_HEALTH_URL_TEMPLATE,
|
||||
}),
|
||||
compose: new ComposeDeploymentAdapter(
|
||||
{
|
||||
secretBroker: new DeploymentSecretBroker(),
|
||||
dockerConfig: env.WF_DEPLOYMENT_DOCKER_CONFIG,
|
||||
directory: env.WF_COMPOSE_DIRECTORY,
|
||||
project: env.WF_COMPOSE_PROJECT,
|
||||
healthUrl: env.WF_HEALTH_URL,
|
||||
dockerContext: env.WF_DOCKER_CONTEXT,
|
||||
candidateHealthUrlTemplate: env.WF_CANDIDATE_HEALTH_URL_TEMPLATE,
|
||||
proxyAdminUrl: env.WF_CADDY_ADMIN_URL,
|
||||
proxyConfigPath: env.WF_CADDY_PROXY_CONFIG_PATH,
|
||||
proxyDialHost: env.WF_CADDY_DIAL_HOST,
|
||||
},
|
||||
undefined,
|
||||
undefined,
|
||||
patience,
|
||||
),
|
||||
kubernetes: new KubernetesDeploymentAdapter(
|
||||
{
|
||||
secretBroker: new DeploymentSecretBroker(),
|
||||
context: env.WF_KUBE_CONTEXT,
|
||||
namespace: env.WF_KUBE_NAMESPACE,
|
||||
healthUrl: env.WF_HEALTH_URL,
|
||||
candidateHealthUrlTemplate: env.WF_CANDIDATE_HEALTH_URL_TEMPLATE,
|
||||
},
|
||||
undefined,
|
||||
undefined,
|
||||
patience,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@ -168,6 +168,8 @@ provisioner 用 `WF_THEIA_IMAGE`(digest)创建工作区;配置 `WF_OCT_WOR
|
||||
| Compose | `WF_COMPOSE_DIRECTORY`、`WF_COMPOSE_PROJECT`、`WF_HEALTH_URL`(稳定入口)、`WF_CANDIDATE_HEALTH_URL_TEMPLATE`(含 `{port}`)、Caddy 的 `WF_CADDY_ADMIN_URL`、`WF_CADDY_PROXY_CONFIG_PATH`(以 `/config/` 开头)与 `WF_CADDY_DIAL_HOST`;可选 `WF_DOCKER_CONTEXT`、`WF_DEPLOYMENT_DOCKER_CONFIG`(拉取镜像的凭据) |
|
||||
| Kubernetes | `WF_KUBE_CONTEXT`、`WF_KUBE_NAMESPACE`、`WF_HEALTH_URL`、`WF_CANDIDATE_HEALTH_URL_TEMPLATE`(含 `{service}`,可含 `{namespace}`);所需权限见 [infra/k8s/README.md](../infra/k8s/README.md#部署目标) |
|
||||
|
||||
新版本先作为候选启动,健康与业务检查通过后才切流,切流后再检查稳定入口,失败时切回旧版本。刚启动的应用可能还没开始监听,刚切换的入口也可能还连着旧连接,所以这两步的检查各试 6 次,间隔 2 至 10 秒,每次重试都写进 Runner 日志。候选始终没有应答时不切流,旧版本继续服务。
|
||||
|
||||
3. API 使用 `WF_RUNNER_SIGNING_KEY_FILE`(Ed25519 私钥,`WF_RUNNER_SIGNING_KEY_ID` 为密钥 ID)签名部署任务。
|
||||
4. 部署密钥通过 Vault KV v2 按固定版本注入,配置与映射文件格式见 [reference/deployment-secrets.md](reference/deployment-secrets.md)。
|
||||
5. 高风险或生产发布需要审批,审批 24 小时内有效,过期后发布失败,继续运行会重新请求发布。
|
||||
|
||||
@ -10,13 +10,26 @@ import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { safeIdentifier } from './git.js';
|
||||
import { executeCommand, type CommandExecutor } from './process.js';
|
||||
import { validateRelease, verifyHttp, deliveryConfiguration } from './deploy.js';
|
||||
import { DEFAULT_PATIENCE, type Patience } from './patience.js';
|
||||
import { validateRelease, verifyHttpPatiently, deliveryConfiguration } from './deploy.js';
|
||||
import { RuntimeError, type DeploymentAdapter, type DeploymentResult, type ReleaseManifest } from './types.js';
|
||||
|
||||
export const environmentResourceName = (release: ReleaseManifest) =>
|
||||
`wf-${createHash('sha256').update(`${release.tenantId}/${release.environmentId}`).digest('hex').slice(0, 20)}`;
|
||||
const revision = (id: string) => createHash('sha256').update(id).digest('hex').slice(0, 12);
|
||||
const candidateName = (release: ReleaseManifest) => `${environmentResourceName(release)}-${revision(release.id)}`;
|
||||
/**
|
||||
* Checks a candidate before it takes any traffic. It may still be starting, or its address still
|
||||
* reaching the network, so the checks are repeated; one that never answers simply did not pass,
|
||||
* and the previous release keeps serving.
|
||||
*/
|
||||
function checkCandidate(release: ReleaseManifest, base: string, fetcher: typeof fetch, patience: Patience) {
|
||||
return verifyHttpPatiently(release, base, fetcher, patience).catch(error => ({
|
||||
passed: false,
|
||||
checks: [] as string[],
|
||||
reason: `Candidate did not answer: ${error instanceof Error ? error.message : String(error)}`,
|
||||
}));
|
||||
}
|
||||
const identity = (release: ReleaseManifest) =>
|
||||
createHash('sha256')
|
||||
.update(
|
||||
@ -86,6 +99,8 @@ export class ComposeDeploymentAdapter implements DeploymentAdapter {
|
||||
private readonly target: ComposeTarget = {},
|
||||
private readonly execute: CommandExecutor = executeCommand,
|
||||
private readonly fetcher: typeof fetch = fetch,
|
||||
/** How long a candidate may take to answer, and the switched entry to settle. */
|
||||
private readonly patience: Patience = DEFAULT_PATIENCE,
|
||||
) {}
|
||||
private project(id: string) {
|
||||
return `${this.target.project ?? 'wishforge'}-${revision(id)}`;
|
||||
@ -291,10 +306,11 @@ export class ComposeDeploymentAdapter implements DeploymentAdapter {
|
||||
reason: 'Candidate identity is invalid; traffic was not switched',
|
||||
});
|
||||
const port = this.port(candidate[0]!, release.containerPort);
|
||||
const checks = await verifyHttp(
|
||||
const checks = await checkCandidate(
|
||||
release,
|
||||
this.target.candidateHealthUrlTemplate!.replaceAll('{port}', port),
|
||||
this.fetcher,
|
||||
this.patience,
|
||||
);
|
||||
if (!checks.passed)
|
||||
return output(release, 'compose', 'applied', { candidate: true, trafficSwitched: false, ...checks });
|
||||
@ -377,7 +393,7 @@ export class ComposeDeploymentAdapter implements DeploymentAdapter {
|
||||
trafficSwitched: false,
|
||||
reason: 'Candidate exists but stable proxy has not selected it',
|
||||
});
|
||||
const http = await verifyHttp(release, this.target.healthUrl, this.fetcher, {
|
||||
const http = await verifyHttpPatiently(release, this.target.healthUrl, this.fetcher, this.patience, {
|
||||
stable: true,
|
||||
});
|
||||
return output(release, 'compose', http.passed ? 'verified' : 'applied', {
|
||||
@ -405,6 +421,8 @@ export class KubernetesDeploymentAdapter implements DeploymentAdapter {
|
||||
private readonly target: KubernetesTarget = {},
|
||||
private readonly execute: CommandExecutor = executeCommand,
|
||||
private readonly fetcher: typeof fetch = fetch,
|
||||
/** How long a candidate may take to answer, and the switched service to settle. */
|
||||
private readonly patience: Patience = DEFAULT_PATIENCE,
|
||||
) {}
|
||||
private args(args: string[]) {
|
||||
return ['--context', this.target.context!, '--namespace', this.target.namespace!, ...args];
|
||||
@ -613,12 +631,13 @@ export class KubernetesDeploymentAdapter implements DeploymentAdapter {
|
||||
const candidate = await this.get('deployment', candidateName(release));
|
||||
if (!candidate || !this.ready(release, candidate))
|
||||
return output(release, 'kubernetes', 'applied', { trafficSwitched: false, reason: 'Candidate is not ready' });
|
||||
const http = await verifyHttp(
|
||||
const http = await checkCandidate(
|
||||
release,
|
||||
this.target
|
||||
.candidateHealthUrlTemplate!.replaceAll('{service}', candidateName(release))
|
||||
.replaceAll('{namespace}', this.target.namespace!),
|
||||
this.fetcher,
|
||||
this.patience,
|
||||
);
|
||||
if (!http.passed)
|
||||
return output(release, 'kubernetes', 'applied', { candidate: true, trafficSwitched: false, ...http });
|
||||
@ -724,7 +743,7 @@ export class KubernetesDeploymentAdapter implements DeploymentAdapter {
|
||||
reason: 'Stable service has not selected this candidate',
|
||||
});
|
||||
const http = this.ready(release, candidate)
|
||||
? await verifyHttp(release, this.target.healthUrl, this.fetcher, {
|
||||
? await verifyHttpPatiently(release, this.target.healthUrl, this.fetcher, this.patience, {
|
||||
stable: true,
|
||||
})
|
||||
: { passed: false, checks: [], reason: 'Candidate readiness pending' };
|
||||
|
||||
@ -195,6 +195,9 @@ export async function verifyHttpPatiently(
|
||||
patience: Patience,
|
||||
options: { stable?: boolean } = {},
|
||||
): ReturnType<typeof verifyHttp> {
|
||||
// A missing address, checks or delivery configuration cannot get better by asking again.
|
||||
if (!base || !release.smokeChecks?.length || (options.stable && deliveryConfiguration(release, base)))
|
||||
return verifyHttp(release, base, fetcher, options);
|
||||
let last: Awaited<ReturnType<typeof verifyHttp>> | undefined;
|
||||
try {
|
||||
return await repeat(
|
||||
|
||||
@ -84,7 +84,8 @@ it('validates a separate Kubernetes candidate before an atomic stable-service sw
|
||||
expect(rendered.items[0].metadata.name).toContain(rev);
|
||||
});
|
||||
|
||||
it('does not switch traffic when candidate smoke checks fail', async () => {
|
||||
/** A Kubernetes target whose candidate is ready and whose stable service still selects the previous release. */
|
||||
function kubernetesTarget(fetcher: typeof fetch, notices: string[] = []) {
|
||||
const commands: Command[] = [];
|
||||
const adapter = new KubernetesDeploymentAdapter(
|
||||
{
|
||||
@ -113,81 +114,164 @@ it('does not switch traffic when candidate smoke checks fail', async () => {
|
||||
stderr: '',
|
||||
};
|
||||
},
|
||||
async () => new Response('failed', { status: 500 }),
|
||||
fetcher,
|
||||
{ attempts: 3, delayMs: 1, onRetry: notice => notices.push(notice) },
|
||||
);
|
||||
return { adapter, commands };
|
||||
}
|
||||
|
||||
it('does not switch traffic when candidate smoke checks fail', async () => {
|
||||
let checks = 0;
|
||||
const { adapter, commands } = kubernetesTarget(async () => {
|
||||
checks++;
|
||||
return new Response('failed', { status: 500 });
|
||||
});
|
||||
expect((await adapter.deploy(release, { execute: true })).status).toBe('applied');
|
||||
expect(commands.some(command => command.args.includes('patch'))).toBe(false);
|
||||
// The candidate was asked as often as the patience allows before it was given up.
|
||||
expect(checks).toBe(3);
|
||||
});
|
||||
|
||||
it('uses a separate Compose project and ETag-guarded Caddy switch after candidate checks', async () => {
|
||||
it('leaves traffic on the previous release when a candidate never answers', async () => {
|
||||
const notices: string[] = [];
|
||||
const { adapter, commands } = kubernetesTarget(async () => {
|
||||
throw new TypeError('fetch failed');
|
||||
}, notices);
|
||||
const result = await adapter.deploy(release, { execute: true });
|
||||
expect(result.status).toBe('applied');
|
||||
expect(result.evidence).toMatchObject({
|
||||
trafficSwitched: false,
|
||||
reason: 'Candidate did not answer: fetch failed',
|
||||
});
|
||||
expect(commands.some(command => command.args.includes('patch'))).toBe(false);
|
||||
expect(notices).toEqual([
|
||||
'fetch failed; trying again in 1 s (attempt 2 of 3)',
|
||||
'fetch failed; trying again in 1 s (attempt 3 of 3)',
|
||||
]);
|
||||
});
|
||||
|
||||
/**
|
||||
* A Compose target behind Caddy: `answer` serves the candidate and, once the proxy selects it,
|
||||
* the stable entry; it gets the URL and whether traffic was switched yet.
|
||||
*/
|
||||
async function composeTarget(answer: (url: string, switched: boolean) => Promise<Response>, notices: string[] = []) {
|
||||
const directory = await mkdtemp(path.join(os.tmpdir(), 'wishforge-bluegreen-'));
|
||||
let selected = '127.0.0.1:41000';
|
||||
let healthy = 0;
|
||||
const commands: Command[] = [];
|
||||
try {
|
||||
const current = (isNew: boolean) => [
|
||||
{
|
||||
Config: {
|
||||
Image: release.image,
|
||||
Labels: {
|
||||
'wishforge.io/release': isNew ? release.id : 'old-release',
|
||||
'wishforge.io/tenant': 'tenant',
|
||||
'wishforge.io/environment': 'env',
|
||||
...(isNew ? JSON.parse(new ComposeDeploymentAdapter().render(release)).services.web.labels : {}),
|
||||
},
|
||||
const patches: string[] = [];
|
||||
const current = (isNew: boolean) => [
|
||||
{
|
||||
Config: {
|
||||
Image: release.image,
|
||||
Labels: {
|
||||
'wishforge.io/release': isNew ? release.id : 'old-release',
|
||||
'wishforge.io/tenant': 'tenant',
|
||||
'wishforge.io/environment': 'env',
|
||||
...(isNew ? JSON.parse(new ComposeDeploymentAdapter().render(release)).services.web.labels : {}),
|
||||
},
|
||||
State: { Running: true },
|
||||
NetworkSettings: { Ports: { '3000/tcp': [{ HostPort: isNew ? '41001' : '41000' }] } },
|
||||
},
|
||||
];
|
||||
const execute = async (command: Command) => {
|
||||
commands.push(command);
|
||||
if (command.args.includes('ps'))
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: command.args.join(' ').includes(createHash('sha256').update(release.id).digest('hex').slice(0, 12))
|
||||
? 'bbbb'
|
||||
: 'aaaa',
|
||||
stderr: '',
|
||||
};
|
||||
if (command.args.includes('inspect'))
|
||||
return { exitCode: 0, stdout: JSON.stringify(current(command.args.includes('bbbb'))), stderr: '' };
|
||||
return { exitCode: 0, stdout: '', stderr: '' };
|
||||
};
|
||||
const fetcher: typeof fetch = async (url, options) => {
|
||||
if (String(url).includes('/config/')) {
|
||||
if (options?.method === 'PATCH') {
|
||||
expect(healthy).toBe(2);
|
||||
expect(new Headers(options.headers).get('if-match')).toBe('"etag-1"');
|
||||
selected = '127.0.0.1:41001';
|
||||
return new Response('', { status: 200 });
|
||||
}
|
||||
return new Response(JSON.stringify({ handler: 'reverse_proxy', upstreams: [{ dial: selected }] }), {
|
||||
headers: { etag: '"etag-1"' },
|
||||
});
|
||||
State: { Running: true },
|
||||
NetworkSettings: { Ports: { '3000/tcp': [{ HostPort: isNew ? '41001' : '41000' }] } },
|
||||
},
|
||||
];
|
||||
const execute = async (command: Command) => {
|
||||
commands.push(command);
|
||||
if (command.args.includes('ps'))
|
||||
return {
|
||||
exitCode: 0,
|
||||
stdout: command.args.join(' ').includes(createHash('sha256').update(release.id).digest('hex').slice(0, 12))
|
||||
? 'bbbb'
|
||||
: 'aaaa',
|
||||
stderr: '',
|
||||
};
|
||||
if (command.args.includes('inspect'))
|
||||
return { exitCode: 0, stdout: JSON.stringify(current(command.args.includes('bbbb'))), stderr: '' };
|
||||
return { exitCode: 0, stdout: '', stderr: '' };
|
||||
};
|
||||
const fetcher: typeof fetch = async (url, options) => {
|
||||
if (String(url).includes('/config/')) {
|
||||
if (options?.method === 'PATCH') {
|
||||
expect(new Headers(options.headers).get('if-match')).toBe('"etag-1"');
|
||||
const body = JSON.parse(String(options.body));
|
||||
selected = body.upstreams[0].dial;
|
||||
patches.push(selected);
|
||||
return new Response('', { status: 200 });
|
||||
}
|
||||
healthy++;
|
||||
return new Response('ready');
|
||||
};
|
||||
const adapter = new ComposeDeploymentAdapter(
|
||||
{
|
||||
directory,
|
||||
project: 'project',
|
||||
healthUrl: 'http://stable.test',
|
||||
candidateHealthUrlTemplate: 'http://127.0.0.1:{port}',
|
||||
proxyAdminUrl: 'http://127.0.0.1:2019',
|
||||
proxyConfigPath: '/config/apps/http/servers/app/routes/0/handle/0',
|
||||
proxyDialHost: '127.0.0.1',
|
||||
},
|
||||
execute,
|
||||
fetcher,
|
||||
);
|
||||
expect((await adapter.deploy(release, { execute: true })).status).toBe('verified');
|
||||
expect(commands.some(command => command.args.includes('down') || command.args.includes('rm'))).toBe(false);
|
||||
expect(commands.find(command => command.args.includes('compose'))!.args).toContain(
|
||||
return new Response(JSON.stringify({ handler: 'reverse_proxy', upstreams: [{ dial: selected }] }), {
|
||||
headers: { etag: '"etag-1"' },
|
||||
});
|
||||
}
|
||||
return answer(String(url), selected === '127.0.0.1:41001');
|
||||
};
|
||||
const adapter = new ComposeDeploymentAdapter(
|
||||
{
|
||||
directory,
|
||||
project: 'project',
|
||||
healthUrl: 'http://stable.test',
|
||||
candidateHealthUrlTemplate: 'http://127.0.0.1:{port}',
|
||||
proxyAdminUrl: 'http://127.0.0.1:2019',
|
||||
proxyConfigPath: '/config/apps/http/servers/app/routes/0/handle/0',
|
||||
proxyDialHost: '127.0.0.1',
|
||||
},
|
||||
execute,
|
||||
fetcher,
|
||||
{ attempts: 4, delayMs: 1, onRetry: notice => notices.push(notice) },
|
||||
);
|
||||
return { adapter, commands, patches, cleanup: () => rm(directory, { recursive: true, force: true }) };
|
||||
}
|
||||
|
||||
it('uses a separate Compose project and ETag-guarded Caddy switch after candidate checks', async () => {
|
||||
const seen: string[] = [];
|
||||
const target = await composeTarget(async (url, switched) => {
|
||||
seen.push(`${switched ? 'switched' : 'candidate'} ${url}`);
|
||||
return new Response('ready');
|
||||
});
|
||||
try {
|
||||
expect((await target.adapter.deploy(release, { execute: true })).status).toBe('verified');
|
||||
// Health and the smoke check pass on the candidate before the proxy is touched, then on the stable entry.
|
||||
expect(seen).toEqual([
|
||||
'candidate http://127.0.0.1:41001/health',
|
||||
'candidate http://127.0.0.1:41001/',
|
||||
'switched http://stable.test/health',
|
||||
'switched http://stable.test/',
|
||||
]);
|
||||
expect(target.patches).toEqual(['127.0.0.1:41001']);
|
||||
expect(target.commands.some(command => command.args.includes('down') || command.args.includes('rm'))).toBe(false);
|
||||
expect(target.commands.find(command => command.args.includes('compose'))!.args).toContain(
|
||||
`project-${createHash('sha256').update(release.id).digest('hex').slice(0, 12)}`,
|
||||
);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
await target.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for a candidate that is still starting, and for the entry to settle after the switch', async () => {
|
||||
const notices: string[] = [];
|
||||
let refused = 0;
|
||||
let settling = 0;
|
||||
const target = await composeTarget(async (url, switched) => {
|
||||
// The container runs before its server listens.
|
||||
if (url.startsWith('http://127.0.0.1:41001') && refused < 2) {
|
||||
refused++;
|
||||
throw new TypeError('fetch failed');
|
||||
}
|
||||
// Right after the switch the entry answers once from a connection that was still closing.
|
||||
if (switched && url.startsWith('http://stable.test') && settling < 1) {
|
||||
settling++;
|
||||
return new Response('', { status: 502 });
|
||||
}
|
||||
return new Response('ready');
|
||||
}, notices);
|
||||
try {
|
||||
expect((await target.adapter.deploy(release, { execute: true })).status).toBe('verified');
|
||||
// Switched once, and not switched back.
|
||||
expect(target.patches).toEqual(['127.0.0.1:41001']);
|
||||
expect(notices).toEqual([
|
||||
'fetch failed; trying again in 1 s (attempt 2 of 4)',
|
||||
'fetch failed; trying again in 1 s (attempt 3 of 4)',
|
||||
'Health returned 502; trying again in 1 s (attempt 2 of 4)',
|
||||
]);
|
||||
} finally {
|
||||
await target.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user