wishforge-ide/packages/runtime/test/poc-release.test.ts
eust-w 91ab5f91e6 🌐 Write run events, waiting details and agent replies in Chinese
The run page mixed English stage events and agent summaries into the
Chinese console. Messages are now written in Simplified Chinese at their
source (control plane, dispatcher, worker, POC build), keeping commands,
SHAs, branches and error codes verbatim; raw runtime errors get a Chinese
prefix. The agent prompt asks for the plan, summary and questions in
Chinese. File-change parsing accepts old and new messages, and the event
label table covers every event kind written.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-04 12:42:04 +08:00

768 lines
36 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { existsSync } from 'node:fs';
import { mkdir, mkdtemp, readFile, rm, utimes, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import { KubectlError, PocDeploymentAdapter, unanswered, writePocJson, type PocRelease } from '../src/poc-release.js';
import { buildPocRelease, createPocBuildPod, pocComponentsToBuild, type PocBuildInput } from '../src/poc-build.js';
import { PocRunnerDeploymentAdapter, pocReleaseDigest } from '../src/poc-runner.js';
import type { ReleaseManifest } from '../src/types.js';
import {
loadPocProfile,
pocComponents,
pocComponentNames,
pocProfileSchema,
profileDigest,
type PocComponent,
type PocProfile,
} from '../src/poc-profile.js';
import { CommandStopped, type Command, type CommandExecutor, type CommandResult } from '../src/process.js';
const roots: string[] = [];
afterEach(async () => {
await Promise.all(roots.splice(0).map(p => rm(p, { recursive: true, force: true })));
});
const profile: PocProfile = pocProfileSchema.parse({
schemaVersion: 1,
id: 'droboticflow-poc',
repository: 'git@gitea.d-robotics.cc:longtao.wu/droboticflow.git',
branch: 'poc',
kubeconfig: '/operator/poc.config',
clusterServer: 'https://120.48.181.105:6443',
namespace: 'droboticflow',
namespaceUid: '15f3b2eb-f6b9-4a7c-89c3-fc5ba38bce71',
deployments: {
api: '6bed36ca-24e0-49a6-8edb-e463a7debf22',
worker: '9b5b0f9b-0211-4a47-85e9-2fec82fef8bd',
coordinator: '454b4c0c-719b-4203-8a1d-2a190a9cd304',
},
registry: 'ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc',
registrySecret: 'droboticflow-registry',
applicationUrl: 'http://120.48.164.19',
kanikoImage: `gcr.io/kaniko-project/executor@sha256:${'a'.repeat(64)}`,
});
function images(letter: string) {
return Object.fromEntries(
pocComponentNames.map(c => [c, `${profile.registry}/${pocComponents[c].repository}@sha256:${letter.repeat(64)}`]),
) as PocRelease['images'];
}
function release(overrides: Partial<PocRelease> = {}): PocRelease {
return {
schemaVersion: 1,
id: 'run-1',
sourceCommit: 'a'.repeat(40),
profileDigest: profileDigest(profile),
images: images('b'),
expectedImages: images('a'),
rollbackCompatible: true,
...overrides,
};
}
async function fixture() {
const root = await mkdtemp(path.join(tmpdir(), 'wishforge-poc-'));
roots.push(root);
const calls: Command[] = [];
const state = Object.fromEntries(
pocComponentNames.map(component => [
component,
{
metadata: { uid: profile.deployments[component], resourceVersion: '1', generation: 1 },
spec: {
replicas: 2,
selector: { matchLabels: { app: pocComponents[component].deployment } },
template: {
metadata: { annotations: { 'existing.example/keep': 'yes' } },
spec: {
containers: [
{ name: pocComponents[component].container, image: images('a')[component] },
{ name: 'untouched-sidecar', image: 'example/sidecar:stable' },
],
volumes: [{ name: 'existing', persistentVolumeClaim: { claimName: 'business-data' } }],
},
},
},
status: { observedGeneration: 1, replicas: 2, updatedReplicas: 2, readyReplicas: 2, availableReplicas: 2 },
},
]),
) as Record<PocComponent, any>;
const failures = {
loseReceipt: false,
failRolloutOnce: false,
wrongPodDigest: false,
externalImageBeforeRollback: false,
};
const execute: CommandExecutor = async cmd => {
calls.push(cmd);
const args = cmd.args.slice(5);
let data: unknown = {};
if (args[0] === 'config') data = { clusters: [{ cluster: { server: profile.clusterServer } }] };
else if (args[0] === 'get' && args[1] === 'namespace') data = { metadata: { uid: profile.namespaceUid } };
else if (args[0] === 'get' && args[1] === 'deployment')
data = state[pocComponentNames.find(c => pocComponents[c].deployment === args[2])!];
else if (args[0] === 'patch') {
const component = pocComponentNames.find(c => pocComponents[c].deployment === args[2])!;
const item = state[component];
const operations = JSON.parse(args[args.indexOf('-p') + 1]!);
for (const operation of operations) {
const keys = operation.path
.slice(1)
.split('/')
.map((k: string) => k.replaceAll('~1', '/').replaceAll('~0', '~'));
let current = item;
for (const key of keys.slice(0, -1)) current = current[key];
if (operation.op === 'test') {
if (current[keys.at(-1)!] !== operation.value)
return { exitCode: 1, stdout: '', stderr: 'JSON patch precondition failed' };
} else current[keys.at(-1)!] = operation.value;
}
item.metadata.resourceVersion = String(Number(item.metadata.resourceVersion) + 1);
item.metadata.generation++;
item.status.observedGeneration++;
if (failures.loseReceipt) {
failures.loseReceipt = false;
throw Error('connection lost after committed patch');
}
data = item;
} else if (args[0] === 'rollout' && failures.failRolloutOnce) {
failures.failRolloutOnce = false;
if (failures.externalImageBeforeRollback) state.api.spec.template.spec.containers[0].image = images('c').api;
return { exitCode: 1, stdout: '', stderr: 'progress deadline exceeded' };
} else if (args[0] === 'get' && args[1] === 'pods') {
const component = pocComponentNames.find(
c => `app=${pocComponents[c].deployment}` === args[args.indexOf('-l') + 1],
)!;
const item = state[component];
data = {
items: Array.from({ length: 2 }, () => ({
metadata: {},
spec: { containers: item.spec.template.spec.containers },
status: {
containerStatuses: [
{
name: pocComponents[component].container,
ready: true,
imageID: failures.wrongPodDigest ? images('c')[component] : item.spec.template.spec.containers[0].image,
},
],
},
})),
};
}
return { exitCode: 0, stdout: JSON.stringify(data), stderr: '' };
};
const adapter = new PocDeploymentAdapter(profile, root, execute, async () =>
Response.json({ buildCommit: 'a'.repeat(40) }),
);
return { root, state, calls, failures, adapter, execute };
}
describe('operator POC release', () => {
it('changes only three named container images and own annotations, then verifies three rollouts and Pod digests', async () => {
const f = await fixture();
const before = structuredClone(f.state);
const result = await f.adapter.deploy(release());
expect(result.state).toBe('succeeded');
expect(result.verification?.checks).toContain('pod-digests');
for (const c of pocComponentNames) {
expect(f.state[c].spec.template.spec.containers[0].image).toBe(images('b')[c]);
expect(f.state[c].spec.template.spec.containers[1]).toEqual(before[c].spec.template.spec.containers[1]);
expect(f.state[c].spec.template.spec.volumes).toEqual(before[c].spec.template.spec.volumes);
expect(f.state[c].spec.template.metadata.annotations['existing.example/keep']).toBe('yes');
}
const patches = f.calls.filter(c => c.args.includes('patch'));
expect(patches).toHaveLength(3);
for (const p of patches)
expect(JSON.parse(p.args.at(-1)!)).toEqual(
expect.arrayContaining([
expect.objectContaining({ op: 'test', path: '/metadata/uid' }),
expect.objectContaining({ op: 'test', path: '/metadata/resourceVersion' }),
expect.objectContaining({ op: 'test', path: '/spec/template/spec/containers/0/image' }),
]),
);
expect(JSON.parse(await readFile(path.join(f.root, 'run-1.json'), 'utf8')).state).toBe('succeeded');
});
it('refuses changed baseline, mismatched profile and other registry without patching', async () => {
const f = await fixture();
f.state.worker.spec.template.spec.containers[0].image = images('c').worker;
await expect(f.adapter.deploy(release())).rejects.toThrow('baseline changed');
await expect(f.adapter.deploy(release({ profileDigest: `sha256:${'b'.repeat(64)}` }))).rejects.toThrow(
'profile digest',
);
await expect(
f.adapter.deploy(release({ images: { ...images('b'), api: `registry.example/api@sha256:${'b'.repeat(64)}` } })),
).rejects.toThrow('repository mismatch');
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(0);
});
it('recovers an uncertain committed patch using observation and compatible rollback', async () => {
const f = await fixture();
f.failures.loseReceipt = true;
const result = await f.adapter.deploy(release());
expect(result.state).toBe('rolled_back');
expect(f.state.coordinator.spec.template.spec.containers[0].image).toBe(images('a').coordinator);
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(2);
});
it('does not roll back when compatibility is not declared', async () => {
const f = await fixture();
f.failures.failRolloutOnce = true;
const result = await f.adapter.deploy(release({ rollbackCompatible: false }));
expect(result.state).toBe('needs_reconciliation');
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(3);
});
it('refuses to overwrite a newer external image during rollback', async () => {
const f = await fixture();
f.failures.failRolloutOnce = true;
f.failures.externalImageBeforeRollback = true;
const result = await f.adapter.deploy(release());
expect(result.state).toBe('needs_reconciliation');
expect(f.state.api.spec.template.spec.containers[0].image).toBe(images('c').api);
});
it('returns repeated completed release without patching and rejects an ID collision', async () => {
const f = await fixture();
await f.adapter.deploy(release());
await f.adapter.deploy(release());
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(3);
await expect(f.adapter.deploy(release({ sourceCommit: 'b'.repeat(40) }))).rejects.toThrow('different content');
});
it('reconciles an interrupted journal without blindly repeating mutations', async () => {
const f = await fixture();
const r = release();
await writePocJson(path.join(f.root, 'run-1.json'), {
schemaVersion: 1,
release: r,
state: 'applying',
startedAt: '',
updatedAt: '',
applied: [],
pending: 'coordinator',
});
await expect(f.adapter.deploy(r)).rejects.toThrow('requires reconcile');
const result = await f.adapter.reconcile('run-1');
expect(result.state).toBe('rolled_back');
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(0);
});
it('fails digest verification even if kubectl rollout reports success', async () => {
const f = await fixture();
f.failures.wrongPodDigest = true;
const result = await f.adapter.deploy(release({ rollbackCompatible: false }));
expect(result.state).toBe('needs_reconciliation');
expect(result.error).toContain('Pod digest');
});
it('keeps unmodified components pinned without restarting them', async () => {
const f = await fixture();
const r = release({ images: { ...images('a'), api: images('b').api } });
expect((await f.adapter.deploy(r)).state).toBe('succeeded');
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(1);
expect(f.calls.filter(c => c.args.includes('rollout'))).toHaveLength(3);
});
it('detects preserved configuration drift before mutation', async () => {
const f = await fixture();
const baseline = await f.adapter.snapshot();
f.state.worker.spec.template.spec.volumes[0].persistentVolumeClaim.claimName = 'changed-by-another-operator';
await expect(f.adapter.deploy(release({ expectedTemplateDigests: baseline.templateDigests }))).rejects.toThrow(
'configuration changed',
);
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(0);
});
it('template fingerprints remain stable across release images and audit annotation changes', async () => {
const f = await fixture();
const before = await f.adapter.snapshot();
expect((await f.adapter.deploy(release({ expectedTemplateDigests: before.templateDigests }))).state).toBe(
'succeeded',
);
expect((await f.adapter.snapshot()).templateDigests).toEqual(before.templateDigests);
});
it('includes protected business checks inside the compatible rollback boundary', async () => {
const f = await fixture();
const result = await f.adapter.deploy(release(), async () => {
throw Error('expected business behavior absent');
});
expect(result.state).toBe('rolled_back');
expect((await f.adapter.snapshot()).images).toEqual(images('a'));
});
it('rejects recovery with a signed manifest different from the stored journal', async () => {
const f = await fixture();
await f.adapter.deploy(release());
await expect(f.adapter.reconcile('run-1', release({ sourceCommit: 'b'.repeat(40) }))).rejects.toThrow(
'persisted signed release',
);
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(3);
});
it('Runner wrapper binds the main source/image to the component bundle and returns broker acceptance evidence', async () => {
const f = await fixture();
const inner = release({ expectedTemplateDigests: (await f.adapter.snapshot()).templateDigests });
const manifest: ReleaseManifest & { pocRelease: PocRelease } = {
schemaVersion: 1,
id: 'outer-release-1',
tenantId: 'tenant',
environmentId: 'poc',
sourceCommit: inner.sourceCommit,
image: inner.images.api,
architecture: 'linux/amd64',
containerPort: 3001,
healthPath: '/api/health',
applicationUrl: profile.applicationUrl,
smokeChecks: [{ path: '/api/ready', expectedStatus: 200 }],
pocRelease: inner,
};
const wrapper = new PocRunnerDeploymentAdapter(profile, f.root, f.execute, async () =>
Response.json({ buildCommit: 'a'.repeat(40) }),
);
expect((await wrapper.deploy(manifest)).status).toBe('dry-run');
expect(f.calls.filter(c => c.args.includes('patch'))).toHaveLength(0);
await expect(wrapper.deploy({ ...manifest, image: images('c').api }, { execute: true })).rejects.toThrow(
'does not match',
);
const result = await wrapper.deploy(manifest, { execute: true });
expect(result.status).toBe('verified');
expect(result.evidence).toMatchObject({
identityMatched: true,
trafficSwitched: true,
bundleDigest: pocReleaseDigest(inner),
pocRelease: inner,
});
expect(result.evidence.checks).toEqual(
expect.arrayContaining(['health', 'business-smoke', 'pod-digests', 'protected-business-smoke']),
);
});
it('requires an operator profile without group/world write access', async () => {
const f = await fixture();
const file = path.join(f.root, 'profile.json');
await writeFile(file, JSON.stringify(profile), { mode: 0o666 });
const { chmod } = await import('node:fs/promises');
await chmod(file, 0o666);
await expect(loadPocProfile(file)).rejects.toThrow('operator-owned');
});
it('builds with emptyDir and only registry secret, pinned executor, no service account or business PVC', () => {
const pod = createPocBuildPod(profile, {
name: 'build',
releaseId: 'run-1',
component: 'api',
sourceCommit: 'a'.repeat(40),
buildTime: '2026-10-22T00:00:00.000Z',
});
expect(pod.spec.automountServiceAccountToken).toBe(false);
expect(pod.spec.volumes).toHaveLength(2);
expect(JSON.stringify(pod)).not.toContain('persistentVolumeClaim');
expect(pod.spec.containers[0]?.args).toContain('--digest-file=/dev/termination-log');
expect(pod.spec.containers[0]?.image).toContain('@sha256:');
expect(pod.spec.activeDeadlineSeconds).toBe(1800);
});
});
const patience = { attempts: 4, delayMs: 1 };
const unreachable = 'Unable to connect to the server: dial tcp 120.48.181.105:6443: connect: operation timed out';
const ok = (stdout = ''): CommandResult => ({ exitCode: 0, stdout, stderr: '' });
/**
* A cluster whose kubectl answers can be lost: each verb's arranged faults are used in turn,
* one that `lands` taking effect before its answer is lost.
*/
async function cluster() {
const f = await fixture();
const workspace = path.join(f.root, 'workspace');
await mkdir(workspace);
const commit = 'c'.repeat(40);
const digest = `sha256:${'d'.repeat(64)}`;
const pods = new Map<string, any>();
const requests: string[][] = [];
const notices: string[] = [];
const progress: string[] = [];
/** The Gitea poc branch: its tip, and whether the commit being built is in its history. */
const branch = { tip: commit, contains: true };
const faults: Record<
string,
{ exitCode?: number; stderr: string; lands?: boolean; when?: (args: string[]) => boolean }[]
> = {};
const execute: CommandExecutor = async command => {
if (command.executable === 'git') {
const [verb] = command.args;
if (verb === 'remote') return ok(profile.repository);
if (verb === 'rev-parse') return ok(command.args.includes('FETCH_HEAD^{commit}') ? branch.tip : commit);
if (verb === 'ls-remote') return ok(`${branch.tip}\trefs/heads/poc`);
if (verb === 'fetch') return ok();
if (verb === 'merge-base') return { ...ok(), exitCode: branch.contains ? 0 : 1 };
await writeFile(command.args.find(arg => arg.startsWith('--output='))!.slice('--output='.length), 'source');
return ok();
}
const args = command.args.slice(5);
requests.push(args);
const queue = faults[args[0]!] ?? [];
const next = queue.findIndex(fault => !fault.when || fault.when(args));
const fault = next < 0 ? undefined : queue.splice(next, 1)[0];
const failure = { exitCode: fault?.exitCode ?? 1, stdout: '', stderr: fault?.stderr ?? '' };
if (fault && !fault.lands) return failure;
let result: CommandResult | undefined;
if (args[0] === 'create') {
const pod = JSON.parse(command.input!);
pods.set(pod.metadata.name, {
...pod,
metadata: { ...pod.metadata, uid: `uid-${pod.metadata.name}` },
status: { containerStatuses: [{ state: { running: {} } }] },
});
result = ok(JSON.stringify(pods.get(pod.metadata.name)));
} else if (args[0] === 'get' && args[1] === 'pod') {
const pod = pods.get(args[2]!);
result = pod
? ok(JSON.stringify(pod))
: args.includes('--ignore-not-found')
? ok()
: { exitCode: 1, stdout: '', stderr: `Error from server (NotFound): pods "${args[2]}" not found` };
} else if (args[0] === 'exec') {
// The source unpacked: kaniko runs and reports its digest.
pods.get(args[1]!).status = {
containerStatuses: [{ state: { terminated: { exitCode: 0, message: digest } } }],
};
result = ok();
} else if (args[0] === 'cp' || args[0] === 'logs') result = ok();
else if (args[0] === 'delete') {
pods.delete(args[2]!.split('/').at(-1)!);
result = ok();
}
result ??= await f.execute(command);
return fault?.lands ? failure : result;
};
const build = (overrides: Partial<PocBuildInput> = {}) =>
buildPocRelease(
{
profile,
workspace,
sourceCommit: commit,
releaseId: 'run-1-build',
stateDirectory: f.root,
components: ['api'],
onProgress: message => {
progress.push(message);
},
patience: { ...patience, onRetry: notice => notices.push(notice) },
...overrides,
},
execute,
);
return {
...f,
execute,
build,
commit,
digest,
pods,
faults,
requests,
notices,
progress,
branch,
count: (verb: string) => requests.filter(args => args[0] === verb).length,
};
}
describe('the POC build and release on a flaky link to the cluster', () => {
it('tells requests the cluster never answered from ones it refused', () => {
expect(unanswered(new KubectlError('get', 1, unreachable))).toBe(true);
expect(
unanswered(new KubectlError('cp', 1, 'error: next reader: read tcp 10.0.0.2:59375->10.0.0.1:6443: i/o timeout')),
).toBe(true);
expect(
unanswered(new CommandStopped('TIMEOUT', 'Command exceeded its time limit', { stdout: '', stderr: '' })),
).toBe(true);
// As kubectl cp reported them on the POC link, without the "Unable to connect" prefix of get.
for (const stderr of [
'error: dial tcp 120.48.181.105:6443: connect: operation timed out',
'E1026 01:06:17.504268 6886 v2.go:129] "Unhandled Error" err="next reader: read tcp 192.168.6.207:59375->120.48.181.105:6443: i/o timeout"',
'error: dial tcp 10.0.0.1:6443: connect: connection timed out',
])
expect(unanswered(new KubectlError('cp', 1, stderr))).toBe(true);
expect(unanswered(new KubectlError('exec', 1, 'error: unable to upgrade connection: container not found'))).toBe(
false,
);
expect(unanswered(new KubectlError('get', 1, 'Error from server (NotFound): pods "build" not found'))).toBe(false);
expect(unanswered(new KubectlError('rollout', 1, 'error: timed out waiting for the condition'))).toBe(false);
// The command in the Pod ran; what it printed does not make its failure a lost answer.
expect(unanswered(new KubectlError('exec', 1, 'tar: unexpected EOF\ncommand terminated with exit code 1'))).toBe(
false,
);
expect(unanswered(new Error('connection lost after committed patch'))).toBe(false);
});
it('builds through lost answers without creating a second Pod or starting kaniko twice', async () => {
const c = await cluster();
c.faults.get = [{ stderr: unreachable }];
// The Pod was created, but the answer never came back.
c.faults.create = [{ stderr: unreachable, lands: true }];
c.faults.cp = [{ stderr: 'error: next reader: read tcp 10.0.0.2:59375->10.0.0.1:6443: i/o timeout' }];
// Unpacking went ahead without an answer; asked again, it is still running, then done.
c.faults.exec = [
{ stderr: unreachable, lands: true },
{ exitCode: 75, stderr: 'command terminated with exit code 75' },
];
const { release } = await c.build();
expect(release.images.api).toBe(`${profile.registry}/${pocComponents.api.repository}@${c.digest}`);
expect(c.count('create')).toBe(1);
expect(c.count('cp')).toBe(2);
expect(c.count('exec')).toBe(3);
expect(c.pods.size).toBe(0);
// Each repeat is told, for the run's timeline.
expect(c.notices).toContain(
`kubectl get 没有得到集群 API 的应答(${unreachable});1 秒后重试(第 2 次,共 4 次)`,
);
expect(c.notices).toHaveLength(4);
});
it('repeats a lookup that never gets through only as often as the patience allows', async () => {
const c = await cluster();
c.faults.get = Array.from({ length: 50 }, () => ({
stderr: unreachable,
when: (args: string[]) => args.includes('--ignore-not-found'),
}));
await expect(c.build()).rejects.toThrow(unreachable);
expect(c.requests.filter(args => args.includes('--ignore-not-found'))).toHaveLength(patience.attempts);
expect(c.count('create')).toBe(0);
});
it('sends the source again when it arrived damaged', async () => {
const c = await cluster();
c.faults.exec = [{ exitCode: 76, stderr: 'command terminated with exit code 76' }];
await c.build();
expect(c.count('cp')).toBe(2);
expect(c.count('exec')).toBe(2);
});
it('gives up once the link stays down, and still removes the Pod', async () => {
const c = await cluster();
c.faults.cp = Array.from({ length: patience.attempts }, () => ({ stderr: unreachable }));
await expect(c.build()).rejects.toThrow(`kubectl cp failed (1): ${unreachable}`);
expect(c.count('cp')).toBe(patience.attempts);
expect(c.pods.size).toBe(0);
});
it('resumes a rollout watch the link dropped instead of rolling back', async () => {
const c = await cluster();
c.faults.get = [{ stderr: 'net/http: TLS handshake timeout' }];
c.faults.rollout = [{ stderr: 'http2: client connection lost' }];
const adapter = new PocDeploymentAdapter(
profile,
c.root,
c.execute,
async () => Response.json({ buildCommit: 'a'.repeat(40) }),
patience,
);
const result = await adapter.deploy(release());
expect(result.state).toBe('succeeded');
expect(c.count('rollout')).toBe(pocComponentNames.length + 1);
// The dropped watch is resumed for what is left of the same rollout's time.
const [dropped, resumed] = c.requests
.filter(args => args[0] === 'rollout')
.map(args => Number(args.at(-1)!.slice('--timeout='.length, -1)));
expect(dropped).toBe(600);
expect(resumed).toBeLessThanOrEqual(600);
});
it('accepts a release once the public address serves the new build, not at its first answer', async () => {
const f = await fixture();
const notices: string[] = [];
let health = 0;
// Right after the rollout the public address first fails, then still reaches the old Pod.
const settling: typeof fetch = async url => {
if (new URL(String(url)).pathname !== '/api/health') return new Response('ok');
health++;
if (health === 1) return new Response('', { status: 502 });
return Response.json({ buildCommit: health === 2 ? 'f'.repeat(40) : 'a'.repeat(40) });
};
const adapter = new PocDeploymentAdapter(profile, f.root, f.execute, settling, {
...patience,
onRetry: notice => notices.push(notice),
});
const result = await adapter.deploy(release());
expect(result.state).toBe('succeeded');
expect(notices).toEqual([
'POC /api/health returned 502;1 秒后重试(第 2 次,共 4 次)',
'POC public API source commit does not match the deployed artifact;1 秒后重试(第 3 次,共 4 次)',
]);
});
it('still rolls back a release whose public API never serves the new build', async () => {
const f = await fixture();
const stale: typeof fetch = async () => Response.json({ buildCommit: 'f'.repeat(40) });
const adapter = new PocDeploymentAdapter(profile, f.root, f.execute, stale, patience);
const result = await adapter.deploy(release());
expect(result.state).toBe('rolled_back');
expect(result.error).toContain('source commit does not match');
});
it('lets the Runner repeat its protected checks while the release settles', async () => {
const f = await fixture();
const inner = release({ expectedTemplateDigests: (await f.adapter.snapshot()).templateDigests });
const manifest: ReleaseManifest & { pocRelease: PocRelease } = {
schemaVersion: 1,
id: 'outer-release-1',
tenantId: 'tenant',
environmentId: 'poc',
sourceCommit: inner.sourceCommit,
image: inner.images.api,
architecture: 'linux/amd64',
containerPort: 3001,
healthPath: '/api/health',
applicationUrl: profile.applicationUrl,
smokeChecks: [{ path: '/api/ready', expectedStatus: 200 }],
pocRelease: inner,
};
let ready = 0;
// The adapter's own readiness check and the Runner's protected smoke check each meet one 503.
const fetcher: typeof fetch = async url => {
if (new URL(String(url)).pathname === '/api/ready' && [1, 3].includes(++ready))
return new Response('', { status: 503 });
return Response.json({ buildCommit: 'a'.repeat(40) });
};
const wrapper = new PocRunnerDeploymentAdapter(profile, f.root, f.execute, fetcher, patience);
expect((await wrapper.deploy(manifest, { execute: true })).status).toBe('verified');
expect(ready).toBe(5);
});
});
describe('what a POC build builds after people changed the branch or the cluster', () => {
/** Leaves a component as a release does: running `image`, built from `commit`. */
const annotate = (item: any, commit: string, image: string) =>
Object.assign(item.spec.template.metadata.annotations, {
'wishforge.d-robotics.cc/source-commit': commit,
'wishforge.d-robotics.cc/image': image,
});
it('builds what the run integrated after people pushed onto it, and says so', async () => {
const c = await cluster();
c.branch.tip = 'e'.repeat(40);
const { release } = await c.build();
expect(release.sourceCommit).toBe(c.commit);
expect(c.count('create')).toBe(1);
expect(c.progress).toContain(
`poc 已前进到 ${'e'.repeat(12)};现在构建这次运行合入的 ${'c'.repeat(12)},更新的提交随之后的运行部署`,
);
});
it('never builds a commit the branch lost to a rewrite', async () => {
const c = await cluster();
c.branch.tip = 'e'.repeat(40);
c.branch.contains = false;
await expect(c.build()).rejects.toMatchObject({ code: 'POC_SOURCE_REWRITTEN' });
expect(c.count('create')).toBe(0);
});
it('knows what each component was built from, until its image changes', async () => {
const f = await fixture();
expect((await f.adapter.snapshot()).deployments.api?.sourceCommit).toBeUndefined();
await f.adapter.deploy(release());
for (const c of pocComponentNames)
expect((await f.adapter.snapshot()).deployments[c]?.sourceCommit).toBe('a'.repeat(40));
// A person sets an image by hand: the annotations stay as they were, but no longer vouch for it.
f.state.worker.spec.template.spec.containers[0].image = images('c').worker;
expect((await f.adapter.snapshot()).deployments.worker?.sourceCommit).toBeUndefined();
// Restored by a rollback, a component runs an image of unknown source again.
const g = await fixture();
g.failures.loseReceipt = true;
expect((await g.adapter.deploy(release())).state).toBe('rolled_back');
expect(g.state.coordinator.spec.template.metadata.annotations['wishforge.d-robotics.cc/image']).toBe(
images('a').coordinator,
);
expect((await g.adapter.snapshot()).deployments.coordinator?.sourceCommit).toBeUndefined();
});
it('plans the build from what each component runs, in the snapshot the release is checked against', async () => {
const c = await cluster();
// The worker runs what a release built from b…; a person set the coordinator's image by hand since.
annotate(c.state.worker, 'b'.repeat(40), images('a').worker);
annotate(c.state.coordinator, 'b'.repeat(40), images('f').coordinator);
let running: Partial<Record<PocComponent, string>> | undefined;
const { release, evidence } = await c.build({
components: undefined,
plan: async seen => {
running = seen;
return { components: ['api', 'coordinator'], rollbackCompatible: true };
},
});
expect(running).toEqual({ worker: 'b'.repeat(40) });
expect(Object.keys(evidence.components).sort()).toEqual(['api', 'coordinator']);
expect(evidence.retainedImages).toEqual({ worker: images('a').worker });
expect(release).toMatchObject({ rollbackCompatible: true, expectedImages: images('a') });
});
it('builds a component someone deployed by a tag again, and never rolls back to the tag', async () => {
const c = await cluster();
const tag = `${profile.registry}/${pocComponents.worker.repository}:30d91677-poc-20261027162554`;
// A person deployed a tag over the worker; even annotations naming it do not vouch for a tag.
annotate(c.state.worker, 'b'.repeat(40), tag);
c.state.worker.spec.template.spec.containers[0].image = tag;
const snapshot = await c.adapter.snapshot();
expect(snapshot.images.worker).toBe(tag);
expect(snapshot.deployments.worker).toMatchObject({ pinned: false });
expect(snapshot.deployments.worker?.sourceCommit).toBeUndefined();
expect(snapshot.deployments.api).toMatchObject({ pinned: true });
// A plan that keeps the tag is refused: a release deploys digests only.
await expect(
c.build({ components: undefined, plan: async () => ({ components: ['api'], rollbackCompatible: true }) }),
).rejects.toThrow('deployed by a tag');
let running: Partial<Record<PocComponent, string>> | undefined;
const { release } = await c.build({
releaseId: 'run-2-build',
components: undefined,
plan: async seen => {
running = seen;
return { components: ['api', 'worker'], rollbackCompatible: true };
},
});
expect(running?.worker).toBeUndefined();
expect(release).toMatchObject({ rollbackCompatible: false, expectedImages: { ...images('a'), worker: tag } });
expect(release.images.worker).toBe(`${profile.registry}/${pocComponents.worker.repository}@${c.digest}`);
expect(
c.progress.some(message => message.startsWith(`worker 运行的 ${tag} 是有人在 WishForge 之外按标签部署的`)),
).toBe(true);
// Deploying replaces exactly the tag that ran, and the components run digests again.
const deployer = new PocDeploymentAdapter(profile, path.join(c.root, 'deployments'), c.execute, async () =>
Response.json({ buildCommit: c.commit }),
);
expect((await deployer.deploy(release)).state).toBe('succeeded');
expect(c.state.worker.spec.template.spec.containers[0].image).toBe(release.images.worker);
expect((await c.adapter.snapshot()).deployments.worker).toMatchObject({ pinned: true, sourceCommit: c.commit });
// A release that would roll back to a tag is refused before anything changes.
await expect(deployer.deploy({ ...release, id: 'run-3', rollbackCompatible: true })).rejects.toThrow(
'cannot roll back',
);
});
it('stops at an image that is neither pinned nor tagged, or not one of the POC repositories', async () => {
const c = await cluster();
c.state.coordinator.spec.template.spec.containers[0].image = `${profile.registry}/${pocComponents.coordinator.repository}`;
await expect(c.adapter.snapshot()).rejects.toThrow('pinned neither by digest nor by tag');
c.state.coordinator.spec.template.spec.containers[0].image = 'docker.io/library/nginx:latest';
await expect(c.build({ components: ['api', 'coordinator'] })).rejects.toThrow('is not an image of');
expect(c.count('create')).toBe(0);
});
it('keeps the records of earlier builds, and only their recent source archives', async () => {
const c = await cluster();
const builds = path.join(c.root, 'builds');
const earlier = async (name: string, hoursAgo: number) => {
await mkdir(path.join(builds, name), { recursive: true });
const archive = path.join(builds, name, 'source.tar.gz');
await writeFile(archive, 'source');
await writeFile(path.join(builds, name, 'build.json'), '{}');
const time = new Date(Date.now() - hoursAgo * 3_600_000);
await utimes(archive, time, time);
return archive;
};
const old = await earlier('run-0-old', 2);
// A build still running, or one that just ended, keeps its archive.
const recent = await earlier('run-0-recent', 0);
await c.build();
expect(existsSync(old)).toBe(false);
expect(existsSync(path.join(builds, 'run-0-old', 'build.json'))).toBe(true);
expect(existsSync(recent)).toBe(true);
expect(existsSync(path.join(builds, 'run-1-build', 'source.tar.gz'))).toBe(true);
});
it('builds worker and coordinator again unless they differ from the source in web files alone', async () => {
const changed: Record<string, string[]> = {
['1'.repeat(40)]: ['src/app.tsx', 'tests/frontend/app.test.ts'],
['2'.repeat(40)]: ['src/app.tsx', 'internal/worker/run.go'],
};
const since = async (commit: string) => changed[commit];
// Worker and coordinator keep their images, so only what the api runs is replaced.
expect(
await pocComponentsToBuild({ api: 'a'.repeat(40), worker: '1'.repeat(40), coordinator: '1'.repeat(40) }, since),
).toEqual({ components: ['api'], reasons: {}, replaced: ['a'.repeat(40)] });
expect(await pocComponentsToBuild({ worker: '2'.repeat(40), coordinator: '3'.repeat(40) }, since)).toEqual({
components: ['api', 'worker', 'coordinator'],
reasons: {
worker: `internal/worker/run.go 在它运行的 ${'2'.repeat(12)} 之后有改动`,
coordinator: `它运行的 ${'3'.repeat(12)} 在仓库里不存在`,
},
replaced: ['2'.repeat(40), '3'.repeat(40)],
});
expect(await pocComponentsToBuild({}, since)).toEqual({
components: ['api', 'worker', 'coordinator'],
reasons: { worker: '它运行的版本来源未知', coordinator: '它运行的版本来源未知' },
replaced: [],
});
});
});