wishforge-ide/apps/provisioner/test/kubernetes.test.mjs
eust-w 3b78e8e536 🐛 Release what superseded run generations left behind
- The local provisioner kept a generation whose workers exited by
  themselves as running, so its run's directories could never be
  purged.
- On Kubernetes, only a run's current generation is ever released: a
  superseded generation's Secrets, with its run tokens, its service
  accounts and network policies stayed forever. Starting a new
  generation now deletes them by name for every earlier generation
  without a running pod, found through the volumes kept for the
  retention period, so the provisioner still cannot list Secrets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 22:22:11 +08:00

497 lines
21 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import test from 'node:test';
import assert from 'node:assert/strict';
import { createHmac } from 'node:crypto';
import {
generationIdentities,
readConfig,
renderRunResources,
provisionRun,
reconcileRunJobs,
} from '../kubernetes.mjs';
import { resourceGrantPayload } from '../resource-grants.mjs';
import { purgeRunStorage, releaseRun } from '../run-release.mjs';
import { createRunOperationQueue } from '../run-operations.mjs';
const env = {
WF_TENANT_ID: 'tenant-a',
WF_KUBE_CONTEXT: 'explicit-test-context',
WF_KUBE_NAMESPACE: 'wishforge-tenant-a',
WF_RUN_WORKER_IMAGE: `registry.example.invalid/worker@sha256:${'a'.repeat(64)}`,
WF_RUN_STORAGE_CLASS: 'reviewed-rwx',
WF_CONTROL_PLANE_URL: 'https://api.wishforge-control.svc:8443',
WF_CONTROL_NAMESPACE: 'wishforge-control',
TEMPORAL_ADDRESS: 'temporal.wishforge-control.svc:7233',
TEMPORAL_NAMESPACE: 'wishforge',
};
env.WF_RESOURCE_GRANT_SECRET = 'test-only-resource-signing-secret-longer-than-32';
const config = readConfig(env);
function grant(overrides = {}) {
const issuedAt = Math.floor(Date.now() / 1000);
const payload = {
schemaVersion: 1,
tenantId: 'tenant-a',
runId: 'run-123',
generation: 'generation-1',
planId: 'pilot',
planVersion: 'pilot-v1',
image: config.image,
cpuMilli: 1500,
memoryMiB: 3072,
storageMiB: 8192,
issuedAt,
expiresAt: issuedAt + 300,
...overrides,
};
return {
...payload,
signature: createHmac('sha256', env.WF_RESOURCE_GRANT_SECRET)
.update(resourceGrantPayload(payload))
.digest('base64url'),
};
}
function token(kind, run = 'run-123') {
return `header.${Buffer.from(JSON.stringify({ tenant_id: 'tenant-a', run_id: run, generation: 'generation-1', kind, iss: 'wishforge-control-plane', aud: 'wishforge-internal', exp: Math.floor(Date.now() / 1000) + 14400 })).toString('base64url')}.signature`;
}
const request = {
tenantId: 'tenant-a',
runId: 'run-123',
generation: 'generation-1',
executorToken: token('worker'),
verifierToken: token('verifier'),
resourceGrant: grant(),
};
test('fails closed without explicit cluster, namespace, image digest and storage', () => {
assert.throws(() => readConfig({}), /unconfigured/);
assert.throws(() => readConfig({ ...env, WF_RUN_WORKER_IMAGE: 'worker:latest' }), /immutable/);
assert.throws(() => renderRunResources(config, { ...request, tenantId: 'tenant-b' }), /outside/);
assert.throws(() => renderRunResources(config, { ...request, executorToken: token('worker', 'other-run') }), /token/);
});
test('renders independent Kata jobs with role-scoped credentials and separate role PVCs', () => {
const { resources } = renderRunResources(config, request);
const jobs = resources.filter(r => r.kind === 'Job');
assert.equal(jobs.length, 2);
assert.notEqual(jobs[0].spec.template.spec.serviceAccountName, jobs[1].spec.template.spec.serviceAccountName);
for (const job of jobs) {
const pod = job.spec.template.spec,
container = pod.containers[0];
assert.equal(pod.runtimeClassName, 'wishforge-kata');
assert.equal(pod.automountServiceAccountToken, false);
assert.equal(container.securityContext.allowPrivilegeEscalation, false);
assert.equal(container.securityContext.readOnlyRootFilesystem, true);
assert.deepEqual(container.securityContext.capabilities.drop, ['ALL']);
assert.ok(!pod.hostNetwork && !pod.hostPID && !pod.hostIPC);
assert.ok(pod.volumes.every(v => !v.hostPath));
assert.ok(!JSON.stringify(pod).includes('docker.sock'));
assert.equal(container.env.filter(e => /TOKEN$/.test(e.name)).length, 1);
assert.ok(container.env.find(e => /TOKEN$/.test(e.name)).valueFrom.secretKeyRef);
assert.ok(!JSON.stringify(pod).includes(request.executorToken));
assert.ok(!JSON.stringify(pod).includes(request.verifierToken));
assert.equal(container.image, request.resourceGrant.image);
assert.equal(container.resources.limits.cpu, '750m');
assert.equal(container.resources.limits.memory, '1536Mi');
}
const pvc = resources.filter(r => r.kind === 'PersistentVolumeClaim');
assert.equal(pvc.length, 2);
assert.ok(pvc.every(claim => claim.spec.accessModes[0] === 'ReadWriteOnce'));
assert.notEqual(
jobs[0].spec.template.spec.volumes[0].persistentVolumeClaim.claimName,
jobs[1].spec.template.spec.volumes[0].persistentVolumeClaim.claimName,
);
const policy = resources.find(r => r.kind === 'NetworkPolicy');
assert.deepEqual(policy.spec.ingress, []);
assert.ok(
policy.spec.egress.every(rule => rule.to.every(to => to.namespaceSelector && to.podSelector && !to.ipBlock)),
);
assert.ok(pvc.every(claim => claim.spec.resources.requests.storage === '4096Mi'));
assert.ok(
jobs.reduce((total, job) => total + parseInt(job.spec.template.spec.containers[0].resources.limits.cpu), 0) <=
request.resourceGrant.cpuMilli,
);
assert.ok(
jobs.reduce((total, job) => total + parseInt(job.spec.template.spec.containers[0].resources.limits.memory), 0) <=
request.resourceGrant.memoryMiB,
);
assert.ok(
pvc.reduce((total, claim) => total + parseInt(claim.spec.resources.requests.storage), 0) <=
request.resourceGrant.storageMiB,
);
const other = renderRunResources(config, {
...request,
runId: 'run-456',
executorToken: token('worker', 'run-456'),
verifierToken: token('verifier', 'run-456'),
resourceGrant: grant({ runId: 'run-456' }),
});
assert.notEqual(other.receipt.workspaceClaim, pvc[0].metadata.name);
});
test('gives sandboxes the tenant’s Temporal client certificate, which production requires', () => {
assert.throws(() => readConfig({ ...env, WF_DEPLOYMENT_MODE: 'production' }), /WF_RUN_TEMPORAL_TLS_SECRET/);
const configured = readConfig({
...env,
WF_DEPLOYMENT_MODE: 'production',
WF_RUN_TEMPORAL_TLS_SECRET: 'tenant-a-temporal-client',
});
for (const job of renderRunResources(configured, request).resources.filter(r => r.kind === 'Job')) {
const pod = job.spec.template.spec;
const values = Object.fromEntries(pod.containers[0].env.map(e => [e.name, e.value]));
assert.equal(values.TEMPORAL_TLS_CERT, '/run/temporal/tls.crt');
assert.equal(values.TEMPORAL_TLS_KEY, '/run/temporal/tls.key');
assert.equal(values.TEMPORAL_TLS_CA, '/run/temporal/ca.crt');
assert.equal(values.TEMPORAL_NAMESPACE, 'wishforge');
assert.deepEqual(pod.volumes.find(v => v.name === 'temporal-tls').secret, {
secretName: 'tenant-a-temporal-client',
defaultMode: 0o440,
});
assert.deepEqual(
pod.containers[0].volumeMounts.find(m => m.name === 'temporal-tls'),
{ name: 'temporal-tls', mountPath: '/run/temporal', readOnly: true },
);
}
assert.throws(() => readConfig({ ...env, WF_RUN_TEMPORAL_TLS_SECRET: 'Not_A_Name' }), /Invalid configured/);
});
test('lets installations reach only the egress proxy and only the verifier reach the image builder', () => {
const configured = readConfig({
...env,
WF_RUN_INSTALL_PROXY_URL: 'http://egress-proxy.wishforge-control.svc:3128',
WF_VERIFIER_DOCKER_HOST: 'tcp://image-builder.wishforge-build.svc:2376',
WF_VERIFIER_DOCKER_TLS_SECRET: 'image-builder-client',
WF_VERIFIER_BUILDER_NAMESPACE: 'wishforge-build',
});
const { resources } = renderRunResources(configured, request);
const [executor, verifier] = ['executor', 'verifier'].map(role =>
resources.find(r => r.kind === 'Job' && r.spec.template.metadata.labels['wishforge.io/worker-role'] === role),
);
const envOf = job => Object.fromEntries(job.spec.template.spec.containers[0].env.map(e => [e.name, e.value]));
assert.equal(envOf(executor).WF_INSTALL_PROXY, 'http://egress-proxy.wishforge-control.svc:3128');
assert.equal(envOf(verifier).WF_INSTALL_PROXY, 'http://egress-proxy.wishforge-control.svc:3128');
assert.equal(envOf(executor).WF_DOCKER_HOST, undefined);
assert.equal(envOf(verifier).WF_DOCKER_HOST, 'tcp://image-builder.wishforge-build.svc:2376');
assert.equal(envOf(verifier).WF_DOCKER_TLS_DIR, '/run/docker-tls');
assert.deepEqual(verifier.spec.template.spec.volumes.find(v => v.name === 'docker-tls').secret, {
secretName: 'image-builder-client',
defaultMode: 0o440,
});
const [isolation, builder] = resources.filter(r => r.kind === 'NetworkPolicy');
assert.deepEqual(isolation.spec.egress.at(-1), {
to: [
{
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'wishforge-control' } },
podSelector: { matchLabels: { 'wishforge.io/component': 'egress-proxy' } },
},
],
ports: [{ protocol: 'TCP', port: 3128 }],
});
assert.equal(builder.metadata.name.endsWith('-builder'), true);
assert.equal(builder.spec.podSelector.matchLabels['wishforge.io/worker-role'], 'verifier');
assert.deepEqual(builder.spec.egress, [
{
to: [
{
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'wishforge-build' } },
podSelector: { matchLabels: { 'wishforge.io/component': 'image-builder' } },
},
],
ports: [{ protocol: 'TCP', port: 2376 }],
},
]);
for (const invalid of [
{ WF_RUN_INSTALL_PROXY_URL: 'http://user:secret@proxy.svc:3128' },
{ WF_RUN_INSTALL_PROXY_URL: 'socks5://proxy.svc:1080' },
{ WF_VERIFIER_DOCKER_HOST: 'unix:///var/run/docker.sock' },
{
WF_VERIFIER_DOCKER_HOST: 'tcp://builder.svc:2376',
WF_DEPLOYMENT_MODE: 'production',
WF_RUN_TEMPORAL_TLS_SECRET: 't',
},
])
assert.throws(() => readConfig({ ...env, ...invalid }));
// Without them nothing beyond the control plane is reachable.
const plain = renderRunResources(config, request).resources.filter(r => r.kind === 'NetworkPolicy');
assert.equal(plain.length, 1);
});
test('runs sandboxes in the provisioner’s deployment mode, and only development clusters without Kata', () => {
const jobEnv = resources =>
resources
.filter(r => r.kind === 'Job')
.map(job => Object.fromEntries(job.spec.template.spec.containers[0].env.map(e => [e.name, e.value])));
for (const values of jobEnv(renderRunResources(config, request).resources)) {
assert.equal(values.WF_DEPLOYMENT_MODE, 'production');
assert.equal(values.NODE_ENV, 'production');
assert.equal(values.WF_SANDBOX_MODE, 'kata');
assert.equal(values.WF_ALLOW_INSECURE_HTTP, undefined);
}
const local = {
...env,
WF_DEPLOYMENT_MODE: 'development',
WF_ALLOW_INSECURE_HTTP: 'true',
WF_RUN_SANDBOX_MODE: 'host',
WF_KATA_RUNTIME_CLASS: 'none',
WF_CONTROL_PLANE_URL: 'http://api.wishforge-control.svc:4000',
};
const { resources } = renderRunResources(readConfig(local), request);
for (const job of resources.filter(r => r.kind === 'Job'))
assert.equal('runtimeClassName' in job.spec.template.spec, false);
for (const values of jobEnv(resources)) {
assert.equal(values.WF_DEPLOYMENT_MODE, 'development');
// Overrides the image's NODE_ENV=production, which development mode refuses.
assert.equal(values.NODE_ENV, 'development');
assert.equal(values.WF_SANDBOX_MODE, 'host');
assert.equal(values.WF_ALLOW_INSECURE_HTTP, 'true');
assert.equal(values.WF_GIT_ALLOW_LOCAL_HTTP, 'true');
assert.equal(values.WF_CONTROL_PLANE_URL, 'http://api.wishforge-control.svc:4000');
}
// None of it is accepted outside an explicit development deployment.
const production = { ...env, WF_DEPLOYMENT_MODE: 'production', WF_RUN_TEMPORAL_TLS_SECRET: 'temporal-client' };
assert.throws(() => readConfig({ ...production, WF_RUN_SANDBOX_MODE: 'host' }), /development/);
assert.throws(() => readConfig({ ...production, WF_KATA_RUNTIME_CLASS: 'none' }), /development/);
assert.throws(() => readConfig({ ...production, WF_ALLOW_INSECURE_HTTP: 'true' }), /development/);
assert.throws(() => readConfig({ ...env, WF_CONTROL_PLANE_URL: 'http://api.wishforge-control.svc:4000' }), /HTTPS/);
});
test('rejects tampered, expired, wrong-scope and above-cap resource grants', () => {
for (const change of [{ cpuMilli: 1600 }, { generation: 'other' }, { tenantId: 'other' }])
assert.throws(
() => renderRunResources(config, { ...request, resourceGrant: { ...request.resourceGrant, ...change } }),
/RESOURCE_GRANT_INVALID/,
);
assert.throws(
() => renderRunResources(config, { ...request, resourceGrant: grant({ runId: 'other' }) }),
/RESOURCE_GRANT_INVALID/,
);
assert.throws(
() => renderRunResources(config, { ...request, resourceGrant: grant({ issuedAt: 1, expiresAt: 301 }) }),
/RESOURCE_GRANT_EXPIRED/,
);
for (const change of [{ cpuMilli: 2001 }, { memoryMiB: 4097 }, { storageMiB: 10241 }])
assert.throws(() => renderRunResources(config, { ...request, resourceGrant: grant(change) }), /RESOURCE_LIMIT/);
});
test('pins signed previous runtime image and requires migration after its retirement', () => {
const previous = `registry.example.invalid/worker@sha256:${'b'.repeat(64)}`;
const requestWithPrevious = { ...request, resourceGrant: grant({ image: previous }) };
const configured = readConfig({ ...env, WF_RUN_PREVIOUS_IMAGES: previous });
assert.ok(
renderRunResources(configured, requestWithPrevious)
.resources.filter(r => r.kind === 'Job')
.every(r => r.spec.template.spec.containers[0].image === previous),
);
assert.throws(() => renderRunResources(config, requestWithPrevious), /RUNTIME_IMAGE_MIGRATION_REQUIRED/);
});
test('cleanup verifies ownership, stops matching jobs, retains PVCs and never fabricates measurement', async () => {
const { resources } = renderRunResources(config, request);
const job = resources.find(r => r.kind === 'Job');
const calls = [];
const release = {
tenantId: request.tenantId,
runId: request.runId,
generation: request.generation,
resourceGrant: grant({
issuedAt: 1,
expiresAt: 301,
image: `registry.example.invalid/retired@sha256:${'c'.repeat(64)}`,
}),
};
const execute = async (_config, args) => {
calls.push(args);
return args[0] === 'get' ? JSON.stringify({ items: args[1] === 'jobs' ? [job] : [] }) : '';
};
const receipt = await releaseRun(config, release, execute);
assert.equal(receipt.status, 'released');
assert.equal(receipt.computeReleased, true);
assert.equal(receipt.persistentStorageRetained, true);
assert.equal(receipt.computeSeconds, undefined);
assert.ok(calls.every(args => !args.includes('pvc') && !args.includes('persistentvolumeclaim')));
const deleted = calls.filter(args => args[0] === 'delete').map(args => args[1]);
assert.deepEqual(deleted, [
'job',
'secret',
'secret',
'serviceaccount',
'serviceaccount',
'networkpolicy',
'networkpolicy',
]);
await assert.rejects(
releaseRun(config, release, async () =>
JSON.stringify({ items: [{ ...job, metadata: { ...job.metadata, namespace: 'other' } }] }),
),
/SCOPE_MISMATCH/,
);
const pending = await releaseRun(config, release, async (_config, args) =>
JSON.stringify({ items: args[1] === 'pods' ? [{ metadata: job.metadata, status: { phase: 'Running' } }] : [] }),
);
assert.equal(pending.status, 'releasing');
assert.equal(pending.computeReleased, false);
});
test('deletes a run’s retained volumes by its labels, never while one of its pods is active', async () => {
const purge = { tenantId: 'tenant-a', runId: 'run-123', generation: 'generation-1', resourceGrant: grant() };
const calls = [];
let phase = 'Running';
const execute = async (_config, args) => {
calls.push(args);
return args[0] === 'get' ? JSON.stringify({ items: [{ status: { phase } }] }) : '';
};
assert.equal((await purgeRunStorage(config, purge, execute)).status, 'busy');
assert.equal(
calls.some(args => args[0] === 'delete'),
false,
);
phase = 'Succeeded';
assert.deepEqual(await purgeRunStorage(config, purge, execute), {
tenantId: 'tenant-a',
runId: 'run-123',
generation: 'generation-1',
status: 'purged',
persistentStorageRetained: false,
});
const deletion = calls.find(args => args[0] === 'delete');
assert.deepEqual(deletion.slice(0, 3), ['delete', 'pvc', '-l']);
assert.match(
deletion[3],
/^app\.kubernetes\.io\/managed-by=wishforge-provisioner,wishforge\.io\/run=[a-f0-9]{24},wishforge\.io\/component=runner$/,
);
await assert.rejects(
purgeRunStorage(config, { ...purge, resourceGrant: { ...purge.resourceGrant, runId: 'other' } }, execute),
);
});
test('serializes delayed provision with cleanup while allowing independent runs', async () => {
const queue = createRunOperationQueue(),
order = [];
let resolve;
const first = queue('tenant-a', 'run-1', async () => {
order.push('provision');
await new Promise(r => {
resolve = r;
});
order.push('provisioned');
});
await Promise.resolve();
const cleanup = queue('tenant-a', 'run-1', async () => {
order.push('cleanup');
});
await queue('tenant-a', 'run-2', async () => {
order.push('independent');
});
assert.deepEqual(order, ['provision', 'independent']);
resolve();
await Promise.all([first, cleanup]);
assert.deepEqual(order, ['provision', 'independent', 'provisioned', 'cleanup']);
await assert.rejects(
queue('tenant-a', 'run-1', async () => {
throw new Error('fixture');
}),
/fixture/,
);
assert.equal(await queue('tenant-a', 'run-1', async () => 'reconciled'), 'reconciled');
});
test('applies policy before jobs, retries deterministic names and propagates infrastructure failure', async () => {
const calls = [];
const apply = async (cfg, resources) => {
assert.equal(cfg.context, 'explicit-test-context');
calls.push(resources);
};
const first = await provisionRun(config, request, apply, async () => ({ terminal: false }));
const second = await provisionRun(config, request, apply, async () => ({ terminal: false }));
assert.deepEqual(first, second);
assert.equal(calls.length, 4);
assert.ok(calls[0].every(r => r.kind !== 'Job'));
assert.ok(calls[1].every(r => r.kind === 'Job'));
await assert.rejects(
provisionRun(
config,
request,
async () => {
throw new Error('KUBERNETES_APPLY_FAILED');
},
async () => ({ terminal: false }),
),
/KUBERNETES_APPLY_FAILED/,
);
});
test('reconciles only old owned run jobs and refuses to pretend a terminal generation resumed', async () => {
const { resources } = renderRunResources(config, request);
const job = resources.find(r => r.kind === 'Job');
const calls = [];
const old = {
...job,
metadata: { ...job.metadata, labels: { ...job.metadata.labels, 'wishforge.io/generation': 'old-generation' } },
};
const execute = async (cfg, args) => {
calls.push(args);
return args[0] === 'get' ? JSON.stringify({ items: [old] }) : '';
};
assert.deepEqual(await reconcileRunJobs(config, request, execute), { terminal: false });
assert.deepEqual(calls[1], [
'delete',
'job',
job.metadata.name,
'--cascade=foreground',
'--wait=true',
'--timeout=30s',
]);
const terminal = await reconcileRunJobs(config, request, async () =>
JSON.stringify({ items: [{ ...job, status: { failed: 1 } }] }),
);
assert.equal(terminal.terminal, true);
let applied = false;
const response = await provisionRun(
config,
request,
async () => {
applied = true;
},
async () => terminal,
);
assert.equal(response.status, 'failed');
assert.equal(applied, false);
await assert.rejects(
reconcileRunJobs(config, request, async () =>
JSON.stringify({ items: [{ ...old, metadata: { ...old.metadata, namespace: 'other-tenant' } }] }),
),
/SCOPE/,
);
});
test('removes what superseded generations left behind, but nothing of the current or a still running one', async () => {
const { resources } = renderRunResources(config, request);
const claim = resources.find(r => r.kind === 'PersistentVolumeClaim');
const generation = (prefix, hash) => ({
metadata: {
...claim.metadata,
name: `${prefix}-executor-workspace`,
labels: { ...claim.metadata.labels, 'wishforge.io/generation': hash },
},
});
const finished = generation(`wf-${'a'.repeat(24)}`, 'finished-generation');
const running = generation(`wf-${'b'.repeat(24)}`, 'running-generation');
const pod = (item, phase) => ({ metadata: item.metadata, status: { phase } });
const calls = [];
const listing = items => async (cfg, args) => {
calls.push(args);
if (args[0] !== 'get') return '';
if (args[1] === 'persistentvolumeclaims') return JSON.stringify({ items });
if (args[1] === 'pods')
return JSON.stringify({ items: [pod(finished, 'Succeeded'), pod(running, 'Running'), pod(claim, 'Running')] });
return JSON.stringify({ items: [] });
};
assert.deepEqual(await reconcileRunJobs(config, request, listing([claim, finished, running])), { terminal: false });
assert.deepEqual(
calls.filter(args => args[0] === 'delete'),
generationIdentities(`wf-${'a'.repeat(24)}`).map(([kind, name]) => [
'delete',
kind,
name,
'--ignore-not-found=true',
'--wait=false',
]),
);
const foreign = { ...finished, metadata: { ...finished.metadata, namespace: 'other-tenant' } };
await assert.rejects(reconcileRunJobs(config, request, listing([foreign])), /SCOPE/);
});