- The local provisioner kept a generation whose workers exited by themselves as running, so its run's directories could never be purged. - On Kubernetes, only a run's current generation is ever released: a superseded generation's Secrets, with its run tokens, its service accounts and network policies stayed forever. Starting a new generation now deletes them by name for every earlier generation without a running pod, found through the volumes kept for the retention period, so the provisioner still cannot list Secrets. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
497 lines
21 KiB
JavaScript
497 lines
21 KiB
JavaScript
import test from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { createHmac } from 'node:crypto';
|
||
import {
|
||
generationIdentities,
|
||
readConfig,
|
||
renderRunResources,
|
||
provisionRun,
|
||
reconcileRunJobs,
|
||
} from '../kubernetes.mjs';
|
||
import { resourceGrantPayload } from '../resource-grants.mjs';
|
||
import { purgeRunStorage, releaseRun } from '../run-release.mjs';
|
||
import { createRunOperationQueue } from '../run-operations.mjs';
|
||
|
||
const env = {
|
||
WF_TENANT_ID: 'tenant-a',
|
||
WF_KUBE_CONTEXT: 'explicit-test-context',
|
||
WF_KUBE_NAMESPACE: 'wishforge-tenant-a',
|
||
WF_RUN_WORKER_IMAGE: `registry.example.invalid/worker@sha256:${'a'.repeat(64)}`,
|
||
WF_RUN_STORAGE_CLASS: 'reviewed-rwx',
|
||
WF_CONTROL_PLANE_URL: 'https://api.wishforge-control.svc:8443',
|
||
WF_CONTROL_NAMESPACE: 'wishforge-control',
|
||
TEMPORAL_ADDRESS: 'temporal.wishforge-control.svc:7233',
|
||
TEMPORAL_NAMESPACE: 'wishforge',
|
||
};
|
||
env.WF_RESOURCE_GRANT_SECRET = 'test-only-resource-signing-secret-longer-than-32';
|
||
const config = readConfig(env);
|
||
function grant(overrides = {}) {
|
||
const issuedAt = Math.floor(Date.now() / 1000);
|
||
const payload = {
|
||
schemaVersion: 1,
|
||
tenantId: 'tenant-a',
|
||
runId: 'run-123',
|
||
generation: 'generation-1',
|
||
planId: 'pilot',
|
||
planVersion: 'pilot-v1',
|
||
image: config.image,
|
||
cpuMilli: 1500,
|
||
memoryMiB: 3072,
|
||
storageMiB: 8192,
|
||
issuedAt,
|
||
expiresAt: issuedAt + 300,
|
||
...overrides,
|
||
};
|
||
return {
|
||
...payload,
|
||
signature: createHmac('sha256', env.WF_RESOURCE_GRANT_SECRET)
|
||
.update(resourceGrantPayload(payload))
|
||
.digest('base64url'),
|
||
};
|
||
}
|
||
function token(kind, run = 'run-123') {
|
||
return `header.${Buffer.from(JSON.stringify({ tenant_id: 'tenant-a', run_id: run, generation: 'generation-1', kind, iss: 'wishforge-control-plane', aud: 'wishforge-internal', exp: Math.floor(Date.now() / 1000) + 14400 })).toString('base64url')}.signature`;
|
||
}
|
||
const request = {
|
||
tenantId: 'tenant-a',
|
||
runId: 'run-123',
|
||
generation: 'generation-1',
|
||
executorToken: token('worker'),
|
||
verifierToken: token('verifier'),
|
||
resourceGrant: grant(),
|
||
};
|
||
|
||
test('fails closed without explicit cluster, namespace, image digest and storage', () => {
|
||
assert.throws(() => readConfig({}), /unconfigured/);
|
||
assert.throws(() => readConfig({ ...env, WF_RUN_WORKER_IMAGE: 'worker:latest' }), /immutable/);
|
||
assert.throws(() => renderRunResources(config, { ...request, tenantId: 'tenant-b' }), /outside/);
|
||
assert.throws(() => renderRunResources(config, { ...request, executorToken: token('worker', 'other-run') }), /token/);
|
||
});
|
||
|
||
test('renders independent Kata jobs with role-scoped credentials and separate role PVCs', () => {
|
||
const { resources } = renderRunResources(config, request);
|
||
const jobs = resources.filter(r => r.kind === 'Job');
|
||
assert.equal(jobs.length, 2);
|
||
assert.notEqual(jobs[0].spec.template.spec.serviceAccountName, jobs[1].spec.template.spec.serviceAccountName);
|
||
for (const job of jobs) {
|
||
const pod = job.spec.template.spec,
|
||
container = pod.containers[0];
|
||
assert.equal(pod.runtimeClassName, 'wishforge-kata');
|
||
assert.equal(pod.automountServiceAccountToken, false);
|
||
assert.equal(container.securityContext.allowPrivilegeEscalation, false);
|
||
assert.equal(container.securityContext.readOnlyRootFilesystem, true);
|
||
assert.deepEqual(container.securityContext.capabilities.drop, ['ALL']);
|
||
assert.ok(!pod.hostNetwork && !pod.hostPID && !pod.hostIPC);
|
||
assert.ok(pod.volumes.every(v => !v.hostPath));
|
||
assert.ok(!JSON.stringify(pod).includes('docker.sock'));
|
||
assert.equal(container.env.filter(e => /TOKEN$/.test(e.name)).length, 1);
|
||
assert.ok(container.env.find(e => /TOKEN$/.test(e.name)).valueFrom.secretKeyRef);
|
||
assert.ok(!JSON.stringify(pod).includes(request.executorToken));
|
||
assert.ok(!JSON.stringify(pod).includes(request.verifierToken));
|
||
assert.equal(container.image, request.resourceGrant.image);
|
||
assert.equal(container.resources.limits.cpu, '750m');
|
||
assert.equal(container.resources.limits.memory, '1536Mi');
|
||
}
|
||
const pvc = resources.filter(r => r.kind === 'PersistentVolumeClaim');
|
||
assert.equal(pvc.length, 2);
|
||
assert.ok(pvc.every(claim => claim.spec.accessModes[0] === 'ReadWriteOnce'));
|
||
assert.notEqual(
|
||
jobs[0].spec.template.spec.volumes[0].persistentVolumeClaim.claimName,
|
||
jobs[1].spec.template.spec.volumes[0].persistentVolumeClaim.claimName,
|
||
);
|
||
const policy = resources.find(r => r.kind === 'NetworkPolicy');
|
||
assert.deepEqual(policy.spec.ingress, []);
|
||
assert.ok(
|
||
policy.spec.egress.every(rule => rule.to.every(to => to.namespaceSelector && to.podSelector && !to.ipBlock)),
|
||
);
|
||
assert.ok(pvc.every(claim => claim.spec.resources.requests.storage === '4096Mi'));
|
||
assert.ok(
|
||
jobs.reduce((total, job) => total + parseInt(job.spec.template.spec.containers[0].resources.limits.cpu), 0) <=
|
||
request.resourceGrant.cpuMilli,
|
||
);
|
||
assert.ok(
|
||
jobs.reduce((total, job) => total + parseInt(job.spec.template.spec.containers[0].resources.limits.memory), 0) <=
|
||
request.resourceGrant.memoryMiB,
|
||
);
|
||
assert.ok(
|
||
pvc.reduce((total, claim) => total + parseInt(claim.spec.resources.requests.storage), 0) <=
|
||
request.resourceGrant.storageMiB,
|
||
);
|
||
const other = renderRunResources(config, {
|
||
...request,
|
||
runId: 'run-456',
|
||
executorToken: token('worker', 'run-456'),
|
||
verifierToken: token('verifier', 'run-456'),
|
||
resourceGrant: grant({ runId: 'run-456' }),
|
||
});
|
||
assert.notEqual(other.receipt.workspaceClaim, pvc[0].metadata.name);
|
||
});
|
||
|
||
test('gives sandboxes the tenant’s Temporal client certificate, which production requires', () => {
|
||
assert.throws(() => readConfig({ ...env, WF_DEPLOYMENT_MODE: 'production' }), /WF_RUN_TEMPORAL_TLS_SECRET/);
|
||
const configured = readConfig({
|
||
...env,
|
||
WF_DEPLOYMENT_MODE: 'production',
|
||
WF_RUN_TEMPORAL_TLS_SECRET: 'tenant-a-temporal-client',
|
||
});
|
||
for (const job of renderRunResources(configured, request).resources.filter(r => r.kind === 'Job')) {
|
||
const pod = job.spec.template.spec;
|
||
const values = Object.fromEntries(pod.containers[0].env.map(e => [e.name, e.value]));
|
||
assert.equal(values.TEMPORAL_TLS_CERT, '/run/temporal/tls.crt');
|
||
assert.equal(values.TEMPORAL_TLS_KEY, '/run/temporal/tls.key');
|
||
assert.equal(values.TEMPORAL_TLS_CA, '/run/temporal/ca.crt');
|
||
assert.equal(values.TEMPORAL_NAMESPACE, 'wishforge');
|
||
assert.deepEqual(pod.volumes.find(v => v.name === 'temporal-tls').secret, {
|
||
secretName: 'tenant-a-temporal-client',
|
||
defaultMode: 0o440,
|
||
});
|
||
assert.deepEqual(
|
||
pod.containers[0].volumeMounts.find(m => m.name === 'temporal-tls'),
|
||
{ name: 'temporal-tls', mountPath: '/run/temporal', readOnly: true },
|
||
);
|
||
}
|
||
assert.throws(() => readConfig({ ...env, WF_RUN_TEMPORAL_TLS_SECRET: 'Not_A_Name' }), /Invalid configured/);
|
||
});
|
||
test('lets installations reach only the egress proxy and only the verifier reach the image builder', () => {
|
||
const configured = readConfig({
|
||
...env,
|
||
WF_RUN_INSTALL_PROXY_URL: 'http://egress-proxy.wishforge-control.svc:3128',
|
||
WF_VERIFIER_DOCKER_HOST: 'tcp://image-builder.wishforge-build.svc:2376',
|
||
WF_VERIFIER_DOCKER_TLS_SECRET: 'image-builder-client',
|
||
WF_VERIFIER_BUILDER_NAMESPACE: 'wishforge-build',
|
||
});
|
||
const { resources } = renderRunResources(configured, request);
|
||
const [executor, verifier] = ['executor', 'verifier'].map(role =>
|
||
resources.find(r => r.kind === 'Job' && r.spec.template.metadata.labels['wishforge.io/worker-role'] === role),
|
||
);
|
||
const envOf = job => Object.fromEntries(job.spec.template.spec.containers[0].env.map(e => [e.name, e.value]));
|
||
assert.equal(envOf(executor).WF_INSTALL_PROXY, 'http://egress-proxy.wishforge-control.svc:3128');
|
||
assert.equal(envOf(verifier).WF_INSTALL_PROXY, 'http://egress-proxy.wishforge-control.svc:3128');
|
||
assert.equal(envOf(executor).WF_DOCKER_HOST, undefined);
|
||
assert.equal(envOf(verifier).WF_DOCKER_HOST, 'tcp://image-builder.wishforge-build.svc:2376');
|
||
assert.equal(envOf(verifier).WF_DOCKER_TLS_DIR, '/run/docker-tls');
|
||
assert.deepEqual(verifier.spec.template.spec.volumes.find(v => v.name === 'docker-tls').secret, {
|
||
secretName: 'image-builder-client',
|
||
defaultMode: 0o440,
|
||
});
|
||
const [isolation, builder] = resources.filter(r => r.kind === 'NetworkPolicy');
|
||
assert.deepEqual(isolation.spec.egress.at(-1), {
|
||
to: [
|
||
{
|
||
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'wishforge-control' } },
|
||
podSelector: { matchLabels: { 'wishforge.io/component': 'egress-proxy' } },
|
||
},
|
||
],
|
||
ports: [{ protocol: 'TCP', port: 3128 }],
|
||
});
|
||
assert.equal(builder.metadata.name.endsWith('-builder'), true);
|
||
assert.equal(builder.spec.podSelector.matchLabels['wishforge.io/worker-role'], 'verifier');
|
||
assert.deepEqual(builder.spec.egress, [
|
||
{
|
||
to: [
|
||
{
|
||
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'wishforge-build' } },
|
||
podSelector: { matchLabels: { 'wishforge.io/component': 'image-builder' } },
|
||
},
|
||
],
|
||
ports: [{ protocol: 'TCP', port: 2376 }],
|
||
},
|
||
]);
|
||
for (const invalid of [
|
||
{ WF_RUN_INSTALL_PROXY_URL: 'http://user:secret@proxy.svc:3128' },
|
||
{ WF_RUN_INSTALL_PROXY_URL: 'socks5://proxy.svc:1080' },
|
||
{ WF_VERIFIER_DOCKER_HOST: 'unix:///var/run/docker.sock' },
|
||
{
|
||
WF_VERIFIER_DOCKER_HOST: 'tcp://builder.svc:2376',
|
||
WF_DEPLOYMENT_MODE: 'production',
|
||
WF_RUN_TEMPORAL_TLS_SECRET: 't',
|
||
},
|
||
])
|
||
assert.throws(() => readConfig({ ...env, ...invalid }));
|
||
// Without them nothing beyond the control plane is reachable.
|
||
const plain = renderRunResources(config, request).resources.filter(r => r.kind === 'NetworkPolicy');
|
||
assert.equal(plain.length, 1);
|
||
});
|
||
test('runs sandboxes in the provisioner’s deployment mode, and only development clusters without Kata', () => {
|
||
const jobEnv = resources =>
|
||
resources
|
||
.filter(r => r.kind === 'Job')
|
||
.map(job => Object.fromEntries(job.spec.template.spec.containers[0].env.map(e => [e.name, e.value])));
|
||
for (const values of jobEnv(renderRunResources(config, request).resources)) {
|
||
assert.equal(values.WF_DEPLOYMENT_MODE, 'production');
|
||
assert.equal(values.NODE_ENV, 'production');
|
||
assert.equal(values.WF_SANDBOX_MODE, 'kata');
|
||
assert.equal(values.WF_ALLOW_INSECURE_HTTP, undefined);
|
||
}
|
||
const local = {
|
||
...env,
|
||
WF_DEPLOYMENT_MODE: 'development',
|
||
WF_ALLOW_INSECURE_HTTP: 'true',
|
||
WF_RUN_SANDBOX_MODE: 'host',
|
||
WF_KATA_RUNTIME_CLASS: 'none',
|
||
WF_CONTROL_PLANE_URL: 'http://api.wishforge-control.svc:4000',
|
||
};
|
||
const { resources } = renderRunResources(readConfig(local), request);
|
||
for (const job of resources.filter(r => r.kind === 'Job'))
|
||
assert.equal('runtimeClassName' in job.spec.template.spec, false);
|
||
for (const values of jobEnv(resources)) {
|
||
assert.equal(values.WF_DEPLOYMENT_MODE, 'development');
|
||
// Overrides the image's NODE_ENV=production, which development mode refuses.
|
||
assert.equal(values.NODE_ENV, 'development');
|
||
assert.equal(values.WF_SANDBOX_MODE, 'host');
|
||
assert.equal(values.WF_ALLOW_INSECURE_HTTP, 'true');
|
||
assert.equal(values.WF_GIT_ALLOW_LOCAL_HTTP, 'true');
|
||
assert.equal(values.WF_CONTROL_PLANE_URL, 'http://api.wishforge-control.svc:4000');
|
||
}
|
||
// None of it is accepted outside an explicit development deployment.
|
||
const production = { ...env, WF_DEPLOYMENT_MODE: 'production', WF_RUN_TEMPORAL_TLS_SECRET: 'temporal-client' };
|
||
assert.throws(() => readConfig({ ...production, WF_RUN_SANDBOX_MODE: 'host' }), /development/);
|
||
assert.throws(() => readConfig({ ...production, WF_KATA_RUNTIME_CLASS: 'none' }), /development/);
|
||
assert.throws(() => readConfig({ ...production, WF_ALLOW_INSECURE_HTTP: 'true' }), /development/);
|
||
assert.throws(() => readConfig({ ...env, WF_CONTROL_PLANE_URL: 'http://api.wishforge-control.svc:4000' }), /HTTPS/);
|
||
});
|
||
test('rejects tampered, expired, wrong-scope and above-cap resource grants', () => {
|
||
for (const change of [{ cpuMilli: 1600 }, { generation: 'other' }, { tenantId: 'other' }])
|
||
assert.throws(
|
||
() => renderRunResources(config, { ...request, resourceGrant: { ...request.resourceGrant, ...change } }),
|
||
/RESOURCE_GRANT_INVALID/,
|
||
);
|
||
assert.throws(
|
||
() => renderRunResources(config, { ...request, resourceGrant: grant({ runId: 'other' }) }),
|
||
/RESOURCE_GRANT_INVALID/,
|
||
);
|
||
assert.throws(
|
||
() => renderRunResources(config, { ...request, resourceGrant: grant({ issuedAt: 1, expiresAt: 301 }) }),
|
||
/RESOURCE_GRANT_EXPIRED/,
|
||
);
|
||
for (const change of [{ cpuMilli: 2001 }, { memoryMiB: 4097 }, { storageMiB: 10241 }])
|
||
assert.throws(() => renderRunResources(config, { ...request, resourceGrant: grant(change) }), /RESOURCE_LIMIT/);
|
||
});
|
||
|
||
test('pins signed previous runtime image and requires migration after its retirement', () => {
|
||
const previous = `registry.example.invalid/worker@sha256:${'b'.repeat(64)}`;
|
||
const requestWithPrevious = { ...request, resourceGrant: grant({ image: previous }) };
|
||
const configured = readConfig({ ...env, WF_RUN_PREVIOUS_IMAGES: previous });
|
||
assert.ok(
|
||
renderRunResources(configured, requestWithPrevious)
|
||
.resources.filter(r => r.kind === 'Job')
|
||
.every(r => r.spec.template.spec.containers[0].image === previous),
|
||
);
|
||
assert.throws(() => renderRunResources(config, requestWithPrevious), /RUNTIME_IMAGE_MIGRATION_REQUIRED/);
|
||
});
|
||
|
||
test('cleanup verifies ownership, stops matching jobs, retains PVCs and never fabricates measurement', async () => {
|
||
const { resources } = renderRunResources(config, request);
|
||
const job = resources.find(r => r.kind === 'Job');
|
||
const calls = [];
|
||
const release = {
|
||
tenantId: request.tenantId,
|
||
runId: request.runId,
|
||
generation: request.generation,
|
||
resourceGrant: grant({
|
||
issuedAt: 1,
|
||
expiresAt: 301,
|
||
image: `registry.example.invalid/retired@sha256:${'c'.repeat(64)}`,
|
||
}),
|
||
};
|
||
const execute = async (_config, args) => {
|
||
calls.push(args);
|
||
return args[0] === 'get' ? JSON.stringify({ items: args[1] === 'jobs' ? [job] : [] }) : '';
|
||
};
|
||
const receipt = await releaseRun(config, release, execute);
|
||
assert.equal(receipt.status, 'released');
|
||
assert.equal(receipt.computeReleased, true);
|
||
assert.equal(receipt.persistentStorageRetained, true);
|
||
assert.equal(receipt.computeSeconds, undefined);
|
||
assert.ok(calls.every(args => !args.includes('pvc') && !args.includes('persistentvolumeclaim')));
|
||
const deleted = calls.filter(args => args[0] === 'delete').map(args => args[1]);
|
||
assert.deepEqual(deleted, [
|
||
'job',
|
||
'secret',
|
||
'secret',
|
||
'serviceaccount',
|
||
'serviceaccount',
|
||
'networkpolicy',
|
||
'networkpolicy',
|
||
]);
|
||
await assert.rejects(
|
||
releaseRun(config, release, async () =>
|
||
JSON.stringify({ items: [{ ...job, metadata: { ...job.metadata, namespace: 'other' } }] }),
|
||
),
|
||
/SCOPE_MISMATCH/,
|
||
);
|
||
const pending = await releaseRun(config, release, async (_config, args) =>
|
||
JSON.stringify({ items: args[1] === 'pods' ? [{ metadata: job.metadata, status: { phase: 'Running' } }] : [] }),
|
||
);
|
||
assert.equal(pending.status, 'releasing');
|
||
assert.equal(pending.computeReleased, false);
|
||
});
|
||
|
||
test('deletes a run’s retained volumes by its labels, never while one of its pods is active', async () => {
|
||
const purge = { tenantId: 'tenant-a', runId: 'run-123', generation: 'generation-1', resourceGrant: grant() };
|
||
const calls = [];
|
||
let phase = 'Running';
|
||
const execute = async (_config, args) => {
|
||
calls.push(args);
|
||
return args[0] === 'get' ? JSON.stringify({ items: [{ status: { phase } }] }) : '';
|
||
};
|
||
assert.equal((await purgeRunStorage(config, purge, execute)).status, 'busy');
|
||
assert.equal(
|
||
calls.some(args => args[0] === 'delete'),
|
||
false,
|
||
);
|
||
phase = 'Succeeded';
|
||
assert.deepEqual(await purgeRunStorage(config, purge, execute), {
|
||
tenantId: 'tenant-a',
|
||
runId: 'run-123',
|
||
generation: 'generation-1',
|
||
status: 'purged',
|
||
persistentStorageRetained: false,
|
||
});
|
||
const deletion = calls.find(args => args[0] === 'delete');
|
||
assert.deepEqual(deletion.slice(0, 3), ['delete', 'pvc', '-l']);
|
||
assert.match(
|
||
deletion[3],
|
||
/^app\.kubernetes\.io\/managed-by=wishforge-provisioner,wishforge\.io\/run=[a-f0-9]{24},wishforge\.io\/component=runner$/,
|
||
);
|
||
await assert.rejects(
|
||
purgeRunStorage(config, { ...purge, resourceGrant: { ...purge.resourceGrant, runId: 'other' } }, execute),
|
||
);
|
||
});
|
||
test('serializes delayed provision with cleanup while allowing independent runs', async () => {
|
||
const queue = createRunOperationQueue(),
|
||
order = [];
|
||
let resolve;
|
||
const first = queue('tenant-a', 'run-1', async () => {
|
||
order.push('provision');
|
||
await new Promise(r => {
|
||
resolve = r;
|
||
});
|
||
order.push('provisioned');
|
||
});
|
||
await Promise.resolve();
|
||
const cleanup = queue('tenant-a', 'run-1', async () => {
|
||
order.push('cleanup');
|
||
});
|
||
await queue('tenant-a', 'run-2', async () => {
|
||
order.push('independent');
|
||
});
|
||
assert.deepEqual(order, ['provision', 'independent']);
|
||
resolve();
|
||
await Promise.all([first, cleanup]);
|
||
assert.deepEqual(order, ['provision', 'independent', 'provisioned', 'cleanup']);
|
||
await assert.rejects(
|
||
queue('tenant-a', 'run-1', async () => {
|
||
throw new Error('fixture');
|
||
}),
|
||
/fixture/,
|
||
);
|
||
assert.equal(await queue('tenant-a', 'run-1', async () => 'reconciled'), 'reconciled');
|
||
});
|
||
|
||
test('applies policy before jobs, retries deterministic names and propagates infrastructure failure', async () => {
|
||
const calls = [];
|
||
const apply = async (cfg, resources) => {
|
||
assert.equal(cfg.context, 'explicit-test-context');
|
||
calls.push(resources);
|
||
};
|
||
const first = await provisionRun(config, request, apply, async () => ({ terminal: false }));
|
||
const second = await provisionRun(config, request, apply, async () => ({ terminal: false }));
|
||
assert.deepEqual(first, second);
|
||
assert.equal(calls.length, 4);
|
||
assert.ok(calls[0].every(r => r.kind !== 'Job'));
|
||
assert.ok(calls[1].every(r => r.kind === 'Job'));
|
||
await assert.rejects(
|
||
provisionRun(
|
||
config,
|
||
request,
|
||
async () => {
|
||
throw new Error('KUBERNETES_APPLY_FAILED');
|
||
},
|
||
async () => ({ terminal: false }),
|
||
),
|
||
/KUBERNETES_APPLY_FAILED/,
|
||
);
|
||
});
|
||
|
||
test('reconciles only old owned run jobs and refuses to pretend a terminal generation resumed', async () => {
|
||
const { resources } = renderRunResources(config, request);
|
||
const job = resources.find(r => r.kind === 'Job');
|
||
const calls = [];
|
||
const old = {
|
||
...job,
|
||
metadata: { ...job.metadata, labels: { ...job.metadata.labels, 'wishforge.io/generation': 'old-generation' } },
|
||
};
|
||
const execute = async (cfg, args) => {
|
||
calls.push(args);
|
||
return args[0] === 'get' ? JSON.stringify({ items: [old] }) : '';
|
||
};
|
||
assert.deepEqual(await reconcileRunJobs(config, request, execute), { terminal: false });
|
||
assert.deepEqual(calls[1], [
|
||
'delete',
|
||
'job',
|
||
job.metadata.name,
|
||
'--cascade=foreground',
|
||
'--wait=true',
|
||
'--timeout=30s',
|
||
]);
|
||
const terminal = await reconcileRunJobs(config, request, async () =>
|
||
JSON.stringify({ items: [{ ...job, status: { failed: 1 } }] }),
|
||
);
|
||
assert.equal(terminal.terminal, true);
|
||
let applied = false;
|
||
const response = await provisionRun(
|
||
config,
|
||
request,
|
||
async () => {
|
||
applied = true;
|
||
},
|
||
async () => terminal,
|
||
);
|
||
assert.equal(response.status, 'failed');
|
||
assert.equal(applied, false);
|
||
await assert.rejects(
|
||
reconcileRunJobs(config, request, async () =>
|
||
JSON.stringify({ items: [{ ...old, metadata: { ...old.metadata, namespace: 'other-tenant' } }] }),
|
||
),
|
||
/SCOPE/,
|
||
);
|
||
});
|
||
|
||
test('removes what superseded generations left behind, but nothing of the current or a still running one', async () => {
|
||
const { resources } = renderRunResources(config, request);
|
||
const claim = resources.find(r => r.kind === 'PersistentVolumeClaim');
|
||
const generation = (prefix, hash) => ({
|
||
metadata: {
|
||
...claim.metadata,
|
||
name: `${prefix}-executor-workspace`,
|
||
labels: { ...claim.metadata.labels, 'wishforge.io/generation': hash },
|
||
},
|
||
});
|
||
const finished = generation(`wf-${'a'.repeat(24)}`, 'finished-generation');
|
||
const running = generation(`wf-${'b'.repeat(24)}`, 'running-generation');
|
||
const pod = (item, phase) => ({ metadata: item.metadata, status: { phase } });
|
||
const calls = [];
|
||
const listing = items => async (cfg, args) => {
|
||
calls.push(args);
|
||
if (args[0] !== 'get') return '';
|
||
if (args[1] === 'persistentvolumeclaims') return JSON.stringify({ items });
|
||
if (args[1] === 'pods')
|
||
return JSON.stringify({ items: [pod(finished, 'Succeeded'), pod(running, 'Running'), pod(claim, 'Running')] });
|
||
return JSON.stringify({ items: [] });
|
||
};
|
||
assert.deepEqual(await reconcileRunJobs(config, request, listing([claim, finished, running])), { terminal: false });
|
||
assert.deepEqual(
|
||
calls.filter(args => args[0] === 'delete'),
|
||
generationIdentities(`wf-${'a'.repeat(24)}`).map(([kind, name]) => [
|
||
'delete',
|
||
kind,
|
||
name,
|
||
'--ignore-not-found=true',
|
||
'--wait=false',
|
||
]),
|
||
);
|
||
const foreign = { ...finished, metadata: { ...finished.metadata, namespace: 'other-tenant' } };
|
||
await assert.rejects(reconcileRunJobs(config, request, listing([foreign])), /SCOPE/);
|
||
});
|