- The local provisioner kept a generation whose workers exited by themselves as running, so its run's directories could never be purged. - On Kubernetes, only a run's current generation is ever released: a superseded generation's Secrets, with its run tokens, its service accounts and network policies stayed forever. Starting a new generation now deletes them by name for every earlier generation without a running pod, found through the volumes kept for the retention period, so the provisioner still cannot list Secrets. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
86 lines
4.7 KiB
JavaScript
86 lines
4.7 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import { verifyResourceGrant } from './resource-grants.mjs';
|
|
import { generationIdentities, lifecycleCommand } from './kubernetes.mjs';
|
|
/** Stops only matching compute Jobs; PVCs are retained and never called free storage. */
|
|
export async function releaseRun(config, request, execute = lifecycleCommand) {
|
|
if (
|
|
!request ||
|
|
Object.keys(request).some(key => !['tenantId', 'runId', 'generation', 'resourceGrant'].includes(key)) ||
|
|
!/^[A-Za-z0-9_-]{1,128}$/.test(request.runId ?? '') ||
|
|
!/^[A-Za-z0-9_-]{1,128}$/.test(request.generation ?? '')
|
|
)
|
|
throw new Error('RESOURCE_GRANT_INVALID');
|
|
verifyResourceGrant(config, request, { allowExpired: true, allowRetiredImage: true, cleanup: true });
|
|
const run = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24),
|
|
generation = createHash('sha256').update(request.generation).digest('hex').slice(0, 24);
|
|
const prefix = `wf-${createHash('sha256').update(`${request.tenantId}\0${request.runId}\0${request.generation}`).digest('hex').slice(0, 24)}`;
|
|
const selector = `app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${run},wishforge.io/generation=${generation},wishforge.io/component=runner`;
|
|
const assertScope = value => {
|
|
if (
|
|
value.metadata?.namespace !== config.namespace ||
|
|
value.metadata?.labels?.['app.kubernetes.io/managed-by'] !== 'wishforge-provisioner' ||
|
|
value.metadata?.labels?.['wishforge.io/component'] !== 'runner' ||
|
|
value.metadata?.labels?.['wishforge.io/run'] !== run ||
|
|
value.metadata?.labels?.['wishforge.io/generation'] !== generation
|
|
)
|
|
throw new Error('KUBERNETES_RELEASE_SCOPE_MISMATCH');
|
|
};
|
|
const jobs = JSON.parse(await execute(config, ['get', 'jobs', '-l', selector, '-o', 'json']));
|
|
if (!Array.isArray(jobs.items)) throw new Error('KUBERNETES_RELEASE_INVALID');
|
|
for (const job of jobs.items) {
|
|
assertScope(job);
|
|
if (![`${prefix}-executor`, `${prefix}-verifier`].includes(job.metadata.name))
|
|
throw new Error('KUBERNETES_RELEASE_SCOPE_MISMATCH');
|
|
await execute(config, [
|
|
'delete',
|
|
'job',
|
|
job.metadata.name,
|
|
'--ignore-not-found=true',
|
|
'--cascade=foreground',
|
|
'--wait=true',
|
|
'--timeout=30s',
|
|
]);
|
|
}
|
|
const pods = JSON.parse(await execute(config, ['get', 'pods', '-l', selector, '-o', 'json']));
|
|
if (!Array.isArray(pods.items)) throw new Error('KUBERNETES_RELEASE_INVALID');
|
|
for (const pod of pods.items) assertScope(pod);
|
|
const active = pods.items.some(pod => !['Succeeded', 'Failed'].includes(pod.status?.phase));
|
|
// Once compute stopped, the generation's credentials, identities and network policy go too.
|
|
// Volumes stay for diagnosis; they carry the same labels for the operator's retention policy.
|
|
if (!active)
|
|
for (const [kind, name] of generationIdentities(prefix))
|
|
await execute(config, ['delete', kind, name, '--ignore-not-found=true', '--wait=false']);
|
|
return {
|
|
tenantId: request.tenantId,
|
|
runId: request.runId,
|
|
generation: request.generation,
|
|
status: active ? 'releasing' : 'released',
|
|
computeReleased: !active,
|
|
persistentStorageRetained: true,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Deletes the volumes every generation of a run left behind, once the control plane's retention
|
|
* period has passed. Nothing is deleted while any of the run's pods is still active.
|
|
*/
|
|
export async function purgeRunStorage(config, request, execute = lifecycleCommand) {
|
|
if (
|
|
!request ||
|
|
Object.keys(request).some(key => !['tenantId', 'runId', 'generation', 'resourceGrant'].includes(key)) ||
|
|
!/^[A-Za-z0-9_-]{1,128}$/.test(request.runId ?? '') ||
|
|
!/^[A-Za-z0-9_-]{1,128}$/.test(request.generation ?? '')
|
|
)
|
|
throw new Error('RESOURCE_GRANT_INVALID');
|
|
verifyResourceGrant(config, request, { allowExpired: true, allowRetiredImage: true, cleanup: true });
|
|
const run = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24);
|
|
const selector = `app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${run},wishforge.io/component=runner`;
|
|
const receipt = { tenantId: request.tenantId, runId: request.runId, generation: request.generation };
|
|
const pods = JSON.parse(await execute(config, ['get', 'pods', '-l', selector, '-o', 'json']));
|
|
if (!Array.isArray(pods.items)) throw new Error('KUBERNETES_RELEASE_INVALID');
|
|
if (pods.items.some(pod => !['Succeeded', 'Failed'].includes(pod.status?.phase)))
|
|
return { ...receipt, status: 'busy', persistentStorageRetained: true };
|
|
await execute(config, ['delete', 'pvc', '-l', selector, '--ignore-not-found=true', '--wait=false']);
|
|
return { ...receipt, status: 'purged', persistentStorageRetained: false };
|
|
}
|