eust-w 3b78e8e536 🐛 Release what superseded run generations left behind
- The local provisioner kept a generation whose workers exited by
  themselves as running, so its run's directories could never be
  purged.
- On Kubernetes, only a run's current generation is ever released: a
  superseded generation's Secrets, with its run tokens, its service
  accounts and network policies stayed forever. Starting a new
  generation now deletes them by name for every earlier generation
  without a running pod, found through the volumes kept for the
  retention period, so the provisioner still cannot list Secrets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 22:22:11 +08:00

86 lines
4.7 KiB
JavaScript

import { createHash } from 'node:crypto';
import { verifyResourceGrant } from './resource-grants.mjs';
import { generationIdentities, lifecycleCommand } from './kubernetes.mjs';
/** Stops only matching compute Jobs; PVCs are retained and never called free storage. */
export async function releaseRun(config, request, execute = lifecycleCommand) {
if (
!request ||
Object.keys(request).some(key => !['tenantId', 'runId', 'generation', 'resourceGrant'].includes(key)) ||
!/^[A-Za-z0-9_-]{1,128}$/.test(request.runId ?? '') ||
!/^[A-Za-z0-9_-]{1,128}$/.test(request.generation ?? '')
)
throw new Error('RESOURCE_GRANT_INVALID');
verifyResourceGrant(config, request, { allowExpired: true, allowRetiredImage: true, cleanup: true });
const run = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24),
generation = createHash('sha256').update(request.generation).digest('hex').slice(0, 24);
const prefix = `wf-${createHash('sha256').update(`${request.tenantId}\0${request.runId}\0${request.generation}`).digest('hex').slice(0, 24)}`;
const selector = `app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${run},wishforge.io/generation=${generation},wishforge.io/component=runner`;
const assertScope = value => {
if (
value.metadata?.namespace !== config.namespace ||
value.metadata?.labels?.['app.kubernetes.io/managed-by'] !== 'wishforge-provisioner' ||
value.metadata?.labels?.['wishforge.io/component'] !== 'runner' ||
value.metadata?.labels?.['wishforge.io/run'] !== run ||
value.metadata?.labels?.['wishforge.io/generation'] !== generation
)
throw new Error('KUBERNETES_RELEASE_SCOPE_MISMATCH');
};
const jobs = JSON.parse(await execute(config, ['get', 'jobs', '-l', selector, '-o', 'json']));
if (!Array.isArray(jobs.items)) throw new Error('KUBERNETES_RELEASE_INVALID');
for (const job of jobs.items) {
assertScope(job);
if (![`${prefix}-executor`, `${prefix}-verifier`].includes(job.metadata.name))
throw new Error('KUBERNETES_RELEASE_SCOPE_MISMATCH');
await execute(config, [
'delete',
'job',
job.metadata.name,
'--ignore-not-found=true',
'--cascade=foreground',
'--wait=true',
'--timeout=30s',
]);
}
const pods = JSON.parse(await execute(config, ['get', 'pods', '-l', selector, '-o', 'json']));
if (!Array.isArray(pods.items)) throw new Error('KUBERNETES_RELEASE_INVALID');
for (const pod of pods.items) assertScope(pod);
const active = pods.items.some(pod => !['Succeeded', 'Failed'].includes(pod.status?.phase));
// Once compute stopped, the generation's credentials, identities and network policy go too.
// Volumes stay for diagnosis; they carry the same labels for the operator's retention policy.
if (!active)
for (const [kind, name] of generationIdentities(prefix))
await execute(config, ['delete', kind, name, '--ignore-not-found=true', '--wait=false']);
return {
tenantId: request.tenantId,
runId: request.runId,
generation: request.generation,
status: active ? 'releasing' : 'released',
computeReleased: !active,
persistentStorageRetained: true,
};
}
/**
* Deletes the volumes every generation of a run left behind, once the control plane's retention
* period has passed. Nothing is deleted while any of the run's pods is still active.
*/
export async function purgeRunStorage(config, request, execute = lifecycleCommand) {
if (
!request ||
Object.keys(request).some(key => !['tenantId', 'runId', 'generation', 'resourceGrant'].includes(key)) ||
!/^[A-Za-z0-9_-]{1,128}$/.test(request.runId ?? '') ||
!/^[A-Za-z0-9_-]{1,128}$/.test(request.generation ?? '')
)
throw new Error('RESOURCE_GRANT_INVALID');
verifyResourceGrant(config, request, { allowExpired: true, allowRetiredImage: true, cleanup: true });
const run = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24);
const selector = `app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${run},wishforge.io/component=runner`;
const receipt = { tenantId: request.tenantId, runId: request.runId, generation: request.generation };
const pods = JSON.parse(await execute(config, ['get', 'pods', '-l', selector, '-o', 'json']));
if (!Array.isArray(pods.items)) throw new Error('KUBERNETES_RELEASE_INVALID');
if (pods.items.some(pod => !['Succeeded', 'Failed'].includes(pod.status?.phase)))
return { ...receipt, status: 'busy', persistentStorageRetained: true };
await execute(config, ['delete', 'pvc', '-l', selector, '--ignore-not-found=true', '--wait=false']);
return { ...receipt, status: 'purged', persistentStorageRetained: false };
}