eust-w 3b78e8e536 🐛 Release what superseded run generations left behind
- The local provisioner kept a generation whose workers exited by
  themselves as running, so its run's directories could never be
  purged.
- On Kubernetes, only a run's current generation is ever released: a
  superseded generation's Secrets, with its run tokens, its service
  accounts and network policies stayed forever. Starting a new
  generation now deletes them by name for every earlier generation
  without a running pod, found through the volumes kept for the
  retention period, so the provisioner still cannot list Secrets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 22:22:11 +08:00

606 lines
26 KiB
JavaScript

import { createHash } from 'node:crypto';
import { spawn } from 'node:child_process';
import { verifyResourceGrant } from './resource-grants.mjs';
const safe = /^[a-zA-Z0-9_-]{1,128}$/;
const dns = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/;
function requireValue(value, name) {
if (!value) throw new Error(`unconfigured: ${name} is required`);
return value;
}
export function readConfig(env = process.env) {
// Sandboxes run in the provisioner's deployment mode; anything but an explicit development
// deployment is production.
const development = env.WF_DEPLOYMENT_MODE === 'development';
const config = {
deploymentMode: development ? 'development' : 'production',
// A local development cluster (see infra/k8s/local) has no Kata and no TLS between services.
insecureHttp: development && env.WF_ALLOW_INSECURE_HTTP === 'true',
sandboxMode: env.WF_RUN_SANDBOX_MODE ?? 'kata',
tenantId: requireValue(env.WF_TENANT_ID, 'WF_TENANT_ID'),
context: requireValue(env.WF_KUBE_CONTEXT, 'WF_KUBE_CONTEXT'),
namespace: requireValue(env.WF_KUBE_NAMESPACE, 'WF_KUBE_NAMESPACE'),
image: requireValue(env.WF_RUN_WORKER_IMAGE, 'WF_RUN_WORKER_IMAGE'),
storageClass: requireValue(env.WF_RUN_STORAGE_CLASS, 'WF_RUN_STORAGE_CLASS'),
storageSize: env.WF_RUN_STORAGE_SIZE ?? '10Gi',
controlPlaneUrl: requireValue(env.WF_CONTROL_PLANE_URL, 'WF_CONTROL_PLANE_URL'),
controlNamespace: requireValue(env.WF_CONTROL_NAMESPACE, 'WF_CONTROL_NAMESPACE'),
temporalAddress: requireValue(env.TEMPORAL_ADDRESS, 'TEMPORAL_ADDRESS'),
temporalNamespace: requireValue(env.TEMPORAL_NAMESPACE, 'TEMPORAL_NAMESPACE'),
gitServicePort: Number(env.WF_GIT_SERVICE_PORT ?? 4504),
// `none` (development only) leaves pods on the cluster's default runtime.
runtimeClass: env.WF_KATA_RUNTIME_CLASS === 'none' ? undefined : (env.WF_KATA_RUNTIME_CLASS ?? 'wishforge-kata'),
activeDeadlineSeconds: Number(env.WF_RUN_MAX_SECONDS ?? 3600),
verifierDockerSecret: env.WF_VERIFIER_DOCKER_CONFIG_SECRET,
// A kubernetes.io/tls Secret (tls.crt, tls.key, ca.crt) with the tenant's Temporal client certificate.
temporalTlsSecret: env.WF_RUN_TEMPORAL_TLS_SECRET,
// The operator's egress proxy (pods labelled wishforge.io/component=egress-proxy) for installations.
installProxy: env.WF_RUN_INSTALL_PROXY_URL,
installProxyNamespace: env.WF_RUN_INSTALL_PROXY_NAMESPACE ?? env.WF_CONTROL_NAMESPACE,
// The remote image builder (pods labelled wishforge.io/component=image-builder) only verifiers reach.
builderHost: env.WF_VERIFIER_DOCKER_HOST,
builderTlsSecret: env.WF_VERIFIER_DOCKER_TLS_SECRET,
builderNamespace: env.WF_VERIFIER_BUILDER_NAMESPACE ?? env.WF_CONTROL_NAMESPACE,
resourceGrantSecret: requireValue(env.WF_RESOURCE_GRANT_SECRET, 'WF_RESOURCE_GRANT_SECRET'),
cpuMilliCap: Number(env.WF_RUN_CPU_MILLI_CAP ?? 2000),
memoryMiBCap: Number(env.WF_RUN_MEMORY_MIB_CAP ?? 4096),
previousImages: (env.WF_RUN_PREVIOUS_IMAGES ?? '')
.split(',')
.map(value => value.trim())
.filter(Boolean),
};
if (
!safe.test(config.tenantId) ||
[
config.namespace,
config.controlNamespace,
...(config.runtimeClass ? [config.runtimeClass] : []),
config.storageClass,
...(config.verifierDockerSecret ? [config.verifierDockerSecret] : []),
...(config.temporalTlsSecret ? [config.temporalTlsSecret] : []),
...(config.installProxy ? [config.installProxyNamespace] : []),
...(config.builderHost ? [config.builderNamespace] : []),
...(config.builderTlsSecret ? [config.builderTlsSecret] : []),
].some(value => !dns.test(value ?? '') || value.length > 63)
)
throw new Error('Invalid configured tenant/namespace/runtime class/storage/secret');
if (config.installProxy) {
const proxy = new URL(config.installProxy);
if (
!['http:', 'https:'].includes(proxy.protocol) ||
proxy.username ||
proxy.password ||
proxy.pathname !== '/' ||
proxy.search ||
proxy.hash
)
throw new Error('WF_RUN_INSTALL_PROXY_URL must be an http(s) origin without credentials');
config.installProxyPort = Number(proxy.port || (proxy.protocol === 'https:' ? 443 : 80));
}
if (config.builderHost) {
const builder = /^tcp:\/\/[a-z0-9.-]+:(\d+)$/.exec(config.builderHost);
if (!builder) throw new Error('WF_VERIFIER_DOCKER_HOST must be tcp://host:port');
config.builderPort = Number(builder[1]);
if (env.WF_DEPLOYMENT_MODE === 'production' && !config.builderTlsSecret)
throw new Error('unconfigured: WF_VERIFIER_DOCKER_TLS_SECRET is required for a production builder');
}
// Sandboxes in production refuse to reach Temporal without a client certificate.
if (env.WF_DEPLOYMENT_MODE === 'production' && !config.temporalTlsSecret)
throw new Error('unconfigured: WF_RUN_TEMPORAL_TLS_SECRET is required in production');
if (!/^[^\s]+@sha256:[a-f0-9]{64}$/.test(config.image))
throw new Error('WF_RUN_WORKER_IMAGE must be an immutable sha256 image digest');
if (
config.resourceGrantSecret.length < 32 ||
!Number.isSafeInteger(config.cpuMilliCap) ||
config.cpuMilliCap < 100 ||
!Number.isSafeInteger(config.memoryMiBCap) ||
config.memoryMiBCap < 512 ||
config.previousImages.length > 10 ||
config.previousImages.some(image => !/^[^\s]+@sha256:[a-f0-9]{64}$/.test(image))
)
throw new Error('Invalid resource grant signing key, static caps or previous runtime image allowlist');
if (!/^\d+Gi$/.test(config.storageSize)) throw new Error('Invalid run storage size');
if (
!Number.isInteger(config.activeDeadlineSeconds) ||
config.activeDeadlineSeconds < 60 ||
config.activeDeadlineSeconds > 10800
)
throw new Error('Run deadline must be 60..10800 seconds and shorter than the scoped token lifetime');
if (env.WF_DEPLOYMENT_MODE === 'production' && env.WF_ALLOW_INSECURE_HTTP === 'true')
throw new Error('WF_ALLOW_INSECURE_HTTP is for local development clusters only');
if (!['kata', 'host'].includes(config.sandboxMode) || (config.sandboxMode === 'host' && !development))
throw new Error('WF_RUN_SANDBOX_MODE must be kata; host is for local development clusters only');
if (!config.runtimeClass && !development)
throw new Error('WF_KATA_RUNTIME_CLASS=none is for local development clusters only');
const api = new URL(config.controlPlaneUrl);
if (
!(api.protocol === 'https:' || (api.protocol === 'http:' && config.insecureHttp)) ||
api.username ||
api.password ||
api.search ||
api.hash
)
throw new Error('The control plane must use an HTTPS URL without embedded credentials');
if (!/^[a-zA-Z0-9.-]+:\d+$/.test(config.temporalAddress))
throw new Error('TEMPORAL_ADDRESS must explicitly include host:port');
if (Number(config.temporalAddress.split(':').at(-1)) < 1 || Number(config.temporalAddress.split(':').at(-1)) > 65535)
throw new Error('Invalid Temporal port');
if (!Number.isInteger(config.gitServicePort) || config.gitServicePort < 1 || config.gitServicePort > 65535)
throw new Error('Invalid WF_GIT_SERVICE_PORT');
return config;
}
function assertToken(token, request, kind, deadline) {
try {
const parts = token.split('.');
if (parts.length !== 3 || token.length > 16384) throw new Error();
const claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
if (
claims.tenant_id !== request.tenantId ||
claims.run_id !== request.runId ||
claims.generation !== request.generation ||
claims.kind !== kind ||
claims.iss !== 'wishforge-control-plane' ||
claims.aud !== 'wishforge-internal' ||
!Number.isFinite(claims.exp) ||
claims.exp * 1000 <= Date.now() + deadline * 1000
)
throw new Error();
} catch {
throw new Error(`Invalid or insufficient-lifetime ${kind} run token`);
}
// The authenticated control plane issues these tokens. This shape/scope check
// is defense-in-depth, not signature verification; the API verifies signatures.
}
export function renderRunResources(config, request) {
if (
!request ||
Object.keys(request).some(
key => !['tenantId', 'runId', 'generation', 'executorToken', 'verifierToken', 'resourceGrant'].includes(key),
) ||
request.tenantId !== config.tenantId ||
!safe.test(request.runId ?? '') ||
!safe.test(request.generation ?? '')
)
throw new Error('Run request is outside the configured tenant or has invalid fields');
const resourceGrant = verifyResourceGrant(config, request);
// One reservation covers both isolated roles; aggregate requested limits must
// stay within the signed run grant, including their separate persistent PVCs.
const roleResources = {
cpuMilli: Math.floor(resourceGrant.cpuMilli / 2),
memoryMiB: Math.floor(resourceGrant.memoryMiB / 2),
storageMiB: Math.floor(resourceGrant.storageMiB / 2),
};
assertToken(request.executorToken, request, 'worker', config.activeDeadlineSeconds);
assertToken(request.verifierToken, request, 'verifier', config.activeDeadlineSeconds);
if (request.executorToken === request.verifierToken) throw new Error('Executor and verifier credentials must differ');
const runHash = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24);
const generationHash = createHash('sha256').update(request.generation).digest('hex').slice(0, 24);
const suffix = createHash('sha256')
.update(`${request.tenantId}\0${request.runId}\0${request.generation}`)
.digest('hex')
.slice(0, 24);
const prefix = `wf-${suffix}`;
const labels = {
'app.kubernetes.io/managed-by': 'wishforge-provisioner',
'wishforge.io/run': runHash,
'wishforge.io/generation': generationHash,
'wishforge.io/component': 'runner',
};
const metadata = name => ({ name, namespace: config.namespace, labels });
const claims = Object.fromEntries(['executor', 'verifier'].map(role => [role, `${prefix}-${role}-workspace`]));
const resources = Object.values(claims).map(claim => ({
apiVersion: 'v1',
kind: 'PersistentVolumeClaim',
metadata: metadata(claim),
spec: {
storageClassName: config.storageClass,
accessModes: ['ReadWriteOnce'],
resources: { requests: { storage: `${roleResources.storageMiB}Mi` } },
},
}));
for (const role of ['executor', 'verifier']) {
const name = `${prefix}-${role}`;
const secretName = `${name}-capability`;
resources.push({
apiVersion: 'v1',
kind: 'Secret',
metadata: metadata(secretName),
type: 'Opaque',
stringData: { token: request[role === 'executor' ? 'executorToken' : 'verifierToken'] },
});
resources.push({
apiVersion: 'v1',
kind: 'ServiceAccount',
metadata: metadata(name),
automountServiceAccountToken: false,
});
const env = [
{ name: 'WF_WORKER_ROLE', value: role },
{ name: 'WF_SANDBOX_RUN_ID', value: request.runId },
{ name: 'WF_SANDBOX_TENANT_ID', value: request.tenantId },
{ name: 'WF_SANDBOX_GENERATION', value: request.generation },
{
name: role === 'executor' ? 'WF_WORKER_TOKEN' : 'WF_VERIFIER_TOKEN',
valueFrom: { secretKeyRef: { name: secretName, key: 'token' } },
},
{ name: 'WF_WORKER_ID', value: name },
{ name: 'WF_WORKSPACE_ROOT', value: '/workspaces' },
{ name: 'WF_PROVIDER_HOME_ROOT', value: '/provider-home' },
{ name: 'WF_CONTROL_PLANE_URL', value: config.controlPlaneUrl },
{ name: 'TEMPORAL_ADDRESS', value: config.temporalAddress },
{ name: 'TEMPORAL_NAMESPACE', value: config.temporalNamespace },
{ name: 'WF_DEPLOYMENT_MODE', value: config.deploymentMode },
// The worker image sets NODE_ENV=production, which a development deployment refuses.
{ name: 'NODE_ENV', value: config.deploymentMode },
{ name: 'WF_SANDBOX_MODE', value: config.sandboxMode },
...(config.insecureHttp
? [
{ name: 'WF_ALLOW_INSECURE_HTTP', value: 'true' },
{ name: 'WF_GIT_ALLOW_LOCAL_HTTP', value: 'true' },
]
: []),
{ name: 'WF_WORKER_CONCURRENCY', value: '1' },
{ name: 'WF_EXECUTION_CPU_MILLI_LIMIT', value: String(roleResources.cpuMilli) },
{ name: 'WF_EXECUTION_MEMORY_MIB_LIMIT', value: String(roleResources.memoryMiB) },
{ name: 'WF_EXECUTION_STORAGE_MIB_LIMIT', value: String(roleResources.storageMiB) },
];
const mounts = [
{ name: 'workspace', mountPath: '/workspaces' },
{ name: 'provider-home', mountPath: '/provider-home' },
{ name: 'temporary', mountPath: '/tmp' },
];
const volumes = [
{ name: 'workspace', persistentVolumeClaim: { claimName: claims[role] } },
{ name: 'provider-home', emptyDir: { medium: 'Memory', sizeLimit: '256Mi' } },
{ name: 'temporary', emptyDir: { sizeLimit: '4Gi' } },
];
if (config.installProxy) env.push({ name: 'WF_INSTALL_PROXY', value: config.installProxy });
if (config.temporalTlsSecret) {
env.push(
{ name: 'TEMPORAL_TLS_CERT', value: '/run/temporal/tls.crt' },
{ name: 'TEMPORAL_TLS_KEY', value: '/run/temporal/tls.key' },
{ name: 'TEMPORAL_TLS_CA', value: '/run/temporal/ca.crt' },
);
mounts.push({ name: 'temporal-tls', mountPath: '/run/temporal', readOnly: true });
volumes.push({ name: 'temporal-tls', secret: { secretName: config.temporalTlsSecret, defaultMode: 0o440 } });
}
if (role === 'verifier') {
env.push({ name: 'WF_VERIFIER_BWRAP', value: '/usr/bin/bwrap' });
if (config.builderHost) {
env.push({ name: 'WF_DOCKER_HOST', value: config.builderHost });
if (config.builderTlsSecret) {
env.push({ name: 'WF_DOCKER_TLS_DIR', value: '/run/docker-tls' });
mounts.push({ name: 'docker-tls', mountPath: '/run/docker-tls', readOnly: true });
volumes.push({ name: 'docker-tls', secret: { secretName: config.builderTlsSecret, defaultMode: 0o440 } });
}
}
if (config.verifierDockerSecret) {
env.push({ name: 'WF_DOCKER_CONFIG', value: '/run/docker' });
mounts.push({ name: 'docker-config', mountPath: '/run/docker', readOnly: true });
volumes.push({ name: 'docker-config', secret: { secretName: config.verifierDockerSecret, defaultMode: 256 } });
}
}
resources.push({
apiVersion: 'batch/v1',
kind: 'Job',
metadata: metadata(name),
spec: {
backoffLimit: 0,
activeDeadlineSeconds: config.activeDeadlineSeconds,
ttlSecondsAfterFinished: 86400,
template: {
metadata: { labels: { ...labels, 'wishforge.io/worker-role': role } },
spec: {
...(config.runtimeClass ? { runtimeClassName: config.runtimeClass } : {}),
serviceAccountName: name,
automountServiceAccountToken: false,
restartPolicy: 'Never',
securityContext: {
runAsNonRoot: true,
runAsUser: 1000,
runAsGroup: 1000,
fsGroup: 1000,
seccompProfile: { type: 'RuntimeDefault' },
},
containers: [
{
name: role,
image: resourceGrant.image,
imagePullPolicy: 'Always',
// The worker image runs the built service (scripts/build-services.mjs).
command: ['node', 'dist/worker/main.mjs'],
env,
securityContext: {
allowPrivilegeEscalation: false,
readOnlyRootFilesystem: true,
capabilities: { drop: ['ALL'] },
},
resources: {
requests: {
cpu: `${Math.min(1000, roleResources.cpuMilli)}m`,
memory: `${Math.min(2048, roleResources.memoryMiB)}Mi`,
'ephemeral-storage': '1Gi',
},
limits: {
cpu: `${roleResources.cpuMilli}m`,
memory: `${roleResources.memoryMiB}Mi`,
'ephemeral-storage': '8Gi',
},
},
volumeMounts: mounts,
},
],
volumes,
},
},
},
});
}
const nsSelector = { namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.controlNamespace } } };
const apiPort = Number(new URL(config.controlPlaneUrl).port || 443);
const temporalPort = Number(config.temporalAddress.split(':').at(-1));
resources.push({
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: metadata(`${prefix}-isolation`),
spec: {
podSelector: { matchLabels: { 'wishforge.io/run': runHash, 'wishforge.io/generation': generationHash } },
policyTypes: ['Ingress', 'Egress'],
ingress: [],
egress: [
{
to: [
{
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' } },
podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } },
},
],
ports: [
{ protocol: 'UDP', port: 53 },
{ protocol: 'TCP', port: 53 },
],
},
{
to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'api' } } }],
ports: [{ protocol: 'TCP', port: apiPort }],
},
{
to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'temporal' } } }],
ports: [{ protocol: 'TCP', port: temporalPort }],
},
// Both roles clone the project from the platform Git service.
{
to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'git-storage' } } }],
ports: [{ protocol: 'TCP', port: config.gitServicePort }],
},
// Dependency installations reach package registries only through the operator's proxy.
...(config.installProxy
? [
{
to: [
{
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.installProxyNamespace } },
podSelector: { matchLabels: { 'wishforge.io/component': 'egress-proxy' } },
},
],
ports: [{ protocol: 'TCP', port: config.installProxyPort }],
},
]
: []),
],
},
});
// Only the verifier builds images, so only its pod may reach the builder.
if (config.builderHost)
resources.push({
apiVersion: 'networking.k8s.io/v1',
kind: 'NetworkPolicy',
metadata: metadata(`${prefix}-builder`),
spec: {
podSelector: {
matchLabels: {
'wishforge.io/run': runHash,
'wishforge.io/generation': generationHash,
'wishforge.io/worker-role': 'verifier',
},
},
policyTypes: ['Egress'],
egress: [
{
to: [
{
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.builderNamespace } },
podSelector: { matchLabels: { 'wishforge.io/component': 'image-builder' } },
},
],
ports: [{ protocol: 'TCP', port: config.builderPort }],
},
],
},
});
return {
resources,
receipt: {
tenantId: request.tenantId,
runId: request.runId,
generation: request.generation,
namespace: config.namespace,
executorJobName: `${prefix}-executor`,
verifierJobName: `${prefix}-verifier`,
workspaceClaim: claims.executor,
verifierWorkspaceClaim: claims.verifier,
status: 'provisioned',
},
};
}
export function runKubectl(config, document) {
return new Promise((resolve, reject) => {
const child = spawn(
'kubectl',
[
'--context',
config.context,
'--namespace',
config.namespace,
'apply',
'--server-side',
'--field-manager=wishforge-provisioner',
'-o',
'name',
'-f',
'-',
],
{ stdio: ['pipe', 'pipe', 'pipe'] },
);
let output = '';
const timeout = setTimeout(() => {
child.kill('SIGTERM');
reject(new Error('KUBERNETES_APPLY_TIMEOUT'));
}, 30000);
child.stdout.on('data', chunk => {
output += chunk;
if (output.length > 65536) child.kill('SIGTERM');
});
// Never expose kubectl diagnostics: API validation errors can contain Secret data.
child.stderr.resume();
child.on('error', () => {
clearTimeout(timeout);
reject(new Error('KUBERNETES_COMMAND_UNAVAILABLE'));
});
child.on('close', code => {
clearTimeout(timeout);
code === 0 ? resolve(output) : reject(new Error('KUBERNETES_APPLY_FAILED'));
});
child.stdin.on('error', () => {});
child.stdin.end(JSON.stringify({ apiVersion: 'v1', kind: 'List', items: document }));
});
}
export async function lifecycleCommand(config, args) {
return new Promise((resolve, reject) => {
const child = spawn('kubectl', ['--context', config.context, '--namespace', config.namespace, ...args], {
stdio: ['ignore', 'pipe', 'pipe'],
});
let stdout = '';
const timer = setTimeout(() => {
child.kill('SIGKILL');
reject(new Error('KUBERNETES_RECONCILE_TIMEOUT'));
}, 45000);
child.stdout.on('data', chunk => {
stdout += chunk;
if (stdout.length > 1024 * 1024) child.kill('SIGKILL');
});
child.stderr.resume();
child.on('error', () => {
clearTimeout(timer);
reject(new Error('KUBERNETES_RECONCILE_FAILED'));
});
child.on('close', code => {
clearTimeout(timer);
code === 0 ? resolve(stdout) : reject(new Error('KUBERNETES_RECONCILE_FAILED'));
});
});
}
/** What a run generation holds besides its Jobs and volumes: credentials, identities, network policy. */
export function generationIdentities(prefix) {
return [
['secret', `${prefix}-executor-capability`],
['secret', `${prefix}-verifier-capability`],
['serviceaccount', `${prefix}-executor`],
['serviceaccount', `${prefix}-verifier`],
['networkpolicy', `${prefix}-isolation`],
['networkpolicy', `${prefix}-builder`],
];
}
export async function reconcileRunJobs(config, request, execute = lifecycleCommand) {
const { resources } = renderRunResources(config, request);
const labels = resources[0].metadata.labels;
const selector = `app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${labels['wishforge.io/run']},wishforge.io/component=runner`;
const listed = JSON.parse(await execute(config, ['get', 'jobs', '-l', selector, '-o', 'json']));
if (!Array.isArray(listed.items)) throw new Error('KUBERNETES_RECONCILE_FAILED');
let terminal = false;
for (const job of listed.items) {
if (
job.metadata?.namespace !== config.namespace ||
job.metadata?.labels?.['app.kubernetes.io/managed-by'] !== 'wishforge-provisioner' ||
job.metadata?.labels?.['wishforge.io/run'] !== labels['wishforge.io/run'] ||
!/^wf-[a-f0-9]{24}-(executor|verifier)$/.test(job.metadata?.name ?? '')
)
throw new Error('KUBERNETES_RECONCILE_SCOPE_MISMATCH');
if (job.metadata.labels['wishforge.io/generation'] === labels['wishforge.io/generation']) {
if (
job.status?.failed > 0 ||
job.status?.succeeded > 0 ||
job.status?.conditions?.some(c => ['Failed', 'Complete'].includes(c.type) && c.status === 'True')
)
terminal = true;
} else
await execute(config, [
'delete',
'job',
job.metadata.name,
'--cascade=foreground',
'--wait=true',
'--timeout=30s',
]);
}
// The generations this one superseded leave their credentials (run tokens), identities and
// network policy behind: only the current generation is ever released. Their volumes, kept for
// the retention period, name every one of them; Secrets cannot be listed, by design.
const owned = item => {
if (
item.metadata?.namespace !== config.namespace ||
item.metadata?.labels?.['app.kubernetes.io/managed-by'] !== 'wishforge-provisioner' ||
item.metadata?.labels?.['wishforge.io/run'] !== labels['wishforge.io/run']
)
throw new Error('KUBERNETES_RECONCILE_SCOPE_MISMATCH');
return item;
};
const claims = JSON.parse(await execute(config, ['get', 'persistentvolumeclaims', '-l', selector, '-o', 'json']));
const pods = JSON.parse(await execute(config, ['get', 'pods', '-l', selector, '-o', 'json']));
if (!Array.isArray(claims.items) || !Array.isArray(pods.items)) throw new Error('KUBERNETES_RECONCILE_FAILED');
const running = new Set(
pods.items
.map(owned)
.filter(pod => !['Succeeded', 'Failed'].includes(pod.status?.phase))
.map(pod => pod.metadata.labels['wishforge.io/generation']),
);
const superseded = new Set();
for (const claim of claims.items.map(owned)) {
const generation = claim.metadata.labels['wishforge.io/generation'];
const prefix = /^(wf-[a-f0-9]{24})-(executor|verifier)-workspace$/.exec(claim.metadata.name)?.[1];
if (prefix && generation !== labels['wishforge.io/generation'] && !running.has(generation)) superseded.add(prefix);
}
for (const prefix of superseded)
for (const [kind, name] of generationIdentities(prefix))
await execute(config, ['delete', kind, name, '--ignore-not-found=true', '--wait=false']);
return { terminal };
}
export async function provisionRun(config, request, apply = runKubectl, reconcile = reconcileRunJobs) {
const rendered = renderRunResources(config, request);
const state = await reconcile(config, request);
if (state.terminal)
return {
...rendered.receipt,
status: 'failed',
reason: 'Run generation has terminal jobs; resume with a new generation',
};
// Provision policy and identities before Jobs. An interrupted call is retried
// with the same deterministic resource names, never with a second workload.
await apply(
config,
rendered.resources.filter(resource => resource.kind !== 'Job'),
);
await apply(
config,
rendered.resources.filter(resource => resource.kind === 'Job'),
);
return rendered.receipt;
}