- The local provisioner kept a generation whose workers exited by themselves as running, so its run's directories could never be purged. - On Kubernetes, only a run's current generation is ever released: a superseded generation's Secrets, with its run tokens, its service accounts and network policies stayed forever. Starting a new generation now deletes them by name for every earlier generation without a running pod, found through the volumes kept for the retention period, so the provisioner still cannot list Secrets. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
606 lines
26 KiB
JavaScript
606 lines
26 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import { spawn } from 'node:child_process';
|
|
import { verifyResourceGrant } from './resource-grants.mjs';
|
|
|
|
const safe = /^[a-zA-Z0-9_-]{1,128}$/;
|
|
const dns = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/;
|
|
function requireValue(value, name) {
|
|
if (!value) throw new Error(`unconfigured: ${name} is required`);
|
|
return value;
|
|
}
|
|
export function readConfig(env = process.env) {
|
|
// Sandboxes run in the provisioner's deployment mode; anything but an explicit development
|
|
// deployment is production.
|
|
const development = env.WF_DEPLOYMENT_MODE === 'development';
|
|
const config = {
|
|
deploymentMode: development ? 'development' : 'production',
|
|
// A local development cluster (see infra/k8s/local) has no Kata and no TLS between services.
|
|
insecureHttp: development && env.WF_ALLOW_INSECURE_HTTP === 'true',
|
|
sandboxMode: env.WF_RUN_SANDBOX_MODE ?? 'kata',
|
|
tenantId: requireValue(env.WF_TENANT_ID, 'WF_TENANT_ID'),
|
|
context: requireValue(env.WF_KUBE_CONTEXT, 'WF_KUBE_CONTEXT'),
|
|
namespace: requireValue(env.WF_KUBE_NAMESPACE, 'WF_KUBE_NAMESPACE'),
|
|
image: requireValue(env.WF_RUN_WORKER_IMAGE, 'WF_RUN_WORKER_IMAGE'),
|
|
storageClass: requireValue(env.WF_RUN_STORAGE_CLASS, 'WF_RUN_STORAGE_CLASS'),
|
|
storageSize: env.WF_RUN_STORAGE_SIZE ?? '10Gi',
|
|
controlPlaneUrl: requireValue(env.WF_CONTROL_PLANE_URL, 'WF_CONTROL_PLANE_URL'),
|
|
controlNamespace: requireValue(env.WF_CONTROL_NAMESPACE, 'WF_CONTROL_NAMESPACE'),
|
|
temporalAddress: requireValue(env.TEMPORAL_ADDRESS, 'TEMPORAL_ADDRESS'),
|
|
temporalNamespace: requireValue(env.TEMPORAL_NAMESPACE, 'TEMPORAL_NAMESPACE'),
|
|
gitServicePort: Number(env.WF_GIT_SERVICE_PORT ?? 4504),
|
|
// `none` (development only) leaves pods on the cluster's default runtime.
|
|
runtimeClass: env.WF_KATA_RUNTIME_CLASS === 'none' ? undefined : (env.WF_KATA_RUNTIME_CLASS ?? 'wishforge-kata'),
|
|
activeDeadlineSeconds: Number(env.WF_RUN_MAX_SECONDS ?? 3600),
|
|
verifierDockerSecret: env.WF_VERIFIER_DOCKER_CONFIG_SECRET,
|
|
// A kubernetes.io/tls Secret (tls.crt, tls.key, ca.crt) with the tenant's Temporal client certificate.
|
|
temporalTlsSecret: env.WF_RUN_TEMPORAL_TLS_SECRET,
|
|
// The operator's egress proxy (pods labelled wishforge.io/component=egress-proxy) for installations.
|
|
installProxy: env.WF_RUN_INSTALL_PROXY_URL,
|
|
installProxyNamespace: env.WF_RUN_INSTALL_PROXY_NAMESPACE ?? env.WF_CONTROL_NAMESPACE,
|
|
// The remote image builder (pods labelled wishforge.io/component=image-builder) only verifiers reach.
|
|
builderHost: env.WF_VERIFIER_DOCKER_HOST,
|
|
builderTlsSecret: env.WF_VERIFIER_DOCKER_TLS_SECRET,
|
|
builderNamespace: env.WF_VERIFIER_BUILDER_NAMESPACE ?? env.WF_CONTROL_NAMESPACE,
|
|
resourceGrantSecret: requireValue(env.WF_RESOURCE_GRANT_SECRET, 'WF_RESOURCE_GRANT_SECRET'),
|
|
cpuMilliCap: Number(env.WF_RUN_CPU_MILLI_CAP ?? 2000),
|
|
memoryMiBCap: Number(env.WF_RUN_MEMORY_MIB_CAP ?? 4096),
|
|
previousImages: (env.WF_RUN_PREVIOUS_IMAGES ?? '')
|
|
.split(',')
|
|
.map(value => value.trim())
|
|
.filter(Boolean),
|
|
};
|
|
if (
|
|
!safe.test(config.tenantId) ||
|
|
[
|
|
config.namespace,
|
|
config.controlNamespace,
|
|
...(config.runtimeClass ? [config.runtimeClass] : []),
|
|
config.storageClass,
|
|
...(config.verifierDockerSecret ? [config.verifierDockerSecret] : []),
|
|
...(config.temporalTlsSecret ? [config.temporalTlsSecret] : []),
|
|
...(config.installProxy ? [config.installProxyNamespace] : []),
|
|
...(config.builderHost ? [config.builderNamespace] : []),
|
|
...(config.builderTlsSecret ? [config.builderTlsSecret] : []),
|
|
].some(value => !dns.test(value ?? '') || value.length > 63)
|
|
)
|
|
throw new Error('Invalid configured tenant/namespace/runtime class/storage/secret');
|
|
if (config.installProxy) {
|
|
const proxy = new URL(config.installProxy);
|
|
if (
|
|
!['http:', 'https:'].includes(proxy.protocol) ||
|
|
proxy.username ||
|
|
proxy.password ||
|
|
proxy.pathname !== '/' ||
|
|
proxy.search ||
|
|
proxy.hash
|
|
)
|
|
throw new Error('WF_RUN_INSTALL_PROXY_URL must be an http(s) origin without credentials');
|
|
config.installProxyPort = Number(proxy.port || (proxy.protocol === 'https:' ? 443 : 80));
|
|
}
|
|
if (config.builderHost) {
|
|
const builder = /^tcp:\/\/[a-z0-9.-]+:(\d+)$/.exec(config.builderHost);
|
|
if (!builder) throw new Error('WF_VERIFIER_DOCKER_HOST must be tcp://host:port');
|
|
config.builderPort = Number(builder[1]);
|
|
if (env.WF_DEPLOYMENT_MODE === 'production' && !config.builderTlsSecret)
|
|
throw new Error('unconfigured: WF_VERIFIER_DOCKER_TLS_SECRET is required for a production builder');
|
|
}
|
|
// Sandboxes in production refuse to reach Temporal without a client certificate.
|
|
if (env.WF_DEPLOYMENT_MODE === 'production' && !config.temporalTlsSecret)
|
|
throw new Error('unconfigured: WF_RUN_TEMPORAL_TLS_SECRET is required in production');
|
|
if (!/^[^\s]+@sha256:[a-f0-9]{64}$/.test(config.image))
|
|
throw new Error('WF_RUN_WORKER_IMAGE must be an immutable sha256 image digest');
|
|
if (
|
|
config.resourceGrantSecret.length < 32 ||
|
|
!Number.isSafeInteger(config.cpuMilliCap) ||
|
|
config.cpuMilliCap < 100 ||
|
|
!Number.isSafeInteger(config.memoryMiBCap) ||
|
|
config.memoryMiBCap < 512 ||
|
|
config.previousImages.length > 10 ||
|
|
config.previousImages.some(image => !/^[^\s]+@sha256:[a-f0-9]{64}$/.test(image))
|
|
)
|
|
throw new Error('Invalid resource grant signing key, static caps or previous runtime image allowlist');
|
|
if (!/^\d+Gi$/.test(config.storageSize)) throw new Error('Invalid run storage size');
|
|
if (
|
|
!Number.isInteger(config.activeDeadlineSeconds) ||
|
|
config.activeDeadlineSeconds < 60 ||
|
|
config.activeDeadlineSeconds > 10800
|
|
)
|
|
throw new Error('Run deadline must be 60..10800 seconds and shorter than the scoped token lifetime');
|
|
if (env.WF_DEPLOYMENT_MODE === 'production' && env.WF_ALLOW_INSECURE_HTTP === 'true')
|
|
throw new Error('WF_ALLOW_INSECURE_HTTP is for local development clusters only');
|
|
if (!['kata', 'host'].includes(config.sandboxMode) || (config.sandboxMode === 'host' && !development))
|
|
throw new Error('WF_RUN_SANDBOX_MODE must be kata; host is for local development clusters only');
|
|
if (!config.runtimeClass && !development)
|
|
throw new Error('WF_KATA_RUNTIME_CLASS=none is for local development clusters only');
|
|
const api = new URL(config.controlPlaneUrl);
|
|
if (
|
|
!(api.protocol === 'https:' || (api.protocol === 'http:' && config.insecureHttp)) ||
|
|
api.username ||
|
|
api.password ||
|
|
api.search ||
|
|
api.hash
|
|
)
|
|
throw new Error('The control plane must use an HTTPS URL without embedded credentials');
|
|
if (!/^[a-zA-Z0-9.-]+:\d+$/.test(config.temporalAddress))
|
|
throw new Error('TEMPORAL_ADDRESS must explicitly include host:port');
|
|
if (Number(config.temporalAddress.split(':').at(-1)) < 1 || Number(config.temporalAddress.split(':').at(-1)) > 65535)
|
|
throw new Error('Invalid Temporal port');
|
|
if (!Number.isInteger(config.gitServicePort) || config.gitServicePort < 1 || config.gitServicePort > 65535)
|
|
throw new Error('Invalid WF_GIT_SERVICE_PORT');
|
|
return config;
|
|
}
|
|
|
|
function assertToken(token, request, kind, deadline) {
|
|
try {
|
|
const parts = token.split('.');
|
|
if (parts.length !== 3 || token.length > 16384) throw new Error();
|
|
const claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
|
|
if (
|
|
claims.tenant_id !== request.tenantId ||
|
|
claims.run_id !== request.runId ||
|
|
claims.generation !== request.generation ||
|
|
claims.kind !== kind ||
|
|
claims.iss !== 'wishforge-control-plane' ||
|
|
claims.aud !== 'wishforge-internal' ||
|
|
!Number.isFinite(claims.exp) ||
|
|
claims.exp * 1000 <= Date.now() + deadline * 1000
|
|
)
|
|
throw new Error();
|
|
} catch {
|
|
throw new Error(`Invalid or insufficient-lifetime ${kind} run token`);
|
|
}
|
|
// The authenticated control plane issues these tokens. This shape/scope check
|
|
// is defense-in-depth, not signature verification; the API verifies signatures.
|
|
}
|
|
|
|
export function renderRunResources(config, request) {
|
|
if (
|
|
!request ||
|
|
Object.keys(request).some(
|
|
key => !['tenantId', 'runId', 'generation', 'executorToken', 'verifierToken', 'resourceGrant'].includes(key),
|
|
) ||
|
|
request.tenantId !== config.tenantId ||
|
|
!safe.test(request.runId ?? '') ||
|
|
!safe.test(request.generation ?? '')
|
|
)
|
|
throw new Error('Run request is outside the configured tenant or has invalid fields');
|
|
const resourceGrant = verifyResourceGrant(config, request);
|
|
// One reservation covers both isolated roles; aggregate requested limits must
|
|
// stay within the signed run grant, including their separate persistent PVCs.
|
|
const roleResources = {
|
|
cpuMilli: Math.floor(resourceGrant.cpuMilli / 2),
|
|
memoryMiB: Math.floor(resourceGrant.memoryMiB / 2),
|
|
storageMiB: Math.floor(resourceGrant.storageMiB / 2),
|
|
};
|
|
assertToken(request.executorToken, request, 'worker', config.activeDeadlineSeconds);
|
|
assertToken(request.verifierToken, request, 'verifier', config.activeDeadlineSeconds);
|
|
if (request.executorToken === request.verifierToken) throw new Error('Executor and verifier credentials must differ');
|
|
const runHash = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24);
|
|
const generationHash = createHash('sha256').update(request.generation).digest('hex').slice(0, 24);
|
|
const suffix = createHash('sha256')
|
|
.update(`${request.tenantId}\0${request.runId}\0${request.generation}`)
|
|
.digest('hex')
|
|
.slice(0, 24);
|
|
const prefix = `wf-${suffix}`;
|
|
const labels = {
|
|
'app.kubernetes.io/managed-by': 'wishforge-provisioner',
|
|
'wishforge.io/run': runHash,
|
|
'wishforge.io/generation': generationHash,
|
|
'wishforge.io/component': 'runner',
|
|
};
|
|
const metadata = name => ({ name, namespace: config.namespace, labels });
|
|
const claims = Object.fromEntries(['executor', 'verifier'].map(role => [role, `${prefix}-${role}-workspace`]));
|
|
const resources = Object.values(claims).map(claim => ({
|
|
apiVersion: 'v1',
|
|
kind: 'PersistentVolumeClaim',
|
|
metadata: metadata(claim),
|
|
spec: {
|
|
storageClassName: config.storageClass,
|
|
accessModes: ['ReadWriteOnce'],
|
|
resources: { requests: { storage: `${roleResources.storageMiB}Mi` } },
|
|
},
|
|
}));
|
|
for (const role of ['executor', 'verifier']) {
|
|
const name = `${prefix}-${role}`;
|
|
const secretName = `${name}-capability`;
|
|
resources.push({
|
|
apiVersion: 'v1',
|
|
kind: 'Secret',
|
|
metadata: metadata(secretName),
|
|
type: 'Opaque',
|
|
stringData: { token: request[role === 'executor' ? 'executorToken' : 'verifierToken'] },
|
|
});
|
|
resources.push({
|
|
apiVersion: 'v1',
|
|
kind: 'ServiceAccount',
|
|
metadata: metadata(name),
|
|
automountServiceAccountToken: false,
|
|
});
|
|
const env = [
|
|
{ name: 'WF_WORKER_ROLE', value: role },
|
|
{ name: 'WF_SANDBOX_RUN_ID', value: request.runId },
|
|
{ name: 'WF_SANDBOX_TENANT_ID', value: request.tenantId },
|
|
{ name: 'WF_SANDBOX_GENERATION', value: request.generation },
|
|
{
|
|
name: role === 'executor' ? 'WF_WORKER_TOKEN' : 'WF_VERIFIER_TOKEN',
|
|
valueFrom: { secretKeyRef: { name: secretName, key: 'token' } },
|
|
},
|
|
{ name: 'WF_WORKER_ID', value: name },
|
|
{ name: 'WF_WORKSPACE_ROOT', value: '/workspaces' },
|
|
{ name: 'WF_PROVIDER_HOME_ROOT', value: '/provider-home' },
|
|
{ name: 'WF_CONTROL_PLANE_URL', value: config.controlPlaneUrl },
|
|
{ name: 'TEMPORAL_ADDRESS', value: config.temporalAddress },
|
|
{ name: 'TEMPORAL_NAMESPACE', value: config.temporalNamespace },
|
|
{ name: 'WF_DEPLOYMENT_MODE', value: config.deploymentMode },
|
|
// The worker image sets NODE_ENV=production, which a development deployment refuses.
|
|
{ name: 'NODE_ENV', value: config.deploymentMode },
|
|
{ name: 'WF_SANDBOX_MODE', value: config.sandboxMode },
|
|
...(config.insecureHttp
|
|
? [
|
|
{ name: 'WF_ALLOW_INSECURE_HTTP', value: 'true' },
|
|
{ name: 'WF_GIT_ALLOW_LOCAL_HTTP', value: 'true' },
|
|
]
|
|
: []),
|
|
{ name: 'WF_WORKER_CONCURRENCY', value: '1' },
|
|
{ name: 'WF_EXECUTION_CPU_MILLI_LIMIT', value: String(roleResources.cpuMilli) },
|
|
{ name: 'WF_EXECUTION_MEMORY_MIB_LIMIT', value: String(roleResources.memoryMiB) },
|
|
{ name: 'WF_EXECUTION_STORAGE_MIB_LIMIT', value: String(roleResources.storageMiB) },
|
|
];
|
|
const mounts = [
|
|
{ name: 'workspace', mountPath: '/workspaces' },
|
|
{ name: 'provider-home', mountPath: '/provider-home' },
|
|
{ name: 'temporary', mountPath: '/tmp' },
|
|
];
|
|
const volumes = [
|
|
{ name: 'workspace', persistentVolumeClaim: { claimName: claims[role] } },
|
|
{ name: 'provider-home', emptyDir: { medium: 'Memory', sizeLimit: '256Mi' } },
|
|
{ name: 'temporary', emptyDir: { sizeLimit: '4Gi' } },
|
|
];
|
|
if (config.installProxy) env.push({ name: 'WF_INSTALL_PROXY', value: config.installProxy });
|
|
if (config.temporalTlsSecret) {
|
|
env.push(
|
|
{ name: 'TEMPORAL_TLS_CERT', value: '/run/temporal/tls.crt' },
|
|
{ name: 'TEMPORAL_TLS_KEY', value: '/run/temporal/tls.key' },
|
|
{ name: 'TEMPORAL_TLS_CA', value: '/run/temporal/ca.crt' },
|
|
);
|
|
mounts.push({ name: 'temporal-tls', mountPath: '/run/temporal', readOnly: true });
|
|
volumes.push({ name: 'temporal-tls', secret: { secretName: config.temporalTlsSecret, defaultMode: 0o440 } });
|
|
}
|
|
if (role === 'verifier') {
|
|
env.push({ name: 'WF_VERIFIER_BWRAP', value: '/usr/bin/bwrap' });
|
|
if (config.builderHost) {
|
|
env.push({ name: 'WF_DOCKER_HOST', value: config.builderHost });
|
|
if (config.builderTlsSecret) {
|
|
env.push({ name: 'WF_DOCKER_TLS_DIR', value: '/run/docker-tls' });
|
|
mounts.push({ name: 'docker-tls', mountPath: '/run/docker-tls', readOnly: true });
|
|
volumes.push({ name: 'docker-tls', secret: { secretName: config.builderTlsSecret, defaultMode: 0o440 } });
|
|
}
|
|
}
|
|
if (config.verifierDockerSecret) {
|
|
env.push({ name: 'WF_DOCKER_CONFIG', value: '/run/docker' });
|
|
mounts.push({ name: 'docker-config', mountPath: '/run/docker', readOnly: true });
|
|
volumes.push({ name: 'docker-config', secret: { secretName: config.verifierDockerSecret, defaultMode: 256 } });
|
|
}
|
|
}
|
|
resources.push({
|
|
apiVersion: 'batch/v1',
|
|
kind: 'Job',
|
|
metadata: metadata(name),
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: config.activeDeadlineSeconds,
|
|
ttlSecondsAfterFinished: 86400,
|
|
template: {
|
|
metadata: { labels: { ...labels, 'wishforge.io/worker-role': role } },
|
|
spec: {
|
|
...(config.runtimeClass ? { runtimeClassName: config.runtimeClass } : {}),
|
|
serviceAccountName: name,
|
|
automountServiceAccountToken: false,
|
|
restartPolicy: 'Never',
|
|
securityContext: {
|
|
runAsNonRoot: true,
|
|
runAsUser: 1000,
|
|
runAsGroup: 1000,
|
|
fsGroup: 1000,
|
|
seccompProfile: { type: 'RuntimeDefault' },
|
|
},
|
|
containers: [
|
|
{
|
|
name: role,
|
|
image: resourceGrant.image,
|
|
imagePullPolicy: 'Always',
|
|
// The worker image runs the built service (scripts/build-services.mjs).
|
|
command: ['node', 'dist/worker/main.mjs'],
|
|
env,
|
|
securityContext: {
|
|
allowPrivilegeEscalation: false,
|
|
readOnlyRootFilesystem: true,
|
|
capabilities: { drop: ['ALL'] },
|
|
},
|
|
resources: {
|
|
requests: {
|
|
cpu: `${Math.min(1000, roleResources.cpuMilli)}m`,
|
|
memory: `${Math.min(2048, roleResources.memoryMiB)}Mi`,
|
|
'ephemeral-storage': '1Gi',
|
|
},
|
|
limits: {
|
|
cpu: `${roleResources.cpuMilli}m`,
|
|
memory: `${roleResources.memoryMiB}Mi`,
|
|
'ephemeral-storage': '8Gi',
|
|
},
|
|
},
|
|
volumeMounts: mounts,
|
|
},
|
|
],
|
|
volumes,
|
|
},
|
|
},
|
|
},
|
|
});
|
|
}
|
|
const nsSelector = { namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.controlNamespace } } };
|
|
const apiPort = Number(new URL(config.controlPlaneUrl).port || 443);
|
|
const temporalPort = Number(config.temporalAddress.split(':').at(-1));
|
|
resources.push({
|
|
apiVersion: 'networking.k8s.io/v1',
|
|
kind: 'NetworkPolicy',
|
|
metadata: metadata(`${prefix}-isolation`),
|
|
spec: {
|
|
podSelector: { matchLabels: { 'wishforge.io/run': runHash, 'wishforge.io/generation': generationHash } },
|
|
policyTypes: ['Ingress', 'Egress'],
|
|
ingress: [],
|
|
egress: [
|
|
{
|
|
to: [
|
|
{
|
|
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' } },
|
|
podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } },
|
|
},
|
|
],
|
|
ports: [
|
|
{ protocol: 'UDP', port: 53 },
|
|
{ protocol: 'TCP', port: 53 },
|
|
],
|
|
},
|
|
{
|
|
to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'api' } } }],
|
|
ports: [{ protocol: 'TCP', port: apiPort }],
|
|
},
|
|
{
|
|
to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'temporal' } } }],
|
|
ports: [{ protocol: 'TCP', port: temporalPort }],
|
|
},
|
|
// Both roles clone the project from the platform Git service.
|
|
{
|
|
to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'git-storage' } } }],
|
|
ports: [{ protocol: 'TCP', port: config.gitServicePort }],
|
|
},
|
|
// Dependency installations reach package registries only through the operator's proxy.
|
|
...(config.installProxy
|
|
? [
|
|
{
|
|
to: [
|
|
{
|
|
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.installProxyNamespace } },
|
|
podSelector: { matchLabels: { 'wishforge.io/component': 'egress-proxy' } },
|
|
},
|
|
],
|
|
ports: [{ protocol: 'TCP', port: config.installProxyPort }],
|
|
},
|
|
]
|
|
: []),
|
|
],
|
|
},
|
|
});
|
|
// Only the verifier builds images, so only its pod may reach the builder.
|
|
if (config.builderHost)
|
|
resources.push({
|
|
apiVersion: 'networking.k8s.io/v1',
|
|
kind: 'NetworkPolicy',
|
|
metadata: metadata(`${prefix}-builder`),
|
|
spec: {
|
|
podSelector: {
|
|
matchLabels: {
|
|
'wishforge.io/run': runHash,
|
|
'wishforge.io/generation': generationHash,
|
|
'wishforge.io/worker-role': 'verifier',
|
|
},
|
|
},
|
|
policyTypes: ['Egress'],
|
|
egress: [
|
|
{
|
|
to: [
|
|
{
|
|
namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.builderNamespace } },
|
|
podSelector: { matchLabels: { 'wishforge.io/component': 'image-builder' } },
|
|
},
|
|
],
|
|
ports: [{ protocol: 'TCP', port: config.builderPort }],
|
|
},
|
|
],
|
|
},
|
|
});
|
|
return {
|
|
resources,
|
|
receipt: {
|
|
tenantId: request.tenantId,
|
|
runId: request.runId,
|
|
generation: request.generation,
|
|
namespace: config.namespace,
|
|
executorJobName: `${prefix}-executor`,
|
|
verifierJobName: `${prefix}-verifier`,
|
|
workspaceClaim: claims.executor,
|
|
verifierWorkspaceClaim: claims.verifier,
|
|
status: 'provisioned',
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runKubectl(config, document) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn(
|
|
'kubectl',
|
|
[
|
|
'--context',
|
|
config.context,
|
|
'--namespace',
|
|
config.namespace,
|
|
'apply',
|
|
'--server-side',
|
|
'--field-manager=wishforge-provisioner',
|
|
'-o',
|
|
'name',
|
|
'-f',
|
|
'-',
|
|
],
|
|
{ stdio: ['pipe', 'pipe', 'pipe'] },
|
|
);
|
|
let output = '';
|
|
const timeout = setTimeout(() => {
|
|
child.kill('SIGTERM');
|
|
reject(new Error('KUBERNETES_APPLY_TIMEOUT'));
|
|
}, 30000);
|
|
child.stdout.on('data', chunk => {
|
|
output += chunk;
|
|
if (output.length > 65536) child.kill('SIGTERM');
|
|
});
|
|
// Never expose kubectl diagnostics: API validation errors can contain Secret data.
|
|
child.stderr.resume();
|
|
child.on('error', () => {
|
|
clearTimeout(timeout);
|
|
reject(new Error('KUBERNETES_COMMAND_UNAVAILABLE'));
|
|
});
|
|
child.on('close', code => {
|
|
clearTimeout(timeout);
|
|
code === 0 ? resolve(output) : reject(new Error('KUBERNETES_APPLY_FAILED'));
|
|
});
|
|
child.stdin.on('error', () => {});
|
|
child.stdin.end(JSON.stringify({ apiVersion: 'v1', kind: 'List', items: document }));
|
|
});
|
|
}
|
|
|
|
export async function lifecycleCommand(config, args) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn('kubectl', ['--context', config.context, '--namespace', config.namespace, ...args], {
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
let stdout = '';
|
|
const timer = setTimeout(() => {
|
|
child.kill('SIGKILL');
|
|
reject(new Error('KUBERNETES_RECONCILE_TIMEOUT'));
|
|
}, 45000);
|
|
child.stdout.on('data', chunk => {
|
|
stdout += chunk;
|
|
if (stdout.length > 1024 * 1024) child.kill('SIGKILL');
|
|
});
|
|
child.stderr.resume();
|
|
child.on('error', () => {
|
|
clearTimeout(timer);
|
|
reject(new Error('KUBERNETES_RECONCILE_FAILED'));
|
|
});
|
|
child.on('close', code => {
|
|
clearTimeout(timer);
|
|
code === 0 ? resolve(stdout) : reject(new Error('KUBERNETES_RECONCILE_FAILED'));
|
|
});
|
|
});
|
|
}
|
|
|
|
/** What a run generation holds besides its Jobs and volumes: credentials, identities, network policy. */
|
|
export function generationIdentities(prefix) {
|
|
return [
|
|
['secret', `${prefix}-executor-capability`],
|
|
['secret', `${prefix}-verifier-capability`],
|
|
['serviceaccount', `${prefix}-executor`],
|
|
['serviceaccount', `${prefix}-verifier`],
|
|
['networkpolicy', `${prefix}-isolation`],
|
|
['networkpolicy', `${prefix}-builder`],
|
|
];
|
|
}
|
|
|
|
export async function reconcileRunJobs(config, request, execute = lifecycleCommand) {
|
|
const { resources } = renderRunResources(config, request);
|
|
const labels = resources[0].metadata.labels;
|
|
const selector = `app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${labels['wishforge.io/run']},wishforge.io/component=runner`;
|
|
const listed = JSON.parse(await execute(config, ['get', 'jobs', '-l', selector, '-o', 'json']));
|
|
if (!Array.isArray(listed.items)) throw new Error('KUBERNETES_RECONCILE_FAILED');
|
|
let terminal = false;
|
|
for (const job of listed.items) {
|
|
if (
|
|
job.metadata?.namespace !== config.namespace ||
|
|
job.metadata?.labels?.['app.kubernetes.io/managed-by'] !== 'wishforge-provisioner' ||
|
|
job.metadata?.labels?.['wishforge.io/run'] !== labels['wishforge.io/run'] ||
|
|
!/^wf-[a-f0-9]{24}-(executor|verifier)$/.test(job.metadata?.name ?? '')
|
|
)
|
|
throw new Error('KUBERNETES_RECONCILE_SCOPE_MISMATCH');
|
|
if (job.metadata.labels['wishforge.io/generation'] === labels['wishforge.io/generation']) {
|
|
if (
|
|
job.status?.failed > 0 ||
|
|
job.status?.succeeded > 0 ||
|
|
job.status?.conditions?.some(c => ['Failed', 'Complete'].includes(c.type) && c.status === 'True')
|
|
)
|
|
terminal = true;
|
|
} else
|
|
await execute(config, [
|
|
'delete',
|
|
'job',
|
|
job.metadata.name,
|
|
'--cascade=foreground',
|
|
'--wait=true',
|
|
'--timeout=30s',
|
|
]);
|
|
}
|
|
// The generations this one superseded leave their credentials (run tokens), identities and
|
|
// network policy behind: only the current generation is ever released. Their volumes, kept for
|
|
// the retention period, name every one of them; Secrets cannot be listed, by design.
|
|
const owned = item => {
|
|
if (
|
|
item.metadata?.namespace !== config.namespace ||
|
|
item.metadata?.labels?.['app.kubernetes.io/managed-by'] !== 'wishforge-provisioner' ||
|
|
item.metadata?.labels?.['wishforge.io/run'] !== labels['wishforge.io/run']
|
|
)
|
|
throw new Error('KUBERNETES_RECONCILE_SCOPE_MISMATCH');
|
|
return item;
|
|
};
|
|
const claims = JSON.parse(await execute(config, ['get', 'persistentvolumeclaims', '-l', selector, '-o', 'json']));
|
|
const pods = JSON.parse(await execute(config, ['get', 'pods', '-l', selector, '-o', 'json']));
|
|
if (!Array.isArray(claims.items) || !Array.isArray(pods.items)) throw new Error('KUBERNETES_RECONCILE_FAILED');
|
|
const running = new Set(
|
|
pods.items
|
|
.map(owned)
|
|
.filter(pod => !['Succeeded', 'Failed'].includes(pod.status?.phase))
|
|
.map(pod => pod.metadata.labels['wishforge.io/generation']),
|
|
);
|
|
const superseded = new Set();
|
|
for (const claim of claims.items.map(owned)) {
|
|
const generation = claim.metadata.labels['wishforge.io/generation'];
|
|
const prefix = /^(wf-[a-f0-9]{24})-(executor|verifier)-workspace$/.exec(claim.metadata.name)?.[1];
|
|
if (prefix && generation !== labels['wishforge.io/generation'] && !running.has(generation)) superseded.add(prefix);
|
|
}
|
|
for (const prefix of superseded)
|
|
for (const [kind, name] of generationIdentities(prefix))
|
|
await execute(config, ['delete', kind, name, '--ignore-not-found=true', '--wait=false']);
|
|
return { terminal };
|
|
}
|
|
|
|
export async function provisionRun(config, request, apply = runKubectl, reconcile = reconcileRunJobs) {
|
|
const rendered = renderRunResources(config, request);
|
|
const state = await reconcile(config, request);
|
|
if (state.terminal)
|
|
return {
|
|
...rendered.receipt,
|
|
status: 'failed',
|
|
reason: 'Run generation has terminal jobs; resume with a new generation',
|
|
};
|
|
// Provision policy and identities before Jobs. An interrupted call is retried
|
|
// with the same deterministic resource names, never with a second workload.
|
|
await apply(
|
|
config,
|
|
rendered.resources.filter(resource => resource.kind !== 'Job'),
|
|
);
|
|
await apply(
|
|
config,
|
|
rendered.resources.filter(resource => resource.kind === 'Job'),
|
|
);
|
|
return rendered.receipt;
|
|
}
|