Imports the previously untracked working tree as-is so that the review changes that follow (requirements -> design -> implementation) can be reviewed as diffs and reverted if needed. No content changes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
175 lines
15 KiB
JavaScript
175 lines
15 KiB
JavaScript
import { createHash } from 'node:crypto';
|
|
import { spawn } from 'node:child_process';
|
|
import { verifyResourceGrant } from './resource-grants.mjs';
|
|
|
|
const safe = /^[a-zA-Z0-9_-]{1,128}$/;
|
|
const dns = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/;
|
|
function requireValue(value, name) { if (!value) throw new Error(`unconfigured: ${name} is required`); return value; }
|
|
export function readConfig(env = process.env) {
|
|
const config = {
|
|
tenantId: requireValue(env.WF_TENANT_ID, 'WF_TENANT_ID'),
|
|
context: requireValue(env.WF_KUBE_CONTEXT, 'WF_KUBE_CONTEXT'),
|
|
namespace: requireValue(env.WF_KUBE_NAMESPACE, 'WF_KUBE_NAMESPACE'),
|
|
image: requireValue(env.WF_RUN_WORKER_IMAGE, 'WF_RUN_WORKER_IMAGE'),
|
|
storageClass: requireValue(env.WF_RUN_STORAGE_CLASS, 'WF_RUN_STORAGE_CLASS'),
|
|
storageSize: env.WF_RUN_STORAGE_SIZE ?? '10Gi',
|
|
controlPlaneUrl: requireValue(env.WF_CONTROL_PLANE_URL, 'WF_CONTROL_PLANE_URL'),
|
|
controlNamespace: requireValue(env.WF_CONTROL_NAMESPACE, 'WF_CONTROL_NAMESPACE'),
|
|
temporalAddress: requireValue(env.TEMPORAL_ADDRESS, 'TEMPORAL_ADDRESS'),
|
|
temporalNamespace: requireValue(env.TEMPORAL_NAMESPACE, 'TEMPORAL_NAMESPACE'),
|
|
runtimeClass: env.WF_KATA_RUNTIME_CLASS ?? 'wishforge-kata',
|
|
activeDeadlineSeconds: Number(env.WF_RUN_MAX_SECONDS ?? 3600),
|
|
verifierDockerSecret: env.WF_VERIFIER_DOCKER_CONFIG_SECRET,
|
|
resourceGrantSecret: requireValue(env.WF_RESOURCE_GRANT_SECRET, 'WF_RESOURCE_GRANT_SECRET'),
|
|
cpuMilliCap: Number(env.WF_RUN_CPU_MILLI_CAP ?? 2000), memoryMiBCap: Number(env.WF_RUN_MEMORY_MIB_CAP ?? 4096),
|
|
previousImages: (env.WF_RUN_PREVIOUS_IMAGES ?? '').split(',').map(value=>value.trim()).filter(Boolean),
|
|
};
|
|
if (!safe.test(config.tenantId) || [config.namespace, config.controlNamespace, config.runtimeClass, config.storageClass, ...(config.verifierDockerSecret ? [config.verifierDockerSecret] : [])].some(value => !dns.test(value) || value.length > 63)) throw new Error('Invalid configured tenant/namespace/runtime class/storage/secret');
|
|
if (!/^[^\s]+@sha256:[a-f0-9]{64}$/.test(config.image)) throw new Error('WF_RUN_WORKER_IMAGE must be an immutable sha256 image digest');
|
|
if (config.resourceGrantSecret.length < 32 || !Number.isSafeInteger(config.cpuMilliCap) || config.cpuMilliCap < 100 || !Number.isSafeInteger(config.memoryMiBCap) || config.memoryMiBCap < 512 || config.previousImages.length > 10 || config.previousImages.some(image=>!/^[^\s]+@sha256:[a-f0-9]{64}$/.test(image))) throw new Error('Invalid resource grant signing key, static caps or previous runtime image allowlist');
|
|
if (!/^\d+Gi$/.test(config.storageSize)) throw new Error('Invalid run storage size');
|
|
if (!Number.isInteger(config.activeDeadlineSeconds) || config.activeDeadlineSeconds < 60 || config.activeDeadlineSeconds > 10800) throw new Error('Run deadline must be 60..10800 seconds and shorter than the scoped token lifetime');
|
|
const api = new URL(config.controlPlaneUrl);
|
|
if (api.protocol !== 'https:' || api.username || api.password || api.search || api.hash) throw new Error('Production control plane must use an HTTPS URL without embedded credentials');
|
|
if (!/^[a-zA-Z0-9.-]+:\d+$/.test(config.temporalAddress)) throw new Error('TEMPORAL_ADDRESS must explicitly include host:port');
|
|
if (Number(config.temporalAddress.split(':').at(-1)) < 1 || Number(config.temporalAddress.split(':').at(-1)) > 65535) throw new Error('Invalid Temporal port');
|
|
return config;
|
|
}
|
|
|
|
function assertToken(token, request, kind, deadline) {
|
|
try {
|
|
const parts = token.split('.');
|
|
if (parts.length !== 3 || token.length > 16384) throw new Error();
|
|
const claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
|
|
if (claims.tenant_id !== request.tenantId || claims.run_id !== request.runId || claims.generation !== request.generation || claims.kind !== kind || claims.iss !== 'wishforge-control-plane' || claims.aud !== 'wishforge-internal' || !Number.isFinite(claims.exp) || claims.exp * 1000 <= Date.now() + deadline * 1000) throw new Error();
|
|
} catch { throw new Error(`Invalid or insufficient-lifetime ${kind} run token`); }
|
|
// The authenticated control plane issues these tokens. This shape/scope check
|
|
// is defense-in-depth, not signature verification; the API verifies signatures.
|
|
}
|
|
|
|
export function renderRunResources(config, request) {
|
|
if (!request || Object.keys(request).some(key => !['tenantId', 'runId', 'generation', 'executorToken', 'verifierToken', 'resourceGrant'].includes(key)) || request.tenantId !== config.tenantId || !safe.test(request.runId ?? '') || !safe.test(request.generation ?? '')) throw new Error('Run request is outside the configured tenant or has invalid fields');
|
|
const resourceGrant = verifyResourceGrant(config, request);
|
|
// One reservation covers both isolated roles; aggregate requested limits must
|
|
// stay within the signed run grant, including their separate persistent PVCs.
|
|
const roleResources = { cpuMilli: Math.floor(resourceGrant.cpuMilli / 2), memoryMiB: Math.floor(resourceGrant.memoryMiB / 2), storageMiB: Math.floor(resourceGrant.storageMiB / 2) };
|
|
assertToken(request.executorToken, request, 'worker', config.activeDeadlineSeconds);
|
|
assertToken(request.verifierToken, request, 'verifier', config.activeDeadlineSeconds);
|
|
if (request.executorToken === request.verifierToken) throw new Error('Executor and verifier credentials must differ');
|
|
const runHash = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24);
|
|
const generationHash = createHash('sha256').update(request.generation).digest('hex').slice(0, 24);
|
|
const suffix = createHash('sha256').update(`${request.tenantId}\0${request.runId}\0${request.generation}`).digest('hex').slice(0, 24);
|
|
const prefix = `wf-${suffix}`;
|
|
const labels = { 'app.kubernetes.io/managed-by': 'wishforge-provisioner', 'wishforge.io/run': runHash, 'wishforge.io/generation': generationHash, 'wishforge.io/component': 'runner' };
|
|
const metadata = name => ({ name, namespace: config.namespace, labels });
|
|
const claims = Object.fromEntries(['executor', 'verifier'].map(role => [role, `${prefix}-${role}-workspace`]));
|
|
const resources = Object.values(claims).map(claim => ({ apiVersion: 'v1', kind: 'PersistentVolumeClaim', metadata: metadata(claim), spec: { storageClassName: config.storageClass, accessModes: ['ReadWriteOnce'], resources: { requests: { storage: `${roleResources.storageMiB}Mi` } } } }));
|
|
for (const role of ['executor', 'verifier']) {
|
|
const name = `${prefix}-${role}`;
|
|
const secretName = `${name}-capability`;
|
|
resources.push({ apiVersion: 'v1', kind: 'Secret', metadata: metadata(secretName), type: 'Opaque', stringData: { token: request[role === 'executor' ? 'executorToken' : 'verifierToken'] } });
|
|
resources.push({ apiVersion: 'v1', kind: 'ServiceAccount', metadata: metadata(name), automountServiceAccountToken: false });
|
|
const env = [
|
|
{ name: 'WF_WORKER_ROLE', value: role },
|
|
{ name: 'WF_SANDBOX_RUN_ID', value: request.runId },
|
|
{ name: 'WF_SANDBOX_TENANT_ID', value: request.tenantId },
|
|
{ name: 'WF_SANDBOX_GENERATION', value: request.generation },
|
|
{ name: role === 'executor' ? 'WF_WORKER_TOKEN' : 'WF_VERIFIER_TOKEN', valueFrom: { secretKeyRef: { name: secretName, key: 'token' } } },
|
|
{ name: 'WF_WORKER_ID', value: name },
|
|
{ name: 'WF_WORKSPACE_ROOT', value: '/workspaces' },
|
|
{ name: 'WF_PROVIDER_HOME_ROOT', value: '/provider-home' },
|
|
{ name: 'WF_CONTROL_PLANE_URL', value: config.controlPlaneUrl },
|
|
{ name: 'TEMPORAL_ADDRESS', value: config.temporalAddress },
|
|
{ name: 'TEMPORAL_NAMESPACE', value: config.temporalNamespace },
|
|
{ name: 'WF_SANDBOX_ENFORCED', value: 'true' },
|
|
{ name: 'WF_WORKER_CONCURRENCY', value: '1' },
|
|
{ name: 'WF_EXECUTION_CPU_MILLI_LIMIT', value: String(roleResources.cpuMilli) },
|
|
{ name: 'WF_EXECUTION_MEMORY_MIB_LIMIT', value: String(roleResources.memoryMiB) },
|
|
{ name: 'WF_EXECUTION_STORAGE_MIB_LIMIT', value: String(roleResources.storageMiB) },
|
|
];
|
|
const mounts = [{ name: 'workspace', mountPath: '/workspaces' }, { name: 'provider-home', mountPath: '/provider-home' }, { name: 'temporary', mountPath: '/tmp' }];
|
|
const volumes = [{ name: 'workspace', persistentVolumeClaim: { claimName: claims[role] } }, { name: 'provider-home', emptyDir: { medium: 'Memory', sizeLimit: '256Mi' } }, { name: 'temporary', emptyDir: { sizeLimit: '4Gi' } }];
|
|
if (role === 'verifier') {
|
|
env.push({ name: 'WF_VERIFIER_BWRAP', value: '/usr/bin/bwrap' });
|
|
if (config.verifierDockerSecret) {
|
|
env.push({ name: 'WF_DOCKER_CONFIG', value: '/run/docker' });
|
|
mounts.push({ name: 'docker-config', mountPath: '/run/docker', readOnly: true });
|
|
volumes.push({ name: 'docker-config', secret: { secretName: config.verifierDockerSecret, defaultMode: 256 } });
|
|
}
|
|
}
|
|
resources.push({ apiVersion: 'batch/v1', kind: 'Job', metadata: metadata(name), spec: {
|
|
backoffLimit: 0, activeDeadlineSeconds: config.activeDeadlineSeconds, ttlSecondsAfterFinished: 86400,
|
|
template: { metadata: { labels: { ...labels, 'wishforge.io/worker-role': role } }, spec: {
|
|
runtimeClassName: config.runtimeClass, serviceAccountName: name, automountServiceAccountToken: false, restartPolicy: 'Never',
|
|
securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000, seccompProfile: { type: 'RuntimeDefault' } },
|
|
containers: [{ name: role, image: resourceGrant.image, imagePullPolicy: 'Always', command: ['npm', 'run', 'dev', '-w', '@wishforge/worker'], env,
|
|
securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: ['ALL'] } },
|
|
resources: { requests: { cpu: `${Math.min(1000,roleResources.cpuMilli)}m`, memory: `${Math.min(2048,roleResources.memoryMiB)}Mi`, 'ephemeral-storage': '1Gi' }, limits: { cpu: `${roleResources.cpuMilli}m`, memory: `${roleResources.memoryMiB}Mi`, 'ephemeral-storage': '8Gi' } }, volumeMounts: mounts }], volumes,
|
|
} },
|
|
} });
|
|
}
|
|
const nsSelector = { namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.controlNamespace } } };
|
|
const apiPort = Number(new URL(config.controlPlaneUrl).port || 443);
|
|
const temporalPort = Number(config.temporalAddress.split(':').at(-1));
|
|
resources.push({ apiVersion: 'networking.k8s.io/v1', kind: 'NetworkPolicy', metadata: metadata(`${prefix}-isolation`), spec: {
|
|
podSelector: { matchLabels: { 'wishforge.io/run': runHash, 'wishforge.io/generation': generationHash } }, policyTypes: ['Ingress', 'Egress'], ingress: [],
|
|
egress: [
|
|
{ to: [{ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' } }, podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } } }], ports: [{ protocol: 'UDP', port: 53 }, { protocol: 'TCP', port: 53 }] },
|
|
{ to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'api' } } }], ports: [{ protocol: 'TCP', port: apiPort }] },
|
|
{ to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'temporal' } } }], ports: [{ protocol: 'TCP', port: temporalPort }] },
|
|
],
|
|
} });
|
|
return { resources, receipt: { tenantId: request.tenantId, runId: request.runId, generation: request.generation, namespace: config.namespace, executorJobName: `${prefix}-executor`, verifierJobName: `${prefix}-verifier`, workspaceClaim: claims.executor, verifierWorkspaceClaim: claims.verifier, status: 'provisioned' } };
|
|
}
|
|
|
|
export function runKubectl(config, document) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn('kubectl', ['--context', config.context, '--namespace', config.namespace, 'apply', '--server-side', '--field-manager=wishforge-provisioner', '-o', 'name', '-f', '-'], { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
let output = '';
|
|
const timeout = setTimeout(() => { child.kill('SIGTERM'); reject(new Error('KUBERNETES_APPLY_TIMEOUT')); }, 30000);
|
|
child.stdout.on('data', chunk => { output += chunk; if (output.length > 65536) child.kill('SIGTERM'); });
|
|
// Never expose kubectl diagnostics: API validation errors can contain Secret data.
|
|
child.stderr.resume();
|
|
child.on('error', () => { clearTimeout(timeout); reject(new Error('KUBERNETES_COMMAND_UNAVAILABLE')); });
|
|
child.on('close', code => { clearTimeout(timeout); code === 0 ? resolve(output) : reject(new Error('KUBERNETES_APPLY_FAILED')); });
|
|
child.stdin.on('error', () => {}); child.stdin.end(JSON.stringify({ apiVersion: 'v1', kind: 'List', items: document }));
|
|
});
|
|
}
|
|
|
|
export async function lifecycleCommand(config, args) {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn('kubectl', ['--context', config.context, '--namespace', config.namespace, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
let stdout=''; const timer=setTimeout(()=>{child.kill('SIGKILL');reject(new Error('KUBERNETES_RECONCILE_TIMEOUT'));},45000);
|
|
child.stdout.on('data', chunk=>{stdout+=chunk; if(stdout.length>1024*1024)child.kill('SIGKILL');});child.stderr.resume();
|
|
child.on('error',()=>{clearTimeout(timer);reject(new Error('KUBERNETES_RECONCILE_FAILED'));});
|
|
child.on('close',code=>{clearTimeout(timer);code===0?resolve(stdout):reject(new Error('KUBERNETES_RECONCILE_FAILED'));});
|
|
});
|
|
}
|
|
|
|
export async function reconcileRunJobs(config, request, execute = lifecycleCommand) {
|
|
const { resources }=renderRunResources(config,request);
|
|
const labels=resources[0].metadata.labels;
|
|
const selector=`app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${labels['wishforge.io/run']},wishforge.io/component=runner`;
|
|
const listed=JSON.parse(await execute(config,['get','jobs','-l',selector,'-o','json']));
|
|
if(!Array.isArray(listed.items))throw new Error('KUBERNETES_RECONCILE_FAILED');
|
|
let terminal=false;
|
|
for(const job of listed.items){
|
|
if(job.metadata?.namespace!==config.namespace || job.metadata?.labels?.['app.kubernetes.io/managed-by']!=='wishforge-provisioner' || job.metadata?.labels?.['wishforge.io/run']!==labels['wishforge.io/run'] || !/^wf-[a-f0-9]{24}-(executor|verifier)$/.test(job.metadata?.name??''))throw new Error('KUBERNETES_RECONCILE_SCOPE_MISMATCH');
|
|
if(job.metadata.labels['wishforge.io/generation']===labels['wishforge.io/generation']){
|
|
if(job.status?.failed>0 || job.status?.succeeded>0 || job.status?.conditions?.some(c=>['Failed','Complete'].includes(c.type)&&c.status==='True'))terminal=true;
|
|
} else await execute(config,['delete','job',job.metadata.name,'--cascade=foreground','--wait=true','--timeout=30s']);
|
|
}
|
|
return {terminal};
|
|
}
|
|
|
|
export async function provisionRun(config, request, apply = runKubectl, reconcile = reconcileRunJobs) {
|
|
const rendered = renderRunResources(config, request);
|
|
const state = await reconcile(config, request);
|
|
if(state.terminal) return { ...rendered.receipt, status:'failed', reason:'Run generation has terminal jobs; resume with a new generation' };
|
|
// Provision policy and identities before Jobs. An interrupted call is retried
|
|
// with the same deterministic resource names, never with a second workload.
|
|
await apply(config, rendered.resources.filter(resource => resource.kind !== 'Job'));
|
|
await apply(config, rendered.resources.filter(resource => resource.kind === 'Job'));
|
|
return rendered.receipt;
|
|
}
|