wishforge-ide/apps/ide/provisioner/provisioner.mjs
eust-w b47a07e8d2 chore: baseline snapshot of the Codex-delivered code before review
Imports the previously untracked working tree as-is so that the review
changes that follow (requirements -> design -> implementation) can be
reviewed as diffs and reverted if needed. No content changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 13:18:04 +08:00

175 lines
15 KiB
JavaScript

import { createHash } from 'node:crypto';
import { spawn } from 'node:child_process';
import { verifyResourceGrant } from './resource-grants.mjs';
const safe = /^[a-zA-Z0-9_-]{1,128}$/;
const dns = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/;
function requireValue(value, name) { if (!value) throw new Error(`unconfigured: ${name} is required`); return value; }
export function readConfig(env = process.env) {
const config = {
tenantId: requireValue(env.WF_TENANT_ID, 'WF_TENANT_ID'),
context: requireValue(env.WF_KUBE_CONTEXT, 'WF_KUBE_CONTEXT'),
namespace: requireValue(env.WF_KUBE_NAMESPACE, 'WF_KUBE_NAMESPACE'),
image: requireValue(env.WF_RUN_WORKER_IMAGE, 'WF_RUN_WORKER_IMAGE'),
storageClass: requireValue(env.WF_RUN_STORAGE_CLASS, 'WF_RUN_STORAGE_CLASS'),
storageSize: env.WF_RUN_STORAGE_SIZE ?? '10Gi',
controlPlaneUrl: requireValue(env.WF_CONTROL_PLANE_URL, 'WF_CONTROL_PLANE_URL'),
controlNamespace: requireValue(env.WF_CONTROL_NAMESPACE, 'WF_CONTROL_NAMESPACE'),
temporalAddress: requireValue(env.TEMPORAL_ADDRESS, 'TEMPORAL_ADDRESS'),
temporalNamespace: requireValue(env.TEMPORAL_NAMESPACE, 'TEMPORAL_NAMESPACE'),
runtimeClass: env.WF_KATA_RUNTIME_CLASS ?? 'wishforge-kata',
activeDeadlineSeconds: Number(env.WF_RUN_MAX_SECONDS ?? 3600),
verifierDockerSecret: env.WF_VERIFIER_DOCKER_CONFIG_SECRET,
resourceGrantSecret: requireValue(env.WF_RESOURCE_GRANT_SECRET, 'WF_RESOURCE_GRANT_SECRET'),
cpuMilliCap: Number(env.WF_RUN_CPU_MILLI_CAP ?? 2000), memoryMiBCap: Number(env.WF_RUN_MEMORY_MIB_CAP ?? 4096),
previousImages: (env.WF_RUN_PREVIOUS_IMAGES ?? '').split(',').map(value=>value.trim()).filter(Boolean),
};
if (!safe.test(config.tenantId) || [config.namespace, config.controlNamespace, config.runtimeClass, config.storageClass, ...(config.verifierDockerSecret ? [config.verifierDockerSecret] : [])].some(value => !dns.test(value) || value.length > 63)) throw new Error('Invalid configured tenant/namespace/runtime class/storage/secret');
if (!/^[^\s]+@sha256:[a-f0-9]{64}$/.test(config.image)) throw new Error('WF_RUN_WORKER_IMAGE must be an immutable sha256 image digest');
if (config.resourceGrantSecret.length < 32 || !Number.isSafeInteger(config.cpuMilliCap) || config.cpuMilliCap < 100 || !Number.isSafeInteger(config.memoryMiBCap) || config.memoryMiBCap < 512 || config.previousImages.length > 10 || config.previousImages.some(image=>!/^[^\s]+@sha256:[a-f0-9]{64}$/.test(image))) throw new Error('Invalid resource grant signing key, static caps or previous runtime image allowlist');
if (!/^\d+Gi$/.test(config.storageSize)) throw new Error('Invalid run storage size');
if (!Number.isInteger(config.activeDeadlineSeconds) || config.activeDeadlineSeconds < 60 || config.activeDeadlineSeconds > 10800) throw new Error('Run deadline must be 60..10800 seconds and shorter than the scoped token lifetime');
const api = new URL(config.controlPlaneUrl);
if (api.protocol !== 'https:' || api.username || api.password || api.search || api.hash) throw new Error('Production control plane must use an HTTPS URL without embedded credentials');
if (!/^[a-zA-Z0-9.-]+:\d+$/.test(config.temporalAddress)) throw new Error('TEMPORAL_ADDRESS must explicitly include host:port');
if (Number(config.temporalAddress.split(':').at(-1)) < 1 || Number(config.temporalAddress.split(':').at(-1)) > 65535) throw new Error('Invalid Temporal port');
return config;
}
function assertToken(token, request, kind, deadline) {
try {
const parts = token.split('.');
if (parts.length !== 3 || token.length > 16384) throw new Error();
const claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
if (claims.tenant_id !== request.tenantId || claims.run_id !== request.runId || claims.generation !== request.generation || claims.kind !== kind || claims.iss !== 'wishforge-control-plane' || claims.aud !== 'wishforge-internal' || !Number.isFinite(claims.exp) || claims.exp * 1000 <= Date.now() + deadline * 1000) throw new Error();
} catch { throw new Error(`Invalid or insufficient-lifetime ${kind} run token`); }
// The authenticated control plane issues these tokens. This shape/scope check
// is defense-in-depth, not signature verification; the API verifies signatures.
}
export function renderRunResources(config, request) {
if (!request || Object.keys(request).some(key => !['tenantId', 'runId', 'generation', 'executorToken', 'verifierToken', 'resourceGrant'].includes(key)) || request.tenantId !== config.tenantId || !safe.test(request.runId ?? '') || !safe.test(request.generation ?? '')) throw new Error('Run request is outside the configured tenant or has invalid fields');
const resourceGrant = verifyResourceGrant(config, request);
// One reservation covers both isolated roles; aggregate requested limits must
// stay within the signed run grant, including their separate persistent PVCs.
const roleResources = { cpuMilli: Math.floor(resourceGrant.cpuMilli / 2), memoryMiB: Math.floor(resourceGrant.memoryMiB / 2), storageMiB: Math.floor(resourceGrant.storageMiB / 2) };
assertToken(request.executorToken, request, 'worker', config.activeDeadlineSeconds);
assertToken(request.verifierToken, request, 'verifier', config.activeDeadlineSeconds);
if (request.executorToken === request.verifierToken) throw new Error('Executor and verifier credentials must differ');
const runHash = createHash('sha256').update(`${request.tenantId}\0${request.runId}`).digest('hex').slice(0, 24);
const generationHash = createHash('sha256').update(request.generation).digest('hex').slice(0, 24);
const suffix = createHash('sha256').update(`${request.tenantId}\0${request.runId}\0${request.generation}`).digest('hex').slice(0, 24);
const prefix = `wf-${suffix}`;
const labels = { 'app.kubernetes.io/managed-by': 'wishforge-provisioner', 'wishforge.io/run': runHash, 'wishforge.io/generation': generationHash, 'wishforge.io/component': 'runner' };
const metadata = name => ({ name, namespace: config.namespace, labels });
const claims = Object.fromEntries(['executor', 'verifier'].map(role => [role, `${prefix}-${role}-workspace`]));
const resources = Object.values(claims).map(claim => ({ apiVersion: 'v1', kind: 'PersistentVolumeClaim', metadata: metadata(claim), spec: { storageClassName: config.storageClass, accessModes: ['ReadWriteOnce'], resources: { requests: { storage: `${roleResources.storageMiB}Mi` } } } }));
for (const role of ['executor', 'verifier']) {
const name = `${prefix}-${role}`;
const secretName = `${name}-capability`;
resources.push({ apiVersion: 'v1', kind: 'Secret', metadata: metadata(secretName), type: 'Opaque', stringData: { token: request[role === 'executor' ? 'executorToken' : 'verifierToken'] } });
resources.push({ apiVersion: 'v1', kind: 'ServiceAccount', metadata: metadata(name), automountServiceAccountToken: false });
const env = [
{ name: 'WF_WORKER_ROLE', value: role },
{ name: 'WF_SANDBOX_RUN_ID', value: request.runId },
{ name: 'WF_SANDBOX_TENANT_ID', value: request.tenantId },
{ name: 'WF_SANDBOX_GENERATION', value: request.generation },
{ name: role === 'executor' ? 'WF_WORKER_TOKEN' : 'WF_VERIFIER_TOKEN', valueFrom: { secretKeyRef: { name: secretName, key: 'token' } } },
{ name: 'WF_WORKER_ID', value: name },
{ name: 'WF_WORKSPACE_ROOT', value: '/workspaces' },
{ name: 'WF_PROVIDER_HOME_ROOT', value: '/provider-home' },
{ name: 'WF_CONTROL_PLANE_URL', value: config.controlPlaneUrl },
{ name: 'TEMPORAL_ADDRESS', value: config.temporalAddress },
{ name: 'TEMPORAL_NAMESPACE', value: config.temporalNamespace },
{ name: 'WF_SANDBOX_ENFORCED', value: 'true' },
{ name: 'WF_WORKER_CONCURRENCY', value: '1' },
{ name: 'WF_EXECUTION_CPU_MILLI_LIMIT', value: String(roleResources.cpuMilli) },
{ name: 'WF_EXECUTION_MEMORY_MIB_LIMIT', value: String(roleResources.memoryMiB) },
{ name: 'WF_EXECUTION_STORAGE_MIB_LIMIT', value: String(roleResources.storageMiB) },
];
const mounts = [{ name: 'workspace', mountPath: '/workspaces' }, { name: 'provider-home', mountPath: '/provider-home' }, { name: 'temporary', mountPath: '/tmp' }];
const volumes = [{ name: 'workspace', persistentVolumeClaim: { claimName: claims[role] } }, { name: 'provider-home', emptyDir: { medium: 'Memory', sizeLimit: '256Mi' } }, { name: 'temporary', emptyDir: { sizeLimit: '4Gi' } }];
if (role === 'verifier') {
env.push({ name: 'WF_VERIFIER_BWRAP', value: '/usr/bin/bwrap' });
if (config.verifierDockerSecret) {
env.push({ name: 'WF_DOCKER_CONFIG', value: '/run/docker' });
mounts.push({ name: 'docker-config', mountPath: '/run/docker', readOnly: true });
volumes.push({ name: 'docker-config', secret: { secretName: config.verifierDockerSecret, defaultMode: 256 } });
}
}
resources.push({ apiVersion: 'batch/v1', kind: 'Job', metadata: metadata(name), spec: {
backoffLimit: 0, activeDeadlineSeconds: config.activeDeadlineSeconds, ttlSecondsAfterFinished: 86400,
template: { metadata: { labels: { ...labels, 'wishforge.io/worker-role': role } }, spec: {
runtimeClassName: config.runtimeClass, serviceAccountName: name, automountServiceAccountToken: false, restartPolicy: 'Never',
securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000, fsGroup: 1000, seccompProfile: { type: 'RuntimeDefault' } },
containers: [{ name: role, image: resourceGrant.image, imagePullPolicy: 'Always', command: ['npm', 'run', 'dev', '-w', '@wishforge/worker'], env,
securityContext: { allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: { drop: ['ALL'] } },
resources: { requests: { cpu: `${Math.min(1000,roleResources.cpuMilli)}m`, memory: `${Math.min(2048,roleResources.memoryMiB)}Mi`, 'ephemeral-storage': '1Gi' }, limits: { cpu: `${roleResources.cpuMilli}m`, memory: `${roleResources.memoryMiB}Mi`, 'ephemeral-storage': '8Gi' } }, volumeMounts: mounts }], volumes,
} },
} });
}
const nsSelector = { namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': config.controlNamespace } } };
const apiPort = Number(new URL(config.controlPlaneUrl).port || 443);
const temporalPort = Number(config.temporalAddress.split(':').at(-1));
resources.push({ apiVersion: 'networking.k8s.io/v1', kind: 'NetworkPolicy', metadata: metadata(`${prefix}-isolation`), spec: {
podSelector: { matchLabels: { 'wishforge.io/run': runHash, 'wishforge.io/generation': generationHash } }, policyTypes: ['Ingress', 'Egress'], ingress: [],
egress: [
{ to: [{ namespaceSelector: { matchLabels: { 'kubernetes.io/metadata.name': 'kube-system' } }, podSelector: { matchLabels: { 'k8s-app': 'kube-dns' } } }], ports: [{ protocol: 'UDP', port: 53 }, { protocol: 'TCP', port: 53 }] },
{ to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'api' } } }], ports: [{ protocol: 'TCP', port: apiPort }] },
{ to: [{ ...nsSelector, podSelector: { matchLabels: { 'wishforge.io/component': 'temporal' } } }], ports: [{ protocol: 'TCP', port: temporalPort }] },
],
} });
return { resources, receipt: { tenantId: request.tenantId, runId: request.runId, generation: request.generation, namespace: config.namespace, executorJobName: `${prefix}-executor`, verifierJobName: `${prefix}-verifier`, workspaceClaim: claims.executor, verifierWorkspaceClaim: claims.verifier, status: 'provisioned' } };
}
export function runKubectl(config, document) {
return new Promise((resolve, reject) => {
const child = spawn('kubectl', ['--context', config.context, '--namespace', config.namespace, 'apply', '--server-side', '--field-manager=wishforge-provisioner', '-o', 'name', '-f', '-'], { stdio: ['pipe', 'pipe', 'pipe'] });
let output = '';
const timeout = setTimeout(() => { child.kill('SIGTERM'); reject(new Error('KUBERNETES_APPLY_TIMEOUT')); }, 30000);
child.stdout.on('data', chunk => { output += chunk; if (output.length > 65536) child.kill('SIGTERM'); });
// Never expose kubectl diagnostics: API validation errors can contain Secret data.
child.stderr.resume();
child.on('error', () => { clearTimeout(timeout); reject(new Error('KUBERNETES_COMMAND_UNAVAILABLE')); });
child.on('close', code => { clearTimeout(timeout); code === 0 ? resolve(output) : reject(new Error('KUBERNETES_APPLY_FAILED')); });
child.stdin.on('error', () => {}); child.stdin.end(JSON.stringify({ apiVersion: 'v1', kind: 'List', items: document }));
});
}
export async function lifecycleCommand(config, args) {
return new Promise((resolve, reject) => {
const child = spawn('kubectl', ['--context', config.context, '--namespace', config.namespace, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
let stdout=''; const timer=setTimeout(()=>{child.kill('SIGKILL');reject(new Error('KUBERNETES_RECONCILE_TIMEOUT'));},45000);
child.stdout.on('data', chunk=>{stdout+=chunk; if(stdout.length>1024*1024)child.kill('SIGKILL');});child.stderr.resume();
child.on('error',()=>{clearTimeout(timer);reject(new Error('KUBERNETES_RECONCILE_FAILED'));});
child.on('close',code=>{clearTimeout(timer);code===0?resolve(stdout):reject(new Error('KUBERNETES_RECONCILE_FAILED'));});
});
}
export async function reconcileRunJobs(config, request, execute = lifecycleCommand) {
const { resources }=renderRunResources(config,request);
const labels=resources[0].metadata.labels;
const selector=`app.kubernetes.io/managed-by=wishforge-provisioner,wishforge.io/run=${labels['wishforge.io/run']},wishforge.io/component=runner`;
const listed=JSON.parse(await execute(config,['get','jobs','-l',selector,'-o','json']));
if(!Array.isArray(listed.items))throw new Error('KUBERNETES_RECONCILE_FAILED');
let terminal=false;
for(const job of listed.items){
if(job.metadata?.namespace!==config.namespace || job.metadata?.labels?.['app.kubernetes.io/managed-by']!=='wishforge-provisioner' || job.metadata?.labels?.['wishforge.io/run']!==labels['wishforge.io/run'] || !/^wf-[a-f0-9]{24}-(executor|verifier)$/.test(job.metadata?.name??''))throw new Error('KUBERNETES_RECONCILE_SCOPE_MISMATCH');
if(job.metadata.labels['wishforge.io/generation']===labels['wishforge.io/generation']){
if(job.status?.failed>0 || job.status?.succeeded>0 || job.status?.conditions?.some(c=>['Failed','Complete'].includes(c.type)&&c.status==='True'))terminal=true;
} else await execute(config,['delete','job',job.metadata.name,'--cascade=foreground','--wait=true','--timeout=30s']);
}
return {terminal};
}
export async function provisionRun(config, request, apply = runKubectl, reconcile = reconcileRunJobs) {
const rendered = renderRunResources(config, request);
const state = await reconcile(config, request);
if(state.terminal) return { ...rendered.receipt, status:'failed', reason:'Run generation has terminal jobs; resume with a new generation' };
// Provision policy and identities before Jobs. An interrupted call is retried
// with the same deterministic resource names, never with a second workload.
await apply(config, rendered.resources.filter(resource => resource.kind !== 'Job'));
await apply(config, rendered.resources.filter(resource => resource.kind === 'Job'));
return rendered.receipt;
}