A start that found the tenant's execution slots taken was deferred like a failed delivery, backing off to two minutes between looks. On the local POC, which allows one run at a time, the next person's wish could sit for two minutes after the run before it finished or stopped to wait, and every look left an error line in the log. Waiting for a slot is ordinary queueing: the dispatcher now looks again every ten seconds, one small transaction each time, and logs nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
336 lines
18 KiB
TypeScript
336 lines
18 KiB
TypeScript
import { resolveGatewayTarget } from './domain/gateway-target.js';
|
|
import { Client, Connection, WorkflowIdReusePolicy } from '@temporalio/client';
|
|
import type { ControlPlane } from './domain/control-plane.js';
|
|
import { ServiceTokens } from './service-tokens.js';
|
|
import { dispatchTenants } from './tenants.js';
|
|
import { clearWaiting, must, setWaiting, touch } from './domain/control-plane.js';
|
|
import { readFile } from 'node:fs/promises';
|
|
import {
|
|
DomainError,
|
|
requireTenantNamespaces,
|
|
temporalNamespace,
|
|
temporalTlsFiles,
|
|
type Project,
|
|
type Run,
|
|
type Workspace,
|
|
} from '@wishforge/contracts';
|
|
import { reserveExecutionGrant, recordAllocation } from './domain/entitlements.js';
|
|
import { resolveProvisionerTarget, type ProvisionerTarget } from './domain/provisioner-target.js';
|
|
|
|
/** How often a start that waits for an execution slot asks again; each ask is one small transaction. */
|
|
const CAPACITY_RECHECK_SECONDS = 10;
|
|
|
|
/** Mutual TLS for the Temporal connection, from the files the environment names. */
|
|
export async function temporalTls(env: NodeJS.ProcessEnv) {
|
|
const files = temporalTlsFiles(env);
|
|
if (!files) return undefined;
|
|
return {
|
|
clientCertPair: { crt: await readFile(files.cert), key: await readFile(files.key) },
|
|
...(files.ca ? { serverRootCACertificate: await readFile(files.ca) } : {}),
|
|
...(files.serverName ? { serverNameOverride: files.serverName } : {}),
|
|
};
|
|
}
|
|
|
|
/** At-least-once outbox delivery; workflow IDs deduplicate starts after an ambiguous response. */
|
|
export class Dispatcher {
|
|
private stopped = false;
|
|
private connection?: Connection;
|
|
constructor(private control: ControlPlane) {}
|
|
async start() {
|
|
requireTenantNamespaces(this.control.env, dispatchTenants(this.control.env));
|
|
while (!this.stopped) {
|
|
try {
|
|
this.connection ??= await Connection.connect({
|
|
address: this.control.env.TEMPORAL_ADDRESS ?? '127.0.0.1:7233',
|
|
tls: await temporalTls(this.control.env),
|
|
});
|
|
for (const tenant of dispatchTenants(this.control.env)) {
|
|
const client = new Client({
|
|
connection: this.connection,
|
|
namespace: temporalNamespace(this.control.env, tenant),
|
|
});
|
|
// Tenant configuration failures must not tear down Temporal or prevent
|
|
// cancellation signals for already-running work.
|
|
let provisioner: ProvisionerTarget | undefined;
|
|
let provisionerError: unknown;
|
|
try {
|
|
provisioner = await resolveProvisionerTarget(this.control.env, tenant);
|
|
} catch (error) {
|
|
provisionerError = error;
|
|
}
|
|
// Claimed items are invisible to other API replicas until acknowledged, deferred or expired.
|
|
const pending = await this.control.store.transaction(tenant, tx =>
|
|
tx.claimOutbox(['run.signal', 'run.start', 'workspace.provision'], 300),
|
|
);
|
|
for (const item of pending) {
|
|
if (item.type === 'workspace.provision') {
|
|
try {
|
|
if (provisionerError) throw provisionerError;
|
|
const gateway = await resolveGatewayTarget(this.control.env, tenant);
|
|
if (!gateway) {
|
|
await this.control.store.transaction(tenant, tx => tx.defer(item.id, 60));
|
|
continue;
|
|
}
|
|
const { currentWorkspace, project } = await this.control.store.transaction(tenant, async tx => {
|
|
const currentWorkspace = await must<Workspace>(tx, 'workspace', String(item.payload.workspaceId));
|
|
return { currentWorkspace, project: await tx.get<Project>('project', currentWorkspace.projectId) };
|
|
});
|
|
if (currentWorkspace.lifecycle) {
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
continue;
|
|
}
|
|
if (!provisioner) {
|
|
await this.control.store.transaction(tenant, tx => tx.defer(item.id, 60));
|
|
continue;
|
|
}
|
|
const response = await fetch(`${provisioner.url}/workspaces`, {
|
|
method: 'POST',
|
|
headers: { Authorization: `Bearer ${provisioner.token}`, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
tenantId: tenant,
|
|
projectId: item.payload.projectId,
|
|
workspaceId: item.payload.workspaceId,
|
|
// The workspace then accepts credentials for this repository only.
|
|
...(project?.repositoryStorage === 'platform' && project.repositoryUrl
|
|
? { repositoryUrl: project.repositoryUrl }
|
|
: {}),
|
|
}),
|
|
signal: AbortSignal.timeout(30000),
|
|
});
|
|
if (!response.ok) throw new Error('WorkspaceProvisioningFailed');
|
|
const state = (await response.json()) as {
|
|
status: 'ready' | 'pending';
|
|
upstreamUrl: string;
|
|
collaborationUpstreamUrl?: string;
|
|
/** The local backend shares one Theia and cannot pause or resume it. */
|
|
lifecycle?: 'unmanaged';
|
|
};
|
|
if (!['ready', 'pending'].includes(state.status)) throw new Error('InvalidWorkspaceReceipt');
|
|
await this.control.store.transaction(tenant, async tx => {
|
|
const w = await must<Workspace>(tx, 'workspace', String(item.payload.workspaceId));
|
|
if (w.lifecycle) {
|
|
await tx.acknowledge(item.id);
|
|
return;
|
|
}
|
|
w.managedBy = state.lifecycle === 'unmanaged' ? 'local-provisioner' : 'wishforge-provisioner';
|
|
if (state.status === 'ready') w.lastActivityAt = new Date().toISOString();
|
|
w.status = state.status === 'ready' ? 'ready' : 'requested';
|
|
w.reason =
|
|
state.status === 'ready' ? undefined : 'Waiting for the isolated Theia instance to become ready.';
|
|
if (state.status === 'ready') {
|
|
const target = new URL(state.upstreamUrl);
|
|
if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password)
|
|
throw new Error('InvalidWorkspaceTarget');
|
|
w.upstreamUrl = target.toString();
|
|
if (state.collaborationUpstreamUrl) {
|
|
const collaboration = new URL(state.collaborationUpstreamUrl);
|
|
if (
|
|
collaboration.hostname !== target.hostname ||
|
|
collaboration.port !== '8100' ||
|
|
collaboration.username ||
|
|
collaboration.password ||
|
|
collaboration.pathname !== '/' ||
|
|
collaboration.search ||
|
|
collaboration.hash
|
|
)
|
|
throw new Error('InvalidCollaborationTarget');
|
|
w.collaborationUpstreamUrl = collaboration.toString();
|
|
}
|
|
w.url = `${gateway.publicUrl}/workspaces/${w.id}/`;
|
|
await tx.acknowledge(item.id);
|
|
}
|
|
await tx.put('workspace', touch(w));
|
|
});
|
|
if (state.status === 'pending')
|
|
await this.control.store.transaction(tenant, tx => tx.defer(item.id, 15));
|
|
} catch (error) {
|
|
await this.control.store.transaction(tenant, tx => tx.defer(item.id, 60));
|
|
console.error(`Workspace provisioning pending: ${error instanceof Error ? error.name : 'failure'}`);
|
|
}
|
|
continue;
|
|
}
|
|
if (item.type !== 'run.start' && item.type !== 'run.signal') continue;
|
|
const workflowId = `wishforge:${tenant}:${String(item.payload.runId)}`;
|
|
try {
|
|
if (item.type === 'run.start') {
|
|
const runId = String(item.payload.runId);
|
|
const run = await this.control.store.transaction(tenant, tx => must<Run>(tx, 'run', runId));
|
|
if (run.pendingStartId && run.pendingStartId !== item.id) {
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
continue;
|
|
}
|
|
if (['canceled', 'failed', 'succeeded', 'paused'].includes(run.status)) {
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
continue;
|
|
}
|
|
if (Number(item.payload.revision) !== run.revision) {
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
continue;
|
|
}
|
|
// Only a provisioning problem is retried here; any other wait needs a new generation.
|
|
if (
|
|
run.status === 'waiting' &&
|
|
run.executionGeneration === item.id &&
|
|
run.waitingReason !== 'sandbox_provisioner'
|
|
) {
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
continue;
|
|
}
|
|
if (provisionerError) throw provisionerError;
|
|
if (!provisioner) {
|
|
await this.control.store.transaction(tenant, async tx => {
|
|
const current = await must<Run>(tx, 'run', runId);
|
|
if (current.waitingReason !== 'sandbox_provisioner') {
|
|
setWaiting(
|
|
current,
|
|
'sandbox_provisioner',
|
|
'An isolated per-run sandbox provisioner must be connected before execution.',
|
|
);
|
|
await tx.put('run', touch(current));
|
|
await tx.event(runId, 'run.waiting', current.waitingDetail!, { reason: current.waitingReason });
|
|
}
|
|
});
|
|
await this.control.store.transaction(tenant, tx => tx.defer(item.id, 60));
|
|
continue;
|
|
}
|
|
const tokens = new ServiceTokens(this.control.env.WF_INTERNAL_TOKEN_KEY);
|
|
const resourceGrant = await this.control.store.transaction(tenant, tx =>
|
|
reserveExecutionGrant(tx, runId, item.id, this.control.env),
|
|
);
|
|
let provision: Response;
|
|
try {
|
|
provision = await fetch(`${provisioner.url}/runs`, {
|
|
method: 'POST',
|
|
headers: { Authorization: `Bearer ${provisioner.token}`, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
tenantId: tenant,
|
|
runId,
|
|
generation: item.id,
|
|
resourceGrant,
|
|
executorToken: await tokens.issue(tenant, runId, 'worker', item.id),
|
|
verifierToken: await tokens.issue(tenant, runId, 'verifier', item.id),
|
|
}),
|
|
signal: AbortSignal.timeout(15000),
|
|
});
|
|
} catch {
|
|
await this.control.store.transaction(tenant, tx => recordAllocation(tx, runId, item.id, 'unknown'));
|
|
throw new Error('SandboxProvisioningUnknown');
|
|
}
|
|
if (!provision.ok) {
|
|
const body = (await provision.json().catch(() => ({}))) as { error?: string };
|
|
await this.control.store.transaction(tenant, tx => recordAllocation(tx, runId, item.id, 'unknown'));
|
|
if (body.error === 'RUNTIME_IMAGE_MIGRATION_REQUIRED')
|
|
throw new DomainError(
|
|
'RUNTIME_IMAGE_MIGRATION_REQUIRED',
|
|
'Pinned runtime image is outside the provisioner current/previous allowlist; authorize checkpoint migration',
|
|
);
|
|
throw new Error('SandboxProvisioningFailed');
|
|
}
|
|
const receipt = (await provision.json()) as {
|
|
status: string;
|
|
tenantId: string;
|
|
runId: string;
|
|
generation: string;
|
|
};
|
|
if (receipt.tenantId !== tenant || receipt.runId !== runId || receipt.generation !== item.id) {
|
|
await this.control.store.transaction(tenant, tx => recordAllocation(tx, runId, item.id, 'unknown'));
|
|
throw new Error('SandboxReceiptScopeMismatch');
|
|
}
|
|
if (receipt.status !== 'provisioned') {
|
|
await this.control.waitRun(
|
|
{ tenantId: tenant, userId: 'worker', role: 'admin', kind: 'worker' },
|
|
runId,
|
|
{
|
|
reason: 'sandbox_provisioner',
|
|
detail: 'The sandbox for this generation already ended; resume to provision a new one.',
|
|
generation: item.id,
|
|
},
|
|
);
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
continue;
|
|
}
|
|
await this.control.store.transaction(tenant, tx => recordAllocation(tx, runId, item.id, 'provisioned'));
|
|
if (run.status === 'waiting' && run.waitingReason === 'sandbox_provisioner')
|
|
await this.control.store.transaction(tenant, async tx => {
|
|
const current = await must<Run>(tx, 'run', runId);
|
|
if (current.status === 'waiting' && current.waitingReason === 'sandbox_provisioner') {
|
|
current.status = 'queued';
|
|
clearWaiting(current);
|
|
await tx.put('run', touch(current));
|
|
}
|
|
});
|
|
await client.workflow.signalWithStart('runWorkflow', {
|
|
workflowId,
|
|
taskQueue: 'wishforge-runs',
|
|
args: [{ tenantId: tenant, runId: item.payload.runId, generation: item.id }],
|
|
signal: 'resume',
|
|
signalArgs: [{ generation: item.id }],
|
|
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
});
|
|
} else if (item.payload.action === 'cancel' || item.payload.action === 'pause') {
|
|
try {
|
|
await client.workflow.getHandle(workflowId).cancel();
|
|
} catch (error) {
|
|
if (!(error instanceof Error && error.name === 'WorkflowNotFoundError')) throw error;
|
|
}
|
|
} else if (item.payload.action === 'completed') {
|
|
try {
|
|
await client.workflow.getHandle(workflowId).signal('resume');
|
|
} catch (error) {
|
|
if (!(error instanceof Error && error.name === 'WorkflowNotFoundError')) throw error;
|
|
}
|
|
}
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
} catch (error) {
|
|
// A newer start replaced this one after it was read: it is done, and the run is not waiting.
|
|
if (item.type === 'run.start' && error instanceof DomainError && error.code === 'START_SUPERSEDED') {
|
|
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
|
continue;
|
|
}
|
|
// Waiting for an execution slot is ordinary queueing, not a failure: looking again soon
|
|
// starts the run within seconds of a slot freeing, where backing off took minutes.
|
|
if (item.type === 'run.start' && error instanceof DomainError && error.code === 'EXECUTION_CAPACITY') {
|
|
await this.control.store.transaction(tenant, tx => tx.defer(item.id, CAPACITY_RECHECK_SECONDS));
|
|
continue;
|
|
}
|
|
if (item.type === 'run.start' && error instanceof DomainError)
|
|
await this.control.store.transaction(tenant, async tx => {
|
|
const current = await must<Run>(tx, 'run', String(item.payload.runId));
|
|
if (
|
|
!['canceled', 'failed', 'succeeded', 'paused'].includes(current.status) &&
|
|
(current.waitingReason !== 'sandbox_provisioner' || current.waitingDetail !== error.message)
|
|
) {
|
|
// Resource grants, runtime images and provisioner errors all block the sandbox.
|
|
setWaiting(current, 'sandbox_provisioner', error.message);
|
|
await tx.put('run', touch(current));
|
|
await tx.event(current.id, 'run.waiting', current.waitingDetail!, {
|
|
reason: current.waitingReason,
|
|
code: error.code,
|
|
});
|
|
}
|
|
});
|
|
await this.control.store.transaction(tenant, tx =>
|
|
tx.defer(item.id, Math.min(120, 5 * 2 ** Math.min(item.attempts, 5))),
|
|
);
|
|
console.error(
|
|
`Outbox ${item.type} pending: ${error instanceof DomainError ? error.code : error instanceof Error ? error.name : 'delivery failure'}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error(
|
|
`Temporal dispatcher reconnecting: ${error instanceof Error ? error.name : 'connection failure'}`,
|
|
);
|
|
await this.connection?.close();
|
|
this.connection = undefined;
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
}
|
|
}
|
|
async close() {
|
|
this.stopped = true;
|
|
await this.connection?.close();
|
|
}
|
|
}
|