wishforge-ide/apps/api/src/domain/runner-broker.ts
eust-w 91ab5f91e6 🌐 Write run events, waiting details and agent replies in Chinese
The run page mixed English stage events and agent summaries into the
Chinese console. Messages are now written in Simplified Chinese at their
source (control plane, dispatcher, worker, POC build), keeping commands,
SHAs, branches and error codes verbatim; raw runtime errors get a Chinese
prefix. The agent prompt asks for the plan, summary and questions in
Chinese. File-change parsing accepts old and new messages, and the event
label table covers every event kind written.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-11-04 12:42:04 +08:00

502 lines
23 KiB
TypeScript

import { canonicalJson, matchesApplicationAccess, isProduction } from '@wishforge/contracts';
import { createHash, createPrivateKey, createPublicKey, randomBytes, randomUUID, sign } from 'node:crypto';
import { readFile } from 'node:fs/promises';
import { z } from 'zod';
import {
DomainError,
type Actor,
type ApplicationAccessEvidence,
type Environment,
type Evidence,
type Release,
type Run,
type RunnerRegistration,
type RunnerTask,
type PocReleaseSet,
} from '@wishforge/contracts';
import type { Store, Transaction } from '../db/store.js';
import { must, projectAccess, record, touch, now } from './control-plane.js';
import { digest, requireHuman, requireRole } from './policy.js';
import { noteReleaseFailure, Releases, releaseBinding } from './releases.js';
interface Envelope {
keyId: string;
signature: string;
payload: {
id: string;
action: string;
target: string;
expiresAt: string;
release: { id: string; sourceCommit: string; image: string; [key: string]: unknown };
[key: string]: unknown;
};
}
type PocRollbackOperation = RunnerTask & { sourceBindingDigest?: string; pocRelease?: PocReleaseSet };
const receipt = z
.object({
result: z
.object({
releaseId: z.string().min(1),
status: z.enum(['dry-run', 'applied', 'verified', 'failed', 'unconfigured', 'not-found', 'conflict']),
target: z.enum(['compose', 'kubernetes']),
evidence: z.record(z.string(), z.unknown()),
rendered: z.string().optional(),
})
.strict(),
})
.strict();
export class RunnerBroker {
constructor(
private store: Store,
private releases: Releases,
private env = process.env,
) {}
async register(actor: Actor, environmentId: string) {
requireRole(actor, ['owner', 'admin']);
requireHuman(actor);
const token = randomBytes(32).toString('base64url');
const tokenHash = createHash('sha256').update(token).digest('hex');
const publicKeys = await this.publicKeys();
const runner = await this.store.transaction(actor.tenantId, async tx => {
const environment = await must<Environment>(tx, 'environment', environmentId);
await projectAccess(tx, actor, environment.projectId);
for (const old of await tx.list<RunnerRegistration>('runner'))
if (old.environmentId === environmentId) await tx.put('runner', { ...touch(old), status: 'revoked' });
const registration = { ...record(actor.tenantId), environmentId, tokenHash, status: 'active' };
await tx.put('runner', registration);
environment.runnerId = registration.id;
await tx.put('environment', touch(environment));
return registration;
});
return {
id: runner.id,
environmentId,
tenantId: actor.tenantId,
token,
publicKeys,
note: 'Save this token once. Runner also requires an operator-issued client certificate; no client certificate is minted here.',
};
}
async publicKeys() {
const key = await this.signingKey();
return {
[this.env.WF_RUNNER_SIGNING_KEY_ID ?? 'v1']: createPublicKey(key)
.export({ type: 'spki', format: 'pem' })
.toString(),
};
}
private async signingKey() {
if (!this.env.WF_RUNNER_SIGNING_KEY_FILE)
throw new DomainError(
'SIGNER_UNCONFIGURED',
'Configure the independent Runner signing key before enabling deployment',
503,
);
const key = createPrivateKey(await readFile(this.env.WF_RUNNER_SIGNING_KEY_FILE));
if (key.asymmetricKeyType !== 'ed25519')
throw new DomainError('INVALID_SIGNER', 'Runner task key must be Ed25519', 503);
return key;
}
private async registration(tx: Transaction, actor: Actor, environment: Environment) {
if (actor.kind !== 'deployer' || actor.userId !== `runner:${environment.id}`)
throw new DomainError('FORBIDDEN', 'Runner is not registered for this environment', 403);
const registration = environment.runnerId
? await tx.get<RunnerRegistration>('runner', environment.runnerId)
: undefined;
if (!registration || registration.status !== 'active' || (actor.clientId && actor.clientId !== registration.id))
throw new DomainError('RUNNER_REVOKED', 'Runner registration is inactive or replaced', 403);
return registration;
}
private async rollbackPoc(
tx: Transaction,
operation: RunnerTask,
target: Release,
): Promise<PocReleaseSet | undefined> {
const stored = operation as PocRollbackOperation;
const current = await must<Release>(tx, 'release', String(operation.expectedReleaseId));
if (current.pocRelease && stored.sourceBindingDigest !== current.bindingDigest)
throw new DomainError('ROLLBACK_STALE', 'The current POC artifact binding changed after rollback was requested');
const recovery = await this.releases.rollbackPocManifest(tx, current, target, operation.id);
if (stored.pocRelease && digest(stored.pocRelease) !== digest(recovery ?? null))
throw new DomainError(
'ROLLBACK_STALE',
'The persisted POC recovery manifest no longer matches its source and target',
);
return recovery;
}
async poll(actor: Actor, environmentId: string) {
const key = await this.signingKey();
return this.store.transaction(actor.tenantId, async tx => {
const environment = await must<Environment>(tx, 'environment', environmentId),
registration = await this.registration(tx, actor, environment);
const tasks = await tx.list<RunnerTask>('runner_task');
const pending = tasks.find(t => t.environmentId === environmentId && t.status === 'pending' && t.envelope);
if (pending) {
const envelope = pending.envelope as Envelope;
if (pending.runnerId === registration.id && Date.parse(envelope.payload.expiresAt) > Date.now()) {
if (environment.activeReleaseId !== pending.releaseId)
throw new DomainError('STALE_RUNNER_TASK', 'Task no longer owns the environment');
const target = await must<Release>(tx, 'release', String(pending.releaseId));
if (!pending.rollbackOperationId) await this.releases.assertCurrent(tx, target);
else {
const operation = await must<RunnerTask>(tx, 'runner_task', String(pending.rollbackOperationId));
if (
operation.lastTaskId !== pending.id ||
operation.expectedReleaseId !== environment.currentReleaseId ||
target.applicationUrl !== environment.applicationUrl ||
target.deploymentSecretPolicyDigest !== environment.deploymentSecretPolicyDigest ||
target.configVersion !== environment.configVersion ||
target.architecture !== environment.architecture ||
digest(target.secretVersionRefs) !== digest(environment.secretVersionRefs)
)
throw new DomainError('ROLLBACK_STALE', 'Rollback state changed');
const recovery = await this.rollbackPoc(tx, operation, target);
if (digest(envelope.payload.release.pocRelease ?? null) !== digest(recovery ?? null))
throw new DomainError(
'ROLLBACK_STALE',
'Signed rollback manifest differs from the current recovery plan',
);
}
return envelope;
}
// An old lease may already have changed the target: the next task may only reconcile it.
pending.status = 'expired';
await tx.put('runner_task', touch(pending));
}
const rollback = tasks.find(
t =>
t.environmentId === environmentId &&
t.action === 'rollback' &&
['pending', 'dispatching', 'unknown'].includes(String(t.status)),
);
let release: Release | undefined;
let action: 'deploy' | 'reconcile';
let previous: string | undefined;
let recoveryPoc: PocReleaseSet | undefined;
if (rollback) {
release = await must<Release>(tx, 'release', String(rollback.targetReleaseId));
if (
environment.currentReleaseId !== rollback.expectedReleaseId ||
environment.activeReleaseId !== release.id ||
rollback.targetBindingDigest !== release.bindingDigest ||
digest(releaseBinding(release)) !== release.bindingDigest ||
release.environmentId !== environment.id ||
release.applicationUrl !== environment.applicationUrl ||
release.deploymentSecretPolicyDigest !== environment.deploymentSecretPolicyDigest ||
release.configVersion !== environment.configVersion ||
release.architecture !== environment.architecture ||
digest(release.secretVersionRefs) !== digest(environment.secretVersionRefs)
)
throw new DomainError(
'ROLLBACK_STALE',
'Rollback no longer matches the environment and approved previous artifact',
);
action = rollback.status === 'pending' ? 'deploy' : 'reconcile';
previous = String(rollback.expectedReleaseId);
recoveryPoc = await this.rollbackPoc(tx, rollback, release);
if (recoveryPoc) Object.assign(rollback, { pocRelease: recoveryPoc });
} else {
const releases = await tx.list<Release>('release');
if (environment.activeReleaseId) {
release = releases.find(
r => r.id === environment.activeReleaseId && ['deploying', 'unknown', 'failed'].includes(r.status),
);
if (!release) return null;
await this.releases.assertCurrent(tx, release);
} else release = await this.nextReady(tx, environmentId, releases);
if (!release) return null;
action = release.status === 'ready' ? 'deploy' : 'reconcile';
previous = release.previousReleaseId;
}
const config = release.deliveryConfigSnapshot as {
containerPort?: number;
healthPath?: string;
smokeChecks?: unknown[];
config?: Record<string, string>;
} | null;
if (!config?.containerPort || !config.healthPath || !config.smokeChecks?.length)
throw new DomainError(
'ACCEPTANCE_UNCONFIGURED',
'Protected port, health path and business smoke checks are required before deployment',
503,
);
const time = Date.now(),
id = randomUUID();
const payload = JSON.parse(
JSON.stringify({
schemaVersion: 1,
id,
tenantId: actor.tenantId,
environmentId,
nonce: randomUUID(),
issuedAt: new Date(time).toISOString(),
expiresAt: new Date(time + 300000).toISOString(),
action,
target: environment.type === 'compose' ? 'compose' : 'kubernetes',
release: {
schemaVersion: 1,
id: release.id,
tenantId: actor.tenantId,
environmentId,
sourceCommit: release.sourceCommit,
image: release.imageDigest,
pocRelease: rollback ? recoveryPoc : release.pocRelease,
architecture: release.architecture,
configVersion: release.configVersion,
deploymentSecretPolicyDigest: release.deploymentSecretPolicyDigest,
secretVersionRefs: release.secretVersionRefs,
bindingDigest: release.bindingDigest,
applicationUrl: release.applicationUrl,
deliveryRequired: release.deliveryRequired,
developmentLoopback: release.developmentLoopback,
config: config.config,
containerPort: config.containerPort,
healthPath: config.healthPath,
smokeChecks: config.smokeChecks,
expectedPreviousReleaseId: previous ?? null,
},
}),
);
const keyId = this.env.WF_RUNNER_SIGNING_KEY_ID ?? 'v1';
const envelope = {
keyId,
payload,
signature: sign(null, Buffer.from(canonicalJson({ keyId, payload })), key).toString('base64url'),
};
const task = {
...record(actor.tenantId),
id,
environmentId,
releaseId: release.id,
runnerId: registration.id,
status: 'pending',
envelope,
...(rollback ? { rollbackOperationId: rollback.id } : {}),
};
await tx.put('runner_task', task);
if (rollback) {
rollback.status = 'dispatching';
rollback.lastTaskId = task.id;
await tx.put('runner_task', touch(rollback));
} else {
environment.activeReleaseId = release.id;
release.status = 'deploying';
await tx.put('environment', touch(environment));
await tx.put('release', touch(release));
}
return envelope;
});
}
async result(actor: Actor, taskId: string, input: unknown) {
const { result } = receipt.parse(input);
return this.store.transaction(actor.tenantId, async tx => {
const task = await must<RunnerTask>(tx, 'runner_task', taskId),
environment = await must<Environment>(tx, 'environment', String(task.environmentId));
const registration = await this.registration(tx, actor, environment);
if (task.runnerId !== registration.id || !task.envelope)
throw new DomainError('STALE_RUNNER_TASK', 'Receipt belongs to a replaced runner or an unsigned operation');
const envelope = task.envelope as Envelope,
payload = envelope.payload;
if (result.releaseId !== payload.release.id || result.target !== payload.target)
throw new DomainError('RESULT_MISMATCH', 'Receipt does not match the signed release and deployment target');
if (task.status === 'completed') {
if (task.resultDigest !== digest(result))
throw new DomainError('RECEIPT_CONFLICT', 'A different receipt has already been committed for this task');
return { acknowledged: true, delivered: task.delivered === true, rolledBack: task.rolledBack === true };
}
if (task.status !== 'pending' || Date.parse(payload.expiresAt) <= Date.now())
throw new DomainError(
'STALE_RUNNER_TASK',
'Task lease expired or was replaced; reconcile the target using a new signed task',
);
const release = await must<Release>(tx, 'release', String(task.releaseId));
if (
environment.activeReleaseId !== release.id ||
release.sourceCommit !== payload.release.sourceCommit ||
release.imageDigest !== payload.release.image ||
release.bindingDigest !== payload.release.bindingDigest
)
throw new DomainError('RESULT_MISMATCH', 'Receipt no longer owns the deployed artifact or environment');
const basicVerified =
result.status === 'verified' &&
result.evidence.identityMatched === true &&
result.evidence.trafficSwitched === true &&
Array.isArray(result.evidence.checks) &&
result.evidence.checks.includes('health') &&
result.evidence.checks.includes('business-smoke');
const rollbackOperation = task.rollbackOperationId
? await must<RunnerTask>(tx, 'runner_task', String(task.rollbackOperationId))
: undefined;
const expectedPoc = rollbackOperation
? await this.rollbackPoc(tx, rollbackOperation, release)
: release.pocRelease;
if (digest(payload.release.pocRelease ?? null) !== digest(expectedPoc ?? null))
throw new DomainError(
'RESULT_MISMATCH',
'The signed component bundle no longer matches this deployment operation',
);
const verified =
basicVerified &&
(!expectedPoc ||
(digest(result.evidence.pocRelease ?? null) === digest(expectedPoc) &&
result.evidence.bundleDigest === digest(expectedPoc))) &&
matchesApplicationAccess(release, result.evidence.accessEvidence) &&
(!isProduction(this.env) || !!release.applicationUrl);
let rolledBack = false;
if (rollbackOperation) {
const operation = rollbackOperation;
if (
operation.status !== 'dispatching' ||
operation.lastTaskId !== task.id ||
environment.currentReleaseId !== operation.expectedReleaseId ||
release.id !== operation.targetReleaseId ||
release.bindingDigest !== operation.targetBindingDigest ||
environment.applicationUrl !== release.applicationUrl ||
environment.deploymentSecretPolicyDigest !== release.deploymentSecretPolicyDigest ||
environment.configVersion !== release.configVersion ||
digest(environment.secretVersionRefs) !== digest(release.secretVersionRefs)
)
throw new DomainError('ROLLBACK_STALE', 'Rollback receipt no longer matches the active operation');
if (verified) {
const original = await must<Release>(tx, 'release', String(operation.releaseId));
original.status = 'rolled_back';
release.status = 'healthy';
if (release.applicationUrl) release.deliveryUrl = release.applicationUrl;
environment.currentReleaseId = release.id;
environment.activeReleaseId = undefined;
operation.status = 'completed';
operation.completedAt = now();
const evidence = this.deploymentEvidence(actor, release, environment, task.id, result.evidence);
await tx.put('evidence', evidence);
release.evidenceIds.push(evidence.id);
release.deployedAt = now();
operation.evidenceId = evidence.id;
await tx.put('release', touch(original));
await tx.put('release', touch(release));
await tx.put('environment', touch(environment));
await tx.event(
original.runId,
'release.rolled_back',
'已恢复上一个不可变制品,并通过了健康检查和业务验收。',
{ operationId: operation.id, targetReleaseId: release.id, evidenceId: evidence.id },
);
rolledBack = true;
} else if (['failed', 'unconfigured'].includes(result.status)) {
// The runner kept or restored the current version: a definite failure that frees the environment.
const reason =
typeof result.evidence.reason === 'string' ? result.evidence.reason.slice(0, 500) : result.status;
operation.status = 'failed';
operation.completedAt = now();
environment.activeReleaseId = undefined;
await tx.put('environment', touch(environment));
const original = await must<Release>(tx, 'release', String(operation.releaseId));
await tx.event(original.runId, 'rollback.failed', `回滚失败(${reason});当前版本继续提供服务。`, {
operationId: operation.id,
});
} else {
operation.status = 'unknown';
const original = await must<Release>(tx, 'release', String(operation.releaseId));
await tx.event(original.runId, 'rollback.unknown', '回滚需要对线上实际状态进行对账。', {
operationId: operation.id,
});
}
await tx.put('runner_task', touch(operation));
} else if (verified) {
// Persist the bound receipt in this same transaction before acceptance checks it.
task.result = result;
task.resultDigest = digest(result);
await tx.put('runner_task', touch(task));
await this.releases.assertCurrent(tx, release);
const evidence = this.deploymentEvidence(actor, release, environment, task.id, result.evidence);
await tx.put('evidence', evidence);
await this.releases.acceptInTransaction(tx, actor, release.id, {
evidenceId: evidence.id,
bindingDigest: release.bindingDigest,
});
} else if (['failed', 'unconfigured'].includes(result.status)) {
// The runner kept or restored the previous version: a definite failure that frees the environment.
if (!['deploying', 'unknown'].includes(release.status))
throw new DomainError('RELEASE_CONFLICT', 'Deployment receipt is obsolete');
const reason =
typeof result.evidence.reason === 'string' ? result.evidence.reason.slice(0, 500) : result.status;
release.status = 'failed';
release.failureReason = reason;
environment.activeReleaseId = undefined;
await tx.put('release', touch(release));
await tx.put('environment', touch(environment));
await noteReleaseFailure(tx, release, `发布 ${release.id} 部署失败(${reason});上一个版本继续提供服务。`);
} else {
if (!['deploying', 'unknown', 'failed'].includes(release.status))
throw new DomainError('RELEASE_CONFLICT', 'Deployment receipt is obsolete');
release.status = 'unknown';
await tx.put('release', touch(release));
await tx.event(
release.runId,
'release.failure',
`环境执行器报告 ${result.status};再次部署之前必须先对线上状态进行对账。`,
{ releaseId: release.id, taskId },
);
}
task.status = 'completed';
task.result = result;
task.resultDigest = digest(result);
task.delivered = verified && !task.rollbackOperationId;
task.rolledBack = rolledBack;
await tx.put('runner_task', touch(task));
return {
acknowledged: true,
delivered: task.delivered,
rolledBack,
...(verified && release.applicationUrl ? { url: release.applicationUrl } : {}),
};
});
}
/**
* The newest ready release of the environment that is still current. One whose run is paused
* may be current again once it continues, and waits; any other stale one fails, so that it does
* not hold back every release after it.
*/
private async nextReady(tx: Transaction, environmentId: string, releases: Release[]) {
for (const release of releases.filter(r => r.environmentId === environmentId && r.status === 'ready')) {
try {
await this.releases.assertCurrent(tx, release);
return release;
} catch (error) {
if (!(error instanceof DomainError) || !['RELEASE_STALE', 'APPROVAL_STALE'].includes(error.code)) throw error;
if ((await tx.get<Run>('run', release.runId))?.status === 'paused') continue;
release.status = 'failed';
release.failureReason = error.message;
await tx.put('release', touch(release));
await noteReleaseFailure(tx, release, `发布 ${release.id} 在部署之前已过期:${error.message}。`);
}
}
}
private deploymentEvidence(
actor: Actor,
release: Release,
environment: Environment,
taskId: string,
resultEvidence: Record<string, unknown>,
): Evidence {
return {
...record(actor.tenantId),
runId: release.runId,
revision: Number(release.revision),
sourceCommit: release.sourceCommit,
imageDigest: release.imageDigest,
type: 'deployment',
passed: true,
...(release.applicationUrl
? { accessEvidence: resultEvidence.accessEvidence as ApplicationAccessEvidence | undefined }
: {}),
checks: ['health', 'business-smoke'],
issuer: actor.userId,
releaseId: release.id,
environmentId: environment.id,
bindingDigest: release.bindingDigest,
runnerTaskId: taskId,
};
}
}