🐛 Drop a run start that a newer one replaced before it takes the run back

The dispatcher read the run, checked that the start it was delivering
was still the pending one, and reserved the execution grant in a later
transaction. A pause and a continue in between enqueued a newer start,
and the older one went ahead anyway: it took the run's execution
generation back and asked the provisioner for a sandbox for a
generation that had already been replaced.

Reserving the grant now checks the pending start again, in the
transaction that takes the generation, and refuses a replaced one with
START_SUPERSEDED. The dispatcher acknowledges such a start without
marking the run as waiting; the newer start proceeds as usual.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
eust-w 2026-10-26 15:56:14 +08:00
parent ad06e82c67
commit df87c5dca6
5 changed files with 99 additions and 18 deletions

View File

@ -279,6 +279,11 @@ export class Dispatcher {
}
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
} catch (error) {
// A newer start replaced this one after it was read: it is done, and the run is not waiting.
if (item.type === 'run.start' && error instanceof DomainError && error.code === 'START_SUPERSEDED') {
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
continue;
}
if (item.type === 'run.start' && error instanceof DomainError && error.code !== 'EXECUTION_CAPACITY')
await this.control.store.transaction(tenant, async tx => {
const current = await must<Run>(tx, 'run', String(item.payload.runId));

View File

@ -187,6 +187,10 @@ export async function reserveExecutionGrant(
);
const run = await must<Run>(tx, 'run', runId);
const plan = effectiveEntitlements(await tx.tenant());
// Checked again here, in the transaction that takes the generation: a pause and a continue since
// the dispatcher looked enqueued a newer start, and this one must not take the run back.
if (run.pendingStartId && run.pendingStartId !== generation)
throw new DomainError('START_SUPERSEDED', 'A newer start of this run replaced this one', 409);
if (['succeeded', 'failed', 'canceled', 'paused'].includes(run.status))
throw new DomainError('RUN_STOPPED', 'Stopped run cannot reserve a sandbox');
const others = (await tx.list<Run>('run')).filter(other => other.id !== run.id && occupiesExecutionSlot(other));

View File

@ -172,6 +172,7 @@ sequenceDiagram
```
- outbox 条目的 ID 即运行的**执行代次**(generation)。每次继续、补充都会产生新代次;旧代次的 worker 令牌立即失效。
- 派发器只启动运行最近一次排入的代次。读取运行之后,如果暂停与继续又排入了更新的代次,预留资源的事务会再核对一次并拒绝旧代次(`START_SUPERSEDED`):旧条目直接确认丢弃,不为它申请沙箱,也不把运行置为等待。
- provisioner 返回失败或超时时,运行进入 `sandbox_provisioner` 等待,派发器指数退避重试;资源预留记为 `unknown`,由资源回收循环在收到释放回执后归还。
### 4.2 执行、验证与修复

View File

@ -135,3 +135,51 @@ it('tells the provisioner the platform repository a new workspace may use', asyn
},
]);
});
it('drops a start that a newer one replaced while it was being dispatched, leaving the run as it is', async () => {
const store = new MemoryStore();
store.seed('alpha');
const run = { ...record('alpha'), status: 'queued', revision: 1, pendingStartId: 'start-fixture' } as Run;
await store.transaction('alpha', async tx => {
await tx.put('run', run);
await tx.enqueue('run.start', { runId: run.id, revision: 1 }, 'start-fixture');
await tx.enqueue('run.signal', { runId: 'other-run', action: 'cancel' }, 'cancel-other');
});
// Right after the dispatcher first reads the run, a pause and a continue enqueue a newer start.
const transaction = store.transaction.bind(store);
let replaced = false;
vi.spyOn(store, 'transaction').mockImplementation(async (tenant, work) => {
const result = await transaction(tenant, work);
if (!replaced && (result as Run | undefined)?.id === run.id) {
replaced = true;
await transaction(tenant, async tx => {
const current = (await tx.get<Run>('run', run.id))!;
await tx.put('run', { ...current, pendingStartId: 'newer-start' });
});
}
return result;
});
const provisioner = vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('No sandbox may be provisioned'));
temporal.connect.mockResolvedValue({ close: temporal.close });
const dispatcher = new Dispatcher(
new ControlPlane(store, {
WF_DISPATCH_TENANTS: 'alpha',
WF_PROVISIONER_TENANT_ID: 'alpha',
WF_PROVISIONER_URL: 'https://fixture.invalid',
WF_PROVISIONER_TOKEN: 'fixture-provisioner-token-only-long-enough',
WF_RESOURCE_GRANT_SECRET: 'resource-fixture-signing-secret-only-12345',
WF_INTERNAL_TOKEN_KEY: 'internal-fixture-signing-key-only-123456789',
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'a'.repeat(64)}`,
}),
);
temporal.cancel.mockImplementation(async () => dispatcher.close());
vi.spyOn(console, 'error').mockImplementation(() => undefined);
await dispatcher.start();
expect(replaced).toBe(true);
expect(provisioner).not.toHaveBeenCalled();
const after = await store.transaction('alpha', tx => tx.get<Run>('run', run.id));
expect(after).toMatchObject({ status: 'queued', pendingStartId: 'newer-start' });
expect(after?.executionGeneration).toBeUndefined();
expect(after?.waitingReason).toBeUndefined();
expect((await store.transaction('alpha', tx => tx.outbox())).map(item => item.id)).not.toContain('start-fixture');
});

View File

@ -12,6 +12,7 @@ import {
recordAllocation,
} from '../apps/api/src/domain/entitlements.js';
import { purgeRetainedStorage, releaseStoppedAllocations } from '../apps/api/src/domain/resource-dispatch.js';
import type { Transaction } from '../apps/api/src/db/store.js';
const owner: Actor = { tenantId: 'alpha', userId: 'owner', kind: 'human', role: 'owner' },
operator: Actor = { ...owner, userId: 'operations-user', role: 'admin' };
const env = {
@ -25,6 +26,12 @@ const env = {
WF_PROVISIONER_TENANT_ID: 'alpha',
};
let db: MemoryStore, control: ControlPlane, entitlements: Entitlements, usage: Usage;
/** Reserves as the dispatcher does: for the start the control plane last enqueued for the run. */
async function reserve(tx: Transaction, runId: string, generation: string, environment: NodeJS.ProcessEnv = env) {
const run = (await tx.get<Run>('run', runId))!;
if (run.pendingStartId !== generation) await tx.put('run', { ...run, pendingStartId: generation });
return reserveExecutionGrant(tx, runId, generation, environment);
}
beforeEach(() => {
db = new MemoryStore();
db.seed('alpha');
@ -81,9 +88,7 @@ describe('fixed pilot entitlements and resource admission', () => {
{ WF_DEFAULT_EXECUTION_STORAGE_MIB: '1' },
])
await expect(
db.transaction('alpha', tx =>
reserveExecutionGrant(tx, created.run_id, 'invalid-default', { ...env, ...defaults }),
),
db.transaction('alpha', tx => reserve(tx, created.run_id, 'invalid-default', { ...env, ...defaults })),
).rejects.toMatchObject({ code: 'RESOURCE_CONFIGURATION_INVALID' });
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0);
});
@ -128,11 +133,11 @@ describe('fixed pilot entitlements and resource admission', () => {
const runs = await queuedRuns(3);
expect(await control.runs(owner)).toHaveLength(3);
const results = await Promise.allSettled(
runs.map(run => db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-1', env))),
runs.map(run => db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env))),
);
expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1);
const first = runs[0].run_id;
const again = await db.transaction('alpha', tx => reserveExecutionGrant(tx, first, 'generation-1', env));
const again = await db.transaction('alpha', tx => reserve(tx, first, 'generation-1', env));
expect(again.image).toBe(env.WF_RUN_WORKER_IMAGE);
expect(again.cpuMilli).toBe(2000);
await db.transaction('alpha', async tx => {
@ -141,17 +146,17 @@ describe('fixed pilot entitlements and resource admission', () => {
run.status = 'canceled';
await tx.put('run', run);
});
await expect(
db.transaction('alpha', tx => reserveExecutionGrant(tx, runs[1].run_id, 'generation-1', env)),
).rejects.toMatchObject({ code: 'EXECUTION_CAPACITY' });
await expect(db.transaction('alpha', tx => reserve(tx, runs[1].run_id, 'generation-1', env))).rejects.toMatchObject(
{ code: 'EXECUTION_CAPACITY' },
);
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1);
});
it('pins the original runtime digest across resumes and rejects legacy checkpoint image guesses', async () => {
const [run] = await queuedRuns(1);
const original = await db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-1', env));
const original = await db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env));
await db.transaction('alpha', tx => recordAllocation(tx, run.run_id, 'generation-1', 'released'));
const resumed = await db.transaction('alpha', tx =>
reserveExecutionGrant(tx, run.run_id, 'generation-2', {
reserve(tx, run.run_id, 'generation-2', {
...env,
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'b'.repeat(64)}`,
}),
@ -163,28 +168,46 @@ describe('fixed pilot entitlements and resource admission', () => {
value.checkpointId = 'legacy-checkpoint';
await tx.put('run', value);
});
await expect(
db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-3', env)),
).rejects.toMatchObject({ code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED' });
await expect(db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-3', env))).rejects.toMatchObject({
code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED',
});
});
it('does not replace an unresolved previous generation on resume', async () => {
const [created] = await queuedRuns(1);
await db.transaction('alpha', async tx => {
await reserveExecutionGrant(tx, created.run_id, 'old-generation', env);
await reserve(tx, created.run_id, 'old-generation', env);
await recordAllocation(tx, created.run_id, 'old-generation', 'unknown');
});
await expect(
db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'new-generation', env)),
db.transaction('alpha', tx => reserve(tx, created.run_id, 'new-generation', env)),
).rejects.toMatchObject({ code: 'RESOURCE_PREVIOUS_GENERATION' });
expect((await control.run(owner, created.run_id)).resourceAllocation).toMatchObject({
generation: 'old-generation',
status: 'unknown',
});
});
it('refuses a start that a newer one replaced, so it cannot take the run back', async () => {
const [created] = await queuedRuns(1);
const first = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!.pendingStartId!;
// A pause and a continue enqueue a newer start before the dispatcher reserves the first.
await db.transaction('alpha', async tx => {
const run = (await tx.get<Run>('run', created.run_id))!;
await tx.put('run', { ...run, pendingStartId: 'newer-start' });
});
await expect(
db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, first, env)),
).rejects.toMatchObject({ code: 'START_SUPERSEDED' });
const untouched = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!;
expect(untouched.executionGeneration).toBeUndefined();
expect(untouched.resourceAllocation).toBeUndefined();
// The newer start reserves as usual.
await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'newer-start', env));
expect((await control.run(owner, created.run_id)).executionGeneration).toBe('newer-start');
});
it('issues a grant accepted by the independent provisioner without allowing changed resources', async () => {
const { verifyResourceGrant } = await import('../apps/provisioner/resource-grants.mjs');
const [created] = await queuedRuns(1);
const grant = await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'generation-1', env));
const grant = await db.transaction('alpha', tx => reserve(tx, created.run_id, 'generation-1', env));
const configuration = {
tenantId: 'alpha',
image: env.WF_RUN_WORKER_IMAGE,
@ -206,7 +229,7 @@ describe('fixed pilot entitlements and resource admission', () => {
const [created] = await queuedRuns(1),
id = created.run_id;
await db.transaction('alpha', async tx => {
await reserveExecutionGrant(tx, id, 'generation-1', env);
await reserve(tx, id, 'generation-1', env);
await recordAllocation(tx, id, 'generation-1', 'unknown');
const run = (await tx.get<Run>('run', id))!;
run.status = 'canceled';
@ -240,7 +263,7 @@ describe('retained run volumes', () => {
const [old, recent] = (await queuedRuns(2)).map(created => created.run_id);
await db.transaction('alpha', async tx => {
for (const id of [old!, recent!]) {
await reserveExecutionGrant(tx, id, `generation-${id}`, env);
await reserve(tx, id, `generation-${id}`, env);
await recordAllocation(tx, id, `generation-${id}`, 'released');
const run = (await tx.get<Run>('run', id))!;
run.status = 'succeeded';