From df87c5dca68ca3a1d1b58c753177bfca0d2ebbbf Mon Sep 17 00:00:00 2001 From: eust-w Date: Mon, 26 Oct 2026 15:56:14 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Drop=20a=20run=20start=20that=20?= =?UTF-8?q?a=20newer=20one=20replaced=20before=20it=20takes=20the=20run=20?= =?UTF-8?q?back?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dispatcher read the run, checked that the start it was delivering was still the pending one, and reserved the execution grant in a later transaction. A pause and a continue in between enqueued a newer start, and the older one went ahead anyway: it took the run's execution generation back and asked the provisioner for a sandbox for a generation that had already been replaced. Reserving the grant now checks the pending start again, in the transaction that takes the generation, and refuses a replaced one with START_SUPERSEDED. The dispatcher acknowledges such a start without marking the run as waiting; the newer start proceeds as usual. Co-Authored-By: Claude Opus 5.5 --- apps/api/src/dispatcher.ts | 5 +++ apps/api/src/domain/entitlements.ts | 4 ++ docs/architecture.md | 1 + tests/dispatcher.test.ts | 48 +++++++++++++++++++++++ tests/entitlements-usage.test.ts | 59 ++++++++++++++++++++--------- 5 files changed, 99 insertions(+), 18 deletions(-) diff --git a/apps/api/src/dispatcher.ts b/apps/api/src/dispatcher.ts index 9657aea..64e6d1e 100644 --- a/apps/api/src/dispatcher.ts +++ b/apps/api/src/dispatcher.ts @@ -279,6 +279,11 @@ export class Dispatcher { } await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id)); } catch (error) { + // A newer start replaced this one after it was read: it is done, and the run is not waiting. + if (item.type === 'run.start' && error instanceof DomainError && error.code === 'START_SUPERSEDED') { + await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id)); + continue; + } if (item.type === 'run.start' && error instanceof DomainError && error.code !== 'EXECUTION_CAPACITY') await this.control.store.transaction(tenant, async tx => { const current = await must(tx, 'run', String(item.payload.runId)); diff --git a/apps/api/src/domain/entitlements.ts b/apps/api/src/domain/entitlements.ts index f8b88c7..c44fcd2 100644 --- a/apps/api/src/domain/entitlements.ts +++ b/apps/api/src/domain/entitlements.ts @@ -187,6 +187,10 @@ export async function reserveExecutionGrant( ); const run = await must(tx, 'run', runId); const plan = effectiveEntitlements(await tx.tenant()); + // Checked again here, in the transaction that takes the generation: a pause and a continue since + // the dispatcher looked enqueued a newer start, and this one must not take the run back. + if (run.pendingStartId && run.pendingStartId !== generation) + throw new DomainError('START_SUPERSEDED', 'A newer start of this run replaced this one', 409); if (['succeeded', 'failed', 'canceled', 'paused'].includes(run.status)) throw new DomainError('RUN_STOPPED', 'Stopped run cannot reserve a sandbox'); const others = (await tx.list('run')).filter(other => other.id !== run.id && occupiesExecutionSlot(other)); diff --git a/docs/architecture.md b/docs/architecture.md index b03538e..a180808 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -172,6 +172,7 @@ sequenceDiagram ``` - outbox 条目的 ID 即运行的**执行代次**(generation)。每次继续、补充都会产生新代次;旧代次的 worker 令牌立即失效。 +- 派发器只启动运行最近一次排入的代次。读取运行之后,如果暂停与继续又排入了更新的代次,预留资源的事务会再核对一次并拒绝旧代次(`START_SUPERSEDED`):旧条目直接确认丢弃,不为它申请沙箱,也不把运行置为等待。 - provisioner 返回失败或超时时,运行进入 `sandbox_provisioner` 等待,派发器指数退避重试;资源预留记为 `unknown`,由资源回收循环在收到释放回执后归还。 ### 4.2 执行、验证与修复 diff --git a/tests/dispatcher.test.ts b/tests/dispatcher.test.ts index aeee899..74496f9 100644 --- a/tests/dispatcher.test.ts +++ b/tests/dispatcher.test.ts @@ -135,3 +135,51 @@ it('tells the provisioner the platform repository a new workspace may use', asyn }, ]); }); + +it('drops a start that a newer one replaced while it was being dispatched, leaving the run as it is', async () => { + const store = new MemoryStore(); + store.seed('alpha'); + const run = { ...record('alpha'), status: 'queued', revision: 1, pendingStartId: 'start-fixture' } as Run; + await store.transaction('alpha', async tx => { + await tx.put('run', run); + await tx.enqueue('run.start', { runId: run.id, revision: 1 }, 'start-fixture'); + await tx.enqueue('run.signal', { runId: 'other-run', action: 'cancel' }, 'cancel-other'); + }); + // Right after the dispatcher first reads the run, a pause and a continue enqueue a newer start. + const transaction = store.transaction.bind(store); + let replaced = false; + vi.spyOn(store, 'transaction').mockImplementation(async (tenant, work) => { + const result = await transaction(tenant, work); + if (!replaced && (result as Run | undefined)?.id === run.id) { + replaced = true; + await transaction(tenant, async tx => { + const current = (await tx.get('run', run.id))!; + await tx.put('run', { ...current, pendingStartId: 'newer-start' }); + }); + } + return result; + }); + const provisioner = vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('No sandbox may be provisioned')); + temporal.connect.mockResolvedValue({ close: temporal.close }); + const dispatcher = new Dispatcher( + new ControlPlane(store, { + WF_DISPATCH_TENANTS: 'alpha', + WF_PROVISIONER_TENANT_ID: 'alpha', + WF_PROVISIONER_URL: 'https://fixture.invalid', + WF_PROVISIONER_TOKEN: 'fixture-provisioner-token-only-long-enough', + WF_RESOURCE_GRANT_SECRET: 'resource-fixture-signing-secret-only-12345', + WF_INTERNAL_TOKEN_KEY: 'internal-fixture-signing-key-only-123456789', + WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'a'.repeat(64)}`, + }), + ); + temporal.cancel.mockImplementation(async () => dispatcher.close()); + vi.spyOn(console, 'error').mockImplementation(() => undefined); + await dispatcher.start(); + expect(replaced).toBe(true); + expect(provisioner).not.toHaveBeenCalled(); + const after = await store.transaction('alpha', tx => tx.get('run', run.id)); + expect(after).toMatchObject({ status: 'queued', pendingStartId: 'newer-start' }); + expect(after?.executionGeneration).toBeUndefined(); + expect(after?.waitingReason).toBeUndefined(); + expect((await store.transaction('alpha', tx => tx.outbox())).map(item => item.id)).not.toContain('start-fixture'); +}); diff --git a/tests/entitlements-usage.test.ts b/tests/entitlements-usage.test.ts index ae7fce9..ef1633a 100644 --- a/tests/entitlements-usage.test.ts +++ b/tests/entitlements-usage.test.ts @@ -12,6 +12,7 @@ import { recordAllocation, } from '../apps/api/src/domain/entitlements.js'; import { purgeRetainedStorage, releaseStoppedAllocations } from '../apps/api/src/domain/resource-dispatch.js'; +import type { Transaction } from '../apps/api/src/db/store.js'; const owner: Actor = { tenantId: 'alpha', userId: 'owner', kind: 'human', role: 'owner' }, operator: Actor = { ...owner, userId: 'operations-user', role: 'admin' }; const env = { @@ -25,6 +26,12 @@ const env = { WF_PROVISIONER_TENANT_ID: 'alpha', }; let db: MemoryStore, control: ControlPlane, entitlements: Entitlements, usage: Usage; +/** Reserves as the dispatcher does: for the start the control plane last enqueued for the run. */ +async function reserve(tx: Transaction, runId: string, generation: string, environment: NodeJS.ProcessEnv = env) { + const run = (await tx.get('run', runId))!; + if (run.pendingStartId !== generation) await tx.put('run', { ...run, pendingStartId: generation }); + return reserveExecutionGrant(tx, runId, generation, environment); +} beforeEach(() => { db = new MemoryStore(); db.seed('alpha'); @@ -81,9 +88,7 @@ describe('fixed pilot entitlements and resource admission', () => { { WF_DEFAULT_EXECUTION_STORAGE_MIB: '1' }, ]) await expect( - db.transaction('alpha', tx => - reserveExecutionGrant(tx, created.run_id, 'invalid-default', { ...env, ...defaults }), - ), + db.transaction('alpha', tx => reserve(tx, created.run_id, 'invalid-default', { ...env, ...defaults })), ).rejects.toMatchObject({ code: 'RESOURCE_CONFIGURATION_INVALID' }); expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0); }); @@ -128,11 +133,11 @@ describe('fixed pilot entitlements and resource admission', () => { const runs = await queuedRuns(3); expect(await control.runs(owner)).toHaveLength(3); const results = await Promise.allSettled( - runs.map(run => db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-1', env))), + runs.map(run => db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env))), ); expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1); const first = runs[0].run_id; - const again = await db.transaction('alpha', tx => reserveExecutionGrant(tx, first, 'generation-1', env)); + const again = await db.transaction('alpha', tx => reserve(tx, first, 'generation-1', env)); expect(again.image).toBe(env.WF_RUN_WORKER_IMAGE); expect(again.cpuMilli).toBe(2000); await db.transaction('alpha', async tx => { @@ -141,17 +146,17 @@ describe('fixed pilot entitlements and resource admission', () => { run.status = 'canceled'; await tx.put('run', run); }); - await expect( - db.transaction('alpha', tx => reserveExecutionGrant(tx, runs[1].run_id, 'generation-1', env)), - ).rejects.toMatchObject({ code: 'EXECUTION_CAPACITY' }); + await expect(db.transaction('alpha', tx => reserve(tx, runs[1].run_id, 'generation-1', env))).rejects.toMatchObject( + { code: 'EXECUTION_CAPACITY' }, + ); expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1); }); it('pins the original runtime digest across resumes and rejects legacy checkpoint image guesses', async () => { const [run] = await queuedRuns(1); - const original = await db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-1', env)); + const original = await db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env)); await db.transaction('alpha', tx => recordAllocation(tx, run.run_id, 'generation-1', 'released')); const resumed = await db.transaction('alpha', tx => - reserveExecutionGrant(tx, run.run_id, 'generation-2', { + reserve(tx, run.run_id, 'generation-2', { ...env, WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'b'.repeat(64)}`, }), @@ -163,28 +168,46 @@ describe('fixed pilot entitlements and resource admission', () => { value.checkpointId = 'legacy-checkpoint'; await tx.put('run', value); }); - await expect( - db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-3', env)), - ).rejects.toMatchObject({ code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED' }); + await expect(db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-3', env))).rejects.toMatchObject({ + code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED', + }); }); it('does not replace an unresolved previous generation on resume', async () => { const [created] = await queuedRuns(1); await db.transaction('alpha', async tx => { - await reserveExecutionGrant(tx, created.run_id, 'old-generation', env); + await reserve(tx, created.run_id, 'old-generation', env); await recordAllocation(tx, created.run_id, 'old-generation', 'unknown'); }); await expect( - db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'new-generation', env)), + db.transaction('alpha', tx => reserve(tx, created.run_id, 'new-generation', env)), ).rejects.toMatchObject({ code: 'RESOURCE_PREVIOUS_GENERATION' }); expect((await control.run(owner, created.run_id)).resourceAllocation).toMatchObject({ generation: 'old-generation', status: 'unknown', }); }); + it('refuses a start that a newer one replaced, so it cannot take the run back', async () => { + const [created] = await queuedRuns(1); + const first = (await db.transaction('alpha', tx => tx.get('run', created.run_id)))!.pendingStartId!; + // A pause and a continue enqueue a newer start before the dispatcher reserves the first. + await db.transaction('alpha', async tx => { + const run = (await tx.get('run', created.run_id))!; + await tx.put('run', { ...run, pendingStartId: 'newer-start' }); + }); + await expect( + db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, first, env)), + ).rejects.toMatchObject({ code: 'START_SUPERSEDED' }); + const untouched = (await db.transaction('alpha', tx => tx.get('run', created.run_id)))!; + expect(untouched.executionGeneration).toBeUndefined(); + expect(untouched.resourceAllocation).toBeUndefined(); + // The newer start reserves as usual. + await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'newer-start', env)); + expect((await control.run(owner, created.run_id)).executionGeneration).toBe('newer-start'); + }); it('issues a grant accepted by the independent provisioner without allowing changed resources', async () => { const { verifyResourceGrant } = await import('../apps/provisioner/resource-grants.mjs'); const [created] = await queuedRuns(1); - const grant = await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'generation-1', env)); + const grant = await db.transaction('alpha', tx => reserve(tx, created.run_id, 'generation-1', env)); const configuration = { tenantId: 'alpha', image: env.WF_RUN_WORKER_IMAGE, @@ -206,7 +229,7 @@ describe('fixed pilot entitlements and resource admission', () => { const [created] = await queuedRuns(1), id = created.run_id; await db.transaction('alpha', async tx => { - await reserveExecutionGrant(tx, id, 'generation-1', env); + await reserve(tx, id, 'generation-1', env); await recordAllocation(tx, id, 'generation-1', 'unknown'); const run = (await tx.get('run', id))!; run.status = 'canceled'; @@ -240,7 +263,7 @@ describe('retained run volumes', () => { const [old, recent] = (await queuedRuns(2)).map(created => created.run_id); await db.transaction('alpha', async tx => { for (const id of [old!, recent!]) { - await reserveExecutionGrant(tx, id, `generation-${id}`, env); + await reserve(tx, id, `generation-${id}`, env); await recordAllocation(tx, id, `generation-${id}`, 'released'); const run = (await tx.get('run', id))!; run.status = 'succeeded';