🐛 Drop a run start that a newer one replaced before it takes the run back
The dispatcher read the run, checked that the start it was delivering was still the pending one, and reserved the execution grant in a later transaction. A pause and a continue in between enqueued a newer start, and the older one went ahead anyway: it took the run's execution generation back and asked the provisioner for a sandbox for a generation that had already been replaced. Reserving the grant now checks the pending start again, in the transaction that takes the generation, and refuses a replaced one with START_SUPERSEDED. The dispatcher acknowledges such a start without marking the run as waiting; the newer start proceeds as usual. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ad06e82c67
commit
df87c5dca6
@ -279,6 +279,11 @@ export class Dispatcher {
|
||||
}
|
||||
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
||||
} catch (error) {
|
||||
// A newer start replaced this one after it was read: it is done, and the run is not waiting.
|
||||
if (item.type === 'run.start' && error instanceof DomainError && error.code === 'START_SUPERSEDED') {
|
||||
await this.control.store.transaction(tenant, tx => tx.acknowledge(item.id));
|
||||
continue;
|
||||
}
|
||||
if (item.type === 'run.start' && error instanceof DomainError && error.code !== 'EXECUTION_CAPACITY')
|
||||
await this.control.store.transaction(tenant, async tx => {
|
||||
const current = await must<Run>(tx, 'run', String(item.payload.runId));
|
||||
|
||||
@ -187,6 +187,10 @@ export async function reserveExecutionGrant(
|
||||
);
|
||||
const run = await must<Run>(tx, 'run', runId);
|
||||
const plan = effectiveEntitlements(await tx.tenant());
|
||||
// Checked again here, in the transaction that takes the generation: a pause and a continue since
|
||||
// the dispatcher looked enqueued a newer start, and this one must not take the run back.
|
||||
if (run.pendingStartId && run.pendingStartId !== generation)
|
||||
throw new DomainError('START_SUPERSEDED', 'A newer start of this run replaced this one', 409);
|
||||
if (['succeeded', 'failed', 'canceled', 'paused'].includes(run.status))
|
||||
throw new DomainError('RUN_STOPPED', 'Stopped run cannot reserve a sandbox');
|
||||
const others = (await tx.list<Run>('run')).filter(other => other.id !== run.id && occupiesExecutionSlot(other));
|
||||
|
||||
@ -172,6 +172,7 @@ sequenceDiagram
|
||||
```
|
||||
|
||||
- outbox 条目的 ID 即运行的**执行代次**(generation)。每次继续、补充都会产生新代次;旧代次的 worker 令牌立即失效。
|
||||
- 派发器只启动运行最近一次排入的代次。读取运行之后,如果暂停与继续又排入了更新的代次,预留资源的事务会再核对一次并拒绝旧代次(`START_SUPERSEDED`):旧条目直接确认丢弃,不为它申请沙箱,也不把运行置为等待。
|
||||
- provisioner 返回失败或超时时,运行进入 `sandbox_provisioner` 等待,派发器指数退避重试;资源预留记为 `unknown`,由资源回收循环在收到释放回执后归还。
|
||||
|
||||
### 4.2 执行、验证与修复
|
||||
|
||||
@ -135,3 +135,51 @@ it('tells the provisioner the platform repository a new workspace may use', asyn
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('drops a start that a newer one replaced while it was being dispatched, leaving the run as it is', async () => {
|
||||
const store = new MemoryStore();
|
||||
store.seed('alpha');
|
||||
const run = { ...record('alpha'), status: 'queued', revision: 1, pendingStartId: 'start-fixture' } as Run;
|
||||
await store.transaction('alpha', async tx => {
|
||||
await tx.put('run', run);
|
||||
await tx.enqueue('run.start', { runId: run.id, revision: 1 }, 'start-fixture');
|
||||
await tx.enqueue('run.signal', { runId: 'other-run', action: 'cancel' }, 'cancel-other');
|
||||
});
|
||||
// Right after the dispatcher first reads the run, a pause and a continue enqueue a newer start.
|
||||
const transaction = store.transaction.bind(store);
|
||||
let replaced = false;
|
||||
vi.spyOn(store, 'transaction').mockImplementation(async (tenant, work) => {
|
||||
const result = await transaction(tenant, work);
|
||||
if (!replaced && (result as Run | undefined)?.id === run.id) {
|
||||
replaced = true;
|
||||
await transaction(tenant, async tx => {
|
||||
const current = (await tx.get<Run>('run', run.id))!;
|
||||
await tx.put('run', { ...current, pendingStartId: 'newer-start' });
|
||||
});
|
||||
}
|
||||
return result;
|
||||
});
|
||||
const provisioner = vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('No sandbox may be provisioned'));
|
||||
temporal.connect.mockResolvedValue({ close: temporal.close });
|
||||
const dispatcher = new Dispatcher(
|
||||
new ControlPlane(store, {
|
||||
WF_DISPATCH_TENANTS: 'alpha',
|
||||
WF_PROVISIONER_TENANT_ID: 'alpha',
|
||||
WF_PROVISIONER_URL: 'https://fixture.invalid',
|
||||
WF_PROVISIONER_TOKEN: 'fixture-provisioner-token-only-long-enough',
|
||||
WF_RESOURCE_GRANT_SECRET: 'resource-fixture-signing-secret-only-12345',
|
||||
WF_INTERNAL_TOKEN_KEY: 'internal-fixture-signing-key-only-123456789',
|
||||
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'a'.repeat(64)}`,
|
||||
}),
|
||||
);
|
||||
temporal.cancel.mockImplementation(async () => dispatcher.close());
|
||||
vi.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
await dispatcher.start();
|
||||
expect(replaced).toBe(true);
|
||||
expect(provisioner).not.toHaveBeenCalled();
|
||||
const after = await store.transaction('alpha', tx => tx.get<Run>('run', run.id));
|
||||
expect(after).toMatchObject({ status: 'queued', pendingStartId: 'newer-start' });
|
||||
expect(after?.executionGeneration).toBeUndefined();
|
||||
expect(after?.waitingReason).toBeUndefined();
|
||||
expect((await store.transaction('alpha', tx => tx.outbox())).map(item => item.id)).not.toContain('start-fixture');
|
||||
});
|
||||
|
||||
@ -12,6 +12,7 @@ import {
|
||||
recordAllocation,
|
||||
} from '../apps/api/src/domain/entitlements.js';
|
||||
import { purgeRetainedStorage, releaseStoppedAllocations } from '../apps/api/src/domain/resource-dispatch.js';
|
||||
import type { Transaction } from '../apps/api/src/db/store.js';
|
||||
const owner: Actor = { tenantId: 'alpha', userId: 'owner', kind: 'human', role: 'owner' },
|
||||
operator: Actor = { ...owner, userId: 'operations-user', role: 'admin' };
|
||||
const env = {
|
||||
@ -25,6 +26,12 @@ const env = {
|
||||
WF_PROVISIONER_TENANT_ID: 'alpha',
|
||||
};
|
||||
let db: MemoryStore, control: ControlPlane, entitlements: Entitlements, usage: Usage;
|
||||
/** Reserves as the dispatcher does: for the start the control plane last enqueued for the run. */
|
||||
async function reserve(tx: Transaction, runId: string, generation: string, environment: NodeJS.ProcessEnv = env) {
|
||||
const run = (await tx.get<Run>('run', runId))!;
|
||||
if (run.pendingStartId !== generation) await tx.put('run', { ...run, pendingStartId: generation });
|
||||
return reserveExecutionGrant(tx, runId, generation, environment);
|
||||
}
|
||||
beforeEach(() => {
|
||||
db = new MemoryStore();
|
||||
db.seed('alpha');
|
||||
@ -81,9 +88,7 @@ describe('fixed pilot entitlements and resource admission', () => {
|
||||
{ WF_DEFAULT_EXECUTION_STORAGE_MIB: '1' },
|
||||
])
|
||||
await expect(
|
||||
db.transaction('alpha', tx =>
|
||||
reserveExecutionGrant(tx, created.run_id, 'invalid-default', { ...env, ...defaults }),
|
||||
),
|
||||
db.transaction('alpha', tx => reserve(tx, created.run_id, 'invalid-default', { ...env, ...defaults })),
|
||||
).rejects.toMatchObject({ code: 'RESOURCE_CONFIGURATION_INVALID' });
|
||||
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(0);
|
||||
});
|
||||
@ -128,11 +133,11 @@ describe('fixed pilot entitlements and resource admission', () => {
|
||||
const runs = await queuedRuns(3);
|
||||
expect(await control.runs(owner)).toHaveLength(3);
|
||||
const results = await Promise.allSettled(
|
||||
runs.map(run => db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-1', env))),
|
||||
runs.map(run => db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env))),
|
||||
);
|
||||
expect(results.filter(r => r.status === 'fulfilled')).toHaveLength(1);
|
||||
const first = runs[0].run_id;
|
||||
const again = await db.transaction('alpha', tx => reserveExecutionGrant(tx, first, 'generation-1', env));
|
||||
const again = await db.transaction('alpha', tx => reserve(tx, first, 'generation-1', env));
|
||||
expect(again.image).toBe(env.WF_RUN_WORKER_IMAGE);
|
||||
expect(again.cpuMilli).toBe(2000);
|
||||
await db.transaction('alpha', async tx => {
|
||||
@ -141,17 +146,17 @@ describe('fixed pilot entitlements and resource admission', () => {
|
||||
run.status = 'canceled';
|
||||
await tx.put('run', run);
|
||||
});
|
||||
await expect(
|
||||
db.transaction('alpha', tx => reserveExecutionGrant(tx, runs[1].run_id, 'generation-1', env)),
|
||||
).rejects.toMatchObject({ code: 'EXECUTION_CAPACITY' });
|
||||
await expect(db.transaction('alpha', tx => reserve(tx, runs[1].run_id, 'generation-1', env))).rejects.toMatchObject(
|
||||
{ code: 'EXECUTION_CAPACITY' },
|
||||
);
|
||||
expect((await entitlements.view(owner)).counts.reservedExecutionSlots).toBe(1);
|
||||
});
|
||||
it('pins the original runtime digest across resumes and rejects legacy checkpoint image guesses', async () => {
|
||||
const [run] = await queuedRuns(1);
|
||||
const original = await db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-1', env));
|
||||
const original = await db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-1', env));
|
||||
await db.transaction('alpha', tx => recordAllocation(tx, run.run_id, 'generation-1', 'released'));
|
||||
const resumed = await db.transaction('alpha', tx =>
|
||||
reserveExecutionGrant(tx, run.run_id, 'generation-2', {
|
||||
reserve(tx, run.run_id, 'generation-2', {
|
||||
...env,
|
||||
WF_RUN_WORKER_IMAGE: `registry.invalid/runtime@sha256:${'b'.repeat(64)}`,
|
||||
}),
|
||||
@ -163,28 +168,46 @@ describe('fixed pilot entitlements and resource admission', () => {
|
||||
value.checkpointId = 'legacy-checkpoint';
|
||||
await tx.put('run', value);
|
||||
});
|
||||
await expect(
|
||||
db.transaction('alpha', tx => reserveExecutionGrant(tx, run.run_id, 'generation-3', env)),
|
||||
).rejects.toMatchObject({ code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED' });
|
||||
await expect(db.transaction('alpha', tx => reserve(tx, run.run_id, 'generation-3', env))).rejects.toMatchObject({
|
||||
code: 'RUNTIME_IMAGE_MIGRATION_REQUIRED',
|
||||
});
|
||||
});
|
||||
it('does not replace an unresolved previous generation on resume', async () => {
|
||||
const [created] = await queuedRuns(1);
|
||||
await db.transaction('alpha', async tx => {
|
||||
await reserveExecutionGrant(tx, created.run_id, 'old-generation', env);
|
||||
await reserve(tx, created.run_id, 'old-generation', env);
|
||||
await recordAllocation(tx, created.run_id, 'old-generation', 'unknown');
|
||||
});
|
||||
await expect(
|
||||
db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'new-generation', env)),
|
||||
db.transaction('alpha', tx => reserve(tx, created.run_id, 'new-generation', env)),
|
||||
).rejects.toMatchObject({ code: 'RESOURCE_PREVIOUS_GENERATION' });
|
||||
expect((await control.run(owner, created.run_id)).resourceAllocation).toMatchObject({
|
||||
generation: 'old-generation',
|
||||
status: 'unknown',
|
||||
});
|
||||
});
|
||||
it('refuses a start that a newer one replaced, so it cannot take the run back', async () => {
|
||||
const [created] = await queuedRuns(1);
|
||||
const first = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!.pendingStartId!;
|
||||
// A pause and a continue enqueue a newer start before the dispatcher reserves the first.
|
||||
await db.transaction('alpha', async tx => {
|
||||
const run = (await tx.get<Run>('run', created.run_id))!;
|
||||
await tx.put('run', { ...run, pendingStartId: 'newer-start' });
|
||||
});
|
||||
await expect(
|
||||
db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, first, env)),
|
||||
).rejects.toMatchObject({ code: 'START_SUPERSEDED' });
|
||||
const untouched = (await db.transaction('alpha', tx => tx.get<Run>('run', created.run_id)))!;
|
||||
expect(untouched.executionGeneration).toBeUndefined();
|
||||
expect(untouched.resourceAllocation).toBeUndefined();
|
||||
// The newer start reserves as usual.
|
||||
await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'newer-start', env));
|
||||
expect((await control.run(owner, created.run_id)).executionGeneration).toBe('newer-start');
|
||||
});
|
||||
it('issues a grant accepted by the independent provisioner without allowing changed resources', async () => {
|
||||
const { verifyResourceGrant } = await import('../apps/provisioner/resource-grants.mjs');
|
||||
const [created] = await queuedRuns(1);
|
||||
const grant = await db.transaction('alpha', tx => reserveExecutionGrant(tx, created.run_id, 'generation-1', env));
|
||||
const grant = await db.transaction('alpha', tx => reserve(tx, created.run_id, 'generation-1', env));
|
||||
const configuration = {
|
||||
tenantId: 'alpha',
|
||||
image: env.WF_RUN_WORKER_IMAGE,
|
||||
@ -206,7 +229,7 @@ describe('fixed pilot entitlements and resource admission', () => {
|
||||
const [created] = await queuedRuns(1),
|
||||
id = created.run_id;
|
||||
await db.transaction('alpha', async tx => {
|
||||
await reserveExecutionGrant(tx, id, 'generation-1', env);
|
||||
await reserve(tx, id, 'generation-1', env);
|
||||
await recordAllocation(tx, id, 'generation-1', 'unknown');
|
||||
const run = (await tx.get<Run>('run', id))!;
|
||||
run.status = 'canceled';
|
||||
@ -240,7 +263,7 @@ describe('retained run volumes', () => {
|
||||
const [old, recent] = (await queuedRuns(2)).map(created => created.run_id);
|
||||
await db.transaction('alpha', async tx => {
|
||||
for (const id of [old!, recent!]) {
|
||||
await reserveExecutionGrant(tx, id, `generation-${id}`, env);
|
||||
await reserve(tx, id, `generation-${id}`, env);
|
||||
await recordAllocation(tx, id, `generation-${id}`, 'released');
|
||||
const run = (await tx.get<Run>('run', id))!;
|
||||
run.status = 'succeeded';
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user