wishforge-ide/tests/provider-proxy.test.ts
eust-w e734bd9846 🐛 Decide which limit stops a stalled model call when its timer is set
The stall timer chose between "idle" and "deadline" by reading the clock
when it fired. A timer armed for the idle limit could fire after the
deadline had also passed on a busy machine, and was then reported as a
deadline stop, which the proxy test caught intermittently. The reason
is now fixed when the timer is armed: whichever limit is nearer.

The two timing tests also get wider margins: the steady stream sends a
frame every 30 ms against a 300 ms idle limit, and the deadline case
can only be stopped by its deadline.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-26 01:53:19 +08:00

563 lines
26 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { beforeEach, describe, expect, it, vi } from 'vitest';
import { SignJWT, decodeJwt } from 'jose';
import type { Actor, Run } from '@wishforge/contracts';
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
import { Connections } from '../apps/api/src/domain/connections.js';
import {
ProviderProxy,
providerProxyConfigured,
type ProxyRequestRecord,
} from '../apps/api/src/domain/provider-proxy.js';
import { ServiceTokens } from '../apps/api/src/service-tokens.js';
import { MemoryStore } from './helpers/memory-store.js';
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' },
worker: Actor = { ...owner, userId: 'worker', kind: 'worker' };
const secret = 'fixture_upstream_key_kept_inside_control_plane';
const env = {
NODE_ENV: 'test',
WF_VAULT_KEY: 'a'.repeat(64),
WF_PROVIDER_PROXY_ENABLED: 'true',
WF_PROVIDER_PROXY_PUBLIC_URL: 'http://127.0.0.1:4000/provider-proxy/v1',
WF_PROVIDER_PROXY_SIGNING_KEY: 'proxy-independent-signing-fixture-key-12345',
WF_PROVIDER_PROXY_ALLOWED_MODELS: 'fixture-model',
};
const encode = (data: unknown) => new TextEncoder().encode(JSON.stringify(data));
const terminal = (input = 10, output = 5) => ({
type: 'response.completed',
response: {
id: 'resp_fixture',
status: 'completed',
usage: { input_tokens: input, output_tokens: output, total_tokens: input + output },
},
});
function sse(events: unknown[], chunkSize = 7) {
const all = Buffer.from(
events.map(value => `event: ${(value as any).type}\r\ndata: ${JSON.stringify(value)}\r\n\r\n`).join(''),
);
let offset = 0;
return new Response(
new ReadableStream({
pull(controller) {
if (offset === all.length) controller.close();
else {
controller.enqueue(all.subarray(offset, offset + chunkSize));
offset = Math.min(all.length, offset + chunkSize);
}
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
);
}
/** An event stream that sends one event every `everyMs`, then sends nothing more and stays open. */
function trickle(events: unknown[], everyMs: number) {
let sent = 0;
return new Response(
new ReadableStream({
async pull(controller) {
if (sent === events.length) return new Promise<void>(() => undefined);
await new Promise(resolve => setTimeout(resolve, everyMs));
const value = events[sent++] as { type: string };
controller.enqueue(new TextEncoder().encode(`event: ${value.type}\ndata: ${JSON.stringify(value)}\n\n`));
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
);
}
const delta = { type: 'response.output_text.delta', delta: 'thinking' };
const operations = () => db.transaction('alpha', tx => tx.list<ProxyRequestRecord>('provider_proxy_request'));
let db: MemoryStore, control: ControlPlane, connections: Connections, run: Run, connectionId: string;
beforeEach(async () => {
db = new MemoryStore();
db.seed('alpha');
db.seed('beta');
control = new ControlPlane(db, env);
connections = new Connections(db, env);
connectionId = (await connections.create(owner, { provider: 'codex', label: 'fixture', apiKey: secret })).id;
const project = await control.createProject(owner, {
name: 'Proxy fixture',
repositoryUrl: 'https://example.org/fixture.git',
});
const wish = await control.submit(
owner,
project.id,
{ goal: 'Test scoped proxy', acceptanceCriteria: ['Credential remains private'] },
'proxy-fixture-submit',
);
run = await control.lease(worker, wish.run_id, 'fixture-worker');
});
async function fixture(fetcher: typeof fetch, timing: { idleMs?: number; deadlineMs?: number } = {}) {
const proxy = new ProviderProxy(db, env, { fetcher, pollMs: 25, ...timing });
const credential = await proxy.credential(worker, run);
const request = (body: Record<string, unknown> = {}, headers?: Record<string, string>, signal?: AbortSignal) =>
proxy.forward({
authorization: `Bearer ${credential.apiKey}`,
method: 'POST',
path: '/responses',
body: encode({ model: 'fixture-model', input: 'Inspect local code', stream: true, ...body }),
headers,
signal,
});
return { proxy, credential, request };
}
async function waitFor(check: () => Promise<boolean>) {
for (let count = 0; count < 100; count++) {
if (await check()) return;
await new Promise(resolve => setTimeout(resolve, 10));
}
throw new Error('Timed out waiting for proxy settlement');
}
describe('run-scoped Codex credential proxy', () => {
it('accepts bounded local namespaces and SDK metadata without treating tool schemas as media', async () => {
const { request } = await fixture((async (_url, init) => {
const data = JSON.parse(String(init?.body));
expect(data.client_metadata).toBeUndefined();
expect(data.tools[0].tools[0].type).toBe('function');
return sse([terminal()]);
}) as typeof fetch);
const body = {
client_metadata: { originator: 'fixture' },
tools: [
{
type: 'namespace',
name: 'local',
tools: [
{
type: 'function',
name: 'inspect',
parameters: { type: 'object', properties: { image_url: { type: 'string' } } },
},
],
},
],
};
expect((await request(body)).status).toBe(200);
await expect(
request({
tools: [{ type: 'namespace', name: 'bad', tools: [{ type: 'mcp', server_url: 'https://untrusted.example' }] }],
}),
).rejects.toMatchObject({ code: 'PROVIDER_REMOTE_TOOL_DENIED' });
await expect(
request({
input: [{ role: 'user', content: [{ type: 'input_image', image_url: 'https://untrusted.example/image' }] }],
}),
).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' });
});
it('retains quota at the instant of cancellation while upstream usage is unresolved', async () => {
db.state.get('alpha')!.tenant.tokenLimit = 100000;
const upstream = vi.fn(
async () =>
new Response(
new ReadableStream({
start(c) {
c.enqueue(
new TextEncoder().encode(
'event: response.created\ndata: {"type":"response.created","response":{"id":"pending"}}\n\n',
),
);
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
),
);
const { request } = await fixture(upstream as typeof fetch);
const response = await request();
const reading = response.text().catch(() => 'aborted');
await control.command(owner, run.id, { action: 'cancel' }, 'cancel-before-settlement');
expect((await control.usage(owner)).reservedTokens).toBe(100000);
await expect(
control.submit(
owner,
run.projectId,
{ goal: 'Another run', acceptanceCriteria: ['Cannot borrow unresolved reservation'] },
'after-cancellation',
),
).rejects.toMatchObject({ code: 'QUOTA_EXCEEDED' });
await reading;
});
it('returns proxy credentials through the actual connection broker without exposing the stored key', async () => {
const connection = await connections.forRun(worker, run);
expect(connection).toMatchObject({ metering: 'proxy', baseUrl: env.WF_PROVIDER_PROXY_PUBLIC_URL });
expect(JSON.stringify(connection)).not.toContain(secret);
});
it('never puts the upstream key in the scoped token and prevents proxy credentials from authenticating to internal services', async () => {
const { credential } = await fixture(vi.fn());
expect(credential.metering).toBe('proxy');
expect(JSON.stringify(credential).includes(secret)).toBe(false);
const claims = decodeJwt(credential.apiKey);
expect(claims).toMatchObject({
tenantId: 'alpha',
runId: run.id,
connectionId,
leaseToken: run.lease!.token,
aud: 'wishforge-openai-proxy',
});
expect(Number(claims.exp) - Number(claims.iat)).toBeLessThanOrEqual(3600);
expect(await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).verify(credential.apiKey)).toBeUndefined();
expect(providerProxyConfigured({ ...env, NODE_ENV: 'production' })).toBe(false);
await expect(new ProviderProxy(db, { NODE_ENV: 'production' }).credential(worker, run)).rejects.toMatchObject({
code: 'PROVIDER_PROXY_UNCONFIGURED',
});
});
it('forwards to the operator’s OpenAI-compatible gateway when one is configured', async () => {
const gateway = { ...env, WF_OPENAI_BASE_URL: 'https://gateway.example/v1/' };
const upstream = vi.fn(async (url: any) => {
expect(String(url)).toBe('https://gateway.example/v1/responses');
return sse([terminal()]);
});
const proxy = new ProviderProxy(db, gateway, { fetcher: upstream as typeof fetch, pollMs: 25 });
const credential = await proxy.credential(worker, run);
const response = await proxy.forward({
authorization: `Bearer ${credential.apiKey}`,
method: 'POST',
path: '/responses',
body: encode({ model: 'fixture-model', input: 'Inspect local code', stream: true }),
});
await response.text();
expect(upstream).toHaveBeenCalledTimes(1);
// A gateway outside HTTPS, or with credentials in it, disables the proxy instead.
expect(providerProxyConfigured({ ...env, WF_OPENAI_BASE_URL: 'https://user:secret@gateway.example/v1' })).toBe(
false,
);
});
it('forwards only trusted endpoint/headers and streams split SSE frames with trusted usage recorded once', async () => {
const upstream = vi.fn(async (url: any, init: any) => {
expect(String(url)).toBe('https://api.openai.com/v1/responses');
expect(init.headers.Authorization === `Bearer ${secret}`).toBe(true);
expect(init.headers['X-Untrusted']).toBeUndefined();
const body = JSON.parse(init.body);
expect(body.store).toBe(false);
expect(body.background).toBe(false);
expect(body.max_output_tokens).toBeLessThanOrEqual(8192);
return sse([
{ type: 'response.created', response: { id: 'resp_fixture' } },
{ type: 'response.output_text.delta', delta: `hello ${secret}` },
terminal(),
]);
});
const { request } = await fixture(upstream as typeof fetch);
const first = await request({}, { 'X-Untrusted': 'not-forwarded' });
const body = await first.text();
expect(body).toContain('response.completed');
expect(body.includes(secret)).toBe(false);
expect(body).toContain('[redacted]');
const repeated = await request();
expect(await repeated.text()).toBe(body);
expect(upstream).toHaveBeenCalledTimes(1);
expect((await control.usage(owner)).tokens).toBe(15);
});
it('rejects tampered tenant tokens, wrong audience, stale generations and rotated provider connections before upstream I/O', async () => {
const upstream = vi.fn();
const { proxy, credential, request } = await fixture(upstream);
const token = await new SignJWT({
...decodeJwt<Record<string, unknown>>(String(credential.apiKey)),
tenantId: 'beta',
})
.setProtectedHeader({ alg: 'HS256' })
.sign(new TextEncoder().encode(env.WF_PROVIDER_PROXY_SIGNING_KEY));
await expect(
proxy.forward({ authorization: `Bearer ${token}`, method: 'GET', path: '/models' }),
).rejects.toMatchObject({ code: 'NOT_FOUND' });
const internal = await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).issue(
'alpha',
run.id,
'worker',
'generation',
);
await expect(
proxy.forward({ authorization: `Bearer ${internal}`, method: 'GET', path: '/models' }),
).rejects.toMatchObject({ code: 'PROVIDER_PROXY_UNAUTHENTICATED' });
await connections.revoke(owner, connectionId);
await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_CONNECTION_REVOKED' });
expect(upstream).not.toHaveBeenCalled();
});
it('denies arbitrary upstream paths, server-side tools/state, unapproved models and multimodal input', async () => {
const upstream = vi.fn();
const { proxy, credential, request } = await fixture(upstream);
await expect(
proxy.forward({
authorization: `Bearer ${credential.apiKey}`,
method: 'POST',
path: 'https://attacker.invalid/v1/responses',
body: encode({}),
}),
).rejects.toMatchObject({ code: 'PROVIDER_ROUTE_DENIED' });
await expect(request({ tools: [{ type: 'mcp', server_url: 'https://attacker.invalid' }] })).rejects.toMatchObject({
code: 'PROVIDER_REMOTE_TOOL_DENIED',
});
await expect(request({ previous_response_id: 'other-tenants-response' })).rejects.toMatchObject({
code: 'PROVIDER_STATEFUL_REQUEST_DENIED',
});
await expect(
request({ input: [{ type: 'input_image', image_url: 'https://attacker.invalid/image' }] }),
).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' });
await expect(request({ model: 'not-approved' })).rejects.toMatchObject({ code: 'PROVIDER_MODEL_DENIED' });
expect(upstream).not.toHaveBeenCalled();
});
it('serializes a run and stops a live stream when the user cancels', async () => {
let canceled = false;
const upstream = vi.fn(
async () =>
new Response(
new ReadableStream({
start(controller) {
controller.enqueue(
new TextEncoder().encode(
'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n',
),
);
},
cancel() {
canceled = true;
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
),
);
const { request } = await fixture(upstream as typeof fetch);
const response = await request();
const reading = response.text().catch(() => 'aborted');
await expect(request({ input: 'another prompt' })).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' });
await control.command(owner, run.id, { action: 'cancel' }, 'cancel-proxy-request');
expect(await reading).toBe('aborted');
await waitFor(async () => canceled);
expect((await control.run(owner, run.id)).status).toBe('canceled');
expect((await control.run(owner, run.id)).reservedTokens).toBeGreaterThan(0);
const operations = await db.transaction('alpha', tx => tx.list<ProxyRequestRecord>('provider_proxy_request'));
expect(operations[0].status).toBe('unknown');
expect(upstream).toHaveBeenCalledTimes(1);
});
it('preserves unknown outcome/reservation after a truncated stream and does not resend it', async () => {
const upstream = vi.fn(async () => sse([{ type: 'response.output_text.delta', delta: 'partial output' }]));
const { request } = await fixture(upstream as typeof fetch);
await expect((await request()).text()).rejects.toThrow();
expect(await control.run(owner, run.id)).toMatchObject({
status: 'waiting',
waitingReason: 'ambiguous_execution',
});
await control.command(owner, run.id, { action: 'resume' }, 'resume-unknown-proxy');
const leased = await control.lease(worker, run.id, 'recovery-worker');
const proxy = new ProviderProxy(db, env, { fetcher: upstream as typeof fetch });
const next = await proxy.credential(worker, leased);
await expect(
proxy.forward({
authorization: `Bearer ${next.apiKey}`,
method: 'POST',
path: '/responses',
body: encode({ model: 'fixture-model', input: 'retry', stream: true }),
}),
).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' });
expect(upstream).toHaveBeenCalledTimes(1);
// An owner reconciles the unknown request; the run can call the model again (FR-C6).
const [unknown] = await proxy.requests(owner, run.id);
expect(unknown).toMatchObject({ status: 'unknown', active: true });
await expect(
proxy.reconcile({ ...owner, kind: 'connector' }, run.id, { requestId: unknown!.id }),
).rejects.toMatchObject({ status: 403 });
expect(await proxy.reconcile(owner, run.id, { requestId: unknown!.id, usageTokens: 42 })).toEqual({
requestId: unknown!.id,
usageTokens: 42,
});
await expect(proxy.reconcile(owner, run.id, { requestId: unknown!.id })).rejects.toMatchObject({
code: 'PROVIDER_REQUEST_SETTLED',
});
const settled = await control.run(owner, run.id);
expect(settled.providerProxyActiveRequest).toBeUndefined();
expect(settled.usedTokens).toBe(42);
upstream.mockImplementation(async () => sse([terminal()]));
const retried = await proxy.forward({
authorization: `Bearer ${next.apiKey}`,
method: 'POST',
path: '/responses',
body: encode({ model: 'fixture-model', input: 'retry after reconciliation', stream: true }),
});
expect(retried.status).toBe(200);
await retried.text();
expect(upstream).toHaveBeenCalledTimes(2);
});
it('rejects an unaffordable request before any model call and meters only actual upstream usage', async () => {
const upstream = vi.fn(async () => sse([terminal(50000, 50000)]));
const { request } = await fixture(upstream as typeof fetch);
await (await request()).text();
expect(await control.run(owner, run.id)).toMatchObject({
status: 'waiting',
waitingReason: 'budget',
usedTokens: 100000,
reservedTokens: 0,
});
await expect(request({ input: 'further work' })).rejects.toThrow();
expect(upstream).toHaveBeenCalledTimes(1);
});
it('refuses insufficient input/output reservation before contacting the provider', async () => {
const upstream = vi.fn();
const { request } = await fixture(upstream);
await db.transaction('alpha', async tx => {
const current = (await tx.get<Run>('run', run.id))!;
current.usedTokens = 99000;
current.reservedTokens = 1000;
await tx.put('run', current);
});
await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_BUDGET_EXHAUSTED' });
expect(upstream).not.toHaveBeenCalled();
// The run says why it stopped instead of leaving the executor to report a provider failure.
const waiting = await control.run(owner, run.id);
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'budget', usedTokens: 99000 });
expect(waiting.waitingDetail).toMatch(/needs \d+ tokens of budget .* but 1000 of 100000 remain/);
expect(Date.parse(waiting.lease!.expiresAt)).toBeLessThanOrEqual(Date.now());
expect((await control.events(owner, run.id)).at(-1)).toMatchObject({ type: 'run.waiting' });
});
it('aborts an in-flight model stream after provider connection rotation', async () => {
let canceled = false;
const upstream = vi.fn(
async () =>
new Response(
new ReadableStream({
start(c) {
c.enqueue(
new TextEncoder().encode(
'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n',
),
);
},
cancel() {
canceled = true;
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
),
);
const { request } = await fixture(upstream as typeof fetch);
const reading = (await request()).text().catch(() => 'aborted');
await connections.create(owner, { provider: 'codex', label: 'rotated', apiKey: 'replacement_fixture_private_key' });
expect(await reading).toBe('aborted');
expect(canceled).toBe(true);
expect((await control.run(owner, run.id)).waitingReason).toBe('ambiguous_execution');
});
it('binds production proxy credentials to execution generation and fails before decrypting if it changes', async () => {
const production = {
...env,
NODE_ENV: 'production',
WF_PROVIDER_PROXY_PUBLIC_URL: 'https://api.example/provider-proxy/v1',
};
const actor = { ...worker, runId: run.id, credentialId: 'fixture-generation' };
await db.transaction('alpha', async tx => {
const current = (await tx.get<Run>('run', run.id))!;
current.executionGeneration = current.pendingStartId = 'fixture-generation';
await tx.put('run', current);
});
const open = vi.fn(async () => secret),
fetcher = vi.fn();
const proxy = new ProviderProxy(db, production, { vault: { open }, fetcher });
const token = await proxy.credential(actor, run);
await db.transaction('alpha', async tx => {
const current = (await tx.get<Run>('run', run.id))!;
current.executionGeneration = 'replacement-generation';
await tx.put('run', current);
});
await expect(
proxy.forward({ authorization: `Bearer ${token.apiKey}`, method: 'GET', path: '/models' }),
).rejects.toMatchObject({ code: 'EXECUTION_REVOKED' });
expect(open).not.toHaveBeenCalled();
expect(fetcher).not.toHaveBeenCalled();
});
it('handles JSON Responses and filters model listings without reflecting upstream secret/error headers', async () => {
const upstream = vi.fn(async (url: any) =>
String(url).endsWith('/models')
? new Response(JSON.stringify({ data: [{ id: 'fixture-model' }, { id: 'expensive-other-model' }] }), {
headers: { 'content-type': 'application/json', 'x-secret': secret },
})
: new Response(JSON.stringify(terminal().response), {
headers: { 'content-type': 'application/json', 'x-secret': secret },
}),
);
const { request, proxy, credential } = await fixture(upstream as typeof fetch);
const response = await request({ stream: false });
expect((await response.json()).usage.total_tokens).toBe(15);
expect(response.headers.has('x-secret')).toBe(false);
const models = await proxy.forward({
authorization: `Bearer ${credential.apiKey}`,
method: 'GET',
path: '/models',
});
expect((await models.json()).data).toEqual([{ id: 'fixture-model' }]);
});
it('rejects changed input with the same idempotency key rather than dispatching again', async () => {
const upstream = vi.fn(async () => sse([terminal()]));
const { request } = await fixture(upstream as typeof fetch);
await (await request({}, { 'idempotency-key': 'same-request-key' })).text();
await expect(request({ input: 'changed' }, { 'idempotency-key': 'same-request-key' })).rejects.toMatchObject({
code: 'PROVIDER_IDEMPOTENCY_CONFLICT',
});
expect(upstream).toHaveBeenCalledTimes(1);
});
it('treats x-client-request-id as a conversation id, not as a key for one request', async () => {
// Codex sends its conversation id in this header on every request of a session.
const upstream = vi.fn(async () => sse([terminal()]));
const { request } = await fixture(upstream as typeof fetch);
await (await request({}, { 'x-client-request-id': 'conversation-0001' })).text();
await (await request({ input: 'the next turn' }, { 'x-client-request-id': 'conversation-0001' })).text();
expect(upstream).toHaveBeenCalledTimes(2);
});
it('keeps a stream open past the idle limit while the provider keeps sending', async () => {
// 450 ms in all, past the idle limit, but never more than 30 ms without a frame.
const upstream = vi.fn(async () => trickle([...Array(14).fill(delta), terminal()], 30));
const { request } = await fixture(upstream as typeof fetch, { idleMs: 300 });
const response = await request();
expect(await response.text()).toContain('response.completed');
expect((await operations())[0]).toMatchObject({ status: 'completed', usageTokens: 15 });
expect((await control.run(owner, run.id)).status).toBe('running');
});
it('stops a stream that falls silent, or runs past the deadline, and holds it for reconciliation', async () => {
const silent = vi.fn(async () => trickle([delta], 10));
const { request } = await fixture(silent as typeof fetch, { idleMs: 100 });
await expect((await request()).text()).rejects.toThrow();
expect((await operations())[0]).toMatchObject({ status: 'unknown', unknownReason: 'upstream_idle' });
expect(await control.run(owner, run.id)).toMatchObject({
status: 'waiting',
waitingReason: 'ambiguous_execution',
});
const [unknown] = await operations();
await new ProviderProxy(db, env).reconcile(owner, run.id, { requestId: unknown!.id, usageTokens: 0 });
await control.command(owner, run.id, { action: 'resume' }, 'resume-after-silent-stream');
run = await control.lease(worker, run.id, 'deadline-worker');
const endless = vi.fn(async () => trickle(Array(100).fill(delta), 20));
// Frames keep coming well within the idle limit, so only the deadline can stop this one.
const deadline = await fixture(endless as typeof fetch, { idleMs: 1000, deadlineMs: 300 });
await expect((await deadline.request({ input: 'a long answer' })).text()).rejects.toThrow();
expect((await operations()).find(op => op.id !== unknown!.id)).toMatchObject({
status: 'unknown',
unknownReason: 'upstream_deadline',
});
});
it('sends a retry again after the provider rejected the request, rather than replaying the rejection', async () => {
const upstream = vi.fn(async () => sse([terminal()]));
upstream.mockImplementationOnce(
async () =>
new Response(JSON.stringify({ error: { message: 'Rate limit reached' } }), {
status: 429,
headers: { 'content-type': 'application/json' },
}),
);
const { request } = await fixture(upstream as typeof fetch);
expect((await request()).status).toBe(429);
const retried = await request();
expect(retried.status).toBe(200);
await retried.text();
expect(await operations()).toEqual([expect.objectContaining({ status: 'completed', usageTokens: 15 })]);
// A request stopped before it reached the provider is sent when retried, too.
const stopped = new AbortController();
stopped.abort();
await expect(request({ input: 'the next turn' }, undefined, stopped.signal)).rejects.toMatchObject({
code: 'PROVIDER_REQUEST_CANCELED',
});
const sent = await request({ input: 'the next turn' });
expect(sent.status).toBe(200);
await sent.text();
expect(upstream).toHaveBeenCalledTimes(3);
expect((await control.run(owner, run.id)).usedTokens).toBe(30);
});
});