The stall timer chose between "idle" and "deadline" by reading the clock when it fired. A timer armed for the idle limit could fire after the deadline had also passed on a busy machine, and was then reported as a deadline stop, which the proxy test caught intermittently. The reason is now fixed when the timer is armed: whichever limit is nearer. The two timing tests also get wider margins: the steady stream sends a frame every 30 ms against a 300 ms idle limit, and the deadline case can only be stopped by its deadline. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
563 lines
26 KiB
TypeScript
563 lines
26 KiB
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||
import { SignJWT, decodeJwt } from 'jose';
|
||
import type { Actor, Run } from '@wishforge/contracts';
|
||
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
|
||
import { Connections } from '../apps/api/src/domain/connections.js';
|
||
import {
|
||
ProviderProxy,
|
||
providerProxyConfigured,
|
||
type ProxyRequestRecord,
|
||
} from '../apps/api/src/domain/provider-proxy.js';
|
||
import { ServiceTokens } from '../apps/api/src/service-tokens.js';
|
||
import { MemoryStore } from './helpers/memory-store.js';
|
||
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' },
|
||
worker: Actor = { ...owner, userId: 'worker', kind: 'worker' };
|
||
const secret = 'fixture_upstream_key_kept_inside_control_plane';
|
||
const env = {
|
||
NODE_ENV: 'test',
|
||
WF_VAULT_KEY: 'a'.repeat(64),
|
||
WF_PROVIDER_PROXY_ENABLED: 'true',
|
||
WF_PROVIDER_PROXY_PUBLIC_URL: 'http://127.0.0.1:4000/provider-proxy/v1',
|
||
WF_PROVIDER_PROXY_SIGNING_KEY: 'proxy-independent-signing-fixture-key-12345',
|
||
WF_PROVIDER_PROXY_ALLOWED_MODELS: 'fixture-model',
|
||
};
|
||
const encode = (data: unknown) => new TextEncoder().encode(JSON.stringify(data));
|
||
const terminal = (input = 10, output = 5) => ({
|
||
type: 'response.completed',
|
||
response: {
|
||
id: 'resp_fixture',
|
||
status: 'completed',
|
||
usage: { input_tokens: input, output_tokens: output, total_tokens: input + output },
|
||
},
|
||
});
|
||
function sse(events: unknown[], chunkSize = 7) {
|
||
const all = Buffer.from(
|
||
events.map(value => `event: ${(value as any).type}\r\ndata: ${JSON.stringify(value)}\r\n\r\n`).join(''),
|
||
);
|
||
let offset = 0;
|
||
return new Response(
|
||
new ReadableStream({
|
||
pull(controller) {
|
||
if (offset === all.length) controller.close();
|
||
else {
|
||
controller.enqueue(all.subarray(offset, offset + chunkSize));
|
||
offset = Math.min(all.length, offset + chunkSize);
|
||
}
|
||
},
|
||
}),
|
||
{ headers: { 'content-type': 'text/event-stream' } },
|
||
);
|
||
}
|
||
/** An event stream that sends one event every `everyMs`, then sends nothing more and stays open. */
|
||
function trickle(events: unknown[], everyMs: number) {
|
||
let sent = 0;
|
||
return new Response(
|
||
new ReadableStream({
|
||
async pull(controller) {
|
||
if (sent === events.length) return new Promise<void>(() => undefined);
|
||
await new Promise(resolve => setTimeout(resolve, everyMs));
|
||
const value = events[sent++] as { type: string };
|
||
controller.enqueue(new TextEncoder().encode(`event: ${value.type}\ndata: ${JSON.stringify(value)}\n\n`));
|
||
},
|
||
}),
|
||
{ headers: { 'content-type': 'text/event-stream' } },
|
||
);
|
||
}
|
||
const delta = { type: 'response.output_text.delta', delta: 'thinking' };
|
||
const operations = () => db.transaction('alpha', tx => tx.list<ProxyRequestRecord>('provider_proxy_request'));
|
||
let db: MemoryStore, control: ControlPlane, connections: Connections, run: Run, connectionId: string;
|
||
beforeEach(async () => {
|
||
db = new MemoryStore();
|
||
db.seed('alpha');
|
||
db.seed('beta');
|
||
control = new ControlPlane(db, env);
|
||
connections = new Connections(db, env);
|
||
connectionId = (await connections.create(owner, { provider: 'codex', label: 'fixture', apiKey: secret })).id;
|
||
const project = await control.createProject(owner, {
|
||
name: 'Proxy fixture',
|
||
repositoryUrl: 'https://example.org/fixture.git',
|
||
});
|
||
const wish = await control.submit(
|
||
owner,
|
||
project.id,
|
||
{ goal: 'Test scoped proxy', acceptanceCriteria: ['Credential remains private'] },
|
||
'proxy-fixture-submit',
|
||
);
|
||
run = await control.lease(worker, wish.run_id, 'fixture-worker');
|
||
});
|
||
async function fixture(fetcher: typeof fetch, timing: { idleMs?: number; deadlineMs?: number } = {}) {
|
||
const proxy = new ProviderProxy(db, env, { fetcher, pollMs: 25, ...timing });
|
||
const credential = await proxy.credential(worker, run);
|
||
const request = (body: Record<string, unknown> = {}, headers?: Record<string, string>, signal?: AbortSignal) =>
|
||
proxy.forward({
|
||
authorization: `Bearer ${credential.apiKey}`,
|
||
method: 'POST',
|
||
path: '/responses',
|
||
body: encode({ model: 'fixture-model', input: 'Inspect local code', stream: true, ...body }),
|
||
headers,
|
||
signal,
|
||
});
|
||
return { proxy, credential, request };
|
||
}
|
||
async function waitFor(check: () => Promise<boolean>) {
|
||
for (let count = 0; count < 100; count++) {
|
||
if (await check()) return;
|
||
await new Promise(resolve => setTimeout(resolve, 10));
|
||
}
|
||
throw new Error('Timed out waiting for proxy settlement');
|
||
}
|
||
describe('run-scoped Codex credential proxy', () => {
|
||
it('accepts bounded local namespaces and SDK metadata without treating tool schemas as media', async () => {
|
||
const { request } = await fixture((async (_url, init) => {
|
||
const data = JSON.parse(String(init?.body));
|
||
expect(data.client_metadata).toBeUndefined();
|
||
expect(data.tools[0].tools[0].type).toBe('function');
|
||
return sse([terminal()]);
|
||
}) as typeof fetch);
|
||
const body = {
|
||
client_metadata: { originator: 'fixture' },
|
||
tools: [
|
||
{
|
||
type: 'namespace',
|
||
name: 'local',
|
||
tools: [
|
||
{
|
||
type: 'function',
|
||
name: 'inspect',
|
||
parameters: { type: 'object', properties: { image_url: { type: 'string' } } },
|
||
},
|
||
],
|
||
},
|
||
],
|
||
};
|
||
expect((await request(body)).status).toBe(200);
|
||
await expect(
|
||
request({
|
||
tools: [{ type: 'namespace', name: 'bad', tools: [{ type: 'mcp', server_url: 'https://untrusted.example' }] }],
|
||
}),
|
||
).rejects.toMatchObject({ code: 'PROVIDER_REMOTE_TOOL_DENIED' });
|
||
await expect(
|
||
request({
|
||
input: [{ role: 'user', content: [{ type: 'input_image', image_url: 'https://untrusted.example/image' }] }],
|
||
}),
|
||
).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' });
|
||
});
|
||
|
||
it('retains quota at the instant of cancellation while upstream usage is unresolved', async () => {
|
||
db.state.get('alpha')!.tenant.tokenLimit = 100000;
|
||
const upstream = vi.fn(
|
||
async () =>
|
||
new Response(
|
||
new ReadableStream({
|
||
start(c) {
|
||
c.enqueue(
|
||
new TextEncoder().encode(
|
||
'event: response.created\ndata: {"type":"response.created","response":{"id":"pending"}}\n\n',
|
||
),
|
||
);
|
||
},
|
||
}),
|
||
{ headers: { 'content-type': 'text/event-stream' } },
|
||
),
|
||
);
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
const response = await request();
|
||
const reading = response.text().catch(() => 'aborted');
|
||
await control.command(owner, run.id, { action: 'cancel' }, 'cancel-before-settlement');
|
||
expect((await control.usage(owner)).reservedTokens).toBe(100000);
|
||
await expect(
|
||
control.submit(
|
||
owner,
|
||
run.projectId,
|
||
{ goal: 'Another run', acceptanceCriteria: ['Cannot borrow unresolved reservation'] },
|
||
'after-cancellation',
|
||
),
|
||
).rejects.toMatchObject({ code: 'QUOTA_EXCEEDED' });
|
||
await reading;
|
||
});
|
||
it('returns proxy credentials through the actual connection broker without exposing the stored key', async () => {
|
||
const connection = await connections.forRun(worker, run);
|
||
expect(connection).toMatchObject({ metering: 'proxy', baseUrl: env.WF_PROVIDER_PROXY_PUBLIC_URL });
|
||
expect(JSON.stringify(connection)).not.toContain(secret);
|
||
});
|
||
it('never puts the upstream key in the scoped token and prevents proxy credentials from authenticating to internal services', async () => {
|
||
const { credential } = await fixture(vi.fn());
|
||
expect(credential.metering).toBe('proxy');
|
||
expect(JSON.stringify(credential).includes(secret)).toBe(false);
|
||
const claims = decodeJwt(credential.apiKey);
|
||
expect(claims).toMatchObject({
|
||
tenantId: 'alpha',
|
||
runId: run.id,
|
||
connectionId,
|
||
leaseToken: run.lease!.token,
|
||
aud: 'wishforge-openai-proxy',
|
||
});
|
||
expect(Number(claims.exp) - Number(claims.iat)).toBeLessThanOrEqual(3600);
|
||
expect(await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).verify(credential.apiKey)).toBeUndefined();
|
||
expect(providerProxyConfigured({ ...env, NODE_ENV: 'production' })).toBe(false);
|
||
await expect(new ProviderProxy(db, { NODE_ENV: 'production' }).credential(worker, run)).rejects.toMatchObject({
|
||
code: 'PROVIDER_PROXY_UNCONFIGURED',
|
||
});
|
||
});
|
||
it('forwards to the operator’s OpenAI-compatible gateway when one is configured', async () => {
|
||
const gateway = { ...env, WF_OPENAI_BASE_URL: 'https://gateway.example/v1/' };
|
||
const upstream = vi.fn(async (url: any) => {
|
||
expect(String(url)).toBe('https://gateway.example/v1/responses');
|
||
return sse([terminal()]);
|
||
});
|
||
const proxy = new ProviderProxy(db, gateway, { fetcher: upstream as typeof fetch, pollMs: 25 });
|
||
const credential = await proxy.credential(worker, run);
|
||
const response = await proxy.forward({
|
||
authorization: `Bearer ${credential.apiKey}`,
|
||
method: 'POST',
|
||
path: '/responses',
|
||
body: encode({ model: 'fixture-model', input: 'Inspect local code', stream: true }),
|
||
});
|
||
await response.text();
|
||
expect(upstream).toHaveBeenCalledTimes(1);
|
||
// A gateway outside HTTPS, or with credentials in it, disables the proxy instead.
|
||
expect(providerProxyConfigured({ ...env, WF_OPENAI_BASE_URL: 'https://user:secret@gateway.example/v1' })).toBe(
|
||
false,
|
||
);
|
||
});
|
||
it('forwards only trusted endpoint/headers and streams split SSE frames with trusted usage recorded once', async () => {
|
||
const upstream = vi.fn(async (url: any, init: any) => {
|
||
expect(String(url)).toBe('https://api.openai.com/v1/responses');
|
||
expect(init.headers.Authorization === `Bearer ${secret}`).toBe(true);
|
||
expect(init.headers['X-Untrusted']).toBeUndefined();
|
||
const body = JSON.parse(init.body);
|
||
expect(body.store).toBe(false);
|
||
expect(body.background).toBe(false);
|
||
expect(body.max_output_tokens).toBeLessThanOrEqual(8192);
|
||
return sse([
|
||
{ type: 'response.created', response: { id: 'resp_fixture' } },
|
||
{ type: 'response.output_text.delta', delta: `hello ${secret}` },
|
||
terminal(),
|
||
]);
|
||
});
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
const first = await request({}, { 'X-Untrusted': 'not-forwarded' });
|
||
const body = await first.text();
|
||
expect(body).toContain('response.completed');
|
||
expect(body.includes(secret)).toBe(false);
|
||
expect(body).toContain('[redacted]');
|
||
const repeated = await request();
|
||
expect(await repeated.text()).toBe(body);
|
||
expect(upstream).toHaveBeenCalledTimes(1);
|
||
expect((await control.usage(owner)).tokens).toBe(15);
|
||
});
|
||
it('rejects tampered tenant tokens, wrong audience, stale generations and rotated provider connections before upstream I/O', async () => {
|
||
const upstream = vi.fn();
|
||
const { proxy, credential, request } = await fixture(upstream);
|
||
const token = await new SignJWT({
|
||
...decodeJwt<Record<string, unknown>>(String(credential.apiKey)),
|
||
tenantId: 'beta',
|
||
})
|
||
.setProtectedHeader({ alg: 'HS256' })
|
||
.sign(new TextEncoder().encode(env.WF_PROVIDER_PROXY_SIGNING_KEY));
|
||
await expect(
|
||
proxy.forward({ authorization: `Bearer ${token}`, method: 'GET', path: '/models' }),
|
||
).rejects.toMatchObject({ code: 'NOT_FOUND' });
|
||
const internal = await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).issue(
|
||
'alpha',
|
||
run.id,
|
||
'worker',
|
||
'generation',
|
||
);
|
||
await expect(
|
||
proxy.forward({ authorization: `Bearer ${internal}`, method: 'GET', path: '/models' }),
|
||
).rejects.toMatchObject({ code: 'PROVIDER_PROXY_UNAUTHENTICATED' });
|
||
await connections.revoke(owner, connectionId);
|
||
await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_CONNECTION_REVOKED' });
|
||
expect(upstream).not.toHaveBeenCalled();
|
||
});
|
||
it('denies arbitrary upstream paths, server-side tools/state, unapproved models and multimodal input', async () => {
|
||
const upstream = vi.fn();
|
||
const { proxy, credential, request } = await fixture(upstream);
|
||
await expect(
|
||
proxy.forward({
|
||
authorization: `Bearer ${credential.apiKey}`,
|
||
method: 'POST',
|
||
path: 'https://attacker.invalid/v1/responses',
|
||
body: encode({}),
|
||
}),
|
||
).rejects.toMatchObject({ code: 'PROVIDER_ROUTE_DENIED' });
|
||
await expect(request({ tools: [{ type: 'mcp', server_url: 'https://attacker.invalid' }] })).rejects.toMatchObject({
|
||
code: 'PROVIDER_REMOTE_TOOL_DENIED',
|
||
});
|
||
await expect(request({ previous_response_id: 'other-tenants-response' })).rejects.toMatchObject({
|
||
code: 'PROVIDER_STATEFUL_REQUEST_DENIED',
|
||
});
|
||
await expect(
|
||
request({ input: [{ type: 'input_image', image_url: 'https://attacker.invalid/image' }] }),
|
||
).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' });
|
||
await expect(request({ model: 'not-approved' })).rejects.toMatchObject({ code: 'PROVIDER_MODEL_DENIED' });
|
||
expect(upstream).not.toHaveBeenCalled();
|
||
});
|
||
it('serializes a run and stops a live stream when the user cancels', async () => {
|
||
let canceled = false;
|
||
const upstream = vi.fn(
|
||
async () =>
|
||
new Response(
|
||
new ReadableStream({
|
||
start(controller) {
|
||
controller.enqueue(
|
||
new TextEncoder().encode(
|
||
'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n',
|
||
),
|
||
);
|
||
},
|
||
cancel() {
|
||
canceled = true;
|
||
},
|
||
}),
|
||
{ headers: { 'content-type': 'text/event-stream' } },
|
||
),
|
||
);
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
const response = await request();
|
||
const reading = response.text().catch(() => 'aborted');
|
||
await expect(request({ input: 'another prompt' })).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' });
|
||
await control.command(owner, run.id, { action: 'cancel' }, 'cancel-proxy-request');
|
||
expect(await reading).toBe('aborted');
|
||
await waitFor(async () => canceled);
|
||
expect((await control.run(owner, run.id)).status).toBe('canceled');
|
||
expect((await control.run(owner, run.id)).reservedTokens).toBeGreaterThan(0);
|
||
const operations = await db.transaction('alpha', tx => tx.list<ProxyRequestRecord>('provider_proxy_request'));
|
||
expect(operations[0].status).toBe('unknown');
|
||
expect(upstream).toHaveBeenCalledTimes(1);
|
||
});
|
||
it('preserves unknown outcome/reservation after a truncated stream and does not resend it', async () => {
|
||
const upstream = vi.fn(async () => sse([{ type: 'response.output_text.delta', delta: 'partial output' }]));
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
await expect((await request()).text()).rejects.toThrow();
|
||
expect(await control.run(owner, run.id)).toMatchObject({
|
||
status: 'waiting',
|
||
waitingReason: 'ambiguous_execution',
|
||
});
|
||
await control.command(owner, run.id, { action: 'resume' }, 'resume-unknown-proxy');
|
||
const leased = await control.lease(worker, run.id, 'recovery-worker');
|
||
const proxy = new ProviderProxy(db, env, { fetcher: upstream as typeof fetch });
|
||
const next = await proxy.credential(worker, leased);
|
||
await expect(
|
||
proxy.forward({
|
||
authorization: `Bearer ${next.apiKey}`,
|
||
method: 'POST',
|
||
path: '/responses',
|
||
body: encode({ model: 'fixture-model', input: 'retry', stream: true }),
|
||
}),
|
||
).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' });
|
||
expect(upstream).toHaveBeenCalledTimes(1);
|
||
|
||
// An owner reconciles the unknown request; the run can call the model again (FR-C6).
|
||
const [unknown] = await proxy.requests(owner, run.id);
|
||
expect(unknown).toMatchObject({ status: 'unknown', active: true });
|
||
await expect(
|
||
proxy.reconcile({ ...owner, kind: 'connector' }, run.id, { requestId: unknown!.id }),
|
||
).rejects.toMatchObject({ status: 403 });
|
||
expect(await proxy.reconcile(owner, run.id, { requestId: unknown!.id, usageTokens: 42 })).toEqual({
|
||
requestId: unknown!.id,
|
||
usageTokens: 42,
|
||
});
|
||
await expect(proxy.reconcile(owner, run.id, { requestId: unknown!.id })).rejects.toMatchObject({
|
||
code: 'PROVIDER_REQUEST_SETTLED',
|
||
});
|
||
const settled = await control.run(owner, run.id);
|
||
expect(settled.providerProxyActiveRequest).toBeUndefined();
|
||
expect(settled.usedTokens).toBe(42);
|
||
upstream.mockImplementation(async () => sse([terminal()]));
|
||
const retried = await proxy.forward({
|
||
authorization: `Bearer ${next.apiKey}`,
|
||
method: 'POST',
|
||
path: '/responses',
|
||
body: encode({ model: 'fixture-model', input: 'retry after reconciliation', stream: true }),
|
||
});
|
||
expect(retried.status).toBe(200);
|
||
await retried.text();
|
||
expect(upstream).toHaveBeenCalledTimes(2);
|
||
});
|
||
it('rejects an unaffordable request before any model call and meters only actual upstream usage', async () => {
|
||
const upstream = vi.fn(async () => sse([terminal(50000, 50000)]));
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
await (await request()).text();
|
||
expect(await control.run(owner, run.id)).toMatchObject({
|
||
status: 'waiting',
|
||
waitingReason: 'budget',
|
||
usedTokens: 100000,
|
||
reservedTokens: 0,
|
||
});
|
||
await expect(request({ input: 'further work' })).rejects.toThrow();
|
||
expect(upstream).toHaveBeenCalledTimes(1);
|
||
});
|
||
it('refuses insufficient input/output reservation before contacting the provider', async () => {
|
||
const upstream = vi.fn();
|
||
const { request } = await fixture(upstream);
|
||
await db.transaction('alpha', async tx => {
|
||
const current = (await tx.get<Run>('run', run.id))!;
|
||
current.usedTokens = 99000;
|
||
current.reservedTokens = 1000;
|
||
await tx.put('run', current);
|
||
});
|
||
await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_BUDGET_EXHAUSTED' });
|
||
expect(upstream).not.toHaveBeenCalled();
|
||
// The run says why it stopped instead of leaving the executor to report a provider failure.
|
||
const waiting = await control.run(owner, run.id);
|
||
expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'budget', usedTokens: 99000 });
|
||
expect(waiting.waitingDetail).toMatch(/needs \d+ tokens of budget .* but 1000 of 100000 remain/);
|
||
expect(Date.parse(waiting.lease!.expiresAt)).toBeLessThanOrEqual(Date.now());
|
||
expect((await control.events(owner, run.id)).at(-1)).toMatchObject({ type: 'run.waiting' });
|
||
});
|
||
it('aborts an in-flight model stream after provider connection rotation', async () => {
|
||
let canceled = false;
|
||
const upstream = vi.fn(
|
||
async () =>
|
||
new Response(
|
||
new ReadableStream({
|
||
start(c) {
|
||
c.enqueue(
|
||
new TextEncoder().encode(
|
||
'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n',
|
||
),
|
||
);
|
||
},
|
||
cancel() {
|
||
canceled = true;
|
||
},
|
||
}),
|
||
{ headers: { 'content-type': 'text/event-stream' } },
|
||
),
|
||
);
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
const reading = (await request()).text().catch(() => 'aborted');
|
||
await connections.create(owner, { provider: 'codex', label: 'rotated', apiKey: 'replacement_fixture_private_key' });
|
||
expect(await reading).toBe('aborted');
|
||
expect(canceled).toBe(true);
|
||
expect((await control.run(owner, run.id)).waitingReason).toBe('ambiguous_execution');
|
||
});
|
||
it('binds production proxy credentials to execution generation and fails before decrypting if it changes', async () => {
|
||
const production = {
|
||
...env,
|
||
NODE_ENV: 'production',
|
||
WF_PROVIDER_PROXY_PUBLIC_URL: 'https://api.example/provider-proxy/v1',
|
||
};
|
||
const actor = { ...worker, runId: run.id, credentialId: 'fixture-generation' };
|
||
await db.transaction('alpha', async tx => {
|
||
const current = (await tx.get<Run>('run', run.id))!;
|
||
current.executionGeneration = current.pendingStartId = 'fixture-generation';
|
||
await tx.put('run', current);
|
||
});
|
||
const open = vi.fn(async () => secret),
|
||
fetcher = vi.fn();
|
||
const proxy = new ProviderProxy(db, production, { vault: { open }, fetcher });
|
||
const token = await proxy.credential(actor, run);
|
||
await db.transaction('alpha', async tx => {
|
||
const current = (await tx.get<Run>('run', run.id))!;
|
||
current.executionGeneration = 'replacement-generation';
|
||
await tx.put('run', current);
|
||
});
|
||
await expect(
|
||
proxy.forward({ authorization: `Bearer ${token.apiKey}`, method: 'GET', path: '/models' }),
|
||
).rejects.toMatchObject({ code: 'EXECUTION_REVOKED' });
|
||
expect(open).not.toHaveBeenCalled();
|
||
expect(fetcher).not.toHaveBeenCalled();
|
||
});
|
||
it('handles JSON Responses and filters model listings without reflecting upstream secret/error headers', async () => {
|
||
const upstream = vi.fn(async (url: any) =>
|
||
String(url).endsWith('/models')
|
||
? new Response(JSON.stringify({ data: [{ id: 'fixture-model' }, { id: 'expensive-other-model' }] }), {
|
||
headers: { 'content-type': 'application/json', 'x-secret': secret },
|
||
})
|
||
: new Response(JSON.stringify(terminal().response), {
|
||
headers: { 'content-type': 'application/json', 'x-secret': secret },
|
||
}),
|
||
);
|
||
const { request, proxy, credential } = await fixture(upstream as typeof fetch);
|
||
const response = await request({ stream: false });
|
||
expect((await response.json()).usage.total_tokens).toBe(15);
|
||
expect(response.headers.has('x-secret')).toBe(false);
|
||
const models = await proxy.forward({
|
||
authorization: `Bearer ${credential.apiKey}`,
|
||
method: 'GET',
|
||
path: '/models',
|
||
});
|
||
expect((await models.json()).data).toEqual([{ id: 'fixture-model' }]);
|
||
});
|
||
it('rejects changed input with the same idempotency key rather than dispatching again', async () => {
|
||
const upstream = vi.fn(async () => sse([terminal()]));
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
await (await request({}, { 'idempotency-key': 'same-request-key' })).text();
|
||
await expect(request({ input: 'changed' }, { 'idempotency-key': 'same-request-key' })).rejects.toMatchObject({
|
||
code: 'PROVIDER_IDEMPOTENCY_CONFLICT',
|
||
});
|
||
expect(upstream).toHaveBeenCalledTimes(1);
|
||
});
|
||
it('treats x-client-request-id as a conversation id, not as a key for one request', async () => {
|
||
// Codex sends its conversation id in this header on every request of a session.
|
||
const upstream = vi.fn(async () => sse([terminal()]));
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
await (await request({}, { 'x-client-request-id': 'conversation-0001' })).text();
|
||
await (await request({ input: 'the next turn' }, { 'x-client-request-id': 'conversation-0001' })).text();
|
||
expect(upstream).toHaveBeenCalledTimes(2);
|
||
});
|
||
it('keeps a stream open past the idle limit while the provider keeps sending', async () => {
|
||
// 450 ms in all, past the idle limit, but never more than 30 ms without a frame.
|
||
const upstream = vi.fn(async () => trickle([...Array(14).fill(delta), terminal()], 30));
|
||
const { request } = await fixture(upstream as typeof fetch, { idleMs: 300 });
|
||
const response = await request();
|
||
expect(await response.text()).toContain('response.completed');
|
||
expect((await operations())[0]).toMatchObject({ status: 'completed', usageTokens: 15 });
|
||
expect((await control.run(owner, run.id)).status).toBe('running');
|
||
});
|
||
it('stops a stream that falls silent, or runs past the deadline, and holds it for reconciliation', async () => {
|
||
const silent = vi.fn(async () => trickle([delta], 10));
|
||
const { request } = await fixture(silent as typeof fetch, { idleMs: 100 });
|
||
await expect((await request()).text()).rejects.toThrow();
|
||
expect((await operations())[0]).toMatchObject({ status: 'unknown', unknownReason: 'upstream_idle' });
|
||
expect(await control.run(owner, run.id)).toMatchObject({
|
||
status: 'waiting',
|
||
waitingReason: 'ambiguous_execution',
|
||
});
|
||
|
||
const [unknown] = await operations();
|
||
await new ProviderProxy(db, env).reconcile(owner, run.id, { requestId: unknown!.id, usageTokens: 0 });
|
||
await control.command(owner, run.id, { action: 'resume' }, 'resume-after-silent-stream');
|
||
run = await control.lease(worker, run.id, 'deadline-worker');
|
||
const endless = vi.fn(async () => trickle(Array(100).fill(delta), 20));
|
||
// Frames keep coming well within the idle limit, so only the deadline can stop this one.
|
||
const deadline = await fixture(endless as typeof fetch, { idleMs: 1000, deadlineMs: 300 });
|
||
await expect((await deadline.request({ input: 'a long answer' })).text()).rejects.toThrow();
|
||
expect((await operations()).find(op => op.id !== unknown!.id)).toMatchObject({
|
||
status: 'unknown',
|
||
unknownReason: 'upstream_deadline',
|
||
});
|
||
});
|
||
it('sends a retry again after the provider rejected the request, rather than replaying the rejection', async () => {
|
||
const upstream = vi.fn(async () => sse([terminal()]));
|
||
upstream.mockImplementationOnce(
|
||
async () =>
|
||
new Response(JSON.stringify({ error: { message: 'Rate limit reached' } }), {
|
||
status: 429,
|
||
headers: { 'content-type': 'application/json' },
|
||
}),
|
||
);
|
||
const { request } = await fixture(upstream as typeof fetch);
|
||
expect((await request()).status).toBe(429);
|
||
const retried = await request();
|
||
expect(retried.status).toBe(200);
|
||
await retried.text();
|
||
expect(await operations()).toEqual([expect.objectContaining({ status: 'completed', usageTokens: 15 })]);
|
||
|
||
// A request stopped before it reached the provider is sent when retried, too.
|
||
const stopped = new AbortController();
|
||
stopped.abort();
|
||
await expect(request({ input: 'the next turn' }, undefined, stopped.signal)).rejects.toMatchObject({
|
||
code: 'PROVIDER_REQUEST_CANCELED',
|
||
});
|
||
const sent = await request({ input: 'the next turn' });
|
||
expect(sent.status).toBe(200);
|
||
await sent.text();
|
||
expect(upstream).toHaveBeenCalledTimes(3);
|
||
expect((await control.run(owner, run.id)).usedTokens).toBe(30);
|
||
});
|
||
});
|