import { beforeEach, describe, expect, it, vi } from 'vitest'; import { SignJWT, decodeJwt } from 'jose'; import type { Actor, Run } from '@wishforge/contracts'; import { ControlPlane } from '../apps/api/src/domain/control-plane.js'; import { Connections } from '../apps/api/src/domain/connections.js'; import { ProviderProxy, providerProxyConfigured, type ProxyRequestRecord, } from '../apps/api/src/domain/provider-proxy.js'; import { ServiceTokens } from '../apps/api/src/service-tokens.js'; import { MemoryStore } from './helpers/memory-store.js'; const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' }, worker: Actor = { ...owner, userId: 'worker', kind: 'worker' }; const secret = 'fixture_upstream_key_kept_inside_control_plane'; const env = { NODE_ENV: 'test', WF_VAULT_KEY: 'a'.repeat(64), WF_PROVIDER_PROXY_ENABLED: 'true', WF_PROVIDER_PROXY_PUBLIC_URL: 'http://127.0.0.1:4000/provider-proxy/v1', WF_PROVIDER_PROXY_SIGNING_KEY: 'proxy-independent-signing-fixture-key-12345', WF_PROVIDER_PROXY_ALLOWED_MODELS: 'fixture-model', }; const encode = (data: unknown) => new TextEncoder().encode(JSON.stringify(data)); const terminal = (input = 10, output = 5) => ({ type: 'response.completed', response: { id: 'resp_fixture', status: 'completed', usage: { input_tokens: input, output_tokens: output, total_tokens: input + output }, }, }); function sse(events: unknown[], chunkSize = 7) { const all = Buffer.from( events.map(value => `event: ${(value as any).type}\r\ndata: ${JSON.stringify(value)}\r\n\r\n`).join(''), ); let offset = 0; return new Response( new ReadableStream({ pull(controller) { if (offset === all.length) controller.close(); else { controller.enqueue(all.subarray(offset, offset + chunkSize)); offset = Math.min(all.length, offset + chunkSize); } }, }), { headers: { 'content-type': 'text/event-stream' } }, ); } /** An event stream that sends one event every `everyMs`, then sends nothing more and stays open. */ function trickle(events: unknown[], everyMs: number) { let sent = 0; return new Response( new ReadableStream({ async pull(controller) { if (sent === events.length) return new Promise(() => undefined); await new Promise(resolve => setTimeout(resolve, everyMs)); const value = events[sent++] as { type: string }; controller.enqueue(new TextEncoder().encode(`event: ${value.type}\ndata: ${JSON.stringify(value)}\n\n`)); }, }), { headers: { 'content-type': 'text/event-stream' } }, ); } const delta = { type: 'response.output_text.delta', delta: 'thinking' }; const operations = () => db.transaction('alpha', tx => tx.list('provider_proxy_request')); let db: MemoryStore, control: ControlPlane, connections: Connections, run: Run, connectionId: string; beforeEach(async () => { db = new MemoryStore(); db.seed('alpha'); db.seed('beta'); control = new ControlPlane(db, env); connections = new Connections(db, env); connectionId = (await connections.create(owner, { provider: 'codex', label: 'fixture', apiKey: secret })).id; const project = await control.createProject(owner, { name: 'Proxy fixture', repositoryUrl: 'https://example.org/fixture.git', }); const wish = await control.submit( owner, project.id, { goal: 'Test scoped proxy', acceptanceCriteria: ['Credential remains private'] }, 'proxy-fixture-submit', ); run = await control.lease(worker, wish.run_id, 'fixture-worker'); }); async function fixture(fetcher: typeof fetch, timing: { idleMs?: number; deadlineMs?: number } = {}) { const proxy = new ProviderProxy(db, env, { fetcher, pollMs: 25, ...timing }); const credential = await proxy.credential(worker, run); const request = (body: Record = {}, headers?: Record, signal?: AbortSignal) => proxy.forward({ authorization: `Bearer ${credential.apiKey}`, method: 'POST', path: '/responses', body: encode({ model: 'fixture-model', input: 'Inspect local code', stream: true, ...body }), headers, signal, }); return { proxy, credential, request }; } async function waitFor(check: () => Promise) { for (let count = 0; count < 100; count++) { if (await check()) return; await new Promise(resolve => setTimeout(resolve, 10)); } throw new Error('Timed out waiting for proxy settlement'); } describe('run-scoped Codex credential proxy', () => { it('accepts bounded local namespaces and SDK metadata without treating tool schemas as media', async () => { const { request } = await fixture((async (_url, init) => { const data = JSON.parse(String(init?.body)); expect(data.client_metadata).toBeUndefined(); expect(data.tools[0].tools[0].type).toBe('function'); return sse([terminal()]); }) as typeof fetch); const body = { client_metadata: { originator: 'fixture' }, tools: [ { type: 'namespace', name: 'local', tools: [ { type: 'function', name: 'inspect', parameters: { type: 'object', properties: { image_url: { type: 'string' } } }, }, ], }, ], }; expect((await request(body)).status).toBe(200); await expect( request({ tools: [{ type: 'namespace', name: 'bad', tools: [{ type: 'mcp', server_url: 'https://untrusted.example' }] }], }), ).rejects.toMatchObject({ code: 'PROVIDER_REMOTE_TOOL_DENIED' }); await expect( request({ input: [{ role: 'user', content: [{ type: 'input_image', image_url: 'https://untrusted.example/image' }] }], }), ).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' }); }); it('retains quota at the instant of cancellation while upstream usage is unresolved', async () => { db.state.get('alpha')!.tenant.tokenLimit = 100000; const upstream = vi.fn( async () => new Response( new ReadableStream({ start(c) { c.enqueue( new TextEncoder().encode( 'event: response.created\ndata: {"type":"response.created","response":{"id":"pending"}}\n\n', ), ); }, }), { headers: { 'content-type': 'text/event-stream' } }, ), ); const { request } = await fixture(upstream as typeof fetch); const response = await request(); const reading = response.text().catch(() => 'aborted'); await control.command(owner, run.id, { action: 'cancel' }, 'cancel-before-settlement'); expect((await control.usage(owner)).reservedTokens).toBe(100000); await expect( control.submit( owner, run.projectId, { goal: 'Another run', acceptanceCriteria: ['Cannot borrow unresolved reservation'] }, 'after-cancellation', ), ).rejects.toMatchObject({ code: 'QUOTA_EXCEEDED' }); await reading; }); it('returns proxy credentials through the actual connection broker without exposing the stored key', async () => { const connection = await connections.forRun(worker, run); expect(connection).toMatchObject({ metering: 'proxy', baseUrl: env.WF_PROVIDER_PROXY_PUBLIC_URL }); expect(JSON.stringify(connection)).not.toContain(secret); }); it('never puts the upstream key in the scoped token and prevents proxy credentials from authenticating to internal services', async () => { const { credential } = await fixture(vi.fn()); expect(credential.metering).toBe('proxy'); expect(JSON.stringify(credential).includes(secret)).toBe(false); const claims = decodeJwt(credential.apiKey); expect(claims).toMatchObject({ tenantId: 'alpha', runId: run.id, connectionId, leaseToken: run.lease!.token, aud: 'wishforge-openai-proxy', }); expect(Number(claims.exp) - Number(claims.iat)).toBeLessThanOrEqual(3600); expect(await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).verify(credential.apiKey)).toBeUndefined(); expect(providerProxyConfigured({ ...env, NODE_ENV: 'production' })).toBe(false); await expect(new ProviderProxy(db, { NODE_ENV: 'production' }).credential(worker, run)).rejects.toMatchObject({ code: 'PROVIDER_PROXY_UNCONFIGURED', }); }); it('forwards to the operator’s OpenAI-compatible gateway when one is configured', async () => { const gateway = { ...env, WF_OPENAI_BASE_URL: 'https://gateway.example/v1/' }; const upstream = vi.fn(async (url: any) => { expect(String(url)).toBe('https://gateway.example/v1/responses'); return sse([terminal()]); }); const proxy = new ProviderProxy(db, gateway, { fetcher: upstream as typeof fetch, pollMs: 25 }); const credential = await proxy.credential(worker, run); const response = await proxy.forward({ authorization: `Bearer ${credential.apiKey}`, method: 'POST', path: '/responses', body: encode({ model: 'fixture-model', input: 'Inspect local code', stream: true }), }); await response.text(); expect(upstream).toHaveBeenCalledTimes(1); // A gateway outside HTTPS, or with credentials in it, disables the proxy instead. expect(providerProxyConfigured({ ...env, WF_OPENAI_BASE_URL: 'https://user:secret@gateway.example/v1' })).toBe( false, ); }); it('forwards only trusted endpoint/headers and streams split SSE frames with trusted usage recorded once', async () => { const upstream = vi.fn(async (url: any, init: any) => { expect(String(url)).toBe('https://api.openai.com/v1/responses'); expect(init.headers.Authorization === `Bearer ${secret}`).toBe(true); expect(init.headers['X-Untrusted']).toBeUndefined(); const body = JSON.parse(init.body); expect(body.store).toBe(false); expect(body.background).toBe(false); expect(body.max_output_tokens).toBeLessThanOrEqual(8192); return sse([ { type: 'response.created', response: { id: 'resp_fixture' } }, { type: 'response.output_text.delta', delta: `hello ${secret}` }, terminal(), ]); }); const { request } = await fixture(upstream as typeof fetch); const first = await request({}, { 'X-Untrusted': 'not-forwarded' }); const body = await first.text(); expect(body).toContain('response.completed'); expect(body.includes(secret)).toBe(false); expect(body).toContain('[redacted]'); const repeated = await request(); expect(await repeated.text()).toBe(body); expect(upstream).toHaveBeenCalledTimes(1); expect((await control.usage(owner)).tokens).toBe(15); }); it('rejects tampered tenant tokens, wrong audience, stale generations and rotated provider connections before upstream I/O', async () => { const upstream = vi.fn(); const { proxy, credential, request } = await fixture(upstream); const token = await new SignJWT({ ...decodeJwt>(String(credential.apiKey)), tenantId: 'beta', }) .setProtectedHeader({ alg: 'HS256' }) .sign(new TextEncoder().encode(env.WF_PROVIDER_PROXY_SIGNING_KEY)); await expect( proxy.forward({ authorization: `Bearer ${token}`, method: 'GET', path: '/models' }), ).rejects.toMatchObject({ code: 'NOT_FOUND' }); const internal = await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).issue( 'alpha', run.id, 'worker', 'generation', ); await expect( proxy.forward({ authorization: `Bearer ${internal}`, method: 'GET', path: '/models' }), ).rejects.toMatchObject({ code: 'PROVIDER_PROXY_UNAUTHENTICATED' }); await connections.revoke(owner, connectionId); await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_CONNECTION_REVOKED' }); expect(upstream).not.toHaveBeenCalled(); }); it('denies arbitrary upstream paths, server-side tools/state, unapproved models and multimodal input', async () => { const upstream = vi.fn(); const { proxy, credential, request } = await fixture(upstream); await expect( proxy.forward({ authorization: `Bearer ${credential.apiKey}`, method: 'POST', path: 'https://attacker.invalid/v1/responses', body: encode({}), }), ).rejects.toMatchObject({ code: 'PROVIDER_ROUTE_DENIED' }); await expect(request({ tools: [{ type: 'mcp', server_url: 'https://attacker.invalid' }] })).rejects.toMatchObject({ code: 'PROVIDER_REMOTE_TOOL_DENIED', }); await expect(request({ previous_response_id: 'other-tenants-response' })).rejects.toMatchObject({ code: 'PROVIDER_STATEFUL_REQUEST_DENIED', }); await expect( request({ input: [{ type: 'input_image', image_url: 'https://attacker.invalid/image' }] }), ).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' }); await expect(request({ model: 'not-approved' })).rejects.toMatchObject({ code: 'PROVIDER_MODEL_DENIED' }); expect(upstream).not.toHaveBeenCalled(); }); it('serializes a run and stops a live stream when the user cancels', async () => { let canceled = false; const upstream = vi.fn( async () => new Response( new ReadableStream({ start(controller) { controller.enqueue( new TextEncoder().encode( 'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n', ), ); }, cancel() { canceled = true; }, }), { headers: { 'content-type': 'text/event-stream' } }, ), ); const { request } = await fixture(upstream as typeof fetch); const response = await request(); const reading = response.text().catch(() => 'aborted'); await expect(request({ input: 'another prompt' })).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' }); await control.command(owner, run.id, { action: 'cancel' }, 'cancel-proxy-request'); expect(await reading).toBe('aborted'); await waitFor(async () => canceled); expect((await control.run(owner, run.id)).status).toBe('canceled'); expect((await control.run(owner, run.id)).reservedTokens).toBeGreaterThan(0); const operations = await db.transaction('alpha', tx => tx.list('provider_proxy_request')); expect(operations[0].status).toBe('unknown'); expect(upstream).toHaveBeenCalledTimes(1); }); it('preserves unknown outcome/reservation after a truncated stream and does not resend it', async () => { const upstream = vi.fn(async () => sse([{ type: 'response.output_text.delta', delta: 'partial output' }])); const { request } = await fixture(upstream as typeof fetch); await expect((await request()).text()).rejects.toThrow(); expect(await control.run(owner, run.id)).toMatchObject({ status: 'waiting', waitingReason: 'ambiguous_execution', }); await control.command(owner, run.id, { action: 'resume' }, 'resume-unknown-proxy'); const leased = await control.lease(worker, run.id, 'recovery-worker'); const proxy = new ProviderProxy(db, env, { fetcher: upstream as typeof fetch }); const next = await proxy.credential(worker, leased); await expect( proxy.forward({ authorization: `Bearer ${next.apiKey}`, method: 'POST', path: '/responses', body: encode({ model: 'fixture-model', input: 'retry', stream: true }), }), ).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' }); expect(upstream).toHaveBeenCalledTimes(1); // An owner reconciles the unknown request; the run can call the model again (FR-C6). const [unknown] = await proxy.requests(owner, run.id); expect(unknown).toMatchObject({ status: 'unknown', active: true }); await expect( proxy.reconcile({ ...owner, kind: 'connector' }, run.id, { requestId: unknown!.id }), ).rejects.toMatchObject({ status: 403 }); expect(await proxy.reconcile(owner, run.id, { requestId: unknown!.id, usageTokens: 42 })).toEqual({ requestId: unknown!.id, usageTokens: 42, }); await expect(proxy.reconcile(owner, run.id, { requestId: unknown!.id })).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_SETTLED', }); const settled = await control.run(owner, run.id); expect(settled.providerProxyActiveRequest).toBeUndefined(); expect(settled.usedTokens).toBe(42); upstream.mockImplementation(async () => sse([terminal()])); const retried = await proxy.forward({ authorization: `Bearer ${next.apiKey}`, method: 'POST', path: '/responses', body: encode({ model: 'fixture-model', input: 'retry after reconciliation', stream: true }), }); expect(retried.status).toBe(200); await retried.text(); expect(upstream).toHaveBeenCalledTimes(2); }); it('rejects an unaffordable request before any model call and meters only actual upstream usage', async () => { const upstream = vi.fn(async () => sse([terminal(50000, 50000)])); const { request } = await fixture(upstream as typeof fetch); await (await request()).text(); expect(await control.run(owner, run.id)).toMatchObject({ status: 'waiting', waitingReason: 'budget', usedTokens: 100000, reservedTokens: 0, }); await expect(request({ input: 'further work' })).rejects.toThrow(); expect(upstream).toHaveBeenCalledTimes(1); }); it('refuses insufficient input/output reservation before contacting the provider', async () => { const upstream = vi.fn(); const { request } = await fixture(upstream); await db.transaction('alpha', async tx => { const current = (await tx.get('run', run.id))!; current.usedTokens = 99000; current.reservedTokens = 1000; await tx.put('run', current); }); await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_BUDGET_EXHAUSTED' }); expect(upstream).not.toHaveBeenCalled(); // The run says why it stopped instead of leaving the executor to report a provider failure. const waiting = await control.run(owner, run.id); expect(waiting).toMatchObject({ status: 'waiting', waitingReason: 'budget', usedTokens: 99000 }); expect(waiting.waitingDetail).toMatch(/需要 \d+ 个 token 的预算.*100000 个中只剩 1000 个/); expect(Date.parse(waiting.lease!.expiresAt)).toBeLessThanOrEqual(Date.now()); expect((await control.events(owner, run.id)).at(-1)).toMatchObject({ type: 'run.waiting' }); }); it('aborts an in-flight model stream after provider connection rotation', async () => { let canceled = false; const upstream = vi.fn( async () => new Response( new ReadableStream({ start(c) { c.enqueue( new TextEncoder().encode( 'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n', ), ); }, cancel() { canceled = true; }, }), { headers: { 'content-type': 'text/event-stream' } }, ), ); const { request } = await fixture(upstream as typeof fetch); const reading = (await request()).text().catch(() => 'aborted'); await connections.create(owner, { provider: 'codex', label: 'rotated', apiKey: 'replacement_fixture_private_key' }); expect(await reading).toBe('aborted'); expect(canceled).toBe(true); expect((await control.run(owner, run.id)).waitingReason).toBe('ambiguous_execution'); }); it('binds production proxy credentials to execution generation and fails before decrypting if it changes', async () => { const production = { ...env, NODE_ENV: 'production', WF_PROVIDER_PROXY_PUBLIC_URL: 'https://api.example/provider-proxy/v1', }; const actor = { ...worker, runId: run.id, credentialId: 'fixture-generation' }; await db.transaction('alpha', async tx => { const current = (await tx.get('run', run.id))!; current.executionGeneration = current.pendingStartId = 'fixture-generation'; await tx.put('run', current); }); const open = vi.fn(async () => secret), fetcher = vi.fn(); const proxy = new ProviderProxy(db, production, { vault: { open }, fetcher }); const token = await proxy.credential(actor, run); await db.transaction('alpha', async tx => { const current = (await tx.get('run', run.id))!; current.executionGeneration = 'replacement-generation'; await tx.put('run', current); }); await expect( proxy.forward({ authorization: `Bearer ${token.apiKey}`, method: 'GET', path: '/models' }), ).rejects.toMatchObject({ code: 'EXECUTION_REVOKED' }); expect(open).not.toHaveBeenCalled(); expect(fetcher).not.toHaveBeenCalled(); }); it('handles JSON Responses and filters model listings without reflecting upstream secret/error headers', async () => { const upstream = vi.fn(async (url: any) => String(url).endsWith('/models') ? new Response(JSON.stringify({ data: [{ id: 'fixture-model' }, { id: 'expensive-other-model' }] }), { headers: { 'content-type': 'application/json', 'x-secret': secret }, }) : new Response(JSON.stringify(terminal().response), { headers: { 'content-type': 'application/json', 'x-secret': secret }, }), ); const { request, proxy, credential } = await fixture(upstream as typeof fetch); const response = await request({ stream: false }); expect((await response.json()).usage.total_tokens).toBe(15); expect(response.headers.has('x-secret')).toBe(false); const models = await proxy.forward({ authorization: `Bearer ${credential.apiKey}`, method: 'GET', path: '/models', }); expect((await models.json()).data).toEqual([{ id: 'fixture-model' }]); }); it('rejects changed input with the same idempotency key rather than dispatching again', async () => { const upstream = vi.fn(async () => sse([terminal()])); const { request } = await fixture(upstream as typeof fetch); await (await request({}, { 'idempotency-key': 'same-request-key' })).text(); await expect(request({ input: 'changed' }, { 'idempotency-key': 'same-request-key' })).rejects.toMatchObject({ code: 'PROVIDER_IDEMPOTENCY_CONFLICT', }); expect(upstream).toHaveBeenCalledTimes(1); }); it('treats x-client-request-id as a conversation id, not as a key for one request', async () => { // Codex sends its conversation id in this header on every request of a session. const upstream = vi.fn(async () => sse([terminal()])); const { request } = await fixture(upstream as typeof fetch); await (await request({}, { 'x-client-request-id': 'conversation-0001' })).text(); await (await request({ input: 'the next turn' }, { 'x-client-request-id': 'conversation-0001' })).text(); expect(upstream).toHaveBeenCalledTimes(2); }); it('keeps a stream open past the idle limit while the provider keeps sending', async () => { // 450 ms in all, past the idle limit, but never more than 30 ms without a frame. const upstream = vi.fn(async () => trickle([...Array(14).fill(delta), terminal()], 30)); const { request } = await fixture(upstream as typeof fetch, { idleMs: 300 }); const response = await request(); expect(await response.text()).toContain('response.completed'); expect((await operations())[0]).toMatchObject({ status: 'completed', usageTokens: 15 }); expect((await control.run(owner, run.id)).status).toBe('running'); }); it('stops a stream that falls silent, or runs past the deadline, and holds it for reconciliation', async () => { const silent = vi.fn(async () => trickle([delta], 10)); const { request } = await fixture(silent as typeof fetch, { idleMs: 100 }); await expect((await request()).text()).rejects.toThrow(); expect((await operations())[0]).toMatchObject({ status: 'unknown', unknownReason: 'upstream_idle' }); expect(await control.run(owner, run.id)).toMatchObject({ status: 'waiting', waitingReason: 'ambiguous_execution', }); const [unknown] = await operations(); await new ProviderProxy(db, env).reconcile(owner, run.id, { requestId: unknown!.id, usageTokens: 0 }); await control.command(owner, run.id, { action: 'resume' }, 'resume-after-silent-stream'); run = await control.lease(worker, run.id, 'deadline-worker'); const endless = vi.fn(async () => trickle(Array(100).fill(delta), 20)); // Frames keep coming well within the idle limit, so only the deadline can stop this one. const deadline = await fixture(endless as typeof fetch, { idleMs: 1000, deadlineMs: 300 }); await expect((await deadline.request({ input: 'a long answer' })).text()).rejects.toThrow(); expect((await operations()).find(op => op.id !== unknown!.id)).toMatchObject({ status: 'unknown', unknownReason: 'upstream_deadline', }); }); it('sends a retry again after the provider rejected the request, rather than replaying the rejection', async () => { const upstream = vi.fn(async () => sse([terminal()])); upstream.mockImplementationOnce( async () => new Response(JSON.stringify({ error: { message: 'Rate limit reached' } }), { status: 429, headers: { 'content-type': 'application/json' }, }), ); const { request } = await fixture(upstream as typeof fetch); expect((await request()).status).toBe(429); const retried = await request(); expect(retried.status).toBe(200); await retried.text(); expect(await operations()).toEqual([expect.objectContaining({ status: 'completed', usageTokens: 15 })]); // A request stopped before it reached the provider is sent when retried, too. const stopped = new AbortController(); stopped.abort(); await expect(request({ input: 'the next turn' }, undefined, stopped.signal)).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_CANCELED', }); const sent = await request({ input: 'the next turn' }); expect(sent.status).toBe(200); await sent.text(); expect(upstream).toHaveBeenCalledTimes(3); expect((await control.run(owner, run.id)).usedTokens).toBe(30); }); });