wishforge-ide/tests/provider-proxy.test.ts
eust-w b90db2cc6f refactor: remove dead code and typecheck the test suite
- Drop the WorkBuddy Cloud executor stub: it always reported
  "unverified" and could never run. WorkBuddy remains an MCP client for
  submitting wishes. Providers are now codex and codebuddy; an unknown
  provider waits with an explicit reason instead of a fake adapter.
- Remove internal routes nobody calls (/internal/releases/:id/begin,
  /accept, /failure and /internal/workspaces/:id/provisioned) with
  their domain methods, the never-consumed release.ready and
  release.rollback outbox events, an unreachable idempotency branch,
  dead helpers in runtime/deploy.ts and an unused RunnerBroker
  dependency.
- Enable noUnusedLocals in the base and web TypeScript configs; the
  runtime package now extends the base config.
- Add tests/tsconfig.json and include it in `npm run typecheck`. This
  surfaced tests still calling the deleted begin/accept path (now
  exercised through the Runner protocol) and five latent type errors.
- Web: provider capability labels match the API's keys and the wish
  form disables unavailable providers.
- Delete the invalid example Runner Job manifest; stop tracking the
  generated apps/web/next-env.d.ts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 14:02:27 +08:00

420 lines
18 KiB
TypeScript

import { beforeEach, describe, expect, it, vi } from 'vitest';
import { SignJWT, decodeJwt } from 'jose';
import type { Actor, Run } from '@wishforge/contracts';
import { ControlPlane } from '../apps/api/src/domain/control-plane.js';
import { Connections } from '../apps/api/src/domain/connections.js';
import { ProviderProxy, providerProxyConfigured } from '../apps/api/src/domain/provider-proxy.js';
import { ServiceTokens } from '../apps/api/src/service-tokens.js';
import { MemoryStore } from './helpers/memory-store.js';
const owner: Actor = { tenantId: 'alpha', userId: 'owner', role: 'owner', kind: 'human' },
worker: Actor = { ...owner, userId: 'worker', kind: 'worker' };
const secret = 'fixture_upstream_key_kept_inside_control_plane';
const env = {
NODE_ENV: 'test',
WF_VAULT_KEY: 'a'.repeat(64),
WF_PROVIDER_PROXY_ENABLED: 'true',
WF_PROVIDER_PROXY_PUBLIC_URL: 'http://127.0.0.1:4000/provider-proxy/v1',
WF_PROVIDER_PROXY_SIGNING_KEY: 'proxy-independent-signing-fixture-key-12345',
WF_PROVIDER_PROXY_ALLOWED_MODELS: 'fixture-model',
};
const encode = (data: unknown) => new TextEncoder().encode(JSON.stringify(data));
const terminal = (input = 10, output = 5) => ({
type: 'response.completed',
response: {
id: 'resp_fixture',
status: 'completed',
usage: { input_tokens: input, output_tokens: output, total_tokens: input + output },
},
});
function sse(events: unknown[], chunkSize = 7) {
const all = Buffer.from(
events.map(value => `event: ${(value as any).type}\r\ndata: ${JSON.stringify(value)}\r\n\r\n`).join(''),
);
let offset = 0;
return new Response(
new ReadableStream({
pull(controller) {
if (offset === all.length) controller.close();
else {
controller.enqueue(all.subarray(offset, offset + chunkSize));
offset = Math.min(all.length, offset + chunkSize);
}
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
);
}
let db: MemoryStore, control: ControlPlane, connections: Connections, run: Run, connectionId: string;
beforeEach(async () => {
db = new MemoryStore();
db.seed('alpha');
db.seed('beta');
control = new ControlPlane(db, env);
connections = new Connections(db, env);
connectionId = (await connections.create(owner, { provider: 'codex', label: 'fixture', apiKey: secret })).id;
const project = await control.createProject(owner, {
name: 'Proxy fixture',
repositoryUrl: 'https://example.org/fixture.git',
});
const wish = await control.submit(
owner,
project.id,
{ goal: 'Test scoped proxy', acceptanceCriteria: ['Credential remains private'] },
'proxy-fixture-submit',
);
run = await control.lease(worker, wish.run_id, 'fixture-worker');
});
async function fixture(fetcher: typeof fetch) {
const proxy = new ProviderProxy(db, env, { fetcher, pollMs: 25 });
const credential = await proxy.credential(worker, run);
const request = (body: Record<string, unknown> = {}, headers?: Record<string, string>, signal?: AbortSignal) =>
proxy.forward({
authorization: `Bearer ${credential.apiKey}`,
method: 'POST',
path: '/responses',
body: encode({ model: 'fixture-model', input: 'Inspect local code', stream: true, ...body }),
headers,
signal,
});
return { proxy, credential, request };
}
async function waitFor(check: () => Promise<boolean>) {
for (let count = 0; count < 100; count++) {
if (await check()) return;
await new Promise(resolve => setTimeout(resolve, 10));
}
throw new Error('Timed out waiting for proxy settlement');
}
describe('run-scoped Codex credential proxy', () => {
it('accepts bounded local namespaces and SDK metadata without treating tool schemas as media', async () => {
const { request } = await fixture((async (_url, init) => {
const data = JSON.parse(String(init?.body));
expect(data.client_metadata).toBeUndefined();
expect(data.tools[0].tools[0].type).toBe('function');
return sse([terminal()]);
}) as typeof fetch);
const body = {
client_metadata: { originator: 'fixture' },
tools: [
{
type: 'namespace',
name: 'local',
tools: [
{
type: 'function',
name: 'inspect',
parameters: { type: 'object', properties: { image_url: { type: 'string' } } },
},
],
},
],
};
expect((await request(body)).status).toBe(200);
await expect(
request({
tools: [{ type: 'namespace', name: 'bad', tools: [{ type: 'mcp', server_url: 'https://untrusted.example' }] }],
}),
).rejects.toMatchObject({ code: 'PROVIDER_REMOTE_TOOL_DENIED' });
await expect(
request({
input: [{ role: 'user', content: [{ type: 'input_image', image_url: 'https://untrusted.example/image' }] }],
}),
).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' });
});
it('retains quota at the instant of cancellation while upstream usage is unresolved', async () => {
db.state.get('alpha')!.tenant.tokenLimit = 100000;
const upstream = vi.fn(
async () =>
new Response(
new ReadableStream({
start(c) {
c.enqueue(
new TextEncoder().encode(
'event: response.created\ndata: {"type":"response.created","response":{"id":"pending"}}\n\n',
),
);
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
),
);
const { request } = await fixture(upstream as typeof fetch);
const response = await request();
const reading = response.text().catch(() => 'aborted');
await control.command(owner, run.id, { action: 'cancel' }, 'cancel-before-settlement');
expect((await control.usage(owner)).reservedTokens).toBe(100000);
await expect(
control.submit(
owner,
run.projectId,
{ goal: 'Another run', acceptanceCriteria: ['Cannot borrow unresolved reservation'] },
'after-cancellation',
),
).rejects.toMatchObject({ code: 'QUOTA_EXCEEDED' });
await reading;
});
it('returns proxy credentials through the actual connection broker without exposing the stored key', async () => {
const connection = await connections.forRun(worker, run);
expect(connection).toMatchObject({ metering: 'proxy', baseUrl: env.WF_PROVIDER_PROXY_PUBLIC_URL });
expect(JSON.stringify(connection)).not.toContain(secret);
});
it('never puts the upstream key in the scoped token and prevents proxy credentials from authenticating to internal services', async () => {
const { credential } = await fixture(vi.fn());
expect(credential.metering).toBe('proxy');
expect(JSON.stringify(credential).includes(secret)).toBe(false);
const claims = decodeJwt(credential.apiKey);
expect(claims).toMatchObject({
tenantId: 'alpha',
runId: run.id,
connectionId,
leaseToken: run.lease!.token,
aud: 'wishforge-openai-proxy',
});
expect(Number(claims.exp) - Number(claims.iat)).toBeLessThanOrEqual(3600);
expect(await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).verify(credential.apiKey)).toBeUndefined();
expect(providerProxyConfigured({ ...env, NODE_ENV: 'production' })).toBe(false);
await expect(new ProviderProxy(db, { NODE_ENV: 'production' }).credential(worker, run)).rejects.toMatchObject({
code: 'PROVIDER_PROXY_UNCONFIGURED',
});
});
it('forwards only trusted endpoint/headers and streams split SSE frames with trusted usage recorded once', async () => {
const upstream = vi.fn(async (url: any, init: any) => {
expect(String(url)).toBe('https://api.openai.com/v1/responses');
expect(init.headers.Authorization === `Bearer ${secret}`).toBe(true);
expect(init.headers['X-Untrusted']).toBeUndefined();
const body = JSON.parse(init.body);
expect(body.store).toBe(false);
expect(body.background).toBe(false);
expect(body.max_output_tokens).toBeLessThanOrEqual(8192);
return sse([
{ type: 'response.created', response: { id: 'resp_fixture' } },
{ type: 'response.output_text.delta', delta: `hello ${secret}` },
terminal(),
]);
});
const { request } = await fixture(upstream as typeof fetch);
const first = await request({}, { 'X-Untrusted': 'not-forwarded' });
const body = await first.text();
expect(body).toContain('response.completed');
expect(body.includes(secret)).toBe(false);
expect(body).toContain('[redacted]');
const repeated = await request();
expect(await repeated.text()).toBe(body);
expect(upstream).toHaveBeenCalledTimes(1);
expect((await control.usage(owner)).tokens).toBe(15);
});
it('rejects tampered tenant tokens, wrong audience, stale generations and rotated provider connections before upstream I/O', async () => {
const upstream = vi.fn();
const { proxy, credential, request } = await fixture(upstream);
const token = await new SignJWT({
...decodeJwt<Record<string, unknown>>(String(credential.apiKey)),
tenantId: 'beta',
})
.setProtectedHeader({ alg: 'HS256' })
.sign(new TextEncoder().encode(env.WF_PROVIDER_PROXY_SIGNING_KEY));
await expect(
proxy.forward({ authorization: `Bearer ${token}`, method: 'GET', path: '/models' }),
).rejects.toMatchObject({ code: 'NOT_FOUND' });
const internal = await new ServiceTokens(env.WF_PROVIDER_PROXY_SIGNING_KEY).issue(
'alpha',
run.id,
'worker',
'generation',
);
await expect(
proxy.forward({ authorization: `Bearer ${internal}`, method: 'GET', path: '/models' }),
).rejects.toMatchObject({ code: 'PROVIDER_PROXY_UNAUTHENTICATED' });
await connections.revoke(owner, connectionId);
await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_CONNECTION_REVOKED' });
expect(upstream).not.toHaveBeenCalled();
});
it('denies arbitrary upstream paths, server-side tools/state, unapproved models and multimodal input', async () => {
const upstream = vi.fn();
const { proxy, credential, request } = await fixture(upstream);
await expect(
proxy.forward({
authorization: `Bearer ${credential.apiKey}`,
method: 'POST',
path: 'https://attacker.invalid/v1/responses',
body: encode({}),
}),
).rejects.toMatchObject({ code: 'PROVIDER_ROUTE_DENIED' });
await expect(request({ tools: [{ type: 'mcp', server_url: 'https://attacker.invalid' }] })).rejects.toMatchObject({
code: 'PROVIDER_REMOTE_TOOL_DENIED',
});
await expect(request({ previous_response_id: 'other-tenants-response' })).rejects.toMatchObject({
code: 'PROVIDER_STATEFUL_REQUEST_DENIED',
});
await expect(
request({ input: [{ type: 'input_image', image_url: 'https://attacker.invalid/image' }] }),
).rejects.toMatchObject({ code: 'PROVIDER_MULTIMODAL_DENIED' });
await expect(request({ model: 'not-approved' })).rejects.toMatchObject({ code: 'PROVIDER_MODEL_DENIED' });
expect(upstream).not.toHaveBeenCalled();
});
it('serializes a run and stops a live stream when the user cancels', async () => {
let canceled = false;
const upstream = vi.fn(
async () =>
new Response(
new ReadableStream({
start(controller) {
controller.enqueue(
new TextEncoder().encode(
'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n',
),
);
},
cancel() {
canceled = true;
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
),
);
const { request } = await fixture(upstream as typeof fetch);
const response = await request();
const reading = response.text().catch(() => 'aborted');
await expect(request({ input: 'another prompt' })).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' });
await control.command(owner, run.id, { action: 'cancel' }, 'cancel-proxy-request');
expect(await reading).toBe('aborted');
await waitFor(async () => canceled);
expect((await control.run(owner, run.id)).status).toBe('canceled');
expect((await control.run(owner, run.id)).reservedTokens).toBeGreaterThan(0);
const operations = await db.transaction('alpha', tx => tx.list('provider_proxy_request'));
expect(operations[0].status).toBe('unknown');
expect(upstream).toHaveBeenCalledTimes(1);
});
it('preserves unknown outcome/reservation after a truncated stream and does not resend it', async () => {
const upstream = vi.fn(async () => sse([{ type: 'response.output_text.delta', delta: 'partial output' }]));
const { request } = await fixture(upstream as typeof fetch);
await expect((await request()).text()).rejects.toThrow();
expect(await control.run(owner, run.id)).toMatchObject({
status: 'waiting',
waitingReason: 'provider_usage_unknown',
});
await control.command(owner, run.id, { action: 'resume' }, 'resume-unknown-proxy');
const leased = await control.lease(worker, run.id, 'recovery-worker');
const proxy = new ProviderProxy(db, env, { fetcher: upstream as typeof fetch });
const next = await proxy.credential(worker, leased);
await expect(
proxy.forward({
authorization: `Bearer ${next.apiKey}`,
method: 'POST',
path: '/responses',
body: encode({ model: 'fixture-model', input: 'retry', stream: true }),
}),
).rejects.toMatchObject({ code: 'PROVIDER_REQUEST_IN_FLIGHT' });
expect(upstream).toHaveBeenCalledTimes(1);
});
it('rejects an unaffordable request before any model call and meters only actual upstream usage', async () => {
const upstream = vi.fn(async () => sse([terminal(50000, 50000)]));
const { request } = await fixture(upstream as typeof fetch);
await (await request()).text();
expect(await control.run(owner, run.id)).toMatchObject({
status: 'waiting',
waitingReason: 'budget',
usedTokens: 100000,
reservedTokens: 0,
});
await expect(request({ input: 'further work' })).rejects.toThrow();
expect(upstream).toHaveBeenCalledTimes(1);
});
it('refuses insufficient input/output reservation before contacting the provider', async () => {
const upstream = vi.fn();
const { request } = await fixture(upstream);
await db.transaction('alpha', async tx => {
const current = (await tx.get<Run>('run', run.id))!;
current.usedTokens = 99000;
current.reservedTokens = 1000;
await tx.put('run', current);
});
await expect(request()).rejects.toMatchObject({ code: 'PROVIDER_BUDGET_EXHAUSTED' });
expect(upstream).not.toHaveBeenCalled();
});
it('aborts an in-flight model stream after provider connection rotation', async () => {
let canceled = false;
const upstream = vi.fn(
async () =>
new Response(
new ReadableStream({
start(c) {
c.enqueue(
new TextEncoder().encode(
'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_fixture"}}\n\n',
),
);
},
cancel() {
canceled = true;
},
}),
{ headers: { 'content-type': 'text/event-stream' } },
),
);
const { request } = await fixture(upstream as typeof fetch);
const reading = (await request()).text().catch(() => 'aborted');
await connections.create(owner, { provider: 'codex', label: 'rotated', apiKey: 'replacement_fixture_private_key' });
expect(await reading).toBe('aborted');
expect(canceled).toBe(true);
expect((await control.run(owner, run.id)).waitingReason).toBe('provider_usage_unknown');
});
it('binds production proxy credentials to execution generation and fails before decrypting if it changes', async () => {
const production = {
...env,
NODE_ENV: 'production',
WF_PROVIDER_PROXY_PUBLIC_URL: 'https://api.example/provider-proxy/v1',
};
const actor = { ...worker, runId: run.id, credentialId: 'fixture-generation' };
await db.transaction('alpha', async tx => {
const current = (await tx.get<Run>('run', run.id))!;
current.executionGeneration = 'fixture-generation';
await tx.put('run', current);
});
const open = vi.fn(async () => secret),
fetcher = vi.fn();
const proxy = new ProviderProxy(db, production, { vault: { open }, fetcher });
const token = await proxy.credential(actor, run);
await db.transaction('alpha', async tx => {
const current = (await tx.get<Run>('run', run.id))!;
current.executionGeneration = 'replacement-generation';
await tx.put('run', current);
});
await expect(
proxy.forward({ authorization: `Bearer ${token.apiKey}`, method: 'GET', path: '/models' }),
).rejects.toMatchObject({ code: 'EXECUTION_REVOKED' });
expect(open).not.toHaveBeenCalled();
expect(fetcher).not.toHaveBeenCalled();
});
it('handles JSON Responses and filters model listings without reflecting upstream secret/error headers', async () => {
const upstream = vi.fn(async (url: any) =>
String(url).endsWith('/models')
? new Response(JSON.stringify({ data: [{ id: 'fixture-model' }, { id: 'expensive-other-model' }] }), {
headers: { 'content-type': 'application/json', 'x-secret': secret },
})
: new Response(JSON.stringify(terminal().response), {
headers: { 'content-type': 'application/json', 'x-secret': secret },
}),
);
const { request, proxy, credential } = await fixture(upstream as typeof fetch);
const response = await request({ stream: false });
expect((await response.json()).usage.total_tokens).toBe(15);
expect(response.headers.has('x-secret')).toBe(false);
const models = await proxy.forward({
authorization: `Bearer ${credential.apiKey}`,
method: 'GET',
path: '/models',
});
expect((await models.json()).data).toEqual([{ id: 'fixture-model' }]);
});
it('rejects changed input with the same idempotency key rather than dispatching again', async () => {
const upstream = vi.fn(async () => sse([terminal()]));
const { request } = await fixture(upstream as typeof fetch);
await (await request({}, { 'idempotency-key': 'same-request-key' })).text();
await expect(request({ input: 'changed' }, { 'idempotency-key': 'same-request-key' })).rejects.toMatchObject({
code: 'PROVIDER_IDEMPOTENCY_CONFLICT',
});
expect(upstream).toHaveBeenCalledTimes(1);
});
});