wishforge-ide/tests/integration/real-model.test.ts
eust-w 3eb8ea03bf feat: run models through an operator's OpenAI- or Anthropic-compatible gateway
The model proxy always forwarded to api.openai.com and development runs gave
Codex no base URL or model, so a company model gateway could not be used.

- WF_OPENAI_BASE_URL points the production proxy (and development Codex runs)
  at an OpenAI-compatible gateway with the Responses API; WF_ANTHROPIC_BASE_URL
  gives Claude Code an Anthropic-compatible one as ANTHROPIC_BASE_URL. Both are
  operator settings in the provider catalog, HTTPS without credentials
  (loopback HTTP outside production); an invalid gateway disables the proxy.
- Development credentials now carry the run's model, so WF_CODEX_DEFAULT_MODEL
  selects a model the gateway serves.
- tests/integration/real-model.test.ts (WF_REAL_MODEL_INTEGRATION=1, key from
  WF_TEST_MODEL_API_KEY) runs one wish through the real Codex executor, the
  verifier and integration into a local repository.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 22:28:33 +08:00

158 lines
7.4 KiB
TypeScript

/**
* Acceptance with a real model: one wish goes through the real Codex executor, the independent
* verifier and integration into a local repository, with the control plane in process.
*
* WF_REAL_MODEL_INTEGRATION=1 WF_TEST_MODEL_API_KEY=<key> \
* WF_OPENAI_BASE_URL=<OpenAI-compatible gateway, optional> WF_CODEX_DEFAULT_MODEL=<model, optional> \
* npx vitest run tests/integration/real-model.test.ts
*
* The key becomes the tenant's BYOK connection in an in-memory store; nothing is written to the
* repository. It spends real tokens (a few thousand for this wish).
*/
import { execFile } from 'node:child_process';
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { promisify } from 'node:util';
import { describe, expect, it } from 'vitest';
import type { Actor, Run } from '@wishforge/contracts';
import { executeCommand, type CommandExecutor } from '@wishforge/runtime';
import { ControlPlane } from '../../apps/api/src/domain/control-plane.js';
import { Services } from '../../apps/api/src/http.js';
import { ServiceTokens } from '../../apps/api/src/service-tokens.js';
import { reserveExecutionGrant } from '../../apps/api/src/domain/entitlements.js';
import { createActivities, type ActivityOutcome, type RunInput } from '../../apps/worker/src/activities.js';
import { MemoryStore } from '../helpers/memory-store.js';
import { internalFetch } from '../helpers/internal-fetch.js';
const enabled = process.env.WF_REAL_MODEL_INTEGRATION === '1' && Boolean(process.env.WF_TEST_MODEL_API_KEY);
const run = promisify(execFile);
const TENANT = 'demo';
const REMOTE_URL = 'http://127.0.0.1:65530/real-model.git';
const owner: Actor = { tenantId: TENANT, userId: 'owner', role: 'owner', kind: 'human' };
const git = (cwd: string, ...args: string[]) =>
run('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', ...args], { cwd }).then(r =>
r.stdout.trim(),
);
describe.skipIf(!enabled)('real model acceptance', () => {
it(
'implements, independently verifies and integrates a small wish',
async () => {
const root = await mkdtemp(path.join(os.tmpdir(), 'wishforge-real-model-'));
try {
const remote = path.join(root, 'remote.git');
await git(root, 'init', '--quiet', '--bare', '--initial-branch=main', remote);
const seed = path.join(root, 'seed');
await git(root, 'clone', '--quiet', remote, seed);
await writeFile(path.join(seed, 'greeting.txt'), 'hello\n');
await writeFile(
path.join(seed, 'check.mjs'),
"import { readFileSync } from 'node:fs';\nif (readFileSync('greeting.txt', 'utf8').trim() !== 'hello world') process.exit(1);\n",
);
await git(seed, 'add', '.');
await git(seed, 'commit', '--quiet', '-m', 'Initial project');
await git(seed, 'push', '--quiet', 'origin', 'main');
const env = {
NODE_ENV: 'test',
WF_DEPLOYMENT_MODE: 'development',
WF_DISPATCH_TENANTS: TENANT,
WF_INTERNAL_TOKEN_KEY: 'internal-token-signing-key-for-tests-only',
WF_WORKER_TOKEN: 'coordinator-static-token-for-tests-only',
WF_RESOURCE_GRANT_SECRET: 'resource-grant-secret-for-tests-only-0000',
WF_RUN_WORKER_IMAGE: `registry.local/wishforge-run@sha256:${'a'.repeat(64)}`,
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
WF_VAULT_KEY: 'a'.repeat(64),
...(process.env.WF_OPENAI_BASE_URL ? { WF_OPENAI_BASE_URL: process.env.WF_OPENAI_BASE_URL } : {}),
...(process.env.WF_CODEX_DEFAULT_MODEL ? { WF_CODEX_DEFAULT_MODEL: process.env.WF_CODEX_DEFAULT_MODEL } : {}),
};
const store = new MemoryStore();
store.seed(TENANT);
const services = new Services(new ControlPlane(store, env));
const project = await services.control.createProject(owner, {
name: 'Greeting',
repositoryUrl: REMOTE_URL,
verificationCommands: [[process.execPath, 'check.mjs']],
});
await services.connections.create(owner, {
provider: 'codex',
label: 'acceptance',
apiKey: process.env.WF_TEST_MODEL_API_KEY!,
});
const { run_id: runId } = await services.control.submit(
owner,
project.id,
{
goal: 'Change greeting.txt so that its whole content is the line "hello world".',
acceptanceCriteria: ['`node check.mjs` exits with code 0'],
provider: 'codex',
budget: { maxTokens: 200000, maxMinutes: 20, maxRepairRounds: 2 },
},
'real-model-0001',
);
// What the dispatcher and the local provisioner do, with the real executor.
const generation = (await services.control.run(owner, runId)).pendingStartId!;
await store.transaction(TENANT, tx => reserveExecutionGrant(tx, runId, generation, env));
const tokens = new ServiceTokens(env.WF_INTERNAL_TOKEN_KEY);
const input: RunInput = { tenantId: TENANT, runId, generation };
const execute: CommandExecutor = command =>
executeCommand(
command.executable === 'git'
? { ...command, args: command.args.map(arg => (arg === REMOTE_URL ? remote : arg)) }
: command,
);
const shared = {
controlPlaneUrl: 'http://127.0.0.1:4000',
workspaceRoot: path.join(root, 'workspaces', generation),
isolatedProviderHomeRoot: path.join(root, 'homes'),
allowLocalGitHttp: true,
fetcher: internalFetch(services),
execute,
};
const sandbox = {
sandboxMode: 'host' as const,
sandboxRunId: runId,
sandboxTenantId: TENANT,
sandboxGeneration: generation,
};
const executor = createActivities({
...shared,
...sandbox,
workerId: `executor-${generation}`,
workerToken: await tokens.issue(TENANT, runId, 'worker', generation),
});
const verifier = createActivities({
...shared,
...sandbox,
workerId: `verifier-${generation}`,
verifierToken: await tokens.issue(TENANT, runId, 'verifier', generation),
});
const coordinator = createActivities({ ...shared, workerId: 'coordinator', workerToken: env.WF_WORKER_TOKEN });
let outcome: ActivityOutcome = { status: 'waiting' };
for (let step = 0; step < 4; step++) {
outcome = await executor.executeRun(input);
if (outcome.status === 'verify') outcome = await verifier.verifyRun(input);
if (outcome.status === 'repair') outcome = await coordinator.prepareRepair(input);
if (outcome.status !== 'repair') break;
}
const finished: Run = await services.control.run(owner, runId);
const events = await services.control.events(owner, runId);
expect(outcome, `${finished.waitingReason ?? ''} ${finished.waitingDetail ?? ''}`).toEqual({
status: 'complete',
});
expect(finished.status).toBe('succeeded');
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
expect(finished.usedTokens).toBeGreaterThan(0);
expect(events.some(event => event.type === 'provider.command' || event.type === 'provider.file_change')).toBe(
true,
);
} finally {
await rm(root, { recursive: true, force: true });
}
},
20 * 60_000,
);
});