The model proxy always forwarded to api.openai.com and development runs gave Codex no base URL or model, so a company model gateway could not be used. - WF_OPENAI_BASE_URL points the production proxy (and development Codex runs) at an OpenAI-compatible gateway with the Responses API; WF_ANTHROPIC_BASE_URL gives Claude Code an Anthropic-compatible one as ANTHROPIC_BASE_URL. Both are operator settings in the provider catalog, HTTPS without credentials (loopback HTTP outside production); an invalid gateway disables the proxy. - Development credentials now carry the run's model, so WF_CODEX_DEFAULT_MODEL selects a model the gateway serves. - tests/integration/real-model.test.ts (WF_REAL_MODEL_INTEGRATION=1, key from WF_TEST_MODEL_API_KEY) runs one wish through the real Codex executor, the verifier and integration into a local repository. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
158 lines
7.4 KiB
TypeScript
158 lines
7.4 KiB
TypeScript
/**
|
|
* Acceptance with a real model: one wish goes through the real Codex executor, the independent
|
|
* verifier and integration into a local repository, with the control plane in process.
|
|
*
|
|
* WF_REAL_MODEL_INTEGRATION=1 WF_TEST_MODEL_API_KEY=<key> \
|
|
* WF_OPENAI_BASE_URL=<OpenAI-compatible gateway, optional> WF_CODEX_DEFAULT_MODEL=<model, optional> \
|
|
* npx vitest run tests/integration/real-model.test.ts
|
|
*
|
|
* The key becomes the tenant's BYOK connection in an in-memory store; nothing is written to the
|
|
* repository. It spends real tokens (a few thousand for this wish).
|
|
*/
|
|
import { execFile } from 'node:child_process';
|
|
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { promisify } from 'node:util';
|
|
import { describe, expect, it } from 'vitest';
|
|
import type { Actor, Run } from '@wishforge/contracts';
|
|
import { executeCommand, type CommandExecutor } from '@wishforge/runtime';
|
|
import { ControlPlane } from '../../apps/api/src/domain/control-plane.js';
|
|
import { Services } from '../../apps/api/src/http.js';
|
|
import { ServiceTokens } from '../../apps/api/src/service-tokens.js';
|
|
import { reserveExecutionGrant } from '../../apps/api/src/domain/entitlements.js';
|
|
import { createActivities, type ActivityOutcome, type RunInput } from '../../apps/worker/src/activities.js';
|
|
import { MemoryStore } from '../helpers/memory-store.js';
|
|
import { internalFetch } from '../helpers/internal-fetch.js';
|
|
|
|
const enabled = process.env.WF_REAL_MODEL_INTEGRATION === '1' && Boolean(process.env.WF_TEST_MODEL_API_KEY);
|
|
const run = promisify(execFile);
|
|
const TENANT = 'demo';
|
|
const REMOTE_URL = 'http://127.0.0.1:65530/real-model.git';
|
|
const owner: Actor = { tenantId: TENANT, userId: 'owner', role: 'owner', kind: 'human' };
|
|
const git = (cwd: string, ...args: string[]) =>
|
|
run('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', ...args], { cwd }).then(r =>
|
|
r.stdout.trim(),
|
|
);
|
|
|
|
describe.skipIf(!enabled)('real model acceptance', () => {
|
|
it(
|
|
'implements, independently verifies and integrates a small wish',
|
|
async () => {
|
|
const root = await mkdtemp(path.join(os.tmpdir(), 'wishforge-real-model-'));
|
|
try {
|
|
const remote = path.join(root, 'remote.git');
|
|
await git(root, 'init', '--quiet', '--bare', '--initial-branch=main', remote);
|
|
const seed = path.join(root, 'seed');
|
|
await git(root, 'clone', '--quiet', remote, seed);
|
|
await writeFile(path.join(seed, 'greeting.txt'), 'hello\n');
|
|
await writeFile(
|
|
path.join(seed, 'check.mjs'),
|
|
"import { readFileSync } from 'node:fs';\nif (readFileSync('greeting.txt', 'utf8').trim() !== 'hello world') process.exit(1);\n",
|
|
);
|
|
await git(seed, 'add', '.');
|
|
await git(seed, 'commit', '--quiet', '-m', 'Initial project');
|
|
await git(seed, 'push', '--quiet', 'origin', 'main');
|
|
|
|
const env = {
|
|
NODE_ENV: 'test',
|
|
WF_DEPLOYMENT_MODE: 'development',
|
|
WF_DISPATCH_TENANTS: TENANT,
|
|
WF_INTERNAL_TOKEN_KEY: 'internal-token-signing-key-for-tests-only',
|
|
WF_WORKER_TOKEN: 'coordinator-static-token-for-tests-only',
|
|
WF_RESOURCE_GRANT_SECRET: 'resource-grant-secret-for-tests-only-0000',
|
|
WF_RUN_WORKER_IMAGE: `registry.local/wishforge-run@sha256:${'a'.repeat(64)}`,
|
|
WF_GIT_ALLOW_LOCAL_HTTP: 'true',
|
|
WF_VAULT_KEY: 'a'.repeat(64),
|
|
...(process.env.WF_OPENAI_BASE_URL ? { WF_OPENAI_BASE_URL: process.env.WF_OPENAI_BASE_URL } : {}),
|
|
...(process.env.WF_CODEX_DEFAULT_MODEL ? { WF_CODEX_DEFAULT_MODEL: process.env.WF_CODEX_DEFAULT_MODEL } : {}),
|
|
};
|
|
const store = new MemoryStore();
|
|
store.seed(TENANT);
|
|
const services = new Services(new ControlPlane(store, env));
|
|
const project = await services.control.createProject(owner, {
|
|
name: 'Greeting',
|
|
repositoryUrl: REMOTE_URL,
|
|
verificationCommands: [[process.execPath, 'check.mjs']],
|
|
});
|
|
await services.connections.create(owner, {
|
|
provider: 'codex',
|
|
label: 'acceptance',
|
|
apiKey: process.env.WF_TEST_MODEL_API_KEY!,
|
|
});
|
|
const { run_id: runId } = await services.control.submit(
|
|
owner,
|
|
project.id,
|
|
{
|
|
goal: 'Change greeting.txt so that its whole content is the line "hello world".',
|
|
acceptanceCriteria: ['`node check.mjs` exits with code 0'],
|
|
provider: 'codex',
|
|
budget: { maxTokens: 200000, maxMinutes: 20, maxRepairRounds: 2 },
|
|
},
|
|
'real-model-0001',
|
|
);
|
|
|
|
// What the dispatcher and the local provisioner do, with the real executor.
|
|
const generation = (await services.control.run(owner, runId)).pendingStartId!;
|
|
await store.transaction(TENANT, tx => reserveExecutionGrant(tx, runId, generation, env));
|
|
const tokens = new ServiceTokens(env.WF_INTERNAL_TOKEN_KEY);
|
|
const input: RunInput = { tenantId: TENANT, runId, generation };
|
|
const execute: CommandExecutor = command =>
|
|
executeCommand(
|
|
command.executable === 'git'
|
|
? { ...command, args: command.args.map(arg => (arg === REMOTE_URL ? remote : arg)) }
|
|
: command,
|
|
);
|
|
const shared = {
|
|
controlPlaneUrl: 'http://127.0.0.1:4000',
|
|
workspaceRoot: path.join(root, 'workspaces', generation),
|
|
isolatedProviderHomeRoot: path.join(root, 'homes'),
|
|
allowLocalGitHttp: true,
|
|
fetcher: internalFetch(services),
|
|
execute,
|
|
};
|
|
const sandbox = {
|
|
sandboxMode: 'host' as const,
|
|
sandboxRunId: runId,
|
|
sandboxTenantId: TENANT,
|
|
sandboxGeneration: generation,
|
|
};
|
|
const executor = createActivities({
|
|
...shared,
|
|
...sandbox,
|
|
workerId: `executor-${generation}`,
|
|
workerToken: await tokens.issue(TENANT, runId, 'worker', generation),
|
|
});
|
|
const verifier = createActivities({
|
|
...shared,
|
|
...sandbox,
|
|
workerId: `verifier-${generation}`,
|
|
verifierToken: await tokens.issue(TENANT, runId, 'verifier', generation),
|
|
});
|
|
const coordinator = createActivities({ ...shared, workerId: 'coordinator', workerToken: env.WF_WORKER_TOKEN });
|
|
let outcome: ActivityOutcome = { status: 'waiting' };
|
|
for (let step = 0; step < 4; step++) {
|
|
outcome = await executor.executeRun(input);
|
|
if (outcome.status === 'verify') outcome = await verifier.verifyRun(input);
|
|
if (outcome.status === 'repair') outcome = await coordinator.prepareRepair(input);
|
|
if (outcome.status !== 'repair') break;
|
|
}
|
|
const finished: Run = await services.control.run(owner, runId);
|
|
const events = await services.control.events(owner, runId);
|
|
expect(outcome, `${finished.waitingReason ?? ''} ${finished.waitingDetail ?? ''}`).toEqual({
|
|
status: 'complete',
|
|
});
|
|
expect(finished.status).toBe('succeeded');
|
|
expect(await git(remote, 'show', 'main:greeting.txt')).toBe('hello world');
|
|
expect(finished.usedTokens).toBeGreaterThan(0);
|
|
expect(events.some(event => event.type === 'provider.command' || event.type === 'provider.file_change')).toBe(
|
|
true,
|
|
);
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true });
|
|
}
|
|
},
|
|
20 * 60_000,
|
|
);
|
|
});
|