data-gen-ebd/tests/test_pi_adapter.mjs
amos b773033658 Record model requests and payloads as prefix deltas in the native trace
Each model_request/model_payload repeated the full context, so the trace
grew with the square of the request count: batch 6 Qwen runs hit 64 MiB
at ~75 requests and 14 attempts (13 graded PASS) were discarded.
Requests now record only items after the common prefix with the previous
one (schema embedded-rollout.pi/2), and pi_run.expand rebuilds full
payloads before the unchanged broker/feedback checks. Needs a new image
(trace6) and recalibration before use.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 10:50:56 +08:00

437 lines
23 KiB
JavaScript

import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import http from 'node:http';
import {test} from 'node:test';
import {proxyUrl, closure, glmPayload, run, main, loadSdk, compactEvent, traceWriter, writeAll, prefixDelta,
MAX_TRACE_BYTES} from '../agent/pi.mjs';
// Mirrors rollout.pi_run.expand: each record holds the items after `from` of the previous rebuilt list.
function rebuild(events, type, body) {
let prior = [];
return events.filter(event => event.type === type).map(event => {
const {field, from} = event.delta;
const full = {...event[body], [field]: [...prior.slice(0, from), ...event[body][field]]};
prior = full[field];
return full;
});
}
test('only a credential-free HTTP proxy address is admitted', () => {
assert.equal(proxyUrl('http://172.20.0.1:8000/v1/'), 'http://172.20.0.1:8000/v1');
for (const value of ['https://open.bigmodel.cn/api/coding/paas/v4',
'http://token@127.0.0.1:1/v1', 'http://127.0.0.1/v1?key=secret',
'http://127.0.0.1/v1#secret', 'not-a-url']) assert.throws(() => proxyUrl(value));
});
test('tool closure requires a unique preceding call and real feedback consumed', () => {
const assistant = {role: 'assistant', content: [{type: 'toolCall', id: 'call-1'}]};
const result = {role: 'toolResult', toolCallId: 'call-1', content: [{type: 'text', text: 'written'}]};
assert.deepEqual(closure([assistant, result], [{messages: [result]}]),
{closed: true, feedback_consumed: true});
assert.deepEqual(closure([assistant, result], [{messages: []}]),
{closed: true, feedback_consumed: false});
assert.equal(closure([assistant], []).closed, false);
assert.equal(closure([result, assistant], []).closed, false);
assert.equal(closure([assistant, result, result], []).closed, false);
assert.equal(closure([assistant, assistant, result], []).closed, false);
});
test('model records keep only items after the common prefix with the previous request', () => {
const first = prefixDelta([], [{role: 'user', content: 'fix'}]);
assert.deepEqual(first, {from: 0, encoded: ['{"role":"user","content":"fix"}']});
const grown = prefixDelta(first.encoded, [{role: 'user', content: 'fix'}, {role: 'assistant', content: 'ok'}]);
assert.equal(grown.from, 1);
assert.equal(prefixDelta(grown.encoded, [{role: 'user', content: 'other'}, {role: 'assistant', content: 'ok'}]).from, 0);
assert.equal(prefixDelta(grown.encoded, [{role: 'user', content: 'fix'}]).from, 1);
});
test('GLM-5.3 payload always enables supported thinking', () => {
const original = {model: 'glm-5.3', messages: [{role: 'user', content: 'fix'}]};
assert.deepEqual(glmPayload(original), {...original,
thinking: {type: 'enabled'}, reasoning_effort: 'high'});
assert.equal(original.thinking, undefined);
});
test('unsupported models and malformed argv fail before opening trace or loading SDK', async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'embedded-pi-model-'));
const traceFile = path.join(directory, 'native.jsonl');
try {
await assert.rejects(() => run('Repair decoder.c.', {traceFile,
baseUrl: 'http://127.0.0.1:8000/v1', modelId: 'other-model'}, undefined),
{message: 'UNSUPPORTED_MODEL'});
await assert.rejects(() => main(['--trace', traceFile, '--model', 'other-model', '--max-tokens', '4096']),
{message: 'UNSUPPORTED_MODEL'});
for (const maxTokens of [undefined, 0, 1.5, '4096', 1048577]) {
await assert.rejects(() => run('Repair decoder.c.', {traceFile, maxTokens,
baseUrl: 'http://127.0.0.1:8000/v1'}, undefined), {message: 'INVALID_MAX_TOKENS'});
}
for (const args of [['--trace', 'relative.jsonl', '--model', 'glm-5.3', '--max-tokens', '4096'],
['--trace', traceFile], ['--trace', traceFile, '--model', 'glm-5.3'],
['--trace', traceFile, '--model', 'glm-5.3', '--max-tokens', '0'],
['--trace', traceFile, '--model', 'glm-5.3', '--provider', 'other']]) {
await assert.rejects(() => main(args), {message: 'USAGE_PI_TRACE_MODEL_MAX_TOKENS'});
}
assert.equal(fs.existsSync(traceFile), false);
} finally {
fs.rmSync(directory, {recursive: true, force: true});
}
});
test('stream deltas omit both cumulative snapshots while complete messages remain intact', () => {
const updates = Array.from({length: 1000}, (_, index) => ({type: 'message_update',
message: {content: [{type: 'text', text: 'x'.repeat(index + 1)}]},
assistantMessageEvent: {type: 'text_delta', contentIndex: 0, delta: 'x',
partial: {content: [{type: 'text', text: 'x'.repeat(index + 1)}]}}}));
const compact = updates.map(compactEvent);
assert.equal(compact.every(event => !Object.hasOwn(event, 'message') &&
!Object.hasOwn(event.assistantMessageEvent, 'partial')), true);
assert.equal(compact.map(event => event.assistantMessageEvent.delta).join(''), 'x'.repeat(1000));
assert.equal(JSON.stringify(compact).length < 150000, true);
assert.equal(Object.hasOwn(updates[0], 'message'), true);
const messageEnd = {type: 'message_end', message: updates.at(-1).message};
assert.equal(compactEvent(messageEnd), messageEnd);
const toolUpdate = {type: 'tool_execution_update', toolCallId: 'call-1',
partialResult: {content: [{type: 'text', text: 'compiler feedback'}]}};
assert.equal(compactEvent(toolUpdate), toolUpdate);
});
test('host mirror survives partial writes and a full nonblocking pipe', () => {
const written = [];
let calls = 0;
const write = (fd, data, offset, length) => {
calls += 1;
if (calls === 2) throw Object.assign(new Error('pipe full'), {code: 'EAGAIN'});
const count = Math.min(3, length);
written.push(data.subarray(offset, offset + count));
return count;
};
writeAll(1, '{"x":"状态"}\n', write);
assert.equal(Buffer.concat(written).toString(), '{"x":"状态"}\n');
assert.throws(() => writeAll(1, 'x', () => {throw Object.assign(new Error('closed'), {code: 'EPIPE'});}),
{code: 'EPIPE'});
});
test('trace quota counts UTF-8 bytes, writes one terminal failure, and refuses further events', () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'embedded-pi-quota-'));
const file = path.join(directory, 'native.jsonl');
const fd = fs.openSync(file, 'wx');
const mirrored = [];
let aborted = 0;
try {
const output = traceWriter(fd, line => mirrored.push(line), () => {aborted += 1;});
const event = {type: 'message_end', message: {content: [{type: 'text', text: '汉'.repeat(Math.ceil(MAX_TRACE_BYTES / 6) + 1)}]}};
output(event);
output(event);
output({type: 'session_end', complete: true});
assert.equal(output.limitExceeded, true);
assert.equal(aborted, 1);
const raw = fs.readFileSync(file, 'utf8');
assert.equal(raw, mirrored.join(''));
assert.equal(Buffer.byteLength(raw) <= MAX_TRACE_BYTES, true);
const records = raw.trim().split('\n').map(JSON.parse);
assert.equal(records.length, 2);
assert.deepEqual(records.at(-1), {type: 'session_end', complete: false, error: 'TRACE_LIMIT_EXCEEDED'});
} finally {
fs.closeSync(fd);
fs.rmSync(directory, {recursive: true, force: true});
}
});
let sdk;
try {
sdk = await loadSdk();
} catch (error) {
if (error.code !== 'ERR_MODULE_NOT_FOUND') throw error;
}
test('actual pinned SDK writes a file, consumes tool feedback, and records wire/native events',
{skip: !sdk}, async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'embedded-pi-test-'));
const workspace = path.join(directory, 'workspace');
fs.mkdirSync(workspace);
// These should never be discovered or executed by the adapter.
fs.mkdirSync(path.join(workspace, '.pi', 'extensions'), {recursive: true});
fs.writeFileSync(path.join(workspace, '.pi', 'extensions', 'attack.js'),
'throw new Error("AUTO_DISCOVERY_OCCURRED")');
fs.writeFileSync(path.join(workspace, 'AGENTS.md'), 'UNTRUSTED_CONTEXT_MARKER');
const requests = [];
const server = http.createServer(async (request, response) => {
let body = '';
for await (const chunk of request) body += chunk;
requests.push(JSON.parse(body));
response.writeHead(200, {'content-type': 'text/event-stream'});
const first = requests.length === 1;
const delta = first ? {role: 'assistant', reasoning_content: 'Fix the parser state.', tool_calls: [{index: 0, id: 'call-write',
type: 'function', function: {name: 'write', arguments: JSON.stringify({path: 'decoder.c',
content: 'int main(void) { return 0; }\n'})}}]} : {role: 'assistant', content: 'Done.'};
const item = (delta, finish_reason = null) => ({id: 'mock-' + requests.length,
object: 'chat.completion.chunk', created: 1, model: 'glm-5.3',
choices: [{index: 0, delta, finish_reason}]});
response.write('data: ' + JSON.stringify(item(delta)) + '\n\n');
response.write('data: ' + JSON.stringify(item({}, first ? 'tool_calls' : 'stop')) + '\n\n');
response.end('data: [DONE]\n\n');
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
try {
const traceFile = path.join(directory, 'native.jsonl');
const mirror = [];
let observedBeforeWrite = false;
const summary = await run('Repair decoder.c. Read, edit and test the local files.',
{cwd: workspace, traceFile, baseUrl: `http://127.0.0.1:${server.address().port}/v1`, maxTokens: 131072,
token: 'attempt-token-must-not-be-logged', mirror: line => {
mirror.push(line);
if (JSON.parse(line).type === 'tool_execution_start') {
observedBeforeWrite = !fs.existsSync(path.join(workspace, 'decoder.c'));
}
}}, sdk);
assert.equal(summary.complete, true);
assert.equal(summary.closed, true);
assert.equal(summary.feedback_consumed, true);
assert.equal(summary.model_requests, 2);
assert.equal(observedBeforeWrite, true);
assert.equal(fs.readFileSync(path.join(workspace, 'decoder.c'), 'utf8'),
'int main(void) { return 0; }\n');
assert.equal(requests[0].thinking.type, 'enabled');
assert.equal(requests[0].reasoning_effort, 'high');
assert.equal(requests[0].max_tokens, 131072);
assert.equal(requests[1].messages.some(message => message.role === 'tool' &&
message.tool_call_id === 'call-write'), true);
assert.equal(requests[1].messages.some(message => message.role === 'assistant' &&
message.reasoning_content === 'Fix the parser state.'), true);
const raw = fs.readFileSync(traceFile, 'utf8');
assert.equal(mirror.join(''), raw);
assert.equal(raw.includes('attempt-token-must-not-be-logged'), false);
assert.equal(raw.includes('UNTRUSTED_CONTEXT_MARKER'), false);
const events = raw.trim().split('\n').map(line => JSON.parse(line));
assert.equal(events.filter(event => event.type === 'message_update').every(event =>
!Object.hasOwn(event, 'message') && !Object.hasOwn(event.assistantMessageEvent, 'partial')), true);
assert.equal(events.some(event => event.type === 'message_end' && event.message?.role === 'assistant' &&
event.message.content.some(block => block.type === 'thinking' && block.thinking === 'Fix the parser state.')), true);
for (const type of ['trace_header', 'model_request', 'model_payload',
'tool_execution_start', 'tool_execution_end', 'message_end', 'session_entries', 'session_end']) {
assert.equal(events.some(event => event.type === type), true, type);
}
assert.equal(events[0].schema, 'embedded-rollout.pi/2');
assert.deepEqual(rebuild(events, 'model_payload', 'payload'), requests);
assert.equal(events.filter(event => event.type === 'model_payload')[1].delta.from > 0, true);
const contexts = rebuild(events, 'model_request', 'context');
assert.equal(events.filter(event => event.type === 'model_request')[1].delta.from > 0, true);
assert.equal(contexts[1].messages.some(message => message.role === 'toolResult' &&
message.toolCallId === 'call-write'), true);
assert.equal(fs.statSync(traceFile).mode & 0o777, 0o644);
await assert.rejects(() => run('again', {cwd: workspace, traceFile, maxTokens: 131072,
baseUrl: `http://127.0.0.1:${server.address().port}/v1`}, sdk), {code: 'EEXIST'});
} finally {
await new Promise(resolve => server.close(resolve));
fs.rmSync(directory, {recursive: true, force: true});
}
});
test('Qwen Responses SDK performs a real tool write, consumes feedback, and keeps model identity',
{skip: !sdk}, async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'embedded-pi-responses-'));
const requests = [];
const server = http.createServer(async (request, response) => {
let body = '';
for await (const chunk of request) body += chunk;
requests.push({url: request.url, body: JSON.parse(body)});
response.writeHead(200, {'content-type': 'text/event-stream'});
const emit = event => response.write('data: ' + JSON.stringify(event) + '\n\n');
const first = requests.length === 1;
const responseId = 'resp-' + requests.length;
emit({type: 'response.created', response: {id: responseId}});
if (first) {
const args = JSON.stringify({path: 'decoder.c', content: 'int main(void) { return 0; }\n'});
const item = {type: 'function_call', id: 'fc-write', call_id: 'call-write', name: 'write'};
emit({type: 'response.output_item.added', output_index: 0, item: {...item, arguments: ''}});
emit({type: 'response.function_call_arguments.delta', item_id: item.id,
output_index: 0, delta: args});
emit({type: 'response.function_call_arguments.done', item_id: item.id,
output_index: 0, arguments: args});
emit({type: 'response.output_item.done', output_index: 0, item: {...item, arguments: args}});
} else {
const item = {type: 'message', id: 'msg-done', role: 'assistant'};
const part = {type: 'output_text', text: 'Done.', annotations: []};
emit({type: 'response.output_item.added', output_index: 0, item: {...item, content: []}});
emit({type: 'response.content_part.added', item_id: item.id, output_index: 0,
content_index: 0, part: {...part, text: ''}});
emit({type: 'response.output_text.delta', item_id: item.id, output_index: 0,
content_index: 0, delta: part.text});
emit({type: 'response.output_item.done', output_index: 0,
item: {...item, status: 'completed', content: [part]}});
}
emit({type: 'response.completed', response: {id: responseId, status: 'completed',
usage: {input_tokens: 30, output_tokens: 10, total_tokens: 40}}});
response.end('data: [DONE]\n\n');
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
try {
const traceFile = path.join(directory, 'native.jsonl');
const mirrored = [];
let observedBeforeWrite = false;
const summary = await run('Repair decoder.c.', {cwd: directory, traceFile,
modelId: 'qwen3.8-27b-nvfp4', maxTokens: 65536, baseUrl: `http://127.0.0.1:${server.address().port}/v1`,
token: 'responses-attempt-token-must-not-be-logged', mirror: line => {
mirrored.push(line);
if (JSON.parse(line).type === 'tool_execution_start') {
observedBeforeWrite = !fs.existsSync(path.join(directory, 'decoder.c'));
}
}}, sdk);
assert.equal(summary.complete, true);
assert.equal(summary.closed, true);
assert.equal(summary.feedback_consumed, true);
assert.equal(summary.model_requests, 2);
assert.equal(observedBeforeWrite, true);
assert.equal(fs.readFileSync(path.join(directory, 'decoder.c'), 'utf8'),
'int main(void) { return 0; }\n');
assert.equal(requests.length, 2);
for (const request of requests) {
assert.equal(request.url, '/v1/responses');
assert.equal(request.body.model, 'qwen3.8-27b-nvfp4');
assert.equal(request.body.stream, true);
assert.equal(request.body.store, false);
assert.equal(Array.isArray(request.body.input), true);
for (const field of ['thinking', 'reasoning_effort', 'reasoning', 'include']) {
assert.equal(Object.hasOwn(request.body, field), false, field);
}
assert.equal(request.body.input.some(message => message.role === 'developer'), false);
}
const input = requests[1].body.input;
const toolCall = input.find(item => item.type === 'function_call');
const toolResult = input.find(item => item.type === 'function_call_output');
assert.equal(toolCall.name, 'write');
assert.equal(toolResult.call_id, toolCall.call_id);
assert.equal(typeof toolResult.output, 'string');
assert.match(toolResult.output, /Successfully wrote/);
const raw = fs.readFileSync(traceFile, 'utf8');
assert.equal(raw, mirrored.join(''));
assert.equal(raw.includes('responses-attempt-token-must-not-be-logged'), false);
const events = raw.trim().split('\n').map(JSON.parse);
const header = events.find(event => event.type === 'trace_header');
assert.equal(header.model.id, 'qwen3.8-27b-nvfp4');
assert.equal(header.model.api, 'openai-responses');
assert.equal(header.model.provider, 'rollout-proxy');
assert.equal(header.model.reasoning, false);
assert.equal(header.model.maxTokens, 65536);
assert.equal(requests[0].body.max_output_tokens, 65536);
assert.equal(events.filter(event => event.type === 'model_payload').length, 2);
assert.deepEqual(rebuild(events, 'model_payload', 'payload'), requests.map(request => request.body));
assert.deepEqual(events.filter(event => event.type === 'model_payload').map(event => event.delta.field),
['input', 'input']);
assert.equal(events.filter(event => event.type === 'model_payload')[1].delta.from > 0, true);
assert.equal(events.some(event => event.type === 'tool_execution_end'), true);
assert.equal(events.at(-1).type, 'session_end');
assert.equal(events.at(-1).complete, true);
} finally {
server.closeAllConnections();
await new Promise(resolve => server.close(resolve));
fs.rmSync(directory, {recursive: true, force: true});
}
});
test('provider failure remains incomplete and keeps the native error evidence',
{skip: !sdk}, async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'embedded-pi-failure-'));
const server = http.createServer((request, response) => {
request.resume();
response.writeHead(401, {'content-type': 'application/json'});
response.end(JSON.stringify({error: {message: 'Unauthorized mock', type: 'authentication_error'}}));
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
try {
const traceFile = path.join(directory, 'native.jsonl');
const summary = await run('Repair decoder.c.', {cwd: directory, traceFile, maxTokens: 131072,
baseUrl: `http://127.0.0.1:${server.address().port}/v1`, mirror: () => {}}, sdk);
assert.equal(summary.complete, false);
assert.equal(summary.final_stop_reason, 'error');
const events = fs.readFileSync(traceFile, 'utf8').trim().split('\n').map(JSON.parse);
assert.equal(events.some(event => event.type === 'message_end' &&
event.message?.stopReason === 'error'), true);
assert.equal(events.at(-1).type, 'session_end');
assert.equal(events.at(-1).complete, false);
assert.equal(summary.model_requests, 1);
} finally {
await new Promise(resolve => server.close(resolve));
fs.rmSync(directory, {recursive: true, force: true});
}
});
test('actual SDK aborts an oversized stream and emits exactly one incomplete terminal event',
{skip: !sdk}, async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'embedded-pi-sdk-quota-'));
let requests = 0, chunks = 0, streamClosed = false;
let closeStream;
const closed = new Promise(resolve => {closeStream = resolve;});
const server = http.createServer((request, response) => {
requests += 1;
request.resume();
response.writeHead(200, {'content-type': 'text/event-stream'});
const timer = setInterval(() => {
chunks += 1;
response.write('data: ' + JSON.stringify({id: 'quota', object: 'chat.completion.chunk',
created: 1, model: 'glm-5.3', choices: [{index: 0, finish_reason: null,
delta: {content: 'x'.repeat(1024 * 1024)}}]}) + '\n\n');
if (chunks >= 128) {clearInterval(timer); response.end('data: [DONE]\n\n');}
}, 5);
response.on('close', () => {streamClosed = true; clearInterval(timer); closeStream();});
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
try {
const traceFile = path.join(directory, 'native.jsonl');
await assert.rejects(() => run('Repair decoder.c.', {cwd: directory, traceFile, maxTokens: 131072,
baseUrl: `http://127.0.0.1:${server.address().port}/v1`, mirror: () => {}}, sdk),
{message: 'TRACE_LIMIT_EXCEEDED'});
let closeTimeout;
try {
await Promise.race([closed, new Promise((resolve, reject) => {
closeTimeout = setTimeout(() => reject(new Error('Provider stream did not close after abort')), 2000);
})]);
} finally {clearTimeout(closeTimeout);}
const records = fs.readFileSync(traceFile, 'utf8').trim().split('\n').map(JSON.parse);
const ends = records.filter(event => event.type === 'session_end');
assert.equal(ends.length, 1);
assert.equal(ends[0].complete, false);
assert.equal(ends[0].error, 'TRACE_LIMIT_EXCEEDED');
assert.equal(records.at(-1), ends[0]);
assert.equal(fs.statSync(traceFile).size <= MAX_TRACE_BYTES, true);
assert.equal(requests, 1);
assert.equal(chunks < 128, true);
assert.equal(streamClosed, true);
} finally {
server.closeAllConnections();
await new Promise(resolve => server.close(resolve));
fs.rmSync(directory, {recursive: true, force: true});
}
});
test('actual SDK rejects a payload that exceeds remaining trace space before sending HTTP',
{skip: !sdk}, async () => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'embedded-pi-payload-quota-'));
let requests = 0;
const server = http.createServer((request, response) => {
requests += 1;
request.resume();
response.writeHead(500);
response.end();
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
try {
const traceFile = path.join(directory, 'native.jsonl');
await assert.rejects(() => run('x'.repeat(MAX_TRACE_BYTES / 4), {cwd: directory, traceFile, maxTokens: 131072,
baseUrl: `http://127.0.0.1:${server.address().port}/v1`, mirror: () => {}}, sdk),
{message: 'TRACE_LIMIT_EXCEEDED'});
const records = fs.readFileSync(traceFile, 'utf8').trim().split('\n').map(JSON.parse);
assert.equal(records.some(event => event.type === 'model_request'), true);
assert.equal(records.some(event => event.type === 'model_payload'), false);
assert.equal(records.at(-1).error, 'TRACE_LIMIT_EXCEEDED');
assert.equal(records.filter(event => event.type === 'session_end').length, 1);
assert.equal(requests, 0);
} finally {
server.closeAllConnections();
await new Promise(resolve => server.close(resolve));
fs.rmSync(directory, {recursive: true, force: true});
}
});