- 工程形态:@d-robotics/physical-byte-agent npm 库包
- 单一 barrel src/extensions/index.ts 装配 21 域扩展(package.json pi.extensions 清单 + .pi/extensions/index.ts 重导出,pi 自动加载)
- src/agent.ts PhysicalByteAgent(launch() 程序化调 createAgentSession + verifyAssembly() 离线自检)
- bin/pba.ts CLI、models.json(glm-5.2 条目)、README.md、AGENTS.md
- 命名:全部域命名(glm-provider/safety-gate/flash/jtag-debugger/loop-orchestrator…),去掉 F 编号
- 架构:内联逻辑 + 顶层仅 import type(esbuild 擦除)+ 工厂内动态 import defineTool/Type
- 逻辑离线可单测(不解析 pi),工厂在真 pi 内装配;fail-closed 语义保留
- 两命门:glm-provider.ts buildGlmModelConfig()(thinkingFormat:qwen + maxTokens:8192 + thinkingLevelMap 全 null)+ detectEmptyOutput();normalizeHexArg() + makeHexNormalizingPrepareArguments()
- 验证:tsc --noEmit 0 错(对 src/types/pi.d.ts 类型镜像)+ vitest 732 用例/21 文件全绿 + tsx 离线装配冒烟 21 扩展全就位
- 密钥铁律:grep sk-nAm 在 src/tests/bin/.pi = 0,key 全程 ${DROBOTICS_LLM_KEY} 占位
Co-Authored-By: Claude <noreply@anthropic.com>
408 lines
14 KiB
TypeScript
408 lines
14 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
||
import {
|
||
chunkPdf,
|
||
buildIndex,
|
||
mergeIndex,
|
||
search,
|
||
formatContext,
|
||
tokenize,
|
||
tokenizeLower,
|
||
pagesFromPdftotextOutput,
|
||
extractAddresses,
|
||
verifyAddressInResults,
|
||
DEFAULT_CHUNK_SIZE,
|
||
MAX_CONTEXT_CHUNK_CHARS,
|
||
type PdfPage,
|
||
} from "../src/extensions/datasheet-rag.js";
|
||
|
||
// ============================================================================
|
||
// 测试用 datasheet 样本(模拟 STM32F4 RM0090 寄存器映射表片段)
|
||
// ============================================================================
|
||
|
||
const SAMPLE_PAGES: PdfPage[] = [
|
||
{
|
||
page: 12,
|
||
filePath: "/datasheets/rm0090.pdf",
|
||
text:
|
||
"GPIOA_BASE = 0x48000000. The GPIOA register base address for port A. " +
|
||
"GPIOA_MODER offset 0x00, GPIOA_OTYPER offset 0x04. " +
|
||
"RCC_AHB1ENR bit 0 = GPIOAEN. Reset value 0x00180000.",
|
||
},
|
||
{
|
||
page: 45,
|
||
filePath: "/datasheets/rm0090.pdf",
|
||
text:
|
||
"RCC_AHB1ENR address 0x40023830. Bit 0 GPIOAEN enables GPIOA clock. " +
|
||
"GPIOA_BASE 0x48000000. Set bit 0 to enable port A clock.",
|
||
},
|
||
{
|
||
page: 200,
|
||
filePath: "/datasheets/rm0090.pdf",
|
||
text:
|
||
"SPI2 base address 0x40003800. CR1 offset 0x00, CR2 offset 0x04. " +
|
||
"SPI2 is on APB1. Baud rate configured via BR bits.",
|
||
},
|
||
];
|
||
|
||
// ============================================================================
|
||
// 分词
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag tokenize", () => {
|
||
it("分词包含 hex 地址和寄存器名", () => {
|
||
const tokens = tokenize("GPIOA_BASE = 0x48000000");
|
||
expect(tokens).toContain("GPIOA_BASE");
|
||
expect(tokens).toContain("0x48000000");
|
||
});
|
||
|
||
it("空字符串返回空数组", () => {
|
||
expect(tokenize("")).toEqual([]);
|
||
expect(tokenizeLower("")).toEqual([]);
|
||
});
|
||
|
||
it("小写化:GPIOA_BASE → gpioa_base", () => {
|
||
const tokens = tokenizeLower("GPIOA_BASE RCC_AHB1ENR");
|
||
expect(tokens).toContain("gpioa_base");
|
||
expect(tokens).toContain("rcc_ahb1enr");
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// pagesFromPdftotextOutput
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag pagesFromPdftotextOutput", () => {
|
||
it("按分页符 \\f 切分页码从 1 起", () => {
|
||
const stdout = "page one text\nGPIOA_BASE\fpage two text\nRCC_AHB1ENR";
|
||
const pages = pagesFromPdftotextOutput(stdout, "/datasheets/rm0090.pdf");
|
||
expect(pages).toHaveLength(2);
|
||
expect(pages[0]!.page).toBe(1);
|
||
expect(pages[1]!.page).toBe(2);
|
||
expect(pages[0]!.text).toContain("GPIOA_BASE");
|
||
expect(pages[1]!.text).toContain("RCC_AHB1ENR");
|
||
});
|
||
|
||
it("filePath 透传到每页", () => {
|
||
const pages = pagesFromPdftotextOutput("a\fb", "/x.pdf");
|
||
expect(pages.every((p) => p.filePath === "/x.pdf")).toBe(true);
|
||
});
|
||
|
||
it("空 stdout 返回空数组(fail-closed)", () => {
|
||
expect(pagesFromPdftotextOutput("", "/x.pdf")).toEqual([]);
|
||
});
|
||
|
||
it("全空白页被过滤", () => {
|
||
const stdout = "real text\f \f\n \n";
|
||
const pages = pagesFromPdftotextOutput(stdout, "/x.pdf");
|
||
expect(pages).toHaveLength(1);
|
||
expect(pages[0]!.page).toBe(1);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// chunkPdf
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag chunkPdf", () => {
|
||
it("每页文本 < chunkSize 产生一个 chunk", () => {
|
||
const chunks = chunkPdf(SAMPLE_PAGES, 5000);
|
||
expect(chunks).toHaveLength(3);
|
||
expect(chunks[0]!.page).toBe(12);
|
||
expect(chunks[1]!.page).toBe(45);
|
||
expect(chunks[2]!.page).toBe(200);
|
||
});
|
||
|
||
it("chunk id 格式 p{page}-c{index}", () => {
|
||
const chunks = chunkPdf(SAMPLE_PAGES, 5000);
|
||
expect(chunks[0]!.id).toBe("p12-c0");
|
||
expect(chunks[1]!.id).toBe("p45-c0");
|
||
expect(chunks[2]!.id).toBe("p200-c0");
|
||
});
|
||
|
||
it("长文本按 chunkSize 切分多个 chunk", () => {
|
||
const longPage: PdfPage = {
|
||
page: 1,
|
||
text: "A".repeat(50) + " " + "B".repeat(50) + " " + "C".repeat(50),
|
||
};
|
||
const chunks = chunkPdf([longPage], 50);
|
||
// 150 字符 + 2 空格 = 152,chunkSize=50 → 3~4 块
|
||
expect(chunks.length).toBeGreaterThanOrEqual(3);
|
||
// 拼回的文本应覆盖全部内容
|
||
const reconstructed = chunks.map((c) => c.text).join(" ");
|
||
expect(reconstructed).toContain("A".repeat(50));
|
||
expect(reconstructed).toContain("B".repeat(50));
|
||
expect(reconstructed).toContain("C".repeat(50));
|
||
});
|
||
|
||
it("chunkSize <= 0 抛错", () => {
|
||
expect(() => chunkPdf(SAMPLE_PAGES, 0)).toThrow();
|
||
expect(() => chunkPdf(SAMPLE_PAGES, -1)).toThrow();
|
||
});
|
||
|
||
it("空页跳过", () => {
|
||
const chunks = chunkPdf([{ page: 1, text: "" }, { page: 2, text: "hello" }]);
|
||
expect(chunks).toHaveLength(1);
|
||
expect(chunks[0]!.page).toBe(2);
|
||
});
|
||
|
||
it("filePath 透传到 chunk", () => {
|
||
const chunks = chunkPdf(SAMPLE_PAGES, 5000);
|
||
expect(chunks[0]!.filePath).toBe("/datasheets/rm0090.pdf");
|
||
});
|
||
|
||
it("默认 chunkSize = 2000", () => {
|
||
expect(DEFAULT_CHUNK_SIZE).toBe(2000);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// buildIndex
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag buildIndex", () => {
|
||
const chunks = chunkPdf(SAMPLE_PAGES, 5000);
|
||
const index = buildIndex(chunks);
|
||
|
||
it("倒排索引 term 小写化", () => {
|
||
expect(index.inverted.has("gpioa_base")).toBe(true);
|
||
expect(index.inverted.has("GPIOA_BASE")).toBe(false);
|
||
});
|
||
|
||
it("倒排索引含 hex 地址 term(小写)", () => {
|
||
expect(index.inverted.has("0x48000000")).toBe(true);
|
||
expect(index.inverted.has("0x40023830")).toBe(true);
|
||
});
|
||
|
||
it("term 出现在多个 chunk 时 Set 含多个 chunkId", () => {
|
||
const set = index.inverted.get("gpioa_base");
|
||
expect(set).toBeDefined();
|
||
expect(set!.size).toBe(2); // p12 + p45
|
||
expect(set!.has("p12-c0")).toBe(true);
|
||
expect(set!.has("p45-c0")).toBe(true);
|
||
});
|
||
|
||
it("termFreq 记录 chunk 内 term 出现次数", () => {
|
||
const freq = index.termFreq.get("p12-c0");
|
||
expect(freq).toBeDefined();
|
||
expect(freq!.get("gpioa_base")).toBe(1);
|
||
expect(freq!.get("0x48000000")).toBe(1);
|
||
// p45 有两次 gpioa_base
|
||
const freq45 = index.termFreq.get("p45-c0");
|
||
expect(freq45!.get("gpioa_base")).toBe(1);
|
||
});
|
||
|
||
it("size 等于 chunk 数", () => {
|
||
expect(index.size).toBe(chunks.length);
|
||
});
|
||
|
||
it("docPaths 收集文档路径", () => {
|
||
expect(index.docPaths.has("/datasheets/rm0090.pdf")).toBe(true);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// mergeIndex
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag mergeIndex", () => {
|
||
it("合并两个文档索引后 chunk/term/docPath 叠加", () => {
|
||
// 用不同页码避免 chunk id 碰撞(id 格式 p{page}-c{index})
|
||
const pagesA: PdfPage[] = [
|
||
{ page: 1, text: "GPIOA_BASE 0x48000000", filePath: "/a.pdf" },
|
||
];
|
||
const pagesB: PdfPage[] = [
|
||
{ page: 2, text: "SPI2_BASE 0x40003800", filePath: "/b.pdf" },
|
||
];
|
||
const target = buildIndex(chunkPdf(pagesA, 5000));
|
||
const addition = buildIndex(chunkPdf(pagesB, 5000));
|
||
const merged = mergeIndex(target, addition);
|
||
|
||
// 两文档各自的 term 仍可查
|
||
expect(merged.inverted.has("gpioa_base")).toBe(true);
|
||
expect(merged.inverted.has("spi2_base")).toBe(true);
|
||
// size 叠加
|
||
expect(merged.size).toBe(target.size);
|
||
// 两文档路径都在
|
||
expect(merged.docPaths.has("/a.pdf")).toBe(true);
|
||
expect(merged.docPaths.has("/b.pdf")).toBe(true);
|
||
// 合并后跨文档检索各自命中正确 chunk
|
||
const resA = search(merged, "GPIOA_BASE", 5);
|
||
expect(resA.length).toBeGreaterThan(0);
|
||
expect(resA[0]!.filePath).toBe("/a.pdf");
|
||
const resB = search(merged, "SPI2_BASE", 5);
|
||
expect(resB.length).toBeGreaterThan(0);
|
||
expect(resB[0]!.filePath).toBe("/b.pdf");
|
||
});
|
||
|
||
it("同 term 出现在两文档时 Set 合并 chunkId", () => {
|
||
// 用不同页码避免 chunk id 碰撞(id 格式 p{page}-c{index})
|
||
const pagesA: PdfPage[] = [{ page: 1, text: "GPIOA_BASE addr", filePath: "/a.pdf" }];
|
||
const pagesB: PdfPage[] = [{ page: 2, text: "GPIOA_BASE again", filePath: "/b.pdf" }];
|
||
const target = buildIndex(chunkPdf(pagesA, 5000));
|
||
const addition = buildIndex(chunkPdf(pagesB, 5000));
|
||
const merged = mergeIndex(target, addition);
|
||
const set = merged.inverted.get("gpioa_base");
|
||
expect(set).toBeDefined();
|
||
expect(set!.size).toBe(2);
|
||
expect(set!.has("p1-c0")).toBe(true);
|
||
expect(set!.has("p2-c0")).toBe(true);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// search
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag search", () => {
|
||
const chunks = chunkPdf(SAMPLE_PAGES, 5000);
|
||
const index = buildIndex(chunks);
|
||
|
||
it("查 GPIOA_BASE 返回含 0x48000000 的 chunk", () => {
|
||
const results = search(index, "GPIOA_BASE", 5);
|
||
expect(results.length).toBeGreaterThan(0);
|
||
// 至少一个结果含 hex 地址
|
||
const hasAddr = results.some((r) => r.text.includes("0x48000000"));
|
||
expect(hasAddr).toBe(true);
|
||
});
|
||
|
||
it("查不存在的寄存器 FOOBAR_Z 返回空(fail-closed 不幻觉)", () => {
|
||
const results = search(index, "FOOBAR_Z", 5);
|
||
expect(results).toHaveLength(0);
|
||
});
|
||
|
||
it("同分按页码升序:gpioa_base 两 chunk tf=1 平分,p12 在前", () => {
|
||
const results = search(index, "gpioa_base", 5);
|
||
expect(results.length).toBeGreaterThanOrEqual(2);
|
||
expect(results[0]!.page).toBe(12); // 页码小的在前
|
||
});
|
||
|
||
it("topK 限制返回数", () => {
|
||
const results = search(index, "GPIOA_BASE 0x48000000", 1);
|
||
expect(results).toHaveLength(1);
|
||
});
|
||
|
||
it("多 term 查询分数叠加", () => {
|
||
// "gpioa_base 0x48000000" → p12 命中两者 score=2, p45 命中两者 score=2
|
||
const results = search(index, "gpioa_base 0x48000000", 5);
|
||
expect(results.length).toBeGreaterThanOrEqual(2);
|
||
expect(results[0]!.score).toBeGreaterThanOrEqual(2);
|
||
});
|
||
|
||
it("filter.docPath 限定文档路径", () => {
|
||
const results = search(index, "GPIOA_BASE", 5, {
|
||
filter: { docPath: "/datasheets/rm0090.pdf" },
|
||
});
|
||
expect(results.length).toBeGreaterThan(0);
|
||
expect(results.every((r) => r.filePath === "/datasheets/rm0090.pdf")).toBe(true);
|
||
});
|
||
|
||
it("filter.docPath 不匹配返回空", () => {
|
||
const results = search(index, "GPIOA_BASE", 5, {
|
||
filter: { docPath: "/other.pdf" },
|
||
});
|
||
expect(results).toHaveLength(0);
|
||
});
|
||
|
||
it("filter.page 限定页码", () => {
|
||
const results = search(index, "GPIOA_BASE", 5, { filter: { page: 200 } });
|
||
// page 200 没有 gpioa_base → 空
|
||
expect(results).toHaveLength(0);
|
||
const results45 = search(index, "RCC_AHB1ENR", 5, { filter: { page: 45 } });
|
||
expect(results45.length).toBeGreaterThan(0);
|
||
expect(results45.every((r) => r.page === 45)).toBe(true);
|
||
});
|
||
|
||
it("空 query 返回空(fail-closed)", () => {
|
||
expect(search(index, "", 5)).toHaveLength(0);
|
||
expect(search(index, " ", 5)).toHaveLength(0);
|
||
});
|
||
|
||
it("返回结果带 page/filePath/score 字段", () => {
|
||
const results = search(index, "SPI2", 5);
|
||
expect(results.length).toBeGreaterThan(0);
|
||
expect(results[0]!.page).toBe(200);
|
||
expect(results[0]!.filePath).toBe("/datasheets/rm0090.pdf");
|
||
expect(typeof results[0]!.score).toBe("number");
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// formatContext
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag formatContext", () => {
|
||
const chunks = chunkPdf(SAMPLE_PAGES, 5000);
|
||
const index = buildIndex(chunks);
|
||
|
||
it("空结果返回 'No matching passages found.'(fail-closed)", () => {
|
||
expect(formatContext([])).toBe("No matching passages found.");
|
||
});
|
||
|
||
it("含寄存器地址 0x48000000", () => {
|
||
const results = search(index, "GPIOA_BASE", 5);
|
||
const text = formatContext(results, "GPIOA_BASE");
|
||
expect(text).toContain("0x48000000");
|
||
});
|
||
|
||
it("含页码和文件路径", () => {
|
||
const results = search(index, "GPIOA_BASE", 5);
|
||
const text = formatContext(results);
|
||
expect(text).toContain("p12");
|
||
expect(text).toContain("rm0090.pdf");
|
||
});
|
||
|
||
it("含 score", () => {
|
||
const results = search(index, "GPIOA_BASE", 5);
|
||
const text = formatContext(results);
|
||
expect(text).toContain("score:");
|
||
});
|
||
|
||
it("含 header 和 End passages 标记", () => {
|
||
const results = search(index, "SPI2", 5);
|
||
const text = formatContext(results, "SPI2");
|
||
expect(text).toContain("=== Datasheet RAG:");
|
||
expect(text).toContain("=== End passages ===");
|
||
expect(text).toContain("SPI2");
|
||
});
|
||
|
||
it("超长 chunk 被截断", () => {
|
||
const longResult = {
|
||
chunkId: "p1-c0",
|
||
page: 1,
|
||
text: "A".repeat(MAX_CONTEXT_CHUNK_CHARS + 500),
|
||
score: 1,
|
||
};
|
||
const text = formatContext([longResult]);
|
||
expect(text).toContain("…");
|
||
expect(text.length).toBeLessThan(longResult.text.length);
|
||
});
|
||
});
|
||
|
||
// ============================================================================
|
||
// extractAddresses / verifyAddressInResults
|
||
// ============================================================================
|
||
|
||
describe("datasheet-rag address verification", () => {
|
||
const chunks = chunkPdf(SAMPLE_PAGES, 5000);
|
||
const index = buildIndex(chunks);
|
||
|
||
it("extractAddresses 提取 hex 地址", () => {
|
||
const results = search(index, "GPIOA_BASE", 5);
|
||
const addrs = extractAddresses(results);
|
||
expect(addrs).toContain("0x48000000");
|
||
expect(addrs).toContain("0x00180000");
|
||
});
|
||
|
||
it("verifyAddressInResults 命中返回 true(大小写不敏感)", () => {
|
||
const results = search(index, "GPIOA_BASE", 5);
|
||
expect(verifyAddressInResults("0x48000000", results)).toBe(true);
|
||
expect(verifyAddressInResults("0X48000000", results)).toBe(true);
|
||
});
|
||
|
||
it("verifyAddressInResults 未命中返回 false(fail-closed)", () => {
|
||
const results = search(index, "GPIOA_BASE", 5);
|
||
expect(verifyAddressInResults("0xdeadbeef", results)).toBe(false);
|
||
expect(verifyAddressInResults("", results)).toBe(false);
|
||
});
|
||
});
|