81 lines
3.1 KiB
Python
81 lines
3.1 KiB
Python
"""Unit tests for deterministic structure-aware token chunking."""
|
|
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
|
|
from ekp_core.chunking import ApproximateTokenCounter, StructureAwareChunker
|
|
from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument
|
|
|
|
|
|
def _document(*blocks: ParsedBlock) -> ParsedDocument:
|
|
return ParsedDocument(document_id=uuid4(), filename="manual.md", blocks=blocks)
|
|
|
|
|
|
def test_approximate_counter_handles_cjk_ascii_and_punctuation() -> None:
|
|
counter = ApproximateTokenCounter()
|
|
|
|
assert counter.count("退款 policy123。") == 6
|
|
assert counter.count("abcdefgh") == 2
|
|
|
|
|
|
def test_chunker_keeps_heading_sections_as_hard_boundaries() -> None:
|
|
parsed = _document(
|
|
ParsedBlock("产品", ParsedBlockType.HEADING, section_path=("产品",)),
|
|
ParsedBlock("产品介绍", ParsedBlockType.PARAGRAPH, section_path=("产品",)),
|
|
ParsedBlock("退款", ParsedBlockType.HEADING, section_path=("产品", "退款")),
|
|
ParsedBlock("退款规则", ParsedBlockType.PARAGRAPH, section_path=("产品", "退款")),
|
|
)
|
|
|
|
chunks = StructureAwareChunker(max_tokens=100, overlap_tokens=10).chunk(parsed)
|
|
|
|
assert [chunk.content for chunk in chunks] == ["产品\n\n产品介绍", "退款\n\n退款规则"]
|
|
assert [chunk.section for chunk in chunks] == ["产品", "产品 / 退款"]
|
|
assert [chunk.chunk_index for chunk in chunks] == [0, 1]
|
|
|
|
|
|
def test_chunker_applies_token_overlap_to_oversized_structural_group() -> None:
|
|
parsed = _document(ParsedBlock("一二三四五六七八九十", ParsedBlockType.PARAGRAPH))
|
|
|
|
chunks = StructureAwareChunker(max_tokens=6, overlap_tokens=2).chunk(parsed)
|
|
|
|
assert [chunk.content for chunk in chunks] == ["一二三四五六", "五六七八九十"]
|
|
assert [chunk.token_count for chunk in chunks] == [6, 6]
|
|
|
|
|
|
def test_chunker_combines_paragraphs_and_preserves_page_range() -> None:
|
|
parsed = _document(
|
|
ParsedBlock("第一页", ParsedBlockType.PARAGRAPH, page_number=1),
|
|
ParsedBlock("第二页", ParsedBlockType.PARAGRAPH, page_number=2),
|
|
)
|
|
|
|
(chunk,) = StructureAwareChunker(max_tokens=100, overlap_tokens=10).chunk(parsed)
|
|
|
|
assert chunk.content == "第一页\n\n第二页"
|
|
assert chunk.page == 1
|
|
assert chunk.page_end == 2
|
|
|
|
|
|
def test_chunker_respects_qianfan_character_limit_as_well_as_tokens() -> None:
|
|
parsed = _document(ParsedBlock(" ".join(["abcd"] * 500), ParsedBlockType.PARAGRAPH))
|
|
|
|
chunks = StructureAwareChunker().chunk(parsed)
|
|
|
|
assert len(chunks) > 1
|
|
assert all(chunk.token_count <= 450 for chunk in chunks)
|
|
assert all(len(chunk.content) <= 2000 for chunk in chunks)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("max_tokens", "overlap_tokens"),
|
|
[(0, 0), (10, -1), (10, 10), (10, 11)],
|
|
)
|
|
def test_chunker_rejects_invalid_windows(max_tokens: int, overlap_tokens: int) -> None:
|
|
with pytest.raises(ValueError):
|
|
StructureAwareChunker(max_tokens=max_tokens, overlap_tokens=overlap_tokens)
|
|
|
|
|
|
def test_chunker_rejects_character_window_smaller_than_one_ascii_subtoken() -> None:
|
|
with pytest.raises(ValueError, match="at least four"):
|
|
StructureAwareChunker(max_characters=3)
|