"""Unit tests for deterministic structure-aware token chunking.""" from uuid import uuid4 import pytest from ekp_core.chunking import ApproximateTokenCounter, StructureAwareChunker from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument def _document(*blocks: ParsedBlock) -> ParsedDocument: return ParsedDocument(document_id=uuid4(), filename="manual.md", blocks=blocks) def test_approximate_counter_handles_cjk_ascii_and_punctuation() -> None: counter = ApproximateTokenCounter() assert counter.count("退款 policy123。") == 6 assert counter.count("abcdefgh") == 2 def test_chunker_keeps_heading_sections_as_hard_boundaries() -> None: parsed = _document( ParsedBlock("产品", ParsedBlockType.HEADING, section_path=("产品",)), ParsedBlock("产品介绍", ParsedBlockType.PARAGRAPH, section_path=("产品",)), ParsedBlock("退款", ParsedBlockType.HEADING, section_path=("产品", "退款")), ParsedBlock("退款规则", ParsedBlockType.PARAGRAPH, section_path=("产品", "退款")), ) chunks = StructureAwareChunker(max_tokens=100, overlap_tokens=10).chunk(parsed) assert [chunk.content for chunk in chunks] == ["产品\n\n产品介绍", "退款\n\n退款规则"] assert [chunk.section for chunk in chunks] == ["产品", "产品 / 退款"] assert [chunk.chunk_index for chunk in chunks] == [0, 1] def test_chunker_applies_token_overlap_to_oversized_structural_group() -> None: parsed = _document(ParsedBlock("一二三四五六七八九十", ParsedBlockType.PARAGRAPH)) chunks = StructureAwareChunker(max_tokens=6, overlap_tokens=2).chunk(parsed) assert [chunk.content for chunk in chunks] == ["一二三四五六", "五六七八九十"] assert [chunk.token_count for chunk in chunks] == [6, 6] def test_chunker_combines_paragraphs_and_preserves_page_range() -> None: parsed = _document( ParsedBlock("第一页", ParsedBlockType.PARAGRAPH, page_number=1), ParsedBlock("第二页", ParsedBlockType.PARAGRAPH, page_number=2), ) (chunk,) = StructureAwareChunker(max_tokens=100, overlap_tokens=10).chunk(parsed) assert chunk.content == "第一页\n\n第二页" assert chunk.page == 1 assert chunk.page_end == 2 def test_chunker_respects_qianfan_character_limit_as_well_as_tokens() -> None: parsed = _document(ParsedBlock(" ".join(["abcd"] * 500), ParsedBlockType.PARAGRAPH)) chunks = StructureAwareChunker().chunk(parsed) assert len(chunks) > 1 assert all(chunk.token_count <= 450 for chunk in chunks) assert all(len(chunk.content) <= 2000 for chunk in chunks) @pytest.mark.parametrize( ("max_tokens", "overlap_tokens"), [(0, 0), (10, -1), (10, 10), (10, 11)], ) def test_chunker_rejects_invalid_windows(max_tokens: int, overlap_tokens: int) -> None: with pytest.raises(ValueError): StructureAwareChunker(max_tokens=max_tokens, overlap_tokens=overlap_tokens) def test_chunker_rejects_character_window_smaller_than_one_ascii_subtoken() -> None: with pytest.raises(ValueError, match="at least four"): StructureAwareChunker(max_characters=3)