knowledge-platform/tests/unit/test_chunking.py
2026-09-01 14:31:30 +08:00

66 lines
2.5 KiB
Python

"""Unit tests for deterministic structure-aware token chunking."""
from uuid import uuid4
import pytest
from ekp_core.chunking import ApproximateTokenCounter, StructureAwareChunker
from ekp_domain import ParsedBlock, ParsedBlockType, ParsedDocument
def _document(*blocks: ParsedBlock) -> ParsedDocument:
return ParsedDocument(document_id=uuid4(), filename="manual.md", blocks=blocks)
def test_approximate_counter_handles_cjk_ascii_and_punctuation() -> None:
counter = ApproximateTokenCounter()
assert counter.count("退款 policy123。") == 6
assert counter.count("abcdefgh") == 2
def test_chunker_keeps_heading_sections_as_hard_boundaries() -> None:
parsed = _document(
ParsedBlock("产品", ParsedBlockType.HEADING, section_path=("产品",)),
ParsedBlock("产品介绍", ParsedBlockType.PARAGRAPH, section_path=("产品",)),
ParsedBlock("退款", ParsedBlockType.HEADING, section_path=("产品", "退款")),
ParsedBlock("退款规则", ParsedBlockType.PARAGRAPH, section_path=("产品", "退款")),
)
chunks = StructureAwareChunker(max_tokens=100, overlap_tokens=10).chunk(parsed)
assert [chunk.content for chunk in chunks] == ["产品\n\n产品介绍", "退款\n\n退款规则"]
assert [chunk.section for chunk in chunks] == ["产品", "产品 / 退款"]
assert [chunk.chunk_index for chunk in chunks] == [0, 1]
def test_chunker_applies_token_overlap_to_oversized_structural_group() -> None:
parsed = _document(ParsedBlock("一二三四五六七八九十", ParsedBlockType.PARAGRAPH))
chunks = StructureAwareChunker(max_tokens=6, overlap_tokens=2).chunk(parsed)
assert [chunk.content for chunk in chunks] == ["一二三四五六", "五六七八九十"]
assert [chunk.token_count for chunk in chunks] == [6, 6]
def test_chunker_combines_paragraphs_and_preserves_page_range() -> None:
parsed = _document(
ParsedBlock("第一页", ParsedBlockType.PARAGRAPH, page_number=1),
ParsedBlock("第二页", ParsedBlockType.PARAGRAPH, page_number=2),
)
(chunk,) = StructureAwareChunker(max_tokens=100, overlap_tokens=10).chunk(parsed)
assert chunk.content == "第一页\n\n第二页"
assert chunk.page == 1
assert chunk.page_end == 2
@pytest.mark.parametrize(
("max_tokens", "overlap_tokens"),
[(0, 0), (10, -1), (10, 10), (10, 11)],
)
def test_chunker_rejects_invalid_windows(max_tokens: int, overlap_tokens: int) -> None:
with pytest.raises(ValueError):
StructureAwareChunker(max_tokens=max_tokens, overlap_tokens=overlap_tokens)