knowledge-platform/tests/evaluation/test_chunking_regression.py
2026-09-01 14:31:30 +08:00

34 lines
1.3 KiB
Python

"""Golden Dataset regression checks for source structure after chunking."""
import json
from io import BytesIO
from pathlib import Path
from uuid import uuid4
from ekp_core.chunking import StructureAwareChunker
from ekp_core.document_cleaner import clean_parsed_document
from ekp_parser import MarkdownParser
def test_golden_sections_remain_traceable_after_chunking() -> None:
"""Ensure synthetic Golden sources retain expected section labels and limits."""
fixture_root = Path("fixtures/documents")
records = [
json.loads(line)
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
if line
]
parser = MarkdownParser()
chunker = StructureAwareChunker()
for record in records:
for filename in record["expected_documents"]:
source_path = fixture_root / filename
parsed = parser.parse(uuid4(), filename, BytesIO(source_path.read_bytes()))
chunks = chunker.chunk(clean_parsed_document(parsed))
assert chunks
assert all(chunk.token_count <= 450 for chunk in chunks)
available_sections = {section for chunk in chunks for section in chunk.section_path}
assert set(record["expected_sections"]).issubset(available_sections)