34 lines
1.3 KiB
Python
34 lines
1.3 KiB
Python
"""Golden Dataset regression checks for source structure after chunking."""
|
|
|
|
import json
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.document_cleaner import clean_parsed_document
|
|
from ekp_parser import MarkdownParser
|
|
|
|
|
|
def test_golden_sections_remain_traceable_after_chunking() -> None:
|
|
"""Ensure synthetic Golden sources retain expected section labels and limits."""
|
|
fixture_root = Path("fixtures/documents")
|
|
records = [
|
|
json.loads(line)
|
|
for line in Path("fixtures/evaluation/golden.jsonl").read_text().splitlines()
|
|
if line
|
|
]
|
|
parser = MarkdownParser()
|
|
chunker = StructureAwareChunker()
|
|
|
|
for record in records:
|
|
for filename in record["expected_documents"]:
|
|
source_path = fixture_root / filename
|
|
parsed = parser.parse(uuid4(), filename, BytesIO(source_path.read_bytes()))
|
|
chunks = chunker.chunk(clean_parsed_document(parsed))
|
|
|
|
assert chunks
|
|
assert all(chunk.token_count <= 450 for chunk in chunks)
|
|
available_sections = {section for chunk in chunks for section in chunk.section_path}
|
|
assert set(record["expected_sections"]).issubset(available_sections)
|