"""Integration tests for upload-to-chunk processing and canonical metadata.""" from io import BytesIO from pathlib import Path import pytest from sqlalchemy import create_engine, select from sqlalchemy.orm import Session, sessionmaker from ekp_core.chunking import StructureAwareChunker from ekp_core.database import create_session_factory from ekp_core.document_chunking import ( DocumentAlreadyChunkedError, DocumentChunkingService, DocumentPipelineService, ) from ekp_core.document_parsing import DocumentParsingService from ekp_core.document_upload import DocumentUploadService from ekp_domain import Base, Chunk, Document, DocumentStatus from ekp_object_storage import LocalObjectStorage from ekp_parser import ParserRouter def _build_services( tmp_path: Path, *, max_tokens: int = 450, overlap_tokens: int = 75, ) -> tuple[ DocumentPipelineService, DocumentUploadService, DocumentParsingService, DocumentChunkingService, LocalObjectStorage, sessionmaker[Session], ]: engine = create_engine("sqlite+pysqlite:///:memory:") Base.metadata.create_all(engine) session_factory = create_session_factory(engine) storage = LocalObjectStorage(tmp_path / "storage") upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024) parsing = DocumentParsingService(session_factory, storage, ParserRouter()) chunking = DocumentChunkingService( session_factory, storage, StructureAwareChunker(max_tokens=max_tokens, overlap_tokens=overlap_tokens), ) pipeline = DocumentPipelineService(upload, parsing, chunking) return pipeline, upload, parsing, chunking, storage, session_factory def test_pipeline_persists_traceable_chunks_without_marking_document_ready( tmp_path: Path, ) -> None: pipeline, _, _, _, _, session_factory = _build_services(tmp_path) result = pipeline.ingest( "policy.md", "text/markdown", BytesIO("# 产品\n产品介绍\n\n## 退款\n退款周期为七天。".encode()), ) assert result.document.status is DocumentStatus.PROCESSING assert len(result.chunks) == 2 assert result.chunks[1].section == "产品 / 退款" assert result.chunks[1].filename == "policy.md" assert result.chunks[1].strategy_version == "structure_v1" with session_factory() as session: stored_chunks = session.scalars(select(Chunk).order_by(Chunk.chunk_index)).all() stored_document = session.get(Document, result.document.id) assert [chunk.chunk_id for chunk in stored_chunks] == [ chunk.chunk_id for chunk in result.chunks ] assert stored_document is not None assert stored_document.status is DocumentStatus.PROCESSING def test_chunking_uses_overlap_and_never_silently_overwrites(tmp_path: Path) -> None: pipeline, _, _, chunking, _, session_factory = _build_services( tmp_path, max_tokens=6, overlap_tokens=2, ) result = pipeline.ingest("numbers.txt", "text/plain", BytesIO("一二三四五六七八九十".encode())) assert [chunk.content for chunk in result.chunks] == ["一二三四五六", "五六七八九十"] with pytest.raises(DocumentAlreadyChunkedError): chunking.process(result.document.id) with session_factory() as session: assert len(session.scalars(select(Chunk)).all()) == 2 def test_invalid_parsed_artifact_sets_failed_without_partial_chunks(tmp_path: Path) -> None: _, upload, parsing, chunking, storage, session_factory = _build_services(tmp_path) document = upload.upload("policy.txt", "text/plain", BytesIO("有效内容".encode())) parsing.process(document.id) assert document.parsed_storage_key is None # Detached upload object is not mutated by parsing. parsed_key = f"parsed/{document.id}/document.json" storage.delete(parsed_key) storage.put(parsed_key, BytesIO(b"invalid-json")) with pytest.raises(ValueError, match="invalid parsed document artifact"): chunking.process(document.id) with session_factory() as session: stored_document = session.get(Document, document.id) chunks = session.scalars(select(Chunk)).all() assert stored_document is not None assert stored_document.status is DocumentStatus.FAILED assert "invalid parsed document artifact" in (stored_document.error_message or "") assert chunks == [] assert storage.exists(document.storage_key)