knowledge-platform/tests/integration/test_async_ingestion.py
2026-09-02 20:18:58 +08:00

432 lines
15 KiB
Python

"""Durable Job, bounded retry and cleanup-before-retry integration tests."""
from collections.abc import Sequence
from datetime import UTC, datetime, timedelta
from io import BytesIO
from pathlib import Path
from typing import BinaryIO
from uuid import UUID
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, sessionmaker
from ekp_core.chunking import StructureAwareChunker
from ekp_core.database import create_session_factory
from ekp_core.document_chunking import DocumentChunkingService
from ekp_core.document_indexing import DocumentIndexingService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
JobRunAction,
dispatch_pending_jobs,
)
from ekp_domain import (
Base,
Chunk,
Document,
IndexProfile,
IngestionAttempt,
IngestionAttemptStatus,
IngestionJob,
IngestionJobStatus,
ParsedDocument,
)
from ekp_embedding import EmbeddingProviderError, FakeEmbeddingProvider
from ekp_object_storage import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_vector_store import FakeVectorStore
class RecordingDispatcher:
"""Capture delivery requests while preserving the dispatcher contract."""
def __init__(self, *, fail: bool = False) -> None:
self.fail = fail
self.job_ids: list[UUID] = []
def enqueue(self, job_id: UUID) -> None:
if self.fail:
raise RuntimeError("simulated broker outage containing secret=must-not-persist")
self.job_ids.append(job_id)
class FailingJobService(IngestionJobService):
"""Inject metadata failure after immutable source upload."""
def create(self, document_id: UUID) -> IngestionJob:
del document_id
raise RuntimeError("simulated Job persistence failure")
class FailOnceEmbeddingProvider:
"""Inject one transient provider failure after parse/chunk persistence."""
def __init__(self) -> None:
self._delegate = FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
self.calls = 0
@property
def model_name(self) -> str:
return self._delegate.model_name
@property
def dimensions(self) -> int:
return self._delegate.dimensions
@property
def max_batch_size(self) -> int:
return self._delegate.max_batch_size
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
self.calls += 1
if self.calls == 1:
raise EmbeddingProviderError("temporary failure token=must-not-persist")
return self._delegate.embed(texts)
class AlwaysFailEmbeddingProvider(FailOnceEmbeddingProvider):
"""Inject a transient error on every attempt to verify the retry bound."""
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
del texts
self.calls += 1
raise EmbeddingProviderError("temporary failure token=must-not-persist")
class RetryObservingParser:
"""Verify cleanup finished before the second provider execution begins."""
def __init__(
self,
session_factory: sessionmaker[Session],
storage: LocalObjectStorage,
vector_store: FakeVectorStore,
) -> None:
self._delegate = ParserRouter()
self._session_factory = session_factory
self._storage = storage
self._vector_store = vector_store
self.calls = 0
self.retry_started_clean = False
def parse(
self,
document_id: UUID,
filename: str,
source: BinaryIO,
) -> ParsedDocument:
self.calls += 1
if self.calls == 2:
with self._session_factory() as session:
document = session.get(Document, document_id)
chunks = session.scalars(
select(Chunk).where(Chunk.document_id == document_id)
).all()
# The source stream is already open here, proving it survived cleanup.
self.retry_started_clean = (
document is not None
and document.parsed_storage_key is None
and not chunks
and not self._vector_store.records
and self._storage.exists(document.storage_key)
)
return self._delegate.parse(document_id, filename, source)
class CleanupFailingVectorStore(FakeVectorStore):
"""Inject a retry cleanup failure without affecting the first attempt."""
fail_delete = False
def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
if self.fail_delete:
raise RuntimeError("cleanup provider failed secret=must-not-persist")
super().delete_document(document_id, index_profile_id)
def _build_runner(
tmp_path: Path,
*,
embedding: FailOnceEmbeddingProvider | FakeEmbeddingProvider | None = None,
vector_store: FakeVectorStore | None = None,
max_attempts: int = 3,
) -> tuple[
DocumentUploadService,
IngestionJobService,
IngestionJobRunner,
sessionmaker[Session],
LocalObjectStorage,
FakeVectorStore,
RetryObservingParser,
]:
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
session_factory = create_session_factory(engine)
storage = LocalObjectStorage(tmp_path / "storage")
selected_store = vector_store or FakeVectorStore()
selected_embedding = embedding or FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
profile = IndexProfile(
profile_id="test-profile",
chunk_strategy_version="structure_v1",
embedding_model=selected_embedding.model_name,
vector_dimension=selected_embedding.dimensions,
)
parser = RetryObservingParser(session_factory, storage, selected_store)
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
parsing = DocumentParsingService(session_factory, storage, parser)
chunking = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
)
indexing = DocumentIndexingService(
session_factory,
selected_embedding,
selected_store,
profile,
)
jobs = IngestionJobService(session_factory, max_attempts=max_attempts)
cleanup = DerivedDocumentCleanupService(
session_factory,
storage,
selected_store,
profile.profile_id,
)
runner = IngestionJobRunner(
session_factory,
parsing,
chunking,
indexing,
cleanup,
retry_backoff_seconds=1,
retry_backoff_max_seconds=4,
lease_seconds=30,
)
return upload, jobs, runner, session_factory, storage, selected_store, parser
def _submit(
upload: DocumentUploadService,
jobs: IngestionJobService,
payload: bytes = b"# Policy\n\nRefunds take seven days.",
) -> tuple[UUID, UUID, RecordingDispatcher]:
dispatcher = RecordingDispatcher()
result = AsyncIngestionSubmissionService(upload, jobs, dispatcher).submit(
"policy.md", "text/markdown", BytesIO(payload)
)
return result.document.id, result.job.id, dispatcher
def _make_retry_due(session_factory: sessionmaker[Session], job_id: UUID) -> None:
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
session.commit()
def test_transient_failure_cleans_derivatives_before_retry(tmp_path: Path) -> None:
embedding = FailOnceEmbeddingProvider()
upload, jobs, runner, session_factory, storage, vector_store, parser = _build_runner(
tmp_path, embedding=embedding
)
document_id, job_id, dispatcher = _submit(upload, jobs)
first = runner.run(job_id)
assert first.action == JobRunAction.RETRY
assert first.retry_delay_seconds == 1
assert dispatcher.job_ids == [job_id]
with session_factory() as session:
stored_job = session.get(IngestionJob, job_id)
assert stored_job is not None
assert stored_job.status == IngestionJobStatus.RETRYING
assert stored_job.attempt_count == 1
assert "must-not-persist" not in (stored_job.error_message or "")
_make_retry_due(session_factory, job_id)
second = runner.run(job_id)
assert second.status == IngestionJobStatus.SUCCEEDED
assert parser.retry_started_clean
with session_factory() as session:
stored_job = session.get(IngestionJob, job_id)
attempts = session.scalars(
select(IngestionAttempt)
.where(IngestionAttempt.job_id == job_id)
.order_by(IngestionAttempt.attempt_number)
).all()
assert stored_job is not None
assert stored_job.attempt_count == 2
assert [attempt.status for attempt in attempts] == [
IngestionAttemptStatus.FAILED,
IngestionAttemptStatus.SUCCEEDED,
]
document = session.get(Document, document_id)
assert document is not None
assert storage.exists(document.storage_key)
assert document.parsed_storage_key is not None
assert storage.exists(document.parsed_storage_key)
assert vector_store.records
def test_permanent_content_failure_does_not_retry(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs, payload=b" \n\t")
result = runner.run(job_id)
assert result.action == JobRunAction.COMPLETE
assert result.status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
assert job.attempt_count == 1
assert job.error_message == "document content could not be processed"
def test_cleanup_failure_is_terminal_and_preserves_source(tmp_path: Path) -> None:
embedding = FailOnceEmbeddingProvider()
vector_store = CleanupFailingVectorStore()
upload, jobs, runner, session_factory, storage, _, _ = _build_runner(
tmp_path,
embedding=embedding,
vector_store=vector_store,
)
document_id, job_id, _ = _submit(upload, jobs)
assert runner.run(job_id).action == JobRunAction.RETRY
_make_retry_due(session_factory, job_id)
vector_store.fail_delete = True
result = runner.run(job_id)
assert result.status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
assert job.attempt_count == 1 # Cleanup failed before attempt 2 was created.
assert job.error_message == "retry cleanup failed"
document = session.get(Document, document_id)
assert document is not None
assert storage.exists(document.storage_key)
def test_duplicate_delivery_defers_while_active_lease_is_valid(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.status = IngestionJobStatus.RUNNING
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
session.commit()
result = runner.run(job_id)
assert result.action == JobRunAction.RETRY
assert result.status == IngestionJobStatus.RUNNING
with session_factory() as session:
assert session.scalars(select(IngestionAttempt)).all() == []
def test_expired_worker_lease_is_recovered_as_a_new_attempt(tmp_path: Path) -> None:
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
job = session.get(IngestionJob, job_id)
assert job is not None
job.status = IngestionJobStatus.RUNNING
job.attempt_count = 1
job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
session.add(IngestionAttempt(job_id=job_id, attempt_number=1))
session.commit()
result = runner.run(job_id)
assert result.status == IngestionJobStatus.SUCCEEDED
with session_factory() as session:
attempts = session.scalars(
select(IngestionAttempt)
.where(IngestionAttempt.job_id == job_id)
.order_by(IngestionAttempt.attempt_number)
).all()
assert [attempt.status for attempt in attempts] == [
IngestionAttemptStatus.FAILED,
IngestionAttemptStatus.SUCCEEDED,
]
assert attempts[0].error_message == "worker execution lease expired"
def test_transient_failures_stop_at_configured_attempt_bound(tmp_path: Path) -> None:
embedding = AlwaysFailEmbeddingProvider()
upload, jobs, runner, session_factory, _, _, _ = _build_runner(
tmp_path,
embedding=embedding,
max_attempts=2,
)
_, job_id, _ = _submit(upload, jobs)
assert runner.run(job_id).action == JobRunAction.RETRY
_make_retry_due(session_factory, job_id)
result = runner.run(job_id)
assert result.action == JobRunAction.COMPLETE
assert result.status == IngestionJobStatus.FAILED
assert runner.run(job_id).status == IngestionJobStatus.FAILED
with session_factory() as session:
job = session.get(IngestionJob, job_id)
attempts = session.scalars(
select(IngestionAttempt).where(IngestionAttempt.job_id == job_id)
).all()
assert job is not None
assert job.attempt_count == job.max_attempts == 2
assert len(attempts) == 2
def test_recovery_dispatches_persisted_job_after_broker_outage(tmp_path: Path) -> None:
upload, jobs, _, _, _, _, _ = _build_runner(tmp_path)
failing = RecordingDispatcher(fail=True)
submission = AsyncIngestionSubmissionService(upload, jobs, failing).submit(
"policy.md", "text/markdown", BytesIO(b"policy")
)
recovered = RecordingDispatcher()
assert dispatch_pending_jobs(jobs, recovered) == 1
assert recovered.job_ids == [submission.job.id]
def test_job_persistence_failure_compensates_document_and_source(tmp_path: Path) -> None:
upload, _, _, session_factory, _, _, _ = _build_runner(tmp_path)
failing_jobs = FailingJobService(session_factory, max_attempts=3)
with pytest.raises(RuntimeError, match="Job persistence failure"):
AsyncIngestionSubmissionService(upload, failing_jobs, RecordingDispatcher()).submit(
"policy.md", "text/markdown", BytesIO(b"policy")
)
with session_factory() as session:
assert session.scalars(select(Document)).all() == []
assert session.scalars(select(IngestionJob)).all() == []
assert not [path for path in tmp_path.rglob("*") if path.is_file()]
def test_database_rejects_two_active_attempts_for_one_job(tmp_path: Path) -> None:
upload, jobs, _, session_factory, _, _, _ = _build_runner(tmp_path)
_, job_id, _ = _submit(upload, jobs)
with session_factory() as session:
session.add_all(
[
IngestionAttempt(job_id=job_id, attempt_number=1),
IngestionAttempt(job_id=job_id, attempt_number=2),
]
)
with pytest.raises(IntegrityError):
session.commit()
session.rollback()