432 lines
15 KiB
Python
432 lines
15 KiB
Python
"""Durable Job, bounded retry and cleanup-before-retry integration tests."""
|
|
|
|
from collections.abc import Sequence
|
|
from datetime import UTC, datetime, timedelta
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from typing import BinaryIO
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.database import create_session_factory
|
|
from ekp_core.document_chunking import DocumentChunkingService
|
|
from ekp_core.document_indexing import DocumentIndexingService
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_core.ingestion_jobs import (
|
|
AsyncIngestionSubmissionService,
|
|
DerivedDocumentCleanupService,
|
|
IngestionJobRunner,
|
|
IngestionJobService,
|
|
JobRunAction,
|
|
dispatch_pending_jobs,
|
|
)
|
|
from ekp_domain import (
|
|
Base,
|
|
Chunk,
|
|
Document,
|
|
IndexProfile,
|
|
IngestionAttempt,
|
|
IngestionAttemptStatus,
|
|
IngestionJob,
|
|
IngestionJobStatus,
|
|
ParsedDocument,
|
|
)
|
|
from ekp_embedding import EmbeddingProviderError, FakeEmbeddingProvider
|
|
from ekp_object_storage import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
from ekp_vector_store import FakeVectorStore
|
|
|
|
|
|
class RecordingDispatcher:
|
|
"""Capture delivery requests while preserving the dispatcher contract."""
|
|
|
|
def __init__(self, *, fail: bool = False) -> None:
|
|
self.fail = fail
|
|
self.job_ids: list[UUID] = []
|
|
|
|
def enqueue(self, job_id: UUID) -> None:
|
|
if self.fail:
|
|
raise RuntimeError("simulated broker outage containing secret=must-not-persist")
|
|
self.job_ids.append(job_id)
|
|
|
|
|
|
class FailingJobService(IngestionJobService):
|
|
"""Inject metadata failure after immutable source upload."""
|
|
|
|
def create(self, document_id: UUID) -> IngestionJob:
|
|
del document_id
|
|
raise RuntimeError("simulated Job persistence failure")
|
|
|
|
|
|
class FailOnceEmbeddingProvider:
|
|
"""Inject one transient provider failure after parse/chunk persistence."""
|
|
|
|
def __init__(self) -> None:
|
|
self._delegate = FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
|
|
self.calls = 0
|
|
|
|
@property
|
|
def model_name(self) -> str:
|
|
return self._delegate.model_name
|
|
|
|
@property
|
|
def dimensions(self) -> int:
|
|
return self._delegate.dimensions
|
|
|
|
@property
|
|
def max_batch_size(self) -> int:
|
|
return self._delegate.max_batch_size
|
|
|
|
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
|
self.calls += 1
|
|
if self.calls == 1:
|
|
raise EmbeddingProviderError("temporary failure token=must-not-persist")
|
|
return self._delegate.embed(texts)
|
|
|
|
|
|
class AlwaysFailEmbeddingProvider(FailOnceEmbeddingProvider):
|
|
"""Inject a transient error on every attempt to verify the retry bound."""
|
|
|
|
def embed(self, texts: Sequence[str]) -> tuple[tuple[float, ...], ...]:
|
|
del texts
|
|
self.calls += 1
|
|
raise EmbeddingProviderError("temporary failure token=must-not-persist")
|
|
|
|
|
|
class RetryObservingParser:
|
|
"""Verify cleanup finished before the second provider execution begins."""
|
|
|
|
def __init__(
|
|
self,
|
|
session_factory: sessionmaker[Session],
|
|
storage: LocalObjectStorage,
|
|
vector_store: FakeVectorStore,
|
|
) -> None:
|
|
self._delegate = ParserRouter()
|
|
self._session_factory = session_factory
|
|
self._storage = storage
|
|
self._vector_store = vector_store
|
|
self.calls = 0
|
|
self.retry_started_clean = False
|
|
|
|
def parse(
|
|
self,
|
|
document_id: UUID,
|
|
filename: str,
|
|
source: BinaryIO,
|
|
) -> ParsedDocument:
|
|
self.calls += 1
|
|
if self.calls == 2:
|
|
with self._session_factory() as session:
|
|
document = session.get(Document, document_id)
|
|
chunks = session.scalars(
|
|
select(Chunk).where(Chunk.document_id == document_id)
|
|
).all()
|
|
# The source stream is already open here, proving it survived cleanup.
|
|
self.retry_started_clean = (
|
|
document is not None
|
|
and document.parsed_storage_key is None
|
|
and not chunks
|
|
and not self._vector_store.records
|
|
and self._storage.exists(document.storage_key)
|
|
)
|
|
return self._delegate.parse(document_id, filename, source)
|
|
|
|
|
|
class CleanupFailingVectorStore(FakeVectorStore):
|
|
"""Inject a retry cleanup failure without affecting the first attempt."""
|
|
|
|
fail_delete = False
|
|
|
|
def delete_document(self, document_id: UUID, index_profile_id: str) -> None:
|
|
if self.fail_delete:
|
|
raise RuntimeError("cleanup provider failed secret=must-not-persist")
|
|
super().delete_document(document_id, index_profile_id)
|
|
|
|
|
|
def _build_runner(
|
|
tmp_path: Path,
|
|
*,
|
|
embedding: FailOnceEmbeddingProvider | FakeEmbeddingProvider | None = None,
|
|
vector_store: FakeVectorStore | None = None,
|
|
max_attempts: int = 3,
|
|
) -> tuple[
|
|
DocumentUploadService,
|
|
IngestionJobService,
|
|
IngestionJobRunner,
|
|
sessionmaker[Session],
|
|
LocalObjectStorage,
|
|
FakeVectorStore,
|
|
RetryObservingParser,
|
|
]:
|
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
|
Base.metadata.create_all(engine)
|
|
session_factory = create_session_factory(engine)
|
|
storage = LocalObjectStorage(tmp_path / "storage")
|
|
selected_store = vector_store or FakeVectorStore()
|
|
selected_embedding = embedding or FakeEmbeddingProvider(model_name="fake-bge", dimensions=8)
|
|
profile = IndexProfile(
|
|
profile_id="test-profile",
|
|
chunk_strategy_version="structure_v1",
|
|
embedding_model=selected_embedding.model_name,
|
|
vector_dimension=selected_embedding.dimensions,
|
|
)
|
|
parser = RetryObservingParser(session_factory, storage, selected_store)
|
|
upload = DocumentUploadService(session_factory, storage, max_upload_bytes=1024 * 1024)
|
|
parsing = DocumentParsingService(session_factory, storage, parser)
|
|
chunking = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(max_tokens=450, overlap_tokens=75),
|
|
)
|
|
indexing = DocumentIndexingService(
|
|
session_factory,
|
|
selected_embedding,
|
|
selected_store,
|
|
profile,
|
|
)
|
|
jobs = IngestionJobService(session_factory, max_attempts=max_attempts)
|
|
cleanup = DerivedDocumentCleanupService(
|
|
session_factory,
|
|
storage,
|
|
selected_store,
|
|
profile.profile_id,
|
|
)
|
|
runner = IngestionJobRunner(
|
|
session_factory,
|
|
parsing,
|
|
chunking,
|
|
indexing,
|
|
cleanup,
|
|
retry_backoff_seconds=1,
|
|
retry_backoff_max_seconds=4,
|
|
lease_seconds=30,
|
|
)
|
|
return upload, jobs, runner, session_factory, storage, selected_store, parser
|
|
|
|
|
|
def _submit(
|
|
upload: DocumentUploadService,
|
|
jobs: IngestionJobService,
|
|
payload: bytes = b"# Policy\n\nRefunds take seven days.",
|
|
) -> tuple[UUID, UUID, RecordingDispatcher]:
|
|
dispatcher = RecordingDispatcher()
|
|
result = AsyncIngestionSubmissionService(upload, jobs, dispatcher).submit(
|
|
"policy.md", "text/markdown", BytesIO(payload)
|
|
)
|
|
return result.document.id, result.job.id, dispatcher
|
|
|
|
|
|
def _make_retry_due(session_factory: sessionmaker[Session], job_id: UUID) -> None:
|
|
with session_factory() as session:
|
|
job = session.get(IngestionJob, job_id)
|
|
assert job is not None
|
|
job.next_run_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
session.commit()
|
|
|
|
|
|
def test_transient_failure_cleans_derivatives_before_retry(tmp_path: Path) -> None:
|
|
embedding = FailOnceEmbeddingProvider()
|
|
upload, jobs, runner, session_factory, storage, vector_store, parser = _build_runner(
|
|
tmp_path, embedding=embedding
|
|
)
|
|
document_id, job_id, dispatcher = _submit(upload, jobs)
|
|
|
|
first = runner.run(job_id)
|
|
assert first.action == JobRunAction.RETRY
|
|
assert first.retry_delay_seconds == 1
|
|
assert dispatcher.job_ids == [job_id]
|
|
with session_factory() as session:
|
|
stored_job = session.get(IngestionJob, job_id)
|
|
assert stored_job is not None
|
|
assert stored_job.status == IngestionJobStatus.RETRYING
|
|
assert stored_job.attempt_count == 1
|
|
assert "must-not-persist" not in (stored_job.error_message or "")
|
|
|
|
_make_retry_due(session_factory, job_id)
|
|
second = runner.run(job_id)
|
|
|
|
assert second.status == IngestionJobStatus.SUCCEEDED
|
|
assert parser.retry_started_clean
|
|
with session_factory() as session:
|
|
stored_job = session.get(IngestionJob, job_id)
|
|
attempts = session.scalars(
|
|
select(IngestionAttempt)
|
|
.where(IngestionAttempt.job_id == job_id)
|
|
.order_by(IngestionAttempt.attempt_number)
|
|
).all()
|
|
assert stored_job is not None
|
|
assert stored_job.attempt_count == 2
|
|
assert [attempt.status for attempt in attempts] == [
|
|
IngestionAttemptStatus.FAILED,
|
|
IngestionAttemptStatus.SUCCEEDED,
|
|
]
|
|
document = session.get(Document, document_id)
|
|
assert document is not None
|
|
assert storage.exists(document.storage_key)
|
|
assert document.parsed_storage_key is not None
|
|
assert storage.exists(document.parsed_storage_key)
|
|
assert vector_store.records
|
|
|
|
|
|
def test_permanent_content_failure_does_not_retry(tmp_path: Path) -> None:
|
|
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
|
|
_, job_id, _ = _submit(upload, jobs, payload=b" \n\t")
|
|
|
|
result = runner.run(job_id)
|
|
|
|
assert result.action == JobRunAction.COMPLETE
|
|
assert result.status == IngestionJobStatus.FAILED
|
|
with session_factory() as session:
|
|
job = session.get(IngestionJob, job_id)
|
|
assert job is not None
|
|
assert job.attempt_count == 1
|
|
assert job.error_message == "document content could not be processed"
|
|
|
|
|
|
def test_cleanup_failure_is_terminal_and_preserves_source(tmp_path: Path) -> None:
|
|
embedding = FailOnceEmbeddingProvider()
|
|
vector_store = CleanupFailingVectorStore()
|
|
upload, jobs, runner, session_factory, storage, _, _ = _build_runner(
|
|
tmp_path,
|
|
embedding=embedding,
|
|
vector_store=vector_store,
|
|
)
|
|
document_id, job_id, _ = _submit(upload, jobs)
|
|
assert runner.run(job_id).action == JobRunAction.RETRY
|
|
_make_retry_due(session_factory, job_id)
|
|
vector_store.fail_delete = True
|
|
|
|
result = runner.run(job_id)
|
|
|
|
assert result.status == IngestionJobStatus.FAILED
|
|
with session_factory() as session:
|
|
job = session.get(IngestionJob, job_id)
|
|
assert job is not None
|
|
assert job.attempt_count == 1 # Cleanup failed before attempt 2 was created.
|
|
assert job.error_message == "retry cleanup failed"
|
|
document = session.get(Document, document_id)
|
|
assert document is not None
|
|
assert storage.exists(document.storage_key)
|
|
|
|
|
|
def test_duplicate_delivery_defers_while_active_lease_is_valid(tmp_path: Path) -> None:
|
|
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
|
|
_, job_id, _ = _submit(upload, jobs)
|
|
with session_factory() as session:
|
|
job = session.get(IngestionJob, job_id)
|
|
assert job is not None
|
|
job.status = IngestionJobStatus.RUNNING
|
|
job.lease_expires_at = datetime.now(UTC) + timedelta(seconds=30)
|
|
session.commit()
|
|
|
|
result = runner.run(job_id)
|
|
|
|
assert result.action == JobRunAction.RETRY
|
|
assert result.status == IngestionJobStatus.RUNNING
|
|
with session_factory() as session:
|
|
assert session.scalars(select(IngestionAttempt)).all() == []
|
|
|
|
|
|
def test_expired_worker_lease_is_recovered_as_a_new_attempt(tmp_path: Path) -> None:
|
|
upload, jobs, runner, session_factory, _, _, _ = _build_runner(tmp_path)
|
|
_, job_id, _ = _submit(upload, jobs)
|
|
with session_factory() as session:
|
|
job = session.get(IngestionJob, job_id)
|
|
assert job is not None
|
|
job.status = IngestionJobStatus.RUNNING
|
|
job.attempt_count = 1
|
|
job.lease_expires_at = datetime.now(UTC) - timedelta(seconds=1)
|
|
session.add(IngestionAttempt(job_id=job_id, attempt_number=1))
|
|
session.commit()
|
|
|
|
result = runner.run(job_id)
|
|
|
|
assert result.status == IngestionJobStatus.SUCCEEDED
|
|
with session_factory() as session:
|
|
attempts = session.scalars(
|
|
select(IngestionAttempt)
|
|
.where(IngestionAttempt.job_id == job_id)
|
|
.order_by(IngestionAttempt.attempt_number)
|
|
).all()
|
|
assert [attempt.status for attempt in attempts] == [
|
|
IngestionAttemptStatus.FAILED,
|
|
IngestionAttemptStatus.SUCCEEDED,
|
|
]
|
|
assert attempts[0].error_message == "worker execution lease expired"
|
|
|
|
|
|
def test_transient_failures_stop_at_configured_attempt_bound(tmp_path: Path) -> None:
|
|
embedding = AlwaysFailEmbeddingProvider()
|
|
upload, jobs, runner, session_factory, _, _, _ = _build_runner(
|
|
tmp_path,
|
|
embedding=embedding,
|
|
max_attempts=2,
|
|
)
|
|
_, job_id, _ = _submit(upload, jobs)
|
|
|
|
assert runner.run(job_id).action == JobRunAction.RETRY
|
|
_make_retry_due(session_factory, job_id)
|
|
result = runner.run(job_id)
|
|
|
|
assert result.action == JobRunAction.COMPLETE
|
|
assert result.status == IngestionJobStatus.FAILED
|
|
assert runner.run(job_id).status == IngestionJobStatus.FAILED
|
|
with session_factory() as session:
|
|
job = session.get(IngestionJob, job_id)
|
|
attempts = session.scalars(
|
|
select(IngestionAttempt).where(IngestionAttempt.job_id == job_id)
|
|
).all()
|
|
assert job is not None
|
|
assert job.attempt_count == job.max_attempts == 2
|
|
assert len(attempts) == 2
|
|
|
|
|
|
def test_recovery_dispatches_persisted_job_after_broker_outage(tmp_path: Path) -> None:
|
|
upload, jobs, _, _, _, _, _ = _build_runner(tmp_path)
|
|
failing = RecordingDispatcher(fail=True)
|
|
submission = AsyncIngestionSubmissionService(upload, jobs, failing).submit(
|
|
"policy.md", "text/markdown", BytesIO(b"policy")
|
|
)
|
|
recovered = RecordingDispatcher()
|
|
|
|
assert dispatch_pending_jobs(jobs, recovered) == 1
|
|
assert recovered.job_ids == [submission.job.id]
|
|
|
|
|
|
def test_job_persistence_failure_compensates_document_and_source(tmp_path: Path) -> None:
|
|
upload, _, _, session_factory, _, _, _ = _build_runner(tmp_path)
|
|
failing_jobs = FailingJobService(session_factory, max_attempts=3)
|
|
|
|
with pytest.raises(RuntimeError, match="Job persistence failure"):
|
|
AsyncIngestionSubmissionService(upload, failing_jobs, RecordingDispatcher()).submit(
|
|
"policy.md", "text/markdown", BytesIO(b"policy")
|
|
)
|
|
|
|
with session_factory() as session:
|
|
assert session.scalars(select(Document)).all() == []
|
|
assert session.scalars(select(IngestionJob)).all() == []
|
|
assert not [path for path in tmp_path.rglob("*") if path.is_file()]
|
|
|
|
|
|
def test_database_rejects_two_active_attempts_for_one_job(tmp_path: Path) -> None:
|
|
upload, jobs, _, session_factory, _, _, _ = _build_runner(tmp_path)
|
|
_, job_id, _ = _submit(upload, jobs)
|
|
|
|
with session_factory() as session:
|
|
session.add_all(
|
|
[
|
|
IngestionAttempt(job_id=job_id, attempt_number=1),
|
|
IngestionAttempt(job_id=job_id, attempt_number=2),
|
|
]
|
|
)
|
|
with pytest.raises(IntegrityError):
|
|
session.commit()
|
|
session.rollback()
|