2026-09-02 20:42:52 +08:00

348 lines
13 KiB
Python

"""Runtime provider and application-service composition from safe settings."""
from dataclasses import dataclass
from sqlalchemy.orm import Session, sessionmaker
from ekp_api.celery_queue import CeleryIngestionDispatcher, create_celery_app
from ekp_core.chunking import StructureAwareChunker
from ekp_core.config import Settings
from ekp_core.database import create_database_engine, create_session_factory
from ekp_core.dense_retrieval import DenseRetrievalService
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
from ekp_core.document_indexing import (
DocumentIndexingPipelineService,
DocumentIndexingService,
)
from ekp_core.document_management import DocumentManagementService
from ekp_core.document_parsing import DocumentParsingService
from ekp_core.document_upload import DocumentUploadService
from ekp_core.index_rebuild import resolve_active_index_profile
from ekp_core.ingestion_batches import IngestionBatchService
from ekp_core.ingestion_jobs import (
AsyncIngestionSubmissionService,
DerivedDocumentCleanupService,
IngestionJobRunner,
IngestionJobService,
)
from ekp_core.search import SearchService
from ekp_domain import IndexProfile
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
from ekp_object_storage import ObjectStorage, create_bos_object_storage
from ekp_object_storage.local import LocalObjectStorage
from ekp_parser import ParserRouter
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
from ekp_vector_store import (
VectorStore,
create_oceanbase_vector_store,
)
BGE_LARGE_ZH_DIMENSIONS = 1024
DEMO_CHUNK_STRATEGY_VERSION = "structure_v1"
class ProviderConfigurationError(ValueError):
"""Raised when a runtime provider lacks required injected configuration."""
@dataclass(frozen=True, slots=True)
class ApiServices:
"""Business services injected into REST routes without provider leakage."""
ingestion: DocumentIndexingPipelineService
submissions: AsyncIngestionSubmissionService
batches: IngestionBatchService
jobs: IngestionJobService
documents: DocumentManagementService
search: SearchService
@dataclass(frozen=True, slots=True)
class WorkerServices:
"""Provider graph required by Celery without REST search-only services."""
runner: IngestionJobRunner
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
@dataclass(frozen=True, slots=True)
class _IngestionComponents:
"""Shared composition result used by API and independent worker processes."""
session_factory: sessionmaker[Session]
storage: ObjectStorage
vector_store: VectorStore
embedding_provider: EmbeddingProvider
index_profile: IndexProfile
upload: DocumentUploadService
parsing: DocumentParsingService
chunking: DocumentChunkingService
indexing: DocumentIndexingService
jobs: IngestionJobService
dispatcher: CeleryIngestionDispatcher
def create_index_profile(
settings: Settings,
session_factory: sessionmaker[Session] | None = None,
) -> IndexProfile:
"""Resolve the canonical active profile or the fresh-deployment baseline."""
configured = IndexProfile(
profile_id=settings.index_profile_id,
chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION,
embedding_model=settings.embedding_model,
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
)
if session_factory is None:
return configured
return resolve_active_index_profile(session_factory, configured)
def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider:
"""Create Qianfan only when its API key is supplied by the runtime."""
if settings.baidu_qianfan_api_key is None:
raise ProviderConfigurationError("BAIDU_QIANFAN_API_KEY is required")
return QianfanEmbeddingProvider(
api_key=settings.baidu_qianfan_api_key.get_secret_value(),
endpoint=settings.embedding_endpoint,
timeout_seconds=settings.embedding_timeout_seconds,
)
def create_object_storage(settings: Settings) -> ObjectStorage:
"""Create local or BOS storage while keeping credentials out of core services."""
if settings.object_storage_provider == "local":
return LocalObjectStorage(settings.object_storage_path)
if settings.bos_endpoint is None:
raise ProviderConfigurationError("BOS_ENDPOINT is required")
if settings.bos_access_key_id is None:
raise ProviderConfigurationError("BOS_ACCESS_KEY_ID is required")
if settings.bos_secret_access_key is None:
raise ProviderConfigurationError("BOS_SECRET_ACCESS_KEY is required")
if settings.bos_bucket is None:
raise ProviderConfigurationError("BOS_BUCKET is required")
return create_bos_object_storage(
endpoint=settings.bos_endpoint,
access_key_id=settings.bos_access_key_id.get_secret_value(),
secret_access_key=settings.bos_secret_access_key.get_secret_value(),
bucket_name=settings.bos_bucket,
key_prefix=settings.bos_key_prefix,
timeout_seconds=settings.bos_timeout_seconds,
)
def create_vector_store(settings: Settings) -> VectorStore:
"""Create the OceanBase adapter from explicit, secret-safe settings."""
if settings.oceanbase_vector_uri is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
if settings.oceanbase_vector_user is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
if settings.oceanbase_vector_password is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
if settings.oceanbase_vector_database is None:
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
return create_oceanbase_vector_store(
uri=settings.oceanbase_vector_uri,
user=settings.oceanbase_vector_user,
password=settings.oceanbase_vector_password.get_secret_value(),
database_name=settings.oceanbase_vector_database,
table_name=settings.oceanbase_vector_table,
index_name=settings.oceanbase_vector_index,
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
batch_size=settings.oceanbase_vector_batch_size,
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
)
def create_dense_retrieval_service(
settings: Settings,
embedding_provider: EmbeddingProvider,
vector_store: VectorStore,
index_profile: IndexProfile,
) -> DenseRetrievalService:
"""Compose the profile-scoped dense stage reused by ingestion search APIs."""
return DenseRetrievalService(
embedding_provider=embedding_provider,
vector_store=vector_store,
index_profile=index_profile,
default_top_n=settings.dense_top_n,
)
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
"""Create the Demo BCE reranker using an explicit or shared Qianfan key."""
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
if secret is None:
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
return QianfanRerankerProvider(
api_key=secret.get_secret_value(),
endpoint=settings.reranker_endpoint,
model_name=settings.reranker_model,
timeout_seconds=settings.reranker_timeout_seconds,
)
def create_search_service(
settings: Settings,
session_factory: sessionmaker[Session],
dense_retriever: DenseRetrievalService,
reranker_provider: RerankerProvider,
) -> SearchService:
"""Compose reranked search independently from REST response schemas."""
return SearchService(
session_factory=session_factory,
dense_retriever=dense_retriever,
reranker_provider=reranker_provider,
default_top_k=settings.search_top_k,
)
def create_runtime_search_service(settings: Settings) -> SearchService:
"""Compose only read-path providers for the independent MCP Gateway."""
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings)
dense_retrieval = create_dense_retrieval_service(
settings,
embedding_provider,
vector_store,
index_profile,
)
return create_search_service(
settings,
session_factory,
dense_retrieval,
create_reranker_provider(settings),
)
def _create_ingestion_components(settings: Settings) -> _IngestionComponents:
"""Compose provider-neutral ingestion stages and the Celery delivery adapter."""
if settings.rabbitmq_url is None or not settings.rabbitmq_url.get_secret_value().strip():
raise ProviderConfigurationError("RABBITMQ_URL is required")
engine = create_database_engine(settings)
session_factory = create_session_factory(engine)
storage = create_object_storage(settings)
index_profile = create_index_profile(settings, session_factory)
embedding_provider = create_embedding_provider(settings)
vector_store = create_vector_store(settings)
upload_service = DocumentUploadService(
session_factory,
storage,
max_upload_bytes=settings.max_upload_bytes,
)
parsing_service = DocumentParsingService(session_factory, storage, ParserRouter())
chunking_service = DocumentChunkingService(
session_factory,
storage,
StructureAwareChunker(
max_tokens=settings.chunk_max_tokens,
overlap_tokens=settings.chunk_overlap_tokens,
max_characters=settings.chunk_max_characters,
),
)
indexing_service = DocumentIndexingService(
session_factory,
embedding_provider,
vector_store,
index_profile,
)
jobs = IngestionJobService(session_factory, settings.ingestion_max_attempts)
dispatcher = CeleryIngestionDispatcher(create_celery_app(settings))
return _IngestionComponents(
session_factory=session_factory,
storage=storage,
vector_store=vector_store,
embedding_provider=embedding_provider,
index_profile=index_profile,
upload=upload_service,
parsing=parsing_service,
chunking=chunking_service,
indexing=indexing_service,
jobs=jobs,
dispatcher=dispatcher,
)
def _create_runner(settings: Settings, components: _IngestionComponents) -> IngestionJobRunner:
"""Compose retry cleanup and execution policy from non-sensitive settings."""
cleanup = DerivedDocumentCleanupService(
components.session_factory,
components.storage,
components.vector_store,
components.index_profile.profile_id,
)
return IngestionJobRunner(
components.session_factory,
components.parsing,
components.chunking,
components.indexing,
cleanup,
retry_backoff_seconds=settings.ingestion_retry_backoff_seconds,
retry_backoff_max_seconds=settings.ingestion_retry_backoff_max_seconds,
lease_seconds=settings.ingestion_worker_lease_seconds,
)
def create_worker_services(settings: Settings) -> WorkerServices:
"""Build the independent worker runtime from the same application boundaries."""
components = _create_ingestion_components(settings)
return WorkerServices(
runner=_create_runner(settings, components),
jobs=components.jobs,
dispatcher=components.dispatcher,
)
def create_api_services(settings: Settings) -> ApiServices:
"""Compose REST services while ingestion execution remains in workers."""
components = _create_ingestion_components(settings)
document_pipeline = DocumentPipelineService(
components.upload,
components.parsing,
components.chunking,
)
ingestion = DocumentIndexingPipelineService(document_pipeline, components.indexing)
submissions = AsyncIngestionSubmissionService(
components.upload,
components.jobs,
components.dispatcher,
)
batches = IngestionBatchService(
components.session_factory,
submissions,
max_files=settings.batch_max_files,
max_total_bytes=settings.batch_max_total_bytes,
)
dense_retrieval = create_dense_retrieval_service(
settings,
components.embedding_provider,
components.vector_store,
components.index_profile,
)
reranker_provider = create_reranker_provider(settings)
search = create_search_service(
settings,
components.session_factory,
dense_retrieval,
reranker_provider,
)
documents = DocumentManagementService(
components.session_factory,
components.storage,
components.vector_store,
active_index_profile_id=components.index_profile.profile_id,
)
return ApiServices(
ingestion=ingestion,
submissions=submissions,
batches=batches,
jobs=components.jobs,
documents=documents,
search=search,
)