220 lines
8.4 KiB
Python
220 lines
8.4 KiB
Python
"""Runtime provider and application-service composition from safe settings."""
|
|
|
|
from dataclasses import dataclass
|
|
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
|
|
from ekp_core.chunking import StructureAwareChunker
|
|
from ekp_core.config import Settings
|
|
from ekp_core.database import create_database_engine, create_session_factory
|
|
from ekp_core.dense_retrieval import DenseRetrievalService
|
|
from ekp_core.document_chunking import DocumentChunkingService, DocumentPipelineService
|
|
from ekp_core.document_indexing import (
|
|
DocumentIndexingPipelineService,
|
|
DocumentIndexingService,
|
|
)
|
|
from ekp_core.document_management import DocumentManagementService
|
|
from ekp_core.document_parsing import DocumentParsingService
|
|
from ekp_core.document_upload import DocumentUploadService
|
|
from ekp_core.index_rebuild import resolve_active_index_profile
|
|
from ekp_core.search import SearchService
|
|
from ekp_domain import IndexProfile
|
|
from ekp_embedding import EmbeddingProvider, QianfanEmbeddingProvider
|
|
from ekp_object_storage import ObjectStorage, create_bos_object_storage
|
|
from ekp_object_storage.local import LocalObjectStorage
|
|
from ekp_parser import ParserRouter
|
|
from ekp_reranker import QianfanRerankerProvider, RerankerProvider
|
|
from ekp_vector_store import (
|
|
VectorStore,
|
|
create_oceanbase_vector_store,
|
|
)
|
|
|
|
BGE_LARGE_ZH_DIMENSIONS = 1024
|
|
DEMO_CHUNK_STRATEGY_VERSION = "structure_v1"
|
|
|
|
|
|
class ProviderConfigurationError(ValueError):
|
|
"""Raised when a runtime provider lacks required injected configuration."""
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ApiServices:
|
|
"""Business services injected into REST routes without provider leakage."""
|
|
|
|
ingestion: DocumentIndexingPipelineService
|
|
documents: DocumentManagementService
|
|
search: SearchService
|
|
|
|
|
|
def create_index_profile(
|
|
settings: Settings,
|
|
session_factory: sessionmaker[Session] | None = None,
|
|
) -> IndexProfile:
|
|
"""Resolve the canonical active profile or the fresh-deployment baseline."""
|
|
configured = IndexProfile(
|
|
profile_id=settings.index_profile_id,
|
|
chunk_strategy_version=DEMO_CHUNK_STRATEGY_VERSION,
|
|
embedding_model=settings.embedding_model,
|
|
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
|
)
|
|
if session_factory is None:
|
|
return configured
|
|
return resolve_active_index_profile(session_factory, configured)
|
|
|
|
|
|
def create_embedding_provider(settings: Settings) -> QianfanEmbeddingProvider:
|
|
"""Create Qianfan only when its API key is supplied by the runtime."""
|
|
if settings.baidu_qianfan_api_key is None:
|
|
raise ProviderConfigurationError("BAIDU_QIANFAN_API_KEY is required")
|
|
return QianfanEmbeddingProvider(
|
|
api_key=settings.baidu_qianfan_api_key.get_secret_value(),
|
|
endpoint=settings.embedding_endpoint,
|
|
timeout_seconds=settings.embedding_timeout_seconds,
|
|
)
|
|
|
|
|
|
def create_object_storage(settings: Settings) -> ObjectStorage:
|
|
"""Create local or BOS storage while keeping credentials out of core services."""
|
|
if settings.object_storage_provider == "local":
|
|
return LocalObjectStorage(settings.object_storage_path)
|
|
if settings.bos_endpoint is None:
|
|
raise ProviderConfigurationError("BOS_ENDPOINT is required")
|
|
if settings.bos_access_key_id is None:
|
|
raise ProviderConfigurationError("BOS_ACCESS_KEY_ID is required")
|
|
if settings.bos_secret_access_key is None:
|
|
raise ProviderConfigurationError("BOS_SECRET_ACCESS_KEY is required")
|
|
if settings.bos_bucket is None:
|
|
raise ProviderConfigurationError("BOS_BUCKET is required")
|
|
return create_bos_object_storage(
|
|
endpoint=settings.bos_endpoint,
|
|
access_key_id=settings.bos_access_key_id.get_secret_value(),
|
|
secret_access_key=settings.bos_secret_access_key.get_secret_value(),
|
|
bucket_name=settings.bos_bucket,
|
|
key_prefix=settings.bos_key_prefix,
|
|
timeout_seconds=settings.bos_timeout_seconds,
|
|
)
|
|
|
|
|
|
def create_vector_store(settings: Settings) -> VectorStore:
|
|
"""Create the OceanBase adapter from explicit, secret-safe settings."""
|
|
if settings.oceanbase_vector_uri is None:
|
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_URI is required")
|
|
if settings.oceanbase_vector_user is None:
|
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_USER is required")
|
|
if settings.oceanbase_vector_password is None:
|
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_PASSWORD is required")
|
|
if settings.oceanbase_vector_database is None:
|
|
raise ProviderConfigurationError("OCEANBASE_VECTOR_DATABASE is required")
|
|
return create_oceanbase_vector_store(
|
|
uri=settings.oceanbase_vector_uri,
|
|
user=settings.oceanbase_vector_user,
|
|
password=settings.oceanbase_vector_password.get_secret_value(),
|
|
database_name=settings.oceanbase_vector_database,
|
|
table_name=settings.oceanbase_vector_table,
|
|
index_name=settings.oceanbase_vector_index,
|
|
vector_dimension=BGE_LARGE_ZH_DIMENSIONS,
|
|
batch_size=settings.oceanbase_vector_batch_size,
|
|
timeout_seconds=settings.oceanbase_vector_timeout_seconds,
|
|
)
|
|
|
|
|
|
def create_dense_retrieval_service(
|
|
settings: Settings,
|
|
embedding_provider: EmbeddingProvider,
|
|
vector_store: VectorStore,
|
|
index_profile: IndexProfile,
|
|
) -> DenseRetrievalService:
|
|
"""Compose the profile-scoped dense stage reused by ingestion search APIs."""
|
|
return DenseRetrievalService(
|
|
embedding_provider=embedding_provider,
|
|
vector_store=vector_store,
|
|
index_profile=index_profile,
|
|
default_top_n=settings.dense_top_n,
|
|
)
|
|
|
|
|
|
def create_reranker_provider(settings: Settings) -> QianfanRerankerProvider:
|
|
"""Create the Demo BCE reranker using an explicit or shared Qianfan key."""
|
|
secret = settings.reranker_api_key or settings.baidu_qianfan_api_key
|
|
if secret is None:
|
|
raise ProviderConfigurationError("RERANKER_API_KEY or BAIDU_QIANFAN_API_KEY is required")
|
|
return QianfanRerankerProvider(
|
|
api_key=secret.get_secret_value(),
|
|
endpoint=settings.reranker_endpoint,
|
|
model_name=settings.reranker_model,
|
|
timeout_seconds=settings.reranker_timeout_seconds,
|
|
)
|
|
|
|
|
|
def create_search_service(
|
|
settings: Settings,
|
|
session_factory: sessionmaker[Session],
|
|
dense_retriever: DenseRetrievalService,
|
|
reranker_provider: RerankerProvider,
|
|
) -> SearchService:
|
|
"""Compose reranked search independently from REST response schemas."""
|
|
return SearchService(
|
|
session_factory=session_factory,
|
|
dense_retriever=dense_retriever,
|
|
reranker_provider=reranker_provider,
|
|
default_top_k=settings.search_top_k,
|
|
)
|
|
|
|
|
|
def create_api_services(settings: Settings) -> ApiServices:
|
|
"""Compose the synchronous Demo runtime once for all business endpoints."""
|
|
engine = create_database_engine(settings)
|
|
session_factory = create_session_factory(engine)
|
|
storage = create_object_storage(settings)
|
|
index_profile = create_index_profile(settings, session_factory)
|
|
embedding_provider = create_embedding_provider(settings)
|
|
vector_store = create_vector_store(settings)
|
|
|
|
upload_service = DocumentUploadService(
|
|
session_factory,
|
|
storage,
|
|
max_upload_bytes=settings.max_upload_bytes,
|
|
)
|
|
parsing_service = DocumentParsingService(session_factory, storage, ParserRouter())
|
|
chunking_service = DocumentChunkingService(
|
|
session_factory,
|
|
storage,
|
|
StructureAwareChunker(
|
|
max_tokens=settings.chunk_max_tokens,
|
|
overlap_tokens=settings.chunk_overlap_tokens,
|
|
max_characters=settings.chunk_max_characters,
|
|
),
|
|
)
|
|
document_pipeline = DocumentPipelineService(
|
|
upload_service,
|
|
parsing_service,
|
|
chunking_service,
|
|
)
|
|
indexing_service = DocumentIndexingService(
|
|
session_factory,
|
|
embedding_provider,
|
|
vector_store,
|
|
index_profile,
|
|
)
|
|
ingestion = DocumentIndexingPipelineService(document_pipeline, indexing_service)
|
|
dense_retrieval = create_dense_retrieval_service(
|
|
settings,
|
|
embedding_provider,
|
|
vector_store,
|
|
index_profile,
|
|
)
|
|
reranker_provider = create_reranker_provider(settings)
|
|
search = create_search_service(
|
|
settings,
|
|
session_factory,
|
|
dense_retrieval,
|
|
reranker_provider,
|
|
)
|
|
documents = DocumentManagementService(
|
|
session_factory,
|
|
storage,
|
|
vector_store,
|
|
active_index_profile_id=index_profile.profile_id,
|
|
)
|
|
return ApiServices(ingestion=ingestion, documents=documents, search=search)
|