commit 069c1964e571e748d164bb88da5d040dc8135722 Author: d-robotics Date: Mon Aug 31 17:57:00 2026 +0800 docs: add reviewed product and architecture specifications diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..64d3bf0 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true + +[*.py] +indent_size = 4 + +[Makefile] +indent_style = tab diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..bef571a --- /dev/null +++ b/.env.example @@ -0,0 +1,45 @@ +APP_ENV=development +APP_NAME=enterprise-knowledge-platform +LOG_LEVEL=INFO +API_HOST=0.0.0.0 +API_PORT=8000 + +DATABASE_URL=postgresql+psycopg://ekp:ekp@localhost:5432/ekp +REDIS_URL=redis://localhost:6379/0 +RABBITMQ_URL=amqp://guest:guest@localhost:5672// + +OBJECT_STORAGE_PROVIDER=s3 +OBJECT_STORAGE_ENDPOINT=http://localhost:9000 +OBJECT_STORAGE_ACCESS_KEY= +OBJECT_STORAGE_SECRET_KEY= +OBJECT_STORAGE_BUCKET=ekp + +VECTOR_STORE_PROVIDER=mochow +MOCHOW_ENDPOINT= +MOCHOW_API_KEY= +MOCHOW_DATABASE= +MOCHOW_TABLE= + +EMBEDDING_PROVIDER=qianfan +EMBEDDING_MODEL=bge-large-zh +EMBEDDING_ENDPOINT= +EMBEDDING_API_KEY= + +RERANKER_PROVIDER=qianfan +RERANKER_MODEL= +RERANKER_ENDPOINT= +RERANKER_API_KEY= + +LLM_PROVIDER=qianfan +LLM_MODEL= +LLM_ENDPOINT= +LLM_API_KEY= + +PARSER_PROVIDER=mineru +OCR_PROVIDER=mineru + +JWT_SECRET= +API_KEY_PEPPER= + +ENABLE_KNOWLEDGE_COMPILATION=true +ENABLE_MCP=true diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fd8df47 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +* text=auto eol=lf +*.sh text eol=lf +*.py text eol=lf +*.md text eol=lf +*.yaml text eol=lf +*.yml text eol=lf +*.json text eol=lf +*.toml text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.png binary +*.jpg binary +*.jpeg binary +*.pdf binary +*.docx binary +*.pptx binary +*.xlsx binary diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1161b43 --- /dev/null +++ b/.gitignore @@ -0,0 +1,60 @@ +# Environment / secrets +.env +.env.* +!.env.example + +# Python +__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.pyright/ +.ruff_cache/ +.coverage +htmlcov/ +.venv/ +venv/ + +# Node / frontend +node_modules/ +dist/ +build/ +.next/ +coverage/ +*.tsbuildinfo + +# IDE / OS +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store +Thumbs.db + +# Runtime / local data +logs/ +tmp/ +data/ +storage/ +uploads/ +models/ +artifacts/ +reports/ + +# Secrets / credentials +*.pem +*.key +*.p12 +credentials.json +secrets.json + +# Model artifacts +*.bin +*.safetensors +*.onnx +*.pt +*.pth + +# Local DB +*.sqlite +*.sqlite3 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ff091c1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,46 @@ +# Enterprise Knowledge Platform — Codex Rules + +## Read first + +Before modifying code, read the documents relevant to the task: + +- `docs/product/PRD.md` +- `docs/architecture/ARCHITECTURE.md` +- `docs/architecture/ADR.md` +- `docs/api/API_CONTRACT.md` +- `docs/api/MCP_CONTRACT.md` +- `docs/plans/V1_EXECUTION_PLAN.md` +- `docs/plans/TASKS.md` +- `docs/development/DEFINITION_OF_DONE.md` + +Do not redesign accepted architecture unless the task explicitly asks for an architecture change. + +## Architecture invariants + +1. API First. +2. V1 uses a modular monolith plus independent async/GPU workers and an MCP gateway. +3. PostgreSQL + object storage hold canonical data. +4. Vector databases are rebuildable indexes, not sources of truth. +5. Business services depend on `VectorStore` abstractions, never directly on Mochow. +6. Embedding, reranking, parsing, OCR, object storage, and LLMs use provider interfaces. +7. Original source documents are immutable. +8. Compiled Knowledge is derived and traceable to source document versions/chunks. +9. ACL is resolved before or during retrieval. Never retrieve global Top-K and filter afterward. +10. MCP exposes business capabilities, not infrastructure/provider details. +11. Index profiles are versioned; model/chunk changes use shadow indexes before activation. +12. Do not split V1 into unnecessary microservices. +13. Never hardcode credentials or production endpoints. +14. Core retrieval changes require tests and RAG regression evaluation. + +## Task discipline + +- Work on one Task ID at a time. +- Keep scope limited to the task. +- Do not opportunistically implement future tasks. +- Update relevant contracts/docs when behavior changes. + +## Before finishing + +Run relevant checks: format, lint, typecheck, unit tests, relevant integration tests, migration checks, API/MCP contract checks, and RAG evaluation for retrieval changes. + +Report changed files, tests/results, unresolved risks, and follow-up tasks. diff --git a/CODEX_START_HERE.md b/CODEX_START_HERE.md new file mode 100644 index 0000000..2067f4c --- /dev/null +++ b/CODEX_START_HERE.md @@ -0,0 +1,19 @@ +# Codex Start Here + +Do not ask Codex to build all of V1 in one task. + +Recommended first prompt: + +> You are taking over the Enterprise Knowledge Platform repository. Do not modify code yet. Read AGENTS.md and the product, architecture, API/MCP, execution-plan, task and Definition-of-Done documents it references. Inspect the repository. Then output: (1) current implementation status, (2) gap analysis against V1, (3) top five technical risks, (4) next 1–3 recommended Task IDs, and (5) blockers. Do not redesign accepted ADRs. + +Preferred cadence: + +1. Plan one task. +2. Review the plan. +3. Implement one task. +4. Run validation. +5. Self-review as a PR. +6. Commit / open PR. +7. Move to the next task. + +One Task = one branch = one PR whenever possible. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c1b31f5 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +.PHONY: help install format lint typecheck test test-unit test-integration eval up down + +help: + @echo "Enterprise Knowledge Platform" + @echo " install Install development dependencies" + @echo " format Format code" + @echo " lint Run linters" + @echo " typecheck Run static type checks" + @echo " test Run default tests" + @echo " test-unit Run unit tests" + @echo " test-integration Run integration tests" + @echo " eval Run RAG evaluation" + @echo " up Start local dependencies" + @echo " down Stop local dependencies" + +install: + @echo "TODO: implement after toolchain selection" +format: + @echo "TODO: wire formatter" +lint: + @echo "TODO: wire linter" +typecheck: + @echo "TODO: wire type checker" +test: + @echo "TODO: wire test suite" +test-unit: + @echo "TODO: wire unit tests" +test-integration: + @echo "TODO: wire integration tests" +eval: + @echo "TODO: wire RAG benchmark" +up: + docker compose -f docker-compose.dev.yml up -d +down: + docker compose -f docker-compose.dev.yml down diff --git a/README.md b/README.md new file mode 100644 index 0000000..64d0ab3 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# Enterprise Knowledge Platform + +Enterprise internal knowledge infrastructure. + +## Core capabilities + +- Multi-format document ingestion +- OCR and document parsing +- Structure-aware chunking +- Multiple embedding providers +- Hybrid retrieval and reranking +- ACL-aware search +- Knowledge Compilation +- REST API +- MCP Server +- Admin Console + +## Architecture baseline + +- API First + Admin Console +- Modular Monolith + async workers + MCP gateway +- PostgreSQL + object storage as canonical source of truth +- Mochow as the initial retrieval index, behind a replaceable adapter +- Embedding / reranker / parser / LLM providers are replaceable +- Raw source is immutable +- Compiled Knowledge is derived and traceable +- ACL is enforced before or during retrieval, never after Top-K + +## Start here + +1. Read `AGENTS.md` +2. Read `CODEX_START_HERE.md` +3. Read `docs/product/PRD.md` +4. Read `docs/architecture/ARCHITECTURE.md` +5. Read `docs/plans/V1_EXECUTION_PLAN.md` +6. Pick exactly one task from `docs/plans/TASKS.md` diff --git a/apps/admin-web/.gitkeep b/apps/admin-web/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/apps/api/.gitkeep b/apps/api/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/apps/mcp-server/.gitkeep b/apps/mcp-server/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deploy/docker/.gitkeep b/deploy/docker/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..3664772 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,30 @@ +services: + postgres: + image: postgres:17 + environment: + POSTGRES_DB: ekp + POSTGRES_USER: ekp + POSTGRES_PASSWORD: ekp + ports: ["5432:5432"] + volumes: ["ekp_postgres:/var/lib/postgresql/data"] + + redis: + image: redis:7 + ports: ["6379:6379"] + + rabbitmq: + image: rabbitmq:4-management + ports: ["5672:5672", "15672:15672"] + + minio: + image: minio/minio + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: ["9000:9000", "9001:9001"] + volumes: ["ekp_minio:/data"] + +volumes: + ekp_postgres: + ekp_minio: diff --git a/docs/api/API_CONTRACT.md b/docs/api/API_CONTRACT.md new file mode 100644 index 0000000..809034b --- /dev/null +++ b/docs/api/API_CONTRACT.md @@ -0,0 +1,7 @@ +# REST API Contract + +Version all public APIs under `/api/v1`. + +Initial domains: auth, users/groups, knowledge-bases, documents, compiled-knowledge, search, evaluations, jobs, audit-logs, providers and MCP administration. + +Use OpenAPI as executable source of truth once implementation starts. Provider-specific concepts must not leak into public contracts. diff --git a/docs/api/MCP_CONTRACT.md b/docs/api/MCP_CONTRACT.md new file mode 100644 index 0000000..1f8dd9b --- /dev/null +++ b/docs/api/MCP_CONTRACT.md @@ -0,0 +1,11 @@ +# MCP Contract + +P0 business tools: + +- `search_knowledge` +- `get_document` +- `get_document_section` +- `get_source` +- `list_knowledge_bases` + +Every request maps to a user/service identity and uses the same authorization/application services as REST. Do not expose Mochow, Qwen, BGE, MinerU or other infrastructure choices. diff --git a/docs/architecture/ADR.md b/docs/architecture/ADR.md new file mode 100644 index 0000000..4d90508 --- /dev/null +++ b/docs/architecture/ADR.md @@ -0,0 +1,10 @@ +# Architecture Decision Records + +- ADR-001: API First + Admin Console — Accepted +- ADR-002: Modular monolith + workers + MCP gateway for V1 — Accepted +- ADR-003: PostgreSQL + object storage are canonical — Accepted +- ADR-004: Mochow is initial retrieval backend but replaceable — Accepted +- ADR-005: Provider abstraction for embedding/reranker/parser/LLM — Accepted +- ADR-006: RAG + Knowledge Compilation — Accepted +- ADR-007: ACL before/during retrieval — Accepted +- ADR-008: MCP hides infrastructure details — Accepted diff --git a/docs/architecture/ARCHITECTURE.md b/docs/architecture/ARCHITECTURE.md new file mode 100644 index 0000000..b31e438 --- /dev/null +++ b/docs/architecture/ARCHITECTURE.md @@ -0,0 +1,749 @@ +# 企业内部智能知识库 V1 技术方案(简版) + +> 版本:v0.1 +> 目标:完成企业内部知识库第一版本,支持多类型文件上传、自动解析清洗、混合检索、Reranker、权限控制、Knowledge Compilation、REST API、MCP 接入及 Admin Console。 + +--- + +## 1. V1 建设目标 + +V1 聚焦形成完整可用闭环: + +```text +创建知识库 + ↓ +上传文件 + ↓ +解析 / OCR / 清洗 + ↓ +结构化 Chunk + ↓ +Embedding + BM25 索引 + ↓ +Hybrid Retrieval + ↓ +Reranker + ↓ +ACL 权限过滤 + ↓ +REST API / MCP + ↓ +WorkBuddy 等客户端调用 +``` + +同时支持第二条知识沉淀链路: + +```text +原始知识 Source + ↓ +结构化知识 Structured Knowledge + ↓ +Knowledge Compilation + ↓ +Wiki / FAQ / SOP / Summary 等派生知识 + ↓ +审核、发布、持续更新 +``` + +--- + +## 2. V1 总体架构 + +```text + ┌─────────────────────┐ + │ Admin Console │ + └──────────┬──────────┘ + │ REST + ▼ +┌────────────┐ ┌─────────────────────┐ +│ WorkBuddy │──── MCP ────▶│ Knowledge API │◀──── REST ─── Other Systems +└────────────┘ │ FastAPI / Python │ + └──────────┬──────────┘ + │ + ┌────────────────────────┼────────────────────────┐ + │ │ │ + ▼ ▼ ▼ + ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ + │ PostgreSQL │ │ Object Store │ │ Mochow │ + │ Canonical DB │ │ S3/MinIO │ │ Search Index │ + └──────────────┘ └──────────────┘ └──────────────┘ + ▲ ▲ + │ │ + ┌───────┴────────────────────────┘ + │ + ┌──────────────┐ + │ Async Worker │ + │ Celery │ + └──────┬───────┘ + │ + ┌──────▼───────┐ + │ GPU Worker │ + │ Parser/OCR │ + │ Embedding │ + │ Reranker │ + └──────────────┘ +``` + +### 架构原则 + +1. **API First**:后台、MCP、其他业务系统均调用统一 API/Service。 +2. **模块化单体优先**:V1 不拆大量微服务,API 服务 + 异步 Worker + MCP Gateway 足够。 +3. **Canonical Data 独立**:PostgreSQL + 对象存储保存事实数据,Vector DB 仅保存可重建索引。 +4. **Provider 可替换**:Embedding、Reranker、Vector DB、Parser、LLM 均通过 Adapter 抽象。 +5. **MCP 不暴露底层实现**:客户端只看到 `search_knowledge` 等业务工具。 + +--- + +## 3. 当前技术选型 + +| 模块 | V1 当前选型 | 说明 | +|---|---|---| +| Backend | Python + FastAPI | AI / 数据处理生态成熟,适合 MCP、异步任务和模型调用 | +| Metadata DB | PostgreSQL | 保存用户、权限、文档、版本、任务、Compiled Knowledge 等 Canonical Data | +| Object Storage | S3 Compatible / MinIO | 保存原始文件、解析产物、中间文件 | +| Task Queue | Celery + RabbitMQ | 文件解析、Embedding、索引、Compilation 等异步任务 | +| PDF / OCR Parser | MinerU 优先 | 中文复杂 PDF、扫描件、表格、版面解析能力较强 | +| Office Parser | Docling 补充 | DOCX/PPTX/XLSX/HTML 等格式统一解析 | +| Vector/Search DB | Mochow | 延续公司现有经验,支持向量、BM25、Hybrid、Filter | +| Embedding | 多模型 Benchmark 决定 | 基线 bge-large-zh,对比 Qwen3 Embedding 等 | +| Reranker | Qwen3 Reranker 等候选 | V1 正式检索链必选能力 | +| LLM | Provider 抽象,云端优先 | 用于 Knowledge Compilation、分类、摘要等 | +| MCP | 独立 MCP Gateway | 面向 WorkBuddy 等客户端,隐藏内部技术栈 | +| Frontend | Admin Console | 知识库、文档、权限、检索调试、Benchmark、任务管理 | + +--- + +## 4. 核心模块 + +### 4.1 Knowledge Base 管理 + +支持: + +- 创建 / 编辑 / 归档知识库; +- Folder 分类; +- Owner / Member; +- 知识库级和文档级权限; +- Retrieval Profile; +- Embedding / Reranker 配置; +- Knowledge Compilation 配置。 + +--- + +### 4.2 文件上传与自动入库 + +V1 支持: + +- PDF +- DOCX +- PPTX +- XLSX +- CSV +- Markdown +- TXT +- HTML +- JPG / PNG + +处理流程: + +```text +Upload +→ Hash 去重 +→ Parser Router +→ OCR(必要时) +→ Cleaning +→ Structure Normalize +→ Chunk +→ Embedding +→ BM25 / Vector Index +→ Completed +``` + +任务状态: + +```text +Queued / Parsing / Cleaning / Chunking / Embedding / Indexing / Completed / Failed +``` + +文件必须支持: + +- Hash 去重; +- Document Version; +- 失败重试; +- Re-parse / Re-embed / Re-index; +- 软删除; +- 来源页码和章节追踪。 + +--- + +### 4.3 文档结构与 Chunk + +解析后形成统一结构: + +```text +Document + ├─ Heading + ├─ Paragraph + ├─ List + ├─ Table + └─ Page +``` + +Chunk 采用 **Structure-aware + Token-aware** 策略,禁止仅按固定字符数粗暴截断。 + +Chunk 至少保存: + +```text +chunk_id +knowledge_base_id +document_id +document_version +content +title +section_path +page_start +page_end +content_type +metadata +acl +index_profile_id +``` + +--- + +## 5. 检索方案 + +V1 正式检索链: + +```text +Query + ↓ +Query Normalize + ↓ +ACL Resolve + ↓ +Dense Search + BM25 + ↓ +Fusion + ↓ +Candidate Top N + ↓ +Reranker + ↓ +Top K + ↓ +Source Enrichment +``` + +### 必须支持 + +- Dense Semantic Search; +- BM25 / Keyword Search; +- Hybrid Search; +- Metadata Filter; +- ACL Filter; +- Reranker; +- Knowledge Base / Folder / Document Scope; +- 来源追踪。 + +禁止使用“全局检索后再删除无权限结果”的方式,权限过滤必须在召回阶段生效。 + +--- + +## 6. Embedding Benchmark + +V1 不锁死单一 Embedding 模型。 + +### 首批候选 + +- bge-large-zh(现有基线); +- Qwen3-Embedding-0.6B; +- Qwen3-Embedding-4B; +- 视 GPU 与效果补充 8B 或其他中文模型。 + +### Benchmark 指标 + +```text +Recall@1 / 3 / 5 / 10 +MRR +NDCG@10 +P50 / P95 Latency +Index Throughput +GPU Memory +API Cost +Storage Cost +``` + +### 测试集 + +建立企业内部 Golden Dataset,V1 建议 200~500 条真实查询,覆盖: + +- 中文自然语言; +- 精确术语 / 编号; +- 产品型号; +- 同义表达; +- 中英文混写; +- 长问题; +- 多条件问题; +- 跨文档问题。 + +最终比较的是完整 Pipeline: + +```text +Embedding × Hybrid Retrieval × Reranker +``` + +而不是只比较 Embedding 排行榜。 + +--- + +## 7. Index Profile 与模型切换 + +Embedding、Chunk、Sparse、Reranker 配置统一形成 `IndexProfile`。 + +示例: + +```text +profile_v1 +- chunk_strategy: structure_v1 +- embedding: bge-large-zh +- sparse: bm25 + +profile_v2 +- chunk_strategy: structure_v2 +- embedding: qwen3-embedding-4b +- sparse: bm25 +``` + +新模型上线时: + +```text +创建新 Profile +→ 构建 Shadow Index +→ Benchmark +→ 切换 Active Index +→ 保留旧索引回滚窗口 +``` + +避免直接原地覆盖现有生产索引。 + +--- + +## 8. Vector Store 设计 + +### 当前默认:Mochow + +选择理由: + +- 公司已有使用经验; +- 已有百度云体系; +- 支持 Dense Vector; +- 支持 BM25 / Full Text; +- 支持 Hybrid Search; +- 支持 Metadata / Scalar Filter; +- 满足 V1 ACL 下推需求。 + +### 保留替换能力 + +统一: + +```text +VectorStoreProvider + ├─ Mochow + ├─ Qdrant + ├─ Milvus + └─ pgvector +``` + +业务 Domain Model 不依赖 Mochow SDK 数据结构。 + +Mochow 中的数据全部视为 **可重建索引**,真实 Canonical Data 保存在 PostgreSQL / Object Storage。 + +--- + +## 9. Knowledge Compilation + +V1 支持 **RAG + Knowledge Compilation**。 + +三层知识模型: + +```text +L0 Source +原始文件,不允许 AI 修改 + +L1 Structured +Parsed / Chunk / Index + +L2 Compiled +Wiki / FAQ / SOP / Summary / Comparison +``` + +### Compilation Pipeline + +```text +Source 新增或更新 +→ Impact Analysis +→ 查找相关 Knowledge Page +→ LLM 生成 Draft +→ Source Verification +→ Conflict Detection +→ Review Required +→ Publish +``` + +### Compiled Knowledge 类型 + +- Wiki Page; +- FAQ; +- SOP; +- Summary; +- Concept; +- Entity; +- Comparison。 + +所有 L2 Knowledge 必须保存 underlying source,并可追溯至 L0。 + +Source 删除、更新或权限变化时,相关 Compiled Knowledge 自动标记 `Stale`。 + +V1 默认 AI 只生成 Draft,重要知识需人工审核后发布。 + +--- + +## 10. 权限模型 + +权限主体: + +```text +User +Group +Role +``` + +资源层级: + +```text +Organization +Knowledge Base +Folder +Document +``` + +权限类型: + +```text +Read +Write +Manage +``` + +检索流程: + +```text +User Identity +→ Permission Resolver +→ Allowed Scope / ACL Filter +→ Mochow Hybrid Retrieval +``` + +MCP / REST / Admin Console 必须复用同一权限服务。 + +--- + +## 11. MCP + +对外提供统一: + +```text +Enterprise Knowledge MCP Server +``` + +V1 建议工具: + +### `search_knowledge` + +搜索当前用户有权限访问的企业知识。 + +### `get_document` + +读取文档 metadata。 + +### `get_document_section` + +获取指定页 / Section / Chunk 的连续上下文。 + +### `get_source` + +获取检索结果的来源信息。 + +### `list_knowledge_bases` + +列出当前身份可访问的知识库。 + +MCP Response 不出现: + +```text +Mochow +Qdrant +bge-large-zh +Qwen3 +MinerU +``` + +等底层实现信息。 + +首要验证客户端:**WorkBuddy Enterprise**。 + +--- + +## 12. REST API + +统一前缀: + +```text +/api/v1/ +``` + +主要 Domain: + +```text +/auth +/users +/groups +/knowledge-bases +/documents +/compiled-knowledge +/search +/evaluations +/jobs +/audit-logs +/providers +/mcp +``` + +核心搜索接口: + +```http +POST /api/v1/search +``` + +返回至少包含: + +```text +request_id +results +source +score +rerank_score +document +page +section +chunk_id +source_type +``` + +--- + +## 13. Admin Console + +V1 一级功能建议: + +```text +Overview +Knowledge Bases +Documents +Compiled Knowledge +Search Lab +Evaluation +Models +MCP & API +Users & Permissions +Jobs +Audit Logs +System Settings +``` + +### Search Lab + +Search Lab 是 V1 P0 工具,用于查看: + +```text +Dense Recall +Sparse Recall +Fusion +Reranker +Final Result +``` + +帮助定位 Parser、Chunk、Embedding、BM25、Reranker 哪一层出现问题。 + +--- + +## 14. 异步任务与 GPU + +异步 Job: + +```text +ParseJob +OCRJob +ChunkJob +EmbeddingJob +IndexJob +CompilationJob +ReindexJob +EvaluationJob +``` + +GPU 节点优先承担: + +- MinerU / OCR; +- Local Embedding; +- Local Reranker。 + +LLM V1 可继续使用云端 Provider,不要求部署本地大模型。 + +--- + +## 15. 数据存储原则 + +### PostgreSQL + +保存: + +- User / Group / Role; +- Knowledge Base; +- Document / Version; +- Chunk metadata; +- ACL; +- Job; +- Index Profile; +- Compiled Knowledge; +- Audit Log; +- Provider Config。 + +### Object Storage + +保存: + +- 原始文件; +- Parser 输出; +- OCR 结果; +- Markdown / JSON 中间产物; +- 需要持久化的大对象。 + +### Mochow + +仅保存: + +- Dense Vector; +- Sparse/BM25 Index; +- Retrieval metadata; +- ACL 所需过滤字段。 + +数据库损坏或更换时,可由 Canonical Data 全量重建。 + +--- + +## 16. 部署建议 + +V1 推荐按以下逻辑角色部署: + +```text +Reverse Proxy / Ingress + │ + ├─ Admin Console + ├─ Knowledge API + └─ MCP Gateway + +PostgreSQL +RabbitMQ +Object Storage + +CPU Worker +GPU Worker + +Mochow + +Observability +``` + +当前几十~百人内部用户规模,不建议一开始拆大量微服务。 + +优先采用: + +```text +模块化单体 API ++ 独立 Async Worker ++ 独立 GPU Worker ++ 独立 MCP Gateway +``` + +未来根据负载再拆服务。 + +--- + +## 17. V1 验收重点 + +### 文件入库 + +P0 文件能够完成: + +```text +Upload → Parse → Clean → Chunk → Embed → Index +``` + +### 检索 + +正式链路支持: + +```text +Dense + BM25 + Hybrid + Reranker + ACL +``` + +### 来源 + +每条结果能够追踪: + +```text +Source File → Version → Page → Section → Chunk +``` + +### 权限 + +Admin Console、REST、MCP 均不能访问无权限知识。 + +### Benchmark + +能够使用同一个 Golden Dataset 对多套 Retrieval Pipeline 进行对比。 + +### Knowledge Compilation + +能够完成: + +```text +Source → Draft → Source Linking → Review → Publish → Stale Detection +``` + +### MCP + +WorkBuddy 可以通过 MCP 完成知识搜索和 Source 获取。 + +--- + +## 18. V1 当前最终选型结论 + +```text +Backend Python + FastAPI +Metadata PostgreSQL +Object Storage S3 Compatible / MinIO +Queue RabbitMQ + Celery +Parser MinerU 主力 + Docling 补充 +Vector Store Mochow Primary,Adapter 可替换 +Embedding bge-large-zh 基线,多模型 Benchmark 后决定生产模型 +Reranker Qwen3 Reranker 等候选,V1 必选 +LLM Provider 抽象,允许云端 +Frontend Admin Console +Protocol REST API + MCP +Architecture Modular Monolith + Workers + MCP Gateway +Knowledge RAG + Knowledge Compilation +``` + +核心原则:**先保证知识可正确解析、可靠检索、严格授权、完整追溯,再逐步增加更高级的知识编译和智能化能力。** diff --git a/docs/development/CODING_RULES.md b/docs/development/CODING_RULES.md new file mode 100644 index 0000000..5c53842 --- /dev/null +++ b/docs/development/CODING_RULES.md @@ -0,0 +1,8 @@ +# Coding Rules + +- Keep infrastructure SDKs behind provider interfaces. +- Do not hardcode credentials, tenant IDs, endpoints, model names or collection names. +- Prefer typed contracts and cohesive modules. +- Make DB migrations forward-only. +- Core tests must work without external SaaS, internet or GPU. +- Avoid premature microservices. diff --git a/docs/development/DEFINITION_OF_DONE.md b/docs/development/DEFINITION_OF_DONE.md new file mode 100644 index 0000000..5f3f431 --- /dev/null +++ b/docs/development/DEFINITION_OF_DONE.md @@ -0,0 +1,3 @@ +# Definition of Done + +A task is done only when scope is implemented, architecture invariants hold, format/lint/typecheck pass, relevant tests pass, migrations/contracts/docs are updated as needed, no secrets are committed, retrieval changes include RAG regression results, and known limitations are documented. diff --git a/docs/development/SECURITY.md b/docs/development/SECURITY.md new file mode 100644 index 0000000..ed17788 --- /dev/null +++ b/docs/development/SECURITY.md @@ -0,0 +1,9 @@ +# Security Baseline + +- Enforce tenant + ACL scope before/during retrieval. +- Never log credentials or raw auth tokens. +- Protect source-document downloads. +- Audit sensitive reads and administrative changes. +- Support `cloud_allowed` and `local_only` data policies per Knowledge Base. +- Never send `local_only` content to cloud AI providers. +- Validate MCP identity before protected operations. diff --git a/docs/development/TEST_STRATEGY.md b/docs/development/TEST_STRATEGY.md new file mode 100644 index 0000000..d8a1ee5 --- /dev/null +++ b/docs/development/TEST_STRATEGY.md @@ -0,0 +1,16 @@ +# Test Strategy + +Unit tests must run without Mochow, Qianfan, internet, GPU or production credentials. + +Provide deterministic fakes: +- FakeEmbeddingProvider +- FakeRerankerProvider +- FakeVectorStore +- FakeLLMProvider +- FakeObjectStorage + +External integrations use separate integration suites/markers. + +E2E covers upload -> parse -> chunk -> embed -> index -> retrieve -> source trace, and source -> compilation draft -> review -> publish -> retrieval. + +Any parsing/chunking/embedding/fusion/retrieval/reranking/query-routing change must run the Golden Dataset evaluation. diff --git a/docs/evaluation/RAG_EVALUATION.md b/docs/evaluation/RAG_EVALUATION.md new file mode 100644 index 0000000..d3cbe14 --- /dev/null +++ b/docs/evaluation/RAG_EVALUATION.md @@ -0,0 +1,9 @@ +# RAG Evaluation + +Maintain a versioned Golden Dataset and report Recall@1/3/5/10, MRR, NDCG@10, latency P50/P95. + +Benchmark the full pipeline: Embedding × Retrieval Strategy × Fusion × Reranker. + +Initial embedding candidates: bge-large-zh, Qwen3-Embedding-0.6B, Qwen3-Embedding-4B, optional 8B. + +Use synthetic/public/anonymized fixtures only. diff --git a/docs/plans/PROGRESS.md b/docs/plans/PROGRESS.md new file mode 100644 index 0000000..37ea99c --- /dev/null +++ b/docs/plans/PROGRESS.md @@ -0,0 +1,7 @@ +# Progress + +Current phase: Phase 0 — Repository / developer harness + +Completed: none +In progress: none +Blocked: none diff --git a/docs/plans/TASKS.md b/docs/plans/TASKS.md new file mode 100644 index 0000000..9f13326 --- /dev/null +++ b/docs/plans/TASKS.md @@ -0,0 +1,61 @@ +# V1 Tasks + +## Phase 0 +- [ ] EKP-001 Repository bootstrap +- [ ] EKP-002 Backend Python toolchain +- [ ] EKP-003 Admin Web toolchain +- [ ] EKP-004 Application configuration +- [ ] EKP-005 Structured logging +- [ ] EKP-006 CI baseline +- [ ] EKP-007 Fake provider framework + +## Canonical domain / ingestion +- [ ] EKP-010 Core domain package +- [ ] EKP-011 PostgreSQL schema and migrations +- [ ] EKP-012 User/group/role model +- [ ] EKP-013 Knowledge Base model +- [ ] EKP-014 Folder model +- [ ] EKP-015 Document + DocumentVersion model +- [ ] EKP-016 ACL model +- [ ] EKP-017 Job + audit models +- [ ] EKP-020 ObjectStorage abstraction + S3 adapter +- [ ] EKP-021 Document upload API +- [ ] EKP-022 Source hash/dedup/version activation +- [ ] EKP-023 Async job framework +- [ ] EKP-024 Parser abstraction +- [ ] EKP-025 MinerU adapter +- [ ] EKP-026 Docling adapter +- [ ] EKP-027 OCR + cleaning pipeline +- [ ] EKP-028 Canonical parsed document model +- [ ] EKP-029 Structure-aware / spreadsheet chunking + +## Indexing / retrieval +- [ ] EKP-040 EmbeddingProvider + fake +- [ ] EKP-041 Qianfan BGE adapter +- [ ] EKP-042 Qwen embedding adapter +- [ ] EKP-043 IndexProfile model +- [ ] EKP-044 VectorStore + fake +- [ ] EKP-045 Mochow adapter +- [ ] EKP-046 Index build + shadow index lifecycle +- [ ] EKP-050 Permission resolver + ACL retrieval filters +- [ ] EKP-051 Dense retrieval +- [ ] EKP-052 BM25/sparse retrieval +- [ ] EKP-053 Hybrid fusion +- [ ] EKP-054 RerankerProvider + fake + Qwen adapter +- [ ] EKP-055 Source enrichment + Search service/API +- [ ] EKP-056 Retrieval diagnostics / Search Lab backend + +## MCP / Admin / Compilation / Evaluation +- [ ] EKP-070 MCP server + identity +- [ ] EKP-071 MCP P0 tools +- [ ] EKP-072 WorkBuddy compatibility test +- [ ] EKP-080 Admin Console core pages +- [ ] EKP-081 Permissions/jobs/chunk inspector/Search Lab UI +- [ ] EKP-090 CompiledKnowledge model/workflow +- [ ] EKP-091 Source linking/conflict/stale/review/publish +- [ ] EKP-092 Compiled retrieval integration +- [ ] EKP-100 Golden Dataset + metrics +- [ ] EKP-101 Multi-embedding benchmark +- [ ] EKP-102 Reranker benchmark +- [ ] EKP-103 Security/performance/rebuild tests +- [ ] EKP-104 V1 release hardening diff --git a/docs/plans/V1_EXECUTION_PLAN.md b/docs/plans/V1_EXECUTION_PLAN.md new file mode 100644 index 0000000..741499b --- /dev/null +++ b/docs/plans/V1_EXECUTION_PLAN.md @@ -0,0 +1,11 @@ +# V1 Execution Plan + +1. Repository / developer harness +2. Canonical domain and ACL +3. Document ingestion and parsing +4. Embedding / Index Profile / Mochow adapter +5. ACL-aware hybrid retrieval + reranker +6. MCP gateway and WorkBuddy verification +7. Admin Console +8. Knowledge Compilation +9. Evaluation, security and release hardening diff --git a/docs/product/PRD.md b/docs/product/PRD.md new file mode 100644 index 0000000..853073e --- /dev/null +++ b/docs/product/PRD.md @@ -0,0 +1,2537 @@ +# 企业内部智能知识库平台 PRD + +**文档版本:** v0.1 +**文档状态:** 需求基线稿 / 待评审 +**产品定位:** API First + Admin Console 的企业级智能知识基础设施 +**目标用户规模:** 企业内部几十人~百人级团队,架构预留更大规模扩展能力 +**主要语言:** 中文优先,兼容英文及中英混合内容 +**部署原则:** 自建部署为主,允许调用云端 AI 服务;支持 NVIDIA GPU +**核心技术原则:** RAG + Knowledge Compilation、Hybrid Retrieval、Reranker、MCP、底层能力可替换 + +--- + +# 1. 项目背景 + +企业内部存在大量分散于 PDF、Word、PPT、Excel、Markdown、图片、扫描件及其他文件中的知识资产。 + +传统文档管理和 Wiki 系统能够解决知识存放、目录组织和人工检索问题,但面对以下场景能力有限: + +- 用户不知道准确关键词; +- 一个问题需要综合多份文档; +- 文档数量增加后,人工维护成本快速上升; +- AI Agent 无法以统一协议安全访问企业知识; +- 文档更新后,知识索引不能自动同步; +- 搜索结果缺乏来源追踪; +- 不同部门之间存在严格知识访问权限; +- 原始资料虽然很多,但没有进一步沉淀成结构化企业知识。 + +本项目计划建设一套企业内部智能知识库平台,将企业原始资料自动转换为: + +1. **可追溯的结构化知识;** +2. **可供 RAG 高质量召回的索引;** +3. **可由 LLM 持续维护的 Compiled Knowledge;** +4. **可通过 REST API / MCP 被 WorkBuddy 等 Agent 调用的知识能力。** + +平台不绑定单一 Embedding 模型、Vector Database、LLM 或 Agent 客户端。 + +--- + +# 2. 产品愿景 + +建设企业内部统一的 **Knowledge Infrastructure**: + +> 任何企业知识进入系统后,都能够自动被解析、清洗、结构化、索引、沉淀和安全调用,并且始终能够追溯到原始事实来源。 + +系统长期形成三层知识: + +```text +L0 Source Knowledge +原始文件 / 原始事实 +不可由 AI 修改 + ↓ +L1 Structured Knowledge +解析文本 / 结构 / Chunk / Index +用于精准检索与 RAG + ↓ +L2 Compiled Knowledge +Wiki / FAQ / SOP / Entity / Summary / Comparison +由 AI 从 L0/L1 派生,可持续维护 +``` + +要求: + +```text +L2 → L1 → L0 +``` + +始终可完整追溯。 + +--- + +# 3. 核心产品原则 + +## 3.1 API First + +所有核心业务能力首先以 Service/API 形式存在。 + +Admin Console、MCP Server 以及未来其他客户端均调用统一业务服务,不允许各自重复实现业务逻辑。 + +```text + Admin Console + │ + REST + │ +WorkBuddy ─ MCP ─ Knowledge Service ─ REST ─ Other Systems + │ + Core Engine +``` + +--- + +## 3.2 Admin Console 不是核心业务逻辑 + +Admin Console 是官方管理客户端,主要解决: + +- 知识库创建; +- 文件上传; +- 文件状态管理; +- 权限管理; +- 检索调试; +- Chunk 查看; +- Knowledge Compilation 管理; +- 模型配置; +- Benchmark; +- MCP 配置; +- 日志和任务监控。 + +所有能力原则上均能够通过 API 操作。 + +--- + +## 3.3 底层实现不得暴露给 MCP Client + +WorkBuddy、Cursor、Claude Code 或其他 Agent 不应该知道: + +- Mochow; +- Qdrant; +- Milvus; +- bge-large-zh; +- Qwen3; +- MinerU; +- 其他 Parser/Model。 + +MCP 只暴露业务语义,例如: + +```text +search_knowledge +get_document +get_document_section +get_source +list_knowledge_bases +``` + +--- + +## 3.4 Source of Truth 原则 + +原始文件是企业知识的最终事实源。 + +LLM 生成的: + +- Wiki; +- FAQ; +- Summary; +- Comparison; +- SOP; +- Entity Page; + +均属于 **Derived Knowledge**。 + +Derived Knowledge 不得覆盖、修改或替代 Source Knowledge。 + +--- + +## 3.5 Provider 可替换原则 + +核心系统不得与单一供应商强耦合。 + +至少抽象: + +```text +EmbeddingProvider +RerankerProvider +VectorStoreProvider +LLMProvider +ParserProvider +OCRProvider +ObjectStorageProvider +``` + +V1 可以优先使用现有百度体系,但必须具备替换路径。 + +--- + +# 4. 产品目标 + +## 4.1 V1 核心目标 + +V1 必须实现以下完整闭环: + +```text +创建知识库 + ↓ +上传文档 + ↓ +自动解析 + ↓ +自动清洗 + ↓ +结构恢复 + ↓ +Chunk + ↓ +Embedding + ↓ +Hybrid Index + ↓ +Reranker + ↓ +带权限检索 + ↓ +来源追踪 + ↓ +REST / MCP +``` + +同时实现: + +```text +Raw Knowledge + ↓ +Knowledge Compilation + ↓ +Compiled Wiki + ↓ +审核 / 发布 + ↓ +RAG 可检索 +``` + +--- + +# 5. 非目标 + +以下能力不作为 V1 核心目标: + +- 自研通用聊天机器人; +- 完整 Agent 平台; +- 通用 Workflow 编排平台; +- GraphRAG; +- 知识图谱编辑平台; +- 视频语义理解; +- 音频会议实时转写; +- 自动互联网爬虫; +- 飞书 / Confluence / Notion 全量同步; +- 多模态图片向量检索; +- 完整文档在线协同编辑器; +- 移动端 App。 + +但架构不得阻碍未来扩展。 + +--- + +# 6. 用户角色 + +## 6.1 System Admin + +系统管理员。 + +权限: + +- 系统配置; +- 用户/组织管理; +- Provider 配置; +- Embedding / Reranker 管理; +- MCP 管理; +- Benchmark; +- 全局审计; +- 系统日志; +- 存储配置。 + +--- + +## 6.2 Knowledge Base Owner + +知识库负责人。 + +权限: + +- 创建/配置知识库; +- 成员管理; +- 文件上传; +- 文件删除; +- 权限设置; +- Compilation 配置; +- 发布 Compiled Knowledge; +- 查看日志; +- 查看检索效果。 + +--- + +## 6.3 Knowledge Editor + +知识维护人员。 + +权限: + +- 上传文档; +- 编辑 metadata; +- 查看解析结果; +- 重新解析; +- 管理 Compiled Knowledge; +- 发起发布审核。 + +--- + +## 6.4 Knowledge Consumer + +普通知识消费者。 + +权限: + +- 检索其有权限访问的知识; +- 查看来源; +- 通过授权 MCP Client 获取知识。 + +不能访问: + +- 无权限知识库; +- 无权限文档; +- 系统管理配置。 + +--- + +## 6.5 AI Agent + +包括: + +- WorkBuddy; +- CodeBuddy; +- Cursor; +- Claude Code; +- 其他 MCP Client; +- 企业内部 Agent。 + +Agent 的实际权限必须绑定用户身份或 Service Account,不得拥有默认全局权限。 + +--- + +# 7. 核心业务对象 + +系统核心对象: + +```text +Organization +User +Group +Role + +KnowledgeBase +Folder +Document +DocumentVersion +DocumentPage +DocumentSection +Chunk + +EmbeddingIndex +SparseIndex +IndexVersion + +CompiledKnowledge +CompiledKnowledgeVersion +CompilationJob + +Permission +ACL + +IngestionJob +RetrievalRequest +RetrievalResult +AuditLog +MCPClient +APIKey +``` + +--- + +# 8. Knowledge Base 模型 + +知识库是逻辑隔离的最上层业务空间。 + +每个知识库至少包含: + +- Name; +- Description; +- Owner; +- Visibility; +- Members; +- ACL; +- Folder Tree; +- Documents; +- Retrieval Configuration; +- Embedding Configuration; +- Reranker Configuration; +- Compilation Configuration; +- MCP Availability; +- Metadata Schema。 + +支持知识库状态: + +```text +Active +Archived +Disabled +``` + +--- + +# 9. 文件类型需求 + +## 9.1 P0 文件类型 + +V1 必须支持: + +- PDF; +- DOCX; +- PPTX; +- XLSX; +- CSV; +- Markdown; +- TXT; +- HTML; +- JPG/JPEG; +- PNG。 + +--- + +## 9.2 PDF 类型 + +必须覆盖: + +- 原生文本 PDF; +- 扫描 PDF; +- 图文混排 PDF; +- 多栏 PDF; +- 含表格 PDF; +- 含页眉页脚 PDF。 + +--- + +# 10. 文件上传 + +**REQ-INGEST-001** + +支持: + +- 单文件上传; +- 多文件批量上传; +- Drag & Drop。 + +**REQ-INGEST-002** + +上传完成必须立即返回 Document ID,不阻塞等待解析。 + +**REQ-INGEST-003** + +Document processing 必须异步执行。 + +**REQ-INGEST-004** + +展示处理阶段: + +```text +Uploading +Queued +Parsing +Cleaning +Chunking +Embedding +Indexing +Completed +Failed +``` + +**REQ-INGEST-005** + +失败必须展示: + +- Failed Stage; +- Error Code; +- Error Message; +- Retry。 + +--- + +# 11. 文件版本和去重 + +## REQ-VERSION-001 + +上传文件时计算 Source Hash。 + +如果: + +```text +相同 Knowledge Base ++ +相同文件 Hash +``` + +系统应提示文件已经存在。 + +--- + +## REQ-VERSION-002 + +同名但 Hash 不同的文件允许: + +```text +Create New Version +``` + +--- + +## REQ-VERSION-003 + +新版本成功建立索引后,旧版本默认退出 Active Retrieval。 + +历史版本仍可追踪。 + +--- + +## REQ-VERSION-004 + +禁止出现: + +```text +新版本已经上传 +但检索结果仍同时返回多个旧版本 +``` + +除非管理员明确允许历史搜索。 + +--- + +# 12. Document Parsing + +解析系统必须使用 Parser Adapter。 + +```text +ParserRouter + ├─ PDFParser + ├─ OfficeParser + ├─ SpreadsheetParser + ├─ MarkdownParser + ├─ HTMLParser + └─ ImageParser +``` + +V1 不规定必须使用单一 Parser。 + +MinerU、Docling 等由下一阶段 Benchmark 决定。 + +--- + +# 13. OCR + +系统必须能够: + +- 自动检测扫描 PDF; +- 自动检测图片中的文字; +- 启动 OCR Pipeline; +- 标记 OCR confidence; +- 保存原始页码。 + +对于低置信度 OCR 页面,应在管理后台显示警告。 + +--- + +# 14. Document Cleaning + +系统必须支持自动清洗: + +- 空页; +- 多余空格; +- 异常换行; +- 连字符断词; +- 重复页眉; +- 重复页脚; +- 页码; +- OCR 噪声; +- 无意义字符; +- 重复内容。 + +要求: + +**不得因为清洗破坏原始 Source。** + +保存: + +```text +Original +Parsed +Cleaned +``` + +三个逻辑层次。 + +--- + +# 15. 文档结构恢复 + +解析结果至少需要识别: + +```text +Document + ├─ Title + ├─ Heading + ├─ Paragraph + ├─ List + ├─ Table + ├─ Image Reference + └─ Page +``` + +每个结构节点至少保留: + +```text +source_document_id +document_version +page_number +section_path +block_type +content +``` + +--- + +# 16. Spreadsheet 处理 + +Excel / CSV 不允许简单拼成连续文本。 + +至少支持: + +- Sheet 识别; +- Header 识别; +- Row/Column 信息; +- Cell 内容; +- Sheet metadata。 + +Chunk 应保留: + +```text +sheet_name +row_range +column_range +``` + +方便来源追踪。 + +--- + +# 17. Chunking + +Chunk Strategy 必须可配置和扩展。 + +支持: + +### Structure-aware Chunk + +优先按照: + +```text +Heading +Paragraph +List +Table +``` + +进行切分。 + +### Token-aware Chunk + +确保满足当前 Embedding Model 最大输入限制。 + +--- + +## REQ-CHUNK-001 + +Chunk 至少保存: + +```text +chunk_id +document_id +document_version +knowledge_base_id +content +title +section_path +page_start +page_end +chunk_index +content_type +source_hash +metadata +ACL +``` + +--- + +# 18. Embedding Provider + +系统必须支持多 Embedding Provider。 + +```text +EmbeddingProvider + ├─ Baidu Qianfan + ├─ Local Model + └─ OpenAI-compatible Provider +``` + +现阶段默认候选: + +- bge-large-zh; +- Qwen3-Embedding-0.6B; +- Qwen3-Embedding-4B; +- Qwen3-Embedding-8B。 + +千帆当前公开模型能力中,bge-large-zh 为 1024 维、512 tokens 上下文;Qwen3 Embedding 提供 0.6B/4B/8B,对应 1024/2560/4096 维,并支持 8192 tokens 上下文。 + +--- + +# 19. Embedding Benchmark + +这是 V1 的正式产品能力,而不仅仅是研发脚本。 + +Admin Console 应提供 Benchmark Workspace。 + +--- + +## 19.1 测试集 + +管理员可以创建: + +```text +Evaluation Dataset +``` + +每条数据包含: + +```text +Query +Expected Document +Expected Section +Expected Chunk +Tags +Difficulty +``` + +--- + +## 19.2 Benchmark 模型 + +V1 至少比较: + +```text +bge-large-zh +Qwen3-Embedding-0.6B +Qwen3-Embedding-4B +``` + +根据 GPU 和成本情况扩展 8B。 + +--- + +## 19.3 Benchmark 指标 + +至少计算: + +```text +Recall@1 +Recall@3 +Recall@5 +Recall@10 +MRR +NDCG@10 +Latency P50 +Latency P95 +Indexing Throughput +GPU Memory +API Cost +Storage Cost +``` + +--- + +## 19.4 中文测试分类 + +测试集必须覆盖: + +- 自然语言问题; +- 精确术语; +- 产品型号; +- 编号; +- 中文简称; +- 中英文混写; +- 长问题; +- 多条件问题; +- 同义表达; +- 跨文档问题。 + +--- + +# 20. Vector Store + +系统采用: + +```text +VectorStoreProvider +``` + +抽象。 + +默认首选: + +```text +Mochow +``` + +原因: + +公司已有使用经验,同时 Mochow 当前官方能力已经支持向量检索、BM25、Hybrid Search 和标量过滤。 + +但系统不得使用 Mochow-specific data model 污染业务 Domain Model。 + +未来可以新增: + +```text +Qdrant +Milvus +pgvector +Elasticsearch/OpenSearch +``` + +--- + +# 21. Retrieval Pipeline + +V1 默认检索 Pipeline: + +```text +Query + ↓ +Query Normalize + ↓ +ACL Resolution + ↓ +┌───────────────┐ +│ Dense Search │ +└───────────────┘ + + +┌───────────────┐ +│ Sparse / BM25 │ +└───────────────┘ + ↓ +Fusion + ↓ +Candidate Top N + ↓ +Reranker + ↓ +Top K + ↓ +Source Enrichment + ↓ +Response +``` + +--- + +# 22. Hybrid Search + +必须支持: + +- Semantic Search; +- Keyword Search; +- Hybrid Search; +- Metadata Filter; +- ACL Filter; +- Knowledge Base Scope; +- Folder Scope; +- Document Scope。 + +管理员可以设置默认: + +```text +Dense Weight +Sparse Weight +Candidate Size +Final TopK +``` + +--- + +# 23. Reranker + +Reranker 是 V1 P0 能力。 + +不允许上线以下单一模式作为默认正式检索链: + +```text +Embedding +→ +TopK +→ +Return +``` + +默认应: + +```text +Hybrid Recall +→ +Candidate Top 20~50 +→ +Reranker +→ +Top 5~10 +``` + +Provider: + +```text +RerankerProvider +``` + +初始候选: + +- Qwen3-Reranker-0.6B; +- Qwen3-Reranker-4B; +- Qwen3-Reranker-8B; +- bce-reranker-base; +- Local Reranker。 + +千帆当前已经提供 Qwen3 Reranker 系列,最大输入支持 32K。 + +--- + +# 24. 权限系统 + +权限必须发生在 **Retrieval 阶段之前/期间**。 + +禁止: + +```text +全局 Search + ↓ +TopK + ↓ +应用层删除用户无权内容 +``` + +必须: + +```text +User + ↓ +Permission Resolver + ↓ +Allowed Scope + ↓ +Retrieval Filter + ↓ +TopK +``` + +--- + +# 25. ACL 模型 + +至少支持: + +```text +Organization +Knowledge Base +Folder +Document +``` + +权限主体: + +```text +User +Group +Role +``` + +权限: + +```text +Read +Write +Manage +``` + +支持: + +```text +Inherited Permission +Explicit Permission +``` + +--- + +# 26. 权限审计 + +所有以下操作记录 Audit Log: + +- 用户登录; +- 查看文档; +- 搜索; +- 下载 Source; +- 上传; +- 删除; +- 权限修改; +- MCP 调用; +- API 调用; +- Compilation; +- 发布 Compiled Knowledge; +- Provider 配置修改。 + +--- + +# 27. Knowledge Compilation + +Knowledge Compilation 是区别于普通 RAG 系统的核心能力。 + +其思想参考 Karpathy LLM Wiki: + +> LLM 不仅在每次查询时临时理解知识,而是将原始资料持续编译为可维护、可积累的结构化知识资产。 + +Karpathy 原始 LLM Wiki 方案明确区分 immutable raw source 与由 LLM 持续维护的 wiki。 + +企业版本必须在这一思想之上增加: + +- 权限; +- 来源追踪; +- 版本; +- 审核; +- 冲突检测; +- 自动过期机制。 + +--- + +# 28. Compiled Knowledge 类型 + +V1 支持: + +```text +Summary +Entity +Concept +FAQ +SOP +Comparison +Overview +Custom Wiki Page +``` + +例如: + +```text +产品/ + ├─ 产品A.md + ├─ 产品B.md + └─ 产品对比.md + +技术/ + ├─ 部署流程.md + ├─ 故障排查.md + └─ 系统架构.md + +企业制度/ + ├─ 报销制度.md + └─ 请假制度.md +``` + +--- + +# 29. Compiled Knowledge Source + +每条 Compiled Knowledge 必须包含: + +```text +compiled_id +title +type +content + +source_document_ids +source_chunk_ids +source_document_versions + +created_by +created_at +updated_at + +status +permissions + +last_verified_at +``` + +--- + +# 30. Knowledge Compilation Workflow + +```text +New / Updated Source + ↓ +Impact Analysis + ↓ +Find Related Wiki Pages + ↓ +Generate Proposed Update + ↓ +Source Verification + ↓ +Conflict Detection + ↓ +Draft + ↓ +Human Review / Auto Publish Policy + ↓ +Published +``` + +--- + +# 31. Compilation 状态 + +```text +Draft +ReviewRequired +Published +Rejected +Stale +Archived +``` + +V1 默认: + +> AI 生成内容先进入 Draft/ReviewRequired,不自动覆盖 Published 内容。 + +管理员未来可对低风险知识库开启 Auto Publish。 + +--- + +# 32. Knowledge Conflict Detection + +例如 Source A: + +```text +退款周期:30 天 +``` + +Source B: + +```text +退款周期:14 天 +``` + +Compilation 不允许自动生成: + +```text +退款周期约 14~30 天 +``` + +必须标记: + +```text +Knowledge Conflict +``` + +并展示: + +- conflicting claims; +- source; +- version; +- date。 + +--- + +# 33. Stale Detection + +如果 Compiled Knowledge 引用的 Source: + +- 被删除; +- 发布新版本; +- 权限发生变化; + +相关 Compiled Knowledge 自动进入: + +```text +Stale +``` + +并触发重新 Compilation。 + +--- + +# 34. Compiled Knowledge 权限 + +默认权限原则: + +> Compiled Knowledge 的可见范围不能高于其引用 Source 的安全范围。 + +系统必须防止: + +```text +Restricted HR Document ++ +Public Document +↓ +AI Summary +↓ +Public Wiki +``` + +造成权限泄漏。 + +具体权限求交/继承算法在技术设计阶段确定。 + +--- + +# 35. Compiled Knowledge 也进入 Retrieval + +检索可以有两类 Source: + +```text +Raw Structured Knowledge +Compiled Knowledge +``` + +Retrieval Response 必须标记: + +```text +source_type = RAW +``` + +或: + +```text +source_type = COMPILED +``` + +Compiled Result 仍必须返回 underlying source。 + +--- + +# 36. Query Routing + +V1 可先采用规则 + LLM Classifier 判断查询: + +```text +Fact Lookup +Exact Search +Conceptual Search +Cross Document Analysis +``` + +简单事实问题优先 Raw Retrieval。 + +复杂综合问题可: + +```text +Compiled Knowledge ++ +Raw Evidence +``` + +联合召回。 + +--- + +# 37. 来源追踪 + +任何 Retrieval Result 必须包含: + +```text +document_id +filename +document_version +page +section_path +chunk_id +content +score +rerank_score +source_type +``` + +对于 Compiled Knowledge: + +```text +compiled_knowledge_id +underlying_sources[] +``` + +--- + +# 38. Admin Console + +产品参考 iWiki / 企业 Wiki 的知识组织思想,但不复制其具体实现。 + +重点吸收: + +- Space/Knowledge Base; +- Folder/Category; +- Content 生命周期; +- 权限; +- 版本; +- 最近更新; +- 审核; +- 内容维护责任人; +- 搜索与浏览结合。 + +腾讯当前企业知识管理体系也将 iWiki 作为可以查询、编辑和管理的企业知识源,并通过 MCP/Connector 与 Agent 连接。 + +--- + +# 39. Admin Console 一级导航 + +建议: + +```text +Overview + +Knowledge Bases + +Documents + +Compiled Knowledge + +Search Lab + +Evaluation + +Models + +MCP & API + +Users & Permissions + +Jobs + +Audit Logs + +System Settings +``` + +--- + +# 40. Knowledge Base 页面 + +包含: + +```text +Overview +Documents +Folders +Compiled Knowledge +Search Test +Evaluation +Permissions +Settings +``` + +--- + +# 41. Document 页面 + +展示: + +- 原始文件; +- 文件信息; +- 当前版本; +- 历史版本; +- Parser; +- Processing Pipeline; +- Parsed Result; +- Chunk; +- Metadata; +- Permissions; +- Related Compiled Knowledge; +- Index Status。 + +--- + +# 42. Chunk Inspector + +管理员可以查看: + +```text +Chunk #23 + +Document: +产品管理制度.pdf + +Page: +17 + +Section: +第三章 / 产品退款 + +Text: +... + +Embedding Model: +... + +Index Version: +... + +ACL: +... + +Retrieval History: +... +``` + +支持: + +```text +Re-chunk +Re-embed +Re-index +``` + +--- + +# 43. Search Lab + +这是 V1 P0 工具。 + +输入 Query 后展示: + +### Query Information + +```text +Normalized Query +Search Scope +ACL +``` + +### Dense Recall + +```text +Rank +Document +Chunk +Score +``` + +### Sparse Recall + +```text +Rank +Document +Chunk +BM25 Score +``` + +### Fusion + +### Reranker + +### Final Result + +通过此功能定位: + +- Parsing 问题; +- Chunk 问题; +- Embedding 问题; +- BM25 问题; +- Reranker 问题。 + +--- + +# 44. MCP Server + +系统提供统一: + +```text +Enterprise Knowledge MCP Server +``` + +不得直接暴露 Vector DB API。 + +--- + +# 45. MCP P0 Tools + +建议 V1: + +### search_knowledge + +```text +query +knowledge_base_ids? +folder_ids? +top_k? +filters? +``` + +返回: + +```text +results[] +sources[] +``` + +--- + +### get_document + +获取指定 Document metadata。 + +--- + +### get_document_section + +获取指定章节/页/Chunk 的连续上下文。 + +--- + +### get_source + +获取完整来源信息。 + +--- + +### list_knowledge_bases + +只返回当前用户有权限访问的知识库。 + +--- + +# 46. MCP P1 Tools + +后续支持: + +```text +search_compiled_knowledge +get_compiled_knowledge +create_knowledge_draft +submit_knowledge +``` + +写操作必须有更严格授权。 + +--- + +# 47. MCP 身份 + +远程 MCP 不得使用一个共享超级管理员 Token。 + +要求: + +```text +Client ++ +User / Service Identity ++ +Permission +``` + +映射到系统 ACL。 + +--- + +# 48. MCP Client Compatibility + +首要目标: + +```text +WorkBuddy Enterprise +``` + +同时尽量兼容标准 MCP Client。 + +WorkBuddy Enterprise 当前官方 MCP 文档已经支持自定义 MCP Server,并提供远程 API Server、权限、安全及 iWiki 等配置示例。 + +--- + +# 49. REST API + +REST API 为正式公开能力。 + +建议 Version: + +```text +/api/v1/ +``` + +核心 Domain: + +```text +/auth +/users +/groups + +/knowledge-bases +/documents +/compiled-knowledge + +/search +/evaluations + +/jobs +/audit-logs + +/mcp +/providers +``` + +--- + +# 50. API Authentication + +V1 支持: + +```text +User Token +Service API Key +``` + +未来预留: + +```text +OIDC +OAuth2 +LDAP +企业微信 / AD / SSO +``` + +--- + +# 51. Search API + +建议核心接口: + +```text +POST /api/v1/search +``` + +Request: + +```json +{ + "query": "...", + "knowledge_base_ids": [], + "top_k": 8, + "retrieval_mode": "hybrid" +} +``` + +Response 逻辑结构: + +```json +{ + "request_id": "...", + "results": [], + "timing": {}, + "retrieval": {} +} +``` + +不得在 API Contract 中暴露: + +```text +mochow.xxx +qdrant.xxx +``` + +--- + +# 52. Provider Management + +管理员可以配置: + +### Embedding Provider + +### Reranker Provider + +### LLM Provider + +### Vector Store Provider + +### Parser Provider + +每个 Provider 具有: + +```text +Name +Type +Endpoint +Credential +Status +Capabilities +Health +``` + +Secret 不允许明文回显。 + +--- + +# 53. LLM Provider + +Knowledge Compilation 可调用: + +- 百度千帆; +- OpenAI-compatible API; +- 本地模型; +- 其他云端模型。 + +系统必须支持: + +```text +LLMProvider +``` + +抽象。 + +--- + +# 54. GPU 使用原则 + +本地 NVIDIA GPU 优先服务于: + +```text +OCR / Document Parsing +Embedding +Reranker +``` + +LLM 可以采用: + +```text +Cloud First +Local Optional +``` + +不强制 V1 本地部署大语言模型。 + +--- + +# 55. 可观测性 + +系统必须记录: + +## Ingestion Metrics + +```text +documents/day +pages/min +parse latency +OCR latency +failed rate +``` + +## Retrieval Metrics + +```text +QPS +P50 +P95 +P99 +Dense Latency +Sparse Latency +Rerank Latency +``` + +## Quality Metrics + +```text +Recall@K +MRR +NDCG +No Result Rate +``` + +## Compilation Metrics + +```text +Jobs +Success Rate +Review Rate +Conflict Rate +Stale Count +``` + +--- + +# 56. Job System + +异步任务至少包括: + +```text +ParseJob +OCRJob +ChunkJob +EmbeddingJob +IndexJob +CompilationJob +ReindexJob +EvaluationJob +``` + +每个 Job: + +```text +Queued +Running +Succeeded +Failed +Cancelled +``` + +必须支持: + +```text +Retry +``` + +--- + +# 57. 删除语义 + +删除 Document 时必须处理: + +```text +Source +Parsed Data +Chunks +Vector Index +Sparse Index +Compiled Knowledge Dependency +``` + +默认采用: + +```text +Soft Delete +``` + +管理员可执行永久删除。 + +--- + +# 58. 搜索质量 Benchmark + +正式上线前必须建立企业自己的 Golden Dataset。 + +建议首批不少于: + +```text +200~500 Queries +``` + +随着使用积累扩展。 + +样本来源: + +- 用户真实问题; +- 产品团队; +- HR; +- 技术团队; +- 销售/客服。 + +--- + +# 59. Retrieval Benchmark Matrix + +至少比较: + +```text +Embedding × Retrieval × Reranker +``` + +例如: + +```text +bge-large-zh ++ Mochow Dense ++ No Reranker + +bge-large-zh ++ Mochow Hybrid ++ Qwen3 Reranker + +Qwen3 Embedding ++ Mochow Hybrid ++ Qwen3 Reranker +``` + +不只 Benchmark Embedding 单模型。 + +--- + +# 60. Benchmark 决策原则 + +最终生产方案根据以下综合指标选择: + +```text +Quality +Latency +Cost +Operational Complexity +Vendor Dependency +GPU Requirement +``` + +而不是只按模型排行榜。 + +--- + +# 61. 安全要求 + +P0: + +- 数据逻辑隔离; +- ACL; +- Secret 加密; +- API Authentication; +- Audit Log; +- MCP Authorization; +- HTTPS; +- 禁止未经授权下载原文件。 + +--- + +# 62. 云端 AI 数据安全 + +因为允许调用云端服务,每个 Provider 必须定义: + +```text +Data Processing Policy +``` + +管理员可以设置: + +```text +Cloud Allowed +Local Only +``` + +Knowledge Base 可覆盖系统默认策略。 + +例如 HR Knowledge Base 可以配置: + +```text +Local Only +``` + +即不得将文档内容发送给任何外部 LLM/Embedding API。 + +--- + +# 63. 性能目标 + +按当前几十~百人团队规模,V1 目标: + +### Search + +正常场景: + +```text +P95 < 2s +``` + +包含 Reranker。 + +不包含最终 LLM Answer Generation。 + +### API + +Metadata API: + +```text +P95 < 500ms +``` + +### Admin Console + +主要页面交互: + +```text +P95 < 2s +``` + +--- + +# 64. 可用性 + +V1 内部生产目标: + +```text +99.5% +``` + +后续视业务重要性提升。 + +--- + +# 65. 数据规模 + +当前尚未明确文档总量,因此 V1 架构最低要求: + +```text +10^6 Chunk 级别 +``` + +不需要大规模重构即可运行。 + +更高规模在架构设计阶段进行 Capacity Planning。 + +--- + +# 66. iWiki 产品设计参考原则 + +不是复制 iWiki,而是吸收传统企业知识系统已经验证的产品理念: + +### 知识是长期资产 + +不能只有“上传文件”。 + +需要: + +```text +Owner +Version +History +Category +Permission +Status +Last Updated +``` + +### Search + Browse + +用户不仅通过搜索进入知识,也需要: + +```text +Knowledge Base +→ Folder +→ Document +→ Knowledge +``` + +浏览。 + +### Knowledge Lifecycle + +```text +Create +Maintain +Review +Publish +Update +Archive +``` + +### AI 不替代治理 + +AI 可以: + +```text +Generate +Summarize +Compile +Detect +Suggest +``` + +但企业重要知识需要: + +```text +Review +Ownership +Audit +``` + +--- + +# 67. RAG + Knowledge Compilation 的产品定位 + +RAG 负责: + +> 找到正确事实。 + +Knowledge Compilation 负责: + +> 让企业知识持续沉淀并减少重复理解成本。 + +二者不存在替代关系。 + +最终查询: + +```text + User Query + │ + Query Router + / \ + ↓ ↓ + Raw Retrieval Compiled Retrieval + ↓ ↓ + \ / + Reranker + ↓ + Evidence + ↓ + MCP / REST / Agent +``` + +--- + +# 68. 核心成功指标 + +V1 上线后重点观察: + +### Adoption + +```text +WAU +Active Knowledge Bases +Search Users +MCP Users +``` + +### Knowledge Coverage + +```text +Indexed Documents % +Successful Parse % +``` + +### Search Quality + +```text +Recall@5 +Recall@10 +No Result Rate +User Positive Feedback +``` + +### Knowledge Maintenance + +```text +Compiled Knowledge Count +Stale Rate +Conflict Rate +Review Completion Rate +``` + +### Reliability + +```text +Index Failure Rate +API Error Rate +MCP Error Rate +``` + +--- + +# 69. V1 P0 Scope + +必须上线: + +1. 用户与权限; +2. Knowledge Base; +3. Folder; +4. 文件上传; +5. PDF/DOCX/PPTX/XLSX/CSV/MD/TXT/HTML/Image; +6. OCR; +7. Parsing; +8. Cleaning; +9. Structure-aware Chunk; +10. Document Version; +11. Hash 去重; +12. Embedding Provider; +13. Mochow Provider; +14. Dense Search; +15. BM25; +16. Hybrid Retrieval; +17. ACL Filter; +18. Reranker; +19. Source Tracking; +20. Search API; +21. MCP Server; +22. WorkBuddy 接入; +23. Admin Console; +24. Search Lab; +25. Job 管理; +26. Audit Log; +27. Knowledge Compilation; +28. Compiled Knowledge 审核; +29. Benchmark; +30. Provider 配置。 + +--- + +# 70. V1.1 / P1 + +后续优先: + +- LDAP/OIDC; +- AD/企业 SSO; +- URL Import; +- Confluence; +- iWiki Import; +- 飞书; +- 企业微信; +- Scheduled Sync; +- MCP 写入知识; +- Auto Compilation Policy; +- Feedback Learning; +- Query Analytics。 + +--- + +# 71. P2 + +未来: + +- GraphRAG; +- Knowledge Graph; +- 多模态 Retrieval; +- 图片 Embedding; +- 视频知识; +- Audio; +- Agent Workflow; +- Enterprise Search; +- Personalized Retrieval; +- Automatic Knowledge Gap Discovery。 + +--- + +# 72. V1 验收标准 + +产品只有同时满足以下条件才认为 V1 完成。 + +## Ingestion + +上传 P0 文件类型能够: + +```text +Upload → Parse → Clean → Chunk → Embed → Index +``` + +全链路完成。 + +--- + +## Retrieval + +支持: + +```text +Dense +BM25 +Hybrid +Reranker +``` + +并通过 Admin Search Lab 查看每个阶段。 + +--- + +## Permission + +用户不能: + +- 搜索; +- 浏览; +- API; +- MCP; + +访问无权限 Document。 + +--- + +## Traceability + +每条 Search Result 可以定位: + +```text +Source File +Version +Page +Section +Chunk +``` + +--- + +## Model + +Embedding 和 Reranker 可以通过 Provider 配置替换。 + +--- + +## Vector DB + +Mochow 可以作为默认生产 Vector Store。 + +业务代码不存在必须依赖 Mochow SDK 数据结构的 API Contract。 + +--- + +## Benchmark + +能够用同一个 Evaluation Dataset 对多个 Embedding Pipeline 进行比较。 + +--- + +## Knowledge Compilation + +能够: + +```text +Source +→ +Generate Draft Knowledge +→ +Source Linking +→ +Review +→ +Publish +``` + +并能够在 Source 更新时把关联知识标记为 Stale。 + +--- + +## MCP + +WorkBuddy 能通过 MCP: + +```text +search_knowledge +get_source +``` + +取得与 REST API 一致、且经过 ACL 的结果。 + +--- + +# 73. 当前技术基线 + +PRD 阶段暂定但不最终锁定: + +```text +Primary Vector Store: +Mochow + +Embedding Baseline: +bge-large-zh + +Embedding Benchmark: +Qwen3 Embedding + +Reranker: +Qwen3 Reranker / equivalent + +Document Parsing: +Benchmark MinerU / Docling + +LLM: +Provider abstraction + +MCP: +Standard MCP Server + +Frontend: +Admin Console + +Backend: +API First +``` + +--- + +# 74. 已确认产品决策 + +当前已经确认: + +**DEC-001** + +采用: + +```text +API First + Admin Console +``` + +--- + +**DEC-002** + +Mochow 继续作为生产首选候选,同时必须保留 Vector Store 替换能力。 + +--- + +**DEC-003** + +不直接确定单一 Embedding 模型。 + +必须进行多模型企业中文 Benchmark。 + +--- + +**DEC-004** + +Reranker 是正式检索链 P0 能力。 + +--- + +**DEC-005** + +产品设计参考 iWiki / 企业 Wiki 的知识组织、权限和知识生命周期思路。 + +--- + +**DEC-006** + +采用: + +```text +RAG + Knowledge Compilation +``` + +而不是二选一。 + +--- + +**DEC-007** + +MCP 只能暴露业务能力,禁止泄露 Mochow、Embedding Model 等底层实现。 + +--- + +**DEC-008** + +企业知识必须始终能够追溯至 immutable Source。 + +--- + +# 75. 下一阶段需要解决的架构问题 + +PRD 评审通过后进入《总体技术架构设计》。 + +重点决策: + +1. Backend 技术栈; +2. Admin Console 技术栈; +3. PostgreSQL 数据模型; +4. Object Storage; +5. Task Queue; +6. MinerU vs Docling; +7. OCR Pipeline; +8. Chunk Strategy; +9. Mochow Schema; +10. VectorStore Adapter; +11. Embedding Provider; +12. Reranker Serving; +13. GPU Resource Scheduling; +14. Knowledge Compilation Engine; +15. Permission Resolver; +16. MCP Server; +17. Authentication; +18. Deployment Topology; +19. Observability; +20. Backup / Disaster Recovery。 + +--- + +# 76. 下一阶段应输出的技术架构方案 + +技术方案至少给出三套: + +## 方案 A:百度云能力优先 + +```text +Qianfan ++ +Mochow ++ +Local Parsing +``` + +优点: + +部署简单、延续现有经验。 + +--- + +## 方案 B:高度自建 + +```text +Local Embedding ++ +Local Reranker ++ +Open-source Vector DB ++ +Local Parsing +``` + +优点: + +数据可控、供应商依赖最低。 + +--- + +## 方案 C:Hybrid 推荐方案 + +```text +Local Parsing / GPU ++ +Replaceable Embedding ++ +Replaceable Reranker ++ +Mochow Primary ++ +Cloud LLM +``` + +目标: + +在开发成本、搜索质量、隐私、运维成本和供应商锁定之间取得平衡。 + +--- + +# 77. 调研依据备注 + +当前百度千帆模型列表表明 bge-large-zh 与 Qwen3 Embedding、Qwen3 Reranker 可以作为同平台 Benchmark 候选,因此早期无需为模型实验同时引入多个云厂商。 + +Mochow 当前官方 API 已支持向量检索与 BM25 全文检索组合的 Hybrid Search,同时可施加标量 Filter,与企业知识库的 Dense + Sparse + ACL Retrieval 需求相匹配。 + +WorkBuddy Enterprise 当前公开 MCP 文档支持自定义 MCP Server,并提供传输、权限、安全、远程 API Server 以及 iWiki 配置示例,因此“企业知识服务 → MCP → WorkBuddy”的产品路线具备正式产品支持基础。 + +Karpathy 的 LLM Wiki 原始思路强调 immutable raw sources 与由 LLM 持续维护的 compiled wiki 分层,这与本产品提出的 L0 Source / L1 Structured / L2 Compiled Knowledge 模型一致;企业版本在此基础上增加 ACL、版本、审计、冲突和审核机制。 + +--- + +**PRD v0.1 END** \ No newline at end of file diff --git a/fixtures/documents/README.md b/fixtures/documents/README.md new file mode 100644 index 0000000..e535b15 --- /dev/null +++ b/fixtures/documents/README.md @@ -0,0 +1,3 @@ +# Test Documents + +Only commit synthetic, public, or properly anonymized fixtures. Never commit real internal enterprise documents, HR data, customer data, contracts, credentials, or confidential material. diff --git a/fixtures/evaluation/golden.jsonl b/fixtures/evaluation/golden.jsonl new file mode 100644 index 0000000..316ab91 --- /dev/null +++ b/fixtures/evaluation/golden.jsonl @@ -0,0 +1,2 @@ +{"id":"sample-001","query":"员工出差住宿标准是多少?","expected_documents":["sample-travel-policy.md"],"expected_sections":["住宿标准"],"tags":["制度","自然语言"]} +{"id":"sample-002","query":"产品A退款周期","expected_documents":["sample-product-a-policy.md"],"expected_sections":["退款"],"tags":["产品","短查询"]} diff --git a/migrations/.gitkeep b/migrations/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/core/.gitkeep b/packages/core/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/domain/.gitkeep b/packages/domain/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/providers/embedding/.gitkeep b/packages/providers/embedding/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/providers/llm/.gitkeep b/packages/providers/llm/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/providers/object_storage/.gitkeep b/packages/providers/object_storage/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/providers/parser/.gitkeep b/packages/providers/parser/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/providers/reranker/.gitkeep b/packages/providers/reranker/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/providers/vector_store/.gitkeep b/packages/providers/vector_store/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/retrieval/.gitkeep b/packages/retrieval/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/packages/shared/.gitkeep b/packages/shared/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/prompts/CODEX_PROMPTS.md b/prompts/CODEX_PROMPTS.md new file mode 100644 index 0000000..92c47cc --- /dev/null +++ b/prompts/CODEX_PROMPTS.md @@ -0,0 +1,37 @@ +# Codex Prompt Library + +## P00 — Repository takeover +Read AGENTS.md and all referenced product/architecture/API/MCP/plan/DoD docs. Do not modify code. Inspect the repository and report current status, V1 gaps, top 5 risks, next 1–3 Task IDs, and blockers. Do not redesign accepted ADRs. + +## P01 — Plan one task +Plan `` only. Read relevant docs/code. Output goal, touched modules, steps, tests, migration/API impact, risks and out-of-scope. Do not modify files. + +## P02 — Implement one task +Implement `` only. Preserve AGENTS.md invariants. Do not implement future tasks. Run format/lint/typecheck/tests and RAG eval if retrieval behavior changes. Report summary, changed files, validation, risks and follow-ups. + +## P03 — Self-review as PR +Review the current branch for correctness, architecture violations, ACL leakage, secrets, provider coupling, migration safety, idempotency, test coverage, API/MCP compatibility and source traceability. Fix Critical/High findings within scope and rerun validation. + +## P04 — Bug fix +Reproduce ``, identify root cause, add failing regression test, implement smallest safe fix, run relevant checks, and report root cause + validation. + +## P05 — Retrieval change +For ``, identify changed retrieval stage. Preserve ACL-before/during-retrieval and source metadata. Run Golden Dataset before/after and report quality + latency deltas. + +## P06 — Embedding benchmark +Compare `` using the same Golden Dataset, corpus, chunk profile and retrieval config. Report Recall@K, MRR, NDCG@10, P50/P95 latency, throughput, dimensions, GPU/API needs, storage/cost and recommendation. + +## P07 — Parser benchmark +Compare MinerU and Docling on anonymized Chinese enterprise docs: native/scanned PDF, tables, multi-column, DOCX, PPTX, XLSX where applicable. Evaluate completeness, order, structure, tables, traceability, OCR, latency and GPU. Recommend routing rules. + +## P08 — MCP task +Implement/review ``. Expose business semantics only; bind identity; reuse REST authorization/application services; preserve traceability; do not expose underlying provider choices. + +## P09 — ACL audit +Verify Knowledge Base, folder inheritance, document overrides, REST, MCP, source download, retrieval, compiled knowledge and jobs. Fail review if unauthorized content can enter candidate Top-K before filtering. + +## P10 — Knowledge Compilation +Implement `` while preserving immutable source, source links, permission inheritance/intersection, stale detection and conflict safety. Never silently merge conflicting claims. + +## P11 — Release readiness +Assess V1 against PRD/DoD/tasks/ADRs: functionality, migrations, ACL/security, secrets, observability, backup/rebuild, performance, RAG quality, MCP, Admin Console and Compilation. Return blockers, high risks, known limitations and go/no-go. diff --git a/scripts/.gitkeep b/scripts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/e2e/.gitkeep b/tests/e2e/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/evaluation/.gitkeep b/tests/evaluation/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/.gitkeep b/tests/integration/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/.gitkeep b/tests/unit/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/workers/compilation/.gitkeep b/workers/compilation/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/workers/embedding/.gitkeep b/workers/embedding/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/workers/ingestion/.gitkeep b/workers/ingestion/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/workers/parser/.gitkeep b/workers/parser/.gitkeep new file mode 100644 index 0000000..e69de29