merge: complete DEPLOY-001 production images

This commit is contained in:
d-robotics 2026-09-03 11:01:58 +08:00
commit d727d0a118
17 changed files with 325 additions and 9 deletions

21
.dockerignore Normal file
View File

@ -0,0 +1,21 @@
# Keep credentials, local state, caches, and developer artifacts out of every image context.
.env
.env.*
!.env.example
.git
.github
.idea
.vscode
.DS_Store
.mypy_cache
.pytest_cache
.ruff_cache
.venv
__pycache__
*.py[cod]
*.sqlite*
dist
storage
tests
docs
docker-compose.dev.yml

View File

@ -4,6 +4,8 @@ LOG_LEVEL=INFO
API_HOST=0.0.0.0
API_PORT=8000
API_PREFIX=/api/v1
# Optional outside source checkouts; production image fixes this to /app/apps/admin-web.
# ADMIN_WEB_ROOT=/absolute/path/to/apps/admin-web
DATABASE_URL=mysql+pymysql://ekp:change-me@localhost:2881/ekp
DATABASE_POOL_SIZE=10

View File

@ -1,4 +1,4 @@
.PHONY: help install run run-mcp run-worker format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
.PHONY: help install run run-mcp run-worker images images-print images-push format format-check lint typecheck test test-unit test-integration migration-check eval live-acceptance up down
UV_CACHE_DIR ?= /tmp/ekp-uv-cache
export UV_CACHE_DIR
@ -9,6 +9,9 @@ help:
@echo " run Start the development API"
@echo " run-mcp Start the authenticated MCP gateway"
@echo " run-worker Start the ingestion worker and recovery scheduler"
@echo " images Build production images for the current builder"
@echo " images-print Render the multi-architecture image build plan"
@echo " images-push Build and push amd64/arm64 production images"
@echo " format Format code"
@echo " format-check Check formatting without modifying files"
@echo " lint Run linters"
@ -30,6 +33,12 @@ run-mcp:
uv run ekp-mcp
run-worker:
uv run ekp-worker
images:
docker buildx bake -f deploy/docker/docker-bake.hcl
images-print:
docker buildx bake -f deploy/docker/docker-bake.hcl --print
images-push:
docker buildx bake -f deploy/docker/docker-bake.hcl --push
format:
uv run ruff format .
format-check:

View File

@ -76,6 +76,10 @@ The Demo Admin Console is available at `http://localhost:8000/admin` and contain
only document upload, document management, and Search Lab pages. See
`docs/development/ADMIN_CONSOLE.md` for the operation and scope boundaries.
DEPLOY-001 provides locked, non-root `api`, `mcp`, and `worker` production image targets with an
amd64/arm64 Buildx Bake plan. See `docs/development/CONTAINER_IMAGES.md`; cluster manifests and
Secret injection remain in DEPLOY-002.
Run the development checks with:
```bash

View File

@ -37,12 +37,13 @@ def create_app(
application.state.settings = resolved_settings
application.state.api_services = api_services
application.include_router(router, prefix=resolved_settings.api_prefix)
admin_web_root = (resolved_settings.admin_web_root or ADMIN_WEB_ROOT).resolve()
# The Demo console is a deliberately thin, same-origin client. It contains
# no business logic and calls only the versioned public REST API.
application.mount(
"/admin/assets",
StaticFiles(directory=ADMIN_WEB_ROOT / "assets"),
StaticFiles(directory=admin_web_root / "assets"),
name="admin-assets",
)
@ -52,7 +53,7 @@ def create_app(
@application.get("/admin/search", include_in_schema=False)
async def admin_console() -> FileResponse:
"""Serve the three-page Demo Admin Console shell."""
return FileResponse(ADMIN_WEB_ROOT / "index.html")
return FileResponse(admin_web_root / "index.html")
@application.get("/healthz", response_model=HealthResponse, tags=["system"])
async def health() -> HealthResponse:

61
deploy/docker/Dockerfile Normal file
View File

@ -0,0 +1,61 @@
# syntax=docker/dockerfile:1.7
# One locked runtime produces three independently deployable process images.
ARG PYTHON_IMAGE=python:3.12.14-slim-bookworm@sha256:782412e85d0f0984994c290652577d4018aff08145c85b262bb63dc0c7522254
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.12.5@sha256:e85be844203885286c60ffad8a858d48afb6c5a5c237ca0e67f12e74b8f174b1
FROM ${UV_IMAGE} AS uv-bin
FROM ${PYTHON_IMAGE} AS runtime
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
VIRTUAL_ENV=/app/.venv \
PATH=/app/.venv/bin:${PATH} \
HOME=/tmp/ekp \
XDG_CACHE_HOME=/tmp/ekp/cache \
APP_ENV=production \
API_HOST=0.0.0.0 \
MCP_HOST=0.0.0.0 \
ADMIN_WEB_ROOT=/app/apps/admin-web
WORKDIR /app
COPY --from=uv-bin /uv /uvx /usr/local/bin/
COPY pyproject.toml uv.lock README.md alembic.ini ./
COPY apps ./apps
COPY packages ./packages
COPY migrations ./migrations
# --frozen prevents lock changes; --no-dev and --no-editable keep the runtime
# independent from source-tree import hooks while retaining migration sources.
RUN uv sync --frozen --no-dev --no-editable \
&& test -f /app/apps/admin-web/index.html \
&& groupadd --gid 10001 ekp \
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin ekp \
&& mkdir -p /tmp/ekp/cache \
&& chown -R 10001:10001 /tmp/ekp \
&& rm -rf /root/.cache/uv
USER 10001:10001
FROM runtime AS api
EXPOSE 8000
# This endpoint is intentionally credential-free and suitable for container liveness.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=2)"]
CMD ["uvicorn", "ekp_api.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
FROM runtime AS mcp
EXPOSE 8766
# MCP requests require authentication, so the image-level check only verifies PID 1.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-mcp"]
FROM runtime AS worker
# Worker health is process-level here; durable Job recovery is validated by the service tests.
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import os; os.kill(1, 0)"]
CMD ["ekp-worker"]

View File

@ -0,0 +1,36 @@
# Reproducible target names consumed by local and CI buildx invocations.
variable "IMAGE_REGISTRY" {
default = "knowledge-platform"
}
variable "IMAGE_TAG" {
default = "dev"
}
group "default" {
targets = ["api", "mcp", "worker"]
}
target "common" {
context = "."
dockerfile = "deploy/docker/Dockerfile"
platforms = ["linux/amd64", "linux/arm64"]
}
target "api" {
inherits = ["common"]
target = "api"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-api:${IMAGE_TAG}"]
}
target "mcp" {
inherits = ["common"]
target = "mcp"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-mcp:${IMAGE_TAG}"]
}
target "worker" {
inherits = ["common"]
target = "worker"
tags = ["${IMAGE_REGISTRY}/enterprise-knowledge-worker:${IMAGE_TAG}"]
}

View File

@ -211,6 +211,9 @@ Celery task 只负责恢复应用上下文并调用 Ingestion Application Servic
清理顺序和业务幂等仍由应用层持有。Broker 消息不是 Job 的 source of truth,Worker 重启或消息
重复投递必须以 OceanBase 中的持久化 Job/Attempt 状态收敛。
DEPLOY-001 将 API/Admin、MCP Gateway 与 Async Worker 构建为三个共享锁定 Python 运行时的
非 root 镜像 target;进程边界与上述模块边界一致,不新增服务或把 Provider 配置烘焙进镜像。
---
### 4.3 文档结构与 Chunk

View File

@ -0,0 +1,90 @@
# 生产容器镜像说明
## 范围
DEPLOY-001 使用同一个锁定运行时构建三个独立镜像目标:
| Target | 默认镜像名 | 入口 | 端口 | 镜像健康检查 |
|---|---|---|---|---|
| `api` | `enterprise-knowledge-api` | Uvicorn + API/Admin | `8000` | `GET /healthz` |
| `mcp` | `enterprise-knowledge-mcp` | `ekp-mcp` | `8766` | PID 1 存活 |
| `worker` | `enterprise-knowledge-worker` | `ekp-worker` | 无 | PID 1 存活 |
镜像没有内置数据库迁移、Helm、Ingress 或 Secret 对象;这些属于 DEPLOY-002。API 镜像已包含
Alembic 和 migrations,可由 DEPLOY-002 的一次性 Migration Job 覆盖命令后复用。
## 可重复构建基线
- Python 基础镜像固定为 `3.12.14-slim-bookworm`,uv 固定为 `0.12.5`,两者同时锁定 manifest
digest;
- Python 依赖只使用提交的 `uv.lock`,以 `uv sync --frozen --no-dev --no-editable` 安装;
- 三个目标共享相同依赖层,不复制不同版本的业务运行时;
- 进程均以固定 UID/GID `10001:10001` 运行,不具备 root 权限;
- `.dockerignore` 排除 `.env`、Git、虚拟环境、缓存、本地存储、测试和开发产物;
- Bake 计划同时声明 `linux/amd64` 与 `linux/arm64`,不下载或打包平台专属模型。
基础镜像 tag/digest 与应用依赖均已固定;镜像发布系统仍应保存本项目镜像的构建 digest,并让
Helm 按 digest 部署。本项目镜像 digest 由实际 Registry 发布产生,不把某个环境的 Registry
地址写入仓库。
## 构建命令
先确认 Docker Buildx builder 支持两个目标平台:
```bash
make images-print
```
构建全部目标:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images
```
只有发布流水线才执行推送:
```bash
IMAGE_REGISTRY=registry.example.internal/knowledge \
IMAGE_TAG=<immutable-release-tag> \
make images-push
```
不要把 Registry 密码放入变量或命令行;构建节点应提前通过其凭据助手登录。禁止使用
`latest` 作为集群部署版本。
## 运行时配置
镜像只内置以下非敏感进程默认值:`APP_ENV=production`、容器监听地址、静态页面绝对路径和
临时缓存目录。OceanBase、BOS、RabbitMQ、Embedding、Reranker 与 MCP Token 必须在容器启动
时由 Secret 注入。镜像构建参数、Layer、标签、健康检查和启动命令都不得包含这些值。
API 的 `/healthz` 只证明进程存活,不读取云端凭据或探测依赖。Kubernetes readiness、资源限制、
只读根文件系统所需临时卷及依赖就绪策略由 DEPLOY-002 配置。MCP endpoint 需要认证,Worker
没有 HTTP server,因此这两个镜像的 Docker 健康检查只验证主进程;业务可恢复性仍以持久化
Job/Attempt 和 DEPLOY-003 故障验收为准。
## 本地单目标检查
需要加载到本地 Docker 时,可直接构建当前平台的一个 target:
```bash
docker build --target api -f deploy/docker/Dockerfile -t ekp-api:test .
docker build --target mcp -f deploy/docker/Dockerfile -t ekp-mcp:test .
docker build --target worker -f deploy/docker/Dockerfile -t ekp-worker:test .
```
启动时使用 `--env-file` 或编排系统的 Secret 引用,不在命令中展开敏感值。API/Admin smoke test
至少检查 `/healthz`、`/admin/upload` 和 `/api/v1/openapi.json`;MCP 与 Worker 的真实依赖启动
验收在具备 RabbitMQ 和目标 Provider 的受控环境执行。
## DEPLOY-001 构建验收
2026-09-02 已使用 Buildx 分别构建并加载三个 `linux/arm64` target 和三个 `linux/amd64`
target。API/Admin 以非 root 用户完成健康、静态页面和 OpenAPI smoke test;MCP/Worker 完成非
root 安装入口检查。构建历史与容器文件系统检查未发现 `.env`、Git、测试文件或敏感配置。
这次验收只生成本地 `deploy001-*` 测试 tag,没有登录或推送 Registry,因此不产生可供集群
引用的项目镜像 digest。DEPLOY-002 必须使用操作者指定的不可变 Registry tag/digest,不得把
本地测试 tag 当作发布版本。

View File

@ -1,12 +1,12 @@
# 项目进度
当前阶段:批量异步上传闭环完成,准备交付生产镜像
当前阶段:生产容器镜像闭环完成,准备实施 Kubernetes/Helm 一键部署
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002
已完成:DEMO-001~DEMO-010、DEMO-012、DEMO-014~DEMO-016、ARCH-001~ARCH-006、PLAN-001~PLAN-002、DEPLOY-001
进行中:无
阻塞:无
下一任务:`DEPLOY-001`。随后严格执行 `DEPLOY-002 → DEPLOY-003`。
下一任务:`DEPLOY-002`,完成并合并后执行 `DEPLOY-003`。
Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → ARCH-005 → ARCH-006`
执行并已完成。DEMO-010 在迁移闭环验收后实施,未扩大为完整 V1 Admin Console。
@ -377,3 +377,22 @@ Provider 架构迁移按 `ARCH-001 → ARCH-002 → ARCH-003 → ARCH-004 → AR
- 未扩展目录上传、压缩包解包、断点续传、手动重试、去重、用户或 ACL;自动重试和清理继续
复用 DEMO-015,Accepted ADR 未变更。
- 下一任务:`DEPLOY-001`。
### DEPLOY-001(已完成)
- 新增共享生产 Dockerfile 的 `api`、`mcp`、`worker` 三个 target,使用精确 Python/uv 版本、
frozen production dependencies、固定非 root UID/GID、exec-form 启动命令和逐 target 健康检查。
- 新增 amd64/arm64 Buildx Bake 计划、镜像构建/渲染/推送 Make 入口和 `.dockerignore`;构建上下文
排除凭据、本地状态、缓存、测试与开发产物,镜像不包含环境 Endpoint 或 Secret。
- Admin 静态目录新增类型化运行时覆盖,使非 editable 安装后的 API 镜像仍能提供同源页面;
源码运行继续使用既有默认目录。
- 新增不依赖 Docker daemon 的镜像契约测试与中文运行文档;基础 Python/uv 镜像同时固定精确
tag 与 manifest digest,Registry 地址和本项目发布 digest 继续由部署环境提供。
- 使用 Docker Buildx `0.36.1` + Colima 分别实际构建并加载全部三个 `linux/arm64` 和三个
`linux/amd64` target;最终 digest 固定版本再次构建命中相同业务 layer。没有执行 Registry
push,也没有在构建参数、history 或镜像文件系统中发现凭据。
- API 镜像以 `10001:10001` 启动并进入 healthy,`/healthz`、`/admin/upload`、OpenAPI 及
DEMO-016 批量路径 smoke test 通过;MCP/Worker 镜像以同一非 root 用户完成安装入口导入,
默认 CMD 与 HEALTHCHECK 元数据核对通过。真实 Provider/Broker 启动仍属于 DEPLOY-003。
- Ruff Format/Lint、Mypy、Docker Bake 渲染、JavaScript/依赖锁/Alembic 与完整测试门禁通过;
Unit + Integration `187 passed`,RAG Evaluation `5 passed`。下一任务:`DEPLOY-002`。

View File

@ -87,7 +87,7 @@ Celery/RabbitMQ。队列调用保持在抽象边界后,不允许业务服务
### DEPLOY-001
交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
**已完成。** 交付 API/Admin、MCP Gateway、Async Worker 的生产容器镜像与启动约定;镜像使用固定依赖、
非 root 用户、健康检查和多架构可构建基线,不内置凭据。
### DEPLOY-002

View File

@ -30,7 +30,7 @@ validation. Provider credentials and internal endpoints are runtime inputs and n
- [x] PLAN-002 公共知识库集群交付范围、任务拆分和一键部署基线
- [x] DEMO-015 Async ingestion Job + Celery/RabbitMQ + bounded retry + pre-retry cleanup
- [x] DEMO-016 Batch async upload API + per-file Job status + Admin Console
- [ ] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [x] DEPLOY-001 Production images for API/Admin, MCP Gateway and Async Worker
- [ ] DEPLOY-002 Kubernetes/Helm one-click deployment + Secret/Migration/Ingress baseline
- [ ] DEPLOY-003 Availability, rolling upgrade and remote WorkBuddy acceptance

View File

@ -18,7 +18,7 @@
1. `PLAN-002`:确定公共知识库交付边界、目标拓扑、Task 拆分与验收门槛;
2. `DEMO-015`(已完成):持久化异步 Job、Celery/RabbitMQ 执行器、自动重试和重试前清理;
3. `DEMO-016`(已完成):批量异步上传、批次/Job 查询和 Admin Console 状态展示;
4. `DEPLOY-001`:API/Admin、MCP Gateway、Async Worker 生产镜像;
4. `DEPLOY-001`(已完成):API/Admin、MCP Gateway、Async Worker 生产镜像;
5. `DEPLOY-002`:Kubernetes/Helm、Secret、Migration、Ingress/TLS 和一键部署入口;
6. `DEPLOY-003`:多副本可用性、滚动升级、Worker 恢复和远程 WorkBuddy 验收。

View File

@ -468,6 +468,9 @@ Document ID 和 Ingestion Job;单个文件处理失败不得阻塞或回滚批
当前 Demo 已实现多文件 `202` 接收、批次聚合查询和逐文件 Job 状态;目录上传、压缩包解包、
断点续传以及完整 V1 的 Hash/Version/ACL 能力仍按各自 Task 后置。
公共知识库集群交付使用分别可扩缩的 API/Admin、MCP Gateway 和 Async Worker 生产镜像;
镜像只包含锁定代码与非敏感默认值,环境凭据必须在部署时注入。
**REQ-INGEST-004**
展示处理阶段:

View File

@ -25,6 +25,7 @@ class Settings(BaseSettings):
api_host: str = "0.0.0.0"
api_port: int = Field(default=8000, ge=1, le=65535)
api_prefix: str = "/api/v1"
admin_web_root: Path | None = None
# SQLAlchemy URLs commonly embed a password, so configuration rendering
# must mask this value and unwrap it only at the database driver boundary.
database_url: SecretStr = SecretStr("mysql+pymysql://localhost:2881/ekp")

View File

@ -14,6 +14,7 @@ def test_settings_have_demo_defaults(isolated_settings_environment: None) -> Non
assert settings.app_env == "development"
assert settings.api_prefix == "/api/v1"
assert settings.api_port == 8000
assert settings.admin_web_root is None
assert settings.database_url.get_secret_value().startswith("mysql+pymysql://")
assert settings.database_pool_size == 10
assert settings.database_max_overflow == 10

View File

@ -0,0 +1,65 @@
"""Static production-image contract tests that do not require a Docker daemon."""
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DOCKERFILE = PROJECT_ROOT / "deploy/docker/Dockerfile"
BAKE_FILE = PROJECT_ROOT / "deploy/docker/docker-bake.hcl"
def test_runtime_is_locked_non_root_and_secret_free() -> None:
"""Guard the supply-chain and least-privilege choices in the shared runtime."""
dockerfile = DOCKERFILE.read_text()
assert "python:3.12.14-slim-bookworm" in dockerfile
assert "ghcr.io/astral-sh/uv:0.12.5" in dockerfile
assert dockerfile.count("@sha256:") == 2
assert "uv sync --frozen --no-dev --no-editable" in dockerfile
assert "USER 10001:10001" in dockerfile
assert "COPY . " not in dockerfile
assert all(
sensitive not in dockerfile
for sensitive in ("DATABASE_URL=", "RABBITMQ_URL=", "API_KEY=", "PASSWORD=")
)
def test_three_process_targets_have_healthchecks_and_exec_commands() -> None:
"""Every independently deployed process must have an explicit runtime contract."""
dockerfile = DOCKERFILE.read_text()
assert dockerfile.count("HEALTHCHECK ") == 3
assert "FROM runtime AS api" in dockerfile
assert "FROM runtime AS mcp" in dockerfile
assert "FROM runtime AS worker" in dockerfile
assert 'CMD ["uvicorn", "ekp_api.main:app"' in dockerfile
assert 'CMD ["ekp-mcp"]' in dockerfile
assert 'CMD ["ekp-worker"]' in dockerfile
def test_bake_plan_covers_both_architectures_and_all_images() -> None:
"""The checked-in build plan must stay suitable for amd64 and arm64 clusters."""
bake = BAKE_FILE.read_text()
assert 'platforms = ["linux/amd64", "linux/arm64"]' in bake
assert 'context = "."' in bake
assert 'targets = ["api", "mcp", "worker"]' in bake
assert bake.count('inherits = ["common"]') == 3
assert all(
image in bake
for image in (
"enterprise-knowledge-api",
"enterprise-knowledge-mcp",
"enterprise-knowledge-worker",
)
)
def test_docker_context_excludes_credentials_and_local_state() -> None:
"""Prevent common developer files from entering any build layer."""
ignored = (PROJECT_ROOT / ".dockerignore").read_text().splitlines()
assert ".env" in ignored
assert ".git" in ignored
assert ".venv" in ignored
assert "storage" in ignored
assert "tests" in ignored