From 71cc6de5e21228aa93dcfc39f6ea66a33d65dfcb Mon Sep 17 00:00:00 2001 From: eust-w Date: Wed, 21 Oct 2026 08:03:41 +0800 Subject: [PATCH] fix: keep the local cluster from filling the Mac's disk Repeated builds grew the VM's disk to 63 GB, until the Mac's disk was full. Writes inside the VM then failed and its file systems broke. - The service image installed dependencies after copying every source file, so each source change rebuilt a 750 MB layer, five images at a time. It now installs from the workspace manifests first, and a test keeps every workspace listed. - The VM's disk defaults to 40 GiB instead of 80. - `up` refuses to create the VM when the Mac has less free space than that, and `build` refuses below 15 GiB free. - After building, superseded images are pruned and the build cache is capped at 10 GB. Co-Authored-By: Claude Opus 5.5 --- docs/local-k3s.md | 3 ++- infra/images/README.md | 3 ++- infra/images/runtime.Dockerfile | 17 ++++++++++++++-- scripts/k3s-local.mjs | 34 +++++++++++++++++++++++++++++-- tests/images.test.ts | 36 +++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 tests/images.test.ts diff --git a/docs/local-k3s.md b/docs/local-k3s.md index b1784a1..09abe9b 100644 --- a/docs/local-k3s.md +++ b/docs/local-k3s.md @@ -19,13 +19,14 @@ ## 前置条件 - macOS 上的 Colima 与 kubectl(`brew install colima kubectl`)。 -- 虚拟机默认 6 CPU、12 GiB 内存、80 GiB 磁盘,可以用 `WF_K3S_CPUS`、`WF_K3S_MEMORY_GIB`、`WF_K3S_DISK_GIB` 调整。 +- 虚拟机默认 6 CPU、12 GiB 内存、40 GiB 磁盘,可以用 `WF_K3S_CPUS`、`WF_K3S_MEMORY_GIB`、`WF_K3S_DISK_GIB` 调整。全部镜像(含 Theia)构建一次约占 10–15 GB。 - 一个 OpenAI 兼容的模型网关,以及它的 Key 和模型名。Codex 使用流式 `/responses` 与函数工具调用,不是每个模型都支持:先用 `GET <网关>/v1/models` 列出模型,再挑一个试。例如经 LiteLLM 的网关上,`deepseek-v4-pro` 可用。 所有 kubectl 与 docker 命令都显式使用 `colima-wishforge` 上下文。生成的密钥与镜像 digest 保存在 `.wishforge/k3s/`(不进 Git)。几点环境上的注意事项: - **当前上下文**:Colima 每次启动都会把它的虚拟机设为 kubectl 与 Docker 的当前上下文。脚本从不依赖当前上下文,`up` 结束时会打印切回原上下文的命令,切不切由你决定。 - **本机代理**:宿主机设置了 `HTTP(S)_PROXY`(例如本机的 Clash)时,Colima 会把代理交给虚拟机里的 k3s 与 Docker,但不排除虚拟机自己的地址。这样 API Server 访问 kubelet 也会走代理,`kubectl logs` 与 `exec` 全部失败。`up` 启动虚拟机时补上 `NO_PROXY`(虚拟机网段 `192.168.5.0/24`、Pod 与 Service 网段、集群域名);不是这样启动的虚拟机,先 `down` 再 `up`。`build` 也把代理作为构建参数交给镜像构建(地址换成虚拟机看到的宿主机 `192.168.5.2`),npm、GitHub 与 open-vsx 的下载因此和宿主机走同一个代理;Docker 不把这些参数写进镜像历史与构建缓存。 +- **磁盘空间**:虚拟机的磁盘是 Mac 上按需增长的文件(`~/.colima/_lima/_disks/colima-wishforge`),每次构建都会让它变大。Mac 的磁盘一旦写满,虚拟机里的写入就会失败,文件系统随之损坏,集群只能删除重建(`colima delete --profile wishforge --data`)。因此 `up` 在 Mac 空闲空间小于磁盘上限时拒绝创建虚拟机,`build` 在空闲空间不足 15 GiB 时拒绝构建;构建结束后会清理不再使用的旧镜像,并把构建缓存限制在 10 GB 以内。服务镜像先按各工作区的 `package.json` 安装依赖,只改源码时依赖层直接复用缓存。 - **Docker 凭据助手**:脚本的 docker 命令使用 `.wishforge/k3s/docker` 中自己的客户端配置:沿用你的 contexts 与插件,但不带凭据助手与已保存的登录。卸载 Docker Desktop 后遗留的 `credsStore: desktop` 因此不会让构建失败。 ## 步骤 diff --git a/infra/images/README.md b/infra/images/README.md index dfbb113..53642a2 100644 --- a/infra/images/README.md +++ b/infra/images/README.md @@ -22,6 +22,7 @@ docker build --file infra/images/runtime.Dockerfile --target worker --build-arg - worker 镜像中的 Docker 客户端必须连接单独配置的构建器(本地用 `WF_DOCKER_CONTEXT`,Kubernetes 中由 provisioner 传入 `WF_DOCKER_HOST` 与 TLS 证书),不要把宿主机 Docker socket 挂进任何 worker。镜像构建由 verifier 完成,只有它挂载构建器的客户端配置(provisioner 的 `WF_VERIFIER_DOCKER_CONFIG_SECRET`),executor 拿不到。 - 受保护验收脚本以 root 所有的镜像内容安装在 `/usr/local/lib/wishforge-verification/` 下,并在 API 中登记它们的摘要(见 [verification-profiles.md](../../docs/reference/verification-profiles.md))。不要把 API、Vault 或部署凭据挂进 executor 或 verifier。 - Runner 的客户环境工具(kubectl、Compose)与 mTLS 证书需要在单独审阅的镜像中补充。 +- 依赖先只按各工作区的 `package.json` 安装,只改源码时这一层直接复用缓存。新增工作区时要在 `runtime.Dockerfile` 中补一行 `COPY`,`tests/images.test.ts` 会检查。 - 构建时由 `scripts/build-services.mjs` 把各服务编译为 `dist/<服务>/main.mjs`:工作区包编入其中,其他依赖从 `node_modules` 加载,Temporal 工作流预先打包为 `dist/worker/workflow-bundle.js`,迁移脚本是 `dist/api/migrate.mjs`。运行时不再转译 TypeScript(FR-K5)。 -这些镜像尚未在运维选定的基础镜像与仓库上构建验证;Node 原生依赖、嵌套的 bubblewrap、远程 BuildKit、Kata/CNI 与多架构都需要在目标集群上验收。 +这些镜像已在本地 k3s 集群(arm64、`node:22-bookworm`,见 [docs/local-k3s.md](../../docs/local-k3s.md))上构建,并完整跑通了愿望与 IDE 的流程。运维选定的基础镜像、x64、嵌套的 bubblewrap、远程 BuildKit 与 Kata/CNI 仍需在目标集群上验收。 diff --git a/infra/images/runtime.Dockerfile b/infra/images/runtime.Dockerfile index 51b14cb..1f4e02c 100644 --- a/infra/images/runtime.Dockerfile +++ b/infra/images/runtime.Dockerfile @@ -14,7 +14,21 @@ USER root RUN apt-get update && apt-get install -y --no-install-recommends git python3 bubblewrap ca-certificates tini \ && rm -rf /var/lib/apt/lists/* WORKDIR /opt/wishforge +# The manifests of every workspace first: dependencies install once and stay cached while only +# sources change. A new workspace needs its line here, or npm ci fails. COPY package.json package-lock.json tsconfig.base.json ./ +COPY apps/api/package.json ./apps/api/ +COPY apps/web/package.json ./apps/web/ +COPY apps/worker/package.json ./apps/worker/ +COPY apps/runner/package.json ./apps/runner/ +COPY apps/git-service/package.json ./apps/git-service/ +COPY apps/provisioner/package.json ./apps/provisioner/ +COPY packages/agent/package.json ./packages/agent/ +COPY packages/artifacts/package.json ./packages/artifacts/ +COPY packages/contracts/package.json ./packages/contracts/ +COPY packages/github/package.json ./packages/github/ +COPY packages/runtime/package.json ./packages/runtime/ +RUN npm ci --ignore-scripts --no-audit --no-fund COPY apps/api ./apps/api COPY apps/web ./apps/web COPY apps/worker ./apps/worker @@ -24,8 +38,7 @@ COPY apps/provisioner ./apps/provisioner COPY packages ./packages COPY scripts/build-services.mjs ./scripts/ # Services run as built JavaScript (dist/), not TypeScript transpiled at start-up. -RUN npm ci --ignore-scripts --no-audit --no-fund \ - && node scripts/build-services.mjs \ +RUN node scripts/build-services.mjs \ && node -e "Promise.all([import('@temporalio/worker'),import('@openai/codex-sdk')]).then(()=>console.log('runtime dependencies loaded'))" ENV NODE_ENV=production USER node diff --git a/scripts/k3s-local.mjs b/scripts/k3s-local.mjs index bb70953..f6fdc8c 100644 --- a/scripts/k3s-local.mjs +++ b/scripts/k3s-local.mjs @@ -16,7 +16,7 @@ // context (which may be a shared cluster) is never touched. import { spawn, spawnSync } from 'node:child_process'; import { createHash, randomBytes } from 'node:crypto'; -import { mkdir, readFile, symlink, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, statfs, symlink, writeFile } from 'node:fs/promises'; import net from 'node:net'; import os from 'node:os'; import path from 'node:path'; @@ -133,8 +133,26 @@ const currentContexts = () => ({ docker: capture('docker', ['context', 'show']).stdout.trim(), }); +/** + * Free space on this Mac's disk, where Colima keeps the VM's disks. They grow with every image + * built; if the Mac's disk fills up, writes inside the VM fail and its file systems break. + */ +async function freeGiB() { + const stats = await statfs(os.homedir()); + return (stats.bavail * stats.bsize) / 2 ** 30; +} +const DISK_GIB = Number(process.env.WF_K3S_DISK_GIB ?? '40'); +const MIN_FREE_TO_BUILD_GIB = 15; + async function up() { const previous = currentContexts(); + if (!profileStatus()) { + const free = await freeGiB(); + if (free < DISK_GIB) + throw new Error( + `The VM's disk may grow to ${DISK_GIB} GiB, but only ${free.toFixed(1)} GiB are free on this Mac. Free some space or set WF_K3S_DISK_GIB lower.`, + ); + } if (!capture('colima', ['status', '--profile', PROFILE]).ok) { // A start that failed while installing k3s (often an interrupted GitHub download) leaves the VM // running without a runtime; stop it so that this start provisions again. @@ -153,7 +171,7 @@ async function up() { '--memory', process.env.WF_K3S_MEMORY_GIB ?? '12', '--disk', - process.env.WF_K3S_DISK_GIB ?? '80', + String(DISK_GIB), ], { env: colimaEnvironment() }, ); @@ -210,6 +228,11 @@ async function build() { const selected = process.argv.slice(3); const unknown = selected.filter(target => !TARGETS.includes(target)); if (unknown.length) throw new Error(`Unknown images ${unknown.join(', ')}; choose from ${TARGETS.join(', ')}`); + const free = await freeGiB(); + if (free < MIN_FREE_TO_BUILD_GIB) + throw new Error( + `Only ${free.toFixed(1)} GiB are free on this Mac; free at least ${MIN_FREE_TO_BUILD_GIB} GiB before building, since the VM's disk grows with every build.`, + ); await prepareDocker(); const buildArgs = (extra = {}) => Object.entries({ ...BASE_IMAGES, ...proxyBuildArgs(), ...extra }).flatMap(([name, value]) => [ @@ -247,6 +270,13 @@ async function build() { } await writeJson('images.json', images); console.log(images); + // Superseded images no container uses, and build cache beyond 10 GB, would only grow the disk. + for (const prune of [ + ['image', 'prune', '--force'], + ['builder', 'prune', '--force', '--max-used-space', '10GB'], + ]) + if (!capture('docker', ['--context', CONTEXT, ...prune], { env: dockerEnv() }).ok) + console.log(`Could not run docker ${prune.join(' ')}; the VM's disk keeps the old layers.`); } async function deploy() { diff --git a/tests/images.test.ts b/tests/images.test.ts new file mode 100644 index 0000000..5687c09 --- /dev/null +++ b/tests/images.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; +import { readdir, readFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; + +// The service image installs dependencies from the workspace manifests alone, so the layer stays +// cached while only sources change. A workspace missing there would fail npm ci in the image. +describe('service image', () => { + it('copies every workspace manifest before installing dependencies', async () => { + const { workspaces } = JSON.parse(await readFile('package.json', 'utf8')) as { workspaces: string[] }; + const directories = ( + await Promise.all( + workspaces.map(async pattern => + pattern.endsWith('/*') + ? (await readdir(pattern.slice(0, -2))).map(name => `${pattern.slice(0, -2)}/${name}`) + : [pattern], + ), + ) + ) + .flat() + .filter(directory => existsSync(`${directory}/package.json`)); + const dockerfile = await readFile('infra/images/runtime.Dockerfile', 'utf8'); + const install = dockerfile.indexOf('RUN npm ci'); + expect(install).toBeGreaterThan(0); + for (const directory of directories) { + const manifest = dockerfile.indexOf(`COPY ${directory}/package.json ./${directory}/`); + expect(manifest, directory).toBeGreaterThan(0); + expect(manifest, directory).toBeLessThan(install); + // Its sources (the workspace, or its whole parent directory) come only after the installation. + const parent = directory.split('/')[0]; + const sources = [`COPY ${directory} ./${directory}`, `COPY ${parent} ./${parent}\n`].map(line => + dockerfile.indexOf(line, install), + ); + expect(Math.max(...sources), directory).toBeGreaterThan(install); + } + }); +});