diff --git a/docs/capacity/source-index-100.json b/docs/capacity/source-index-100.json index c69ff09c..28fbb70f 100644 --- a/docs/capacity/source-index-100.json +++ b/docs/capacity/source-index-100.json @@ -3,7 +3,7 @@ "profile": "fanyu-source-index-100-v1", "namespace": "droboticflow-fanyu", "budget": { - "cpu": 36, + "cpu": 54, "memoryBytes": 120000000000 }, "deployments": { @@ -47,20 +47,20 @@ }, "droboticflow-annotation-validator": { "container": "processor", - "replicas": 2, + "replicas": 4, "requests": { - "cpu": "2", + "cpu": "6", "memory": "4Gi" }, "limits": { - "cpu": "4", + "cpu": "8", "memory": "8Gi" }, "env": { "ANNOTATION_SOURCE_INDEX": "true", "ANNOTATION_PROCESSOR_ROLE": "validation", "ANNOTATION_PROCESSOR_CONCURRENCY": "1", - "ANNOTATION_INDEX_CONCURRENCY": "1", + "ANNOTATION_INDEX_CONCURRENCY": "4", "ANNOTATION_INDEX_BACKFILL": "0", "DROBOTICFLOW_PROJECT_WRITE_LOCK_MODE": "striped", "DROBOTICFLOW_DATABASE_BOOTSTRAP": "verify-existing" @@ -106,7 +106,8 @@ }, "constraints": [ "Offline rendering only; the baseline was deployed on 2026-10-17, but future plans still require separate deployment authorization.", - "Six CPU cores and remaining memory stay unallocated after the proposed steady services and legacy-export budget.", + "2026-10-26 user-approved validator scale-up (validator-scale-20261026.md): 4 x 4 validation slots, 8 CPU limit per replica, preferred node 10.189.16.33. Resident limits budget raised to 48 CPU; budget.cpu 54 includes the 6 CPU export reservation.", + "No CPU spare remains after the resident 48 CPU and the 6 CPU legacy-export reservation; memory keeps its remaining headroom.", "Operator Job Pods have independent resource specifications. The export allowance is a planning reservation, not a namespace quota or implemented global admission limit.", "Do not enable source-index output until API, revision workers, validators, workflow workers and the package operator all understand the new schemas.", "Mongo and BOS are external; this budget does not expand their capacity or network bandwidth.", diff --git a/docs/capacity/source-index-100.md b/docs/capacity/source-index-100.md index f3372fe4..dcdeab68 100644 --- a/docs/capacity/source-index-100.md +++ b/docs/capacity/source-index-100.md @@ -4,19 +4,19 @@ ## 资源与并行 -预算按 36 CPU、120,000,000,000 字节计算。配置权威源为 `source-index-100.json`。 +预算按 54 CPU(常驻 48 + 导出预留 6,2026-10-26 经用户批准由 36 上调)、120,000,000,000 字节计算。配置权威源为 `source-index-100.json`。 | 组件 | 副本 × 单副本 limit | 总上限 | 用途 | | --- | --- | --- | --- | | API | 4 × 2 CPU / 4 GiB | 8 CPU / 16 GiB | 登录、列表、保存、任务状态 | | 切片修订 | 2 × 2 CPU / 4 GiB | 4 CPU / 8 GiB | 每副本 8 路,共 16 路 JSON 修订 | -| 独立源校验 | 2 × 4 CPU / 8 GiB | 8 CPU / 16 GiB | 每副本 1 路完整校验,共 2 路 | +| 独立源校验 | 4 × 8 CPU / 8 GiB | 32 CPU / 32 GiB | 每副本 4 路完整校验,共 16 路(2026-10-26 扩容,见 validator-scale-20261026.md) | | Worker | 2 × 1 CPU / 8 GiB | 2 CPU / 16 GiB | 调度、交付发布 | | Coordinator | 2 × 1 CPU / 2 GiB | 2 CPU / 4 GiB | 同步协调 | -| 常驻合计 | | 24 CPU / 60 GiB | | +| 常驻合计 | | 48 CPU / 76 GiB | | | 旧媒体导出规划预留 | | 6 CPU / 12 GiB | 独立 Job;尚非强制准入上限 | -余下 6 CPU 与约 42.7 GB 留作余量。各算子 Job 的资源单独定义;该表不是命名空间配额,也不代表已有 GPU/normalize/export Jobs 已受 36 CPU 总限额约束。未改变它们的配置。 +CPU 无余量,内存余约 25.5 GB。各算子 Job 的资源单独定义;该表不是命名空间配额,也不代表已有 GPU/normalize/export Jobs 已受 36 CPU 总限额约束。未改变它们的配置。 提高切片并行只针对完成源校验后的轻量修订。冷源排入独立验证队列,不能把 16 路全部用于解码。更多副本不解除 Mongo 项目写锁,也不增加外部 Mongo、BOS 或客户端网络容量。 diff --git a/docs/capacity/validator-scale-20261026.md b/docs/capacity/validator-scale-20261026.md index acdcd53b..5a65a1ef 100644 --- a/docs/capacity/validator-scale-20261026.md +++ b/docs/capacity/validator-scale-20261026.md @@ -1,6 +1,6 @@ -# 源校验 validator 扩容计划:100+ 人并发(2026-10-26,待批准,未执行) +# 源校验 validator 扩容:100+ 人并发(2026-10-26,已批准并执行) -本文是独立配置变更方案,不是部署授权。`scripts/release.py` 只覆盖 API / processor 同镜像更新,不检查也不修改 validator;本变更按 `docs/release/README.md` 单独审查,执行前须在紧邻部署时取得用户明确批准。 +用户于 2026-10-26 批准本方案及常驻 CPU 预算上调到 48,并已执行,记录见文末。后续再改仍需单独批准。`scripts/release.py` 只覆盖 API / processor 同镜像更新,不检查也不修改 validator;本变更按 `docs/release/README.md` 单独审查,执行前须在紧邻部署时取得用户明确批准。 ## 需求实测(只读,Mongo `annotationIndexJobs` / `humanEvents`,UTC 小时) @@ -89,3 +89,11 @@ kubectl -n droboticflow-fanyu rollout status deploy/droboticflow-annotation-vali - `10.189.16.33` 同时运行 GPU 与导出类算子 Job,大批 Job 并发时会争用 CPU。limits 只是上限,不是独占资源。 - 容量估算基于 10-24 与 10-26 两天的数据,人均需求可能随项目视频长度变化。 - **扩容只消除排队,每次确认仍要等约 1 分钟**。要让用户几乎不等,下一步需要改代码:在领取或打开任务时就提前排入校验(标注期间完成),并且左右目并行解码。这项改动需另行评审。 + +## 执行记录(2026-10-26 19:06 北京时间) + +- 前置只读检查:context `2c1284bae51b4bf3a2e623557844c6d6@kubernetes`、namespace uid、Deployment uid 和 resourceVersion `29531042` 都与快照一致;2/2 Ready、无重启;执行前 10 分钟没有校验任务,队列为空。执行前另存备份 `before-20261026T190545.yaml`。 +- 执行:补丁附带 `resourceVersion` 前置条件(期间若有他人修改会被拒绝),滚动约 2 分钟完成,generation 6→7,resourceVersion `29573013`。 +- 核验:4/4 Ready、0 重启,全部落在 `10.189.16.33`;线上 spec 与离线渲染完全一致;镜像未变(`sha256:b7655059…`);启动日志正常,无 Warning/OOM 事件;API 4/4、processor 2/2、Worker 2/2 未受影响。节点 `10.189.16.33` 的 requests 8.3→30.3 CPU(55%),与估算一致。队列:1901 个完成、2 个历史失败、0 个排队。 +- 同步:`source-index-100.json` / `.md` 的 validator 条目与预算(`budget.cpu` 54 = 常驻 48 + 导出预留 6),`tests/release` 39 项通过。 +- **尚未验证**:执行时没有负载,16 路并发下的吞吐、单任务耗时、CPU 与 BOS 表现需要在下一个工作日开工高峰(09:00 起)按上文“部署后核验”取证;业务验收由用户确认。 diff --git a/tests/release/test_source_capacity.py b/tests/release/test_source_capacity.py index cca76490..b96f8d07 100644 --- a/tests/release/test_source_capacity.py +++ b/tests/release/test_source_capacity.py @@ -33,9 +33,9 @@ class SourceCapacityTests(unittest.TestCase): snapshot, profile, evidence = self.fixture() result = capacity.render(snapshot, "c" * 40, profile, "registry.test/package@sha256:" + "b" * 64, evidence) self.assertEqual(result["state"], "review_only_not_authorized") - self.assertEqual(result["resourceTotals"]["steadyLimitCPU"], 24) - self.assertEqual(result["resourceTotals"]["unallocatedCPUAfterExportAllowance"], 6) - self.assertEqual(result["createValidator"]["spec"]["replicas"], 2) + self.assertEqual(result["resourceTotals"]["steadyLimitCPU"], 48) + self.assertEqual(result["resourceTotals"]["unallocatedCPUAfterExportAllowance"], 0) + self.assertEqual(result["createValidator"]["spec"]["replicas"], 4) validator_env = {e["name"]: e.get("value") for e in result["createValidator"]["spec"]["template"]["spec"]["containers"][0]["env"]} self.assertEqual(validator_env["ANNOTATION_PROCESSOR_MODE"], "processor") self.assertEqual(validator_env["DROBOTICFLOW_DATABASE_BOOTSTRAP"], "verify-existing") @@ -52,8 +52,8 @@ class SourceCapacityTests(unittest.TestCase): processor["spec"]["template"]["spec"]["containers"][0]["env"][-1] = port result = capacity.render(snapshot, "c" * 40, profile, "registry.test/package@sha256:" + "b" * 64, evidence) self.assertEqual(profile["deployments"]["droboticflow-worker"]["limits"]["memory"], "8Gi") - self.assertEqual(result["resourceTotals"]["steadyLimitMemoryBytes"], 60 * 2**30) - self.assertEqual(result["resourceTotals"]["unallocatedMemoryBytesAfterExportAllowance"], 120000000000 - 72 * 2**30) + self.assertEqual(result["resourceTotals"]["steadyLimitMemoryBytes"], 76 * 2**30) + self.assertEqual(result["resourceTotals"]["unallocatedMemoryBytesAfterExportAllowance"], 120000000000 - 88 * 2**30) env = result["createValidator"]["spec"]["template"]["spec"]["containers"][0]["env"] self.assertEqual([e for e in env if e["name"] == port["name"]], [port]) self.assertEqual(sum(e["name"] == "DROBOTICFLOW_DATABASE_BOOTSTRAP" for e in env), 1)