forked from longtao.wu/droboticflow
🛡️ Guard work availability and preserve release verification evidence
This commit is contained in:
parent
8d4b672649
commit
a35e84163a
@ -7,7 +7,7 @@ name: web-ci
|
||||
# didn't boot the dev server.
|
||||
on:
|
||||
push:
|
||||
branches: ['**']
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
@ -41,7 +41,7 @@ jobs:
|
||||
- run: go test ./internal/api -run '^TestReviewPoolSearchBrowserReplica$' -count=1 -timeout=3m -v
|
||||
if: ${{ !cancelled() }}
|
||||
env:
|
||||
REVIEW_POOL_BROWSER_TEST: '1'
|
||||
REVIEW_POOL_BROWSER_TEST: "1"
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
@ -69,3 +69,39 @@ jobs:
|
||||
- run: go test ./internal/store -run '^TestHumanPoolSearchMongo' -count=1 -timeout=3m -v
|
||||
env:
|
||||
POOL_LOCAL_MONGO_URI: mongodb://127.0.0.1:27018
|
||||
|
||||
work-availability:
|
||||
# Current source/media paths require FFmpeg >=5.1 (-fps_mode). Keep the
|
||||
# existing Ubuntu 22 checks intact and run this mandatory matrix on 24.04.
|
||||
runs-on: ubuntu-24.04
|
||||
needs: pool-search-mongo
|
||||
if: ${{ !cancelled() }}
|
||||
services:
|
||||
mongo:
|
||||
image: mongo:8
|
||||
ports:
|
||||
- 27018:27017
|
||||
options: >-
|
||||
--health-cmd "mongosh --quiet --eval 'db.adminCommand({ping:1})'"
|
||||
--health-interval 5s --health-timeout 5s --health-retries 12
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: sudo apt-get update && sudo apt-get install -y ffmpeg
|
||||
- run: npx playwright install --with-deps chromium
|
||||
- run: npm run test:work-availability
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: work-availability-logs
|
||||
path: local-data/
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
20
.scratch/reliability-guardrails/work.md
Normal file
20
.scratch/reliability-guardrails/work.md
Normal file
@ -0,0 +1,20 @@
|
||||
# Reliability guardrails
|
||||
|
||||
Base: dev 8d4b6726. User authorized local implementation of retrospective improvements.
|
||||
|
||||
Scope: mandatory source-admission and annotation-preservation checks wired to CI/release; strict-mode readiness and explicit source-mode plans; private redacted failure logs and immutable check evidence; bounded request/DB/lock observations; reproducible runtime builds and reviewed combined image/config plans.
|
||||
|
||||
Compatibility: keep current advisory policy, permissions, optimistic versions, annotations, legacy reads and historical release records. No production configuration/data changes, backfill, index/backup writes, push or deployment in this task. Existing unrelated admin pool test edits are protected.
|
||||
|
||||
Verification: synthetic media and isolated local Mongo, existing baseline gates, release state-machine/CLI and security regression tests. No test may silently skip critical cases. Keep failures and unverified coverage visible.
|
||||
|
||||
Acceptance:
|
||||
|
||||
1. New releases require explicit effective source modes; a reviewed combined image/mode patch preserves all other configuration. Strict activation requires a complete, current deployment inventory under the candidate rules, without enqueueing validation or changing work.
|
||||
2. Existing annotation preservation, submission/deletion races, restart recovery, permissions and legacy-data regressions become mandatory, with missing/skipped cases rejected.
|
||||
3. The same gate exercises real local Mongo equivalence and advisory browser work/recovery in CI and release checks.
|
||||
4. Local checks preserve bounded private redacted output and immutable evidence; browser failures retain unique artifacts. Reuse is explicit and limited to unchanged, unexpired successful pure checks.
|
||||
5. Request-scoped numeric Mongo/query/lock observations reveal phase costs without customer data, extra queries or production explain.
|
||||
6. Authoritative image builds pin runtime dependencies and record recipe provenance. Image/source-mode changes form one reviewable processor-before-API plan. Other production configuration, backups and data mutations remain outside this task.
|
||||
|
||||
Limitations: inventory is point-in-time, not a data freeze or certificate; strict activation requires a controlled cutover window. Local tests do not prove production write acceptance, capacity or backup recovery. Remote CI awaits a separately authorized push.
|
||||
@ -1,7 +1,8 @@
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
|
||||
ARG DROBOTICFLOW_BUILD_TIME=unknown
|
||||
ARG API_RUNTIME_IMAGE=debian:bookworm-slim
|
||||
|
||||
FROM node:22-alpine AS web-build
|
||||
FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 AS web-build
|
||||
WORKDIR /app
|
||||
|
||||
COPY package*.json ./
|
||||
@ -15,7 +16,7 @@ COPY src ./src
|
||||
COPY public ./public
|
||||
RUN npm run build:web
|
||||
|
||||
FROM golang:1.24-bookworm AS go-build
|
||||
FROM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS go-build
|
||||
WORKDIR /src
|
||||
ARG GOPROXY=https://proxy.golang.org,direct
|
||||
|
||||
@ -26,23 +27,31 @@ COPY cmd ./cmd
|
||||
COPY internal ./internal
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-api ./cmd/api
|
||||
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
FROM ${API_RUNTIME_IMAGE} AS runtime
|
||||
# Production release script supplies a pinned prepared runtime. Direct local/POC
|
||||
# builds retain the original Debian setup unless explicitly opted in.
|
||||
ARG API_RUNTIME_PREPARED=0
|
||||
USER root
|
||||
WORKDIR /app
|
||||
RUN if [ "$API_RUNTIME_PREPARED" = "1" ]; then \
|
||||
test -x /usr/bin/ffprobe && getent passwd 10001; \
|
||||
else \
|
||||
apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* && \
|
||||
groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow; \
|
||||
fi
|
||||
# A compatibility runtime may be an older API image. Never retain its old app.
|
||||
RUN rm -rf /app/dist /usr/local/bin/droboticflow-api
|
||||
COPY --from=web-build --chown=10001:10001 /app/dist ./dist
|
||||
COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api
|
||||
|
||||
# Application identity must not invalidate the OS dependency layers.
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT
|
||||
ARG DROBOTICFLOW_BUILD_TIME
|
||||
WORKDIR /app
|
||||
ENV PORT=3001
|
||||
ENV DIST_DIR=/app/dist
|
||||
ENV PORT=3001 DIST_DIR=/app/dist
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
|
||||
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/*
|
||||
RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow
|
||||
|
||||
COPY --from=web-build --chown=10001:10001 /app/dist ./dist
|
||||
COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api
|
||||
|
||||
EXPOSE 3001
|
||||
USER 10001:10001
|
||||
CMD ["droboticflow-api"]
|
||||
|
||||
9
Dockerfile.api-runtime
Normal file
9
Dockerfile.api-runtime
Normal file
@ -0,0 +1,9 @@
|
||||
# Build/publish separately after approval; record this image's immutable digest
|
||||
# in docs/release/api-runtime.json. No application or customer data is copied.
|
||||
FROM docker.io/library/debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && \
|
||||
dpkg-query -W > /runtime-packages.txt && rm -rf /var/lib/apt/lists/* && \
|
||||
groupadd --system --gid 10001 droboticflow && \
|
||||
useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow
|
||||
WORKDIR /app
|
||||
USER 10001:10001
|
||||
@ -26,7 +26,7 @@ import (
|
||||
|
||||
func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) {
|
||||
return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) {
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI))
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI).SetMonitor(store.ReadCommandMonitor()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
48
docs/operations/reliability-guardrails.md
Normal file
48
docs/operations/reliability-guardrails.md
Normal file
@ -0,0 +1,48 @@
|
||||
# 工作流程可靠性防护
|
||||
|
||||
本批修改把已存在的恢复测试和发布约束接入执行入口,不改变 advisory 的业务准入,
|
||||
不改变权限、版本、领取、送检、审核或验收规则,不自动修改生产记录。
|
||||
|
||||
## 必跑回归与证据
|
||||
|
||||
`npm run test:work-availability` 分阶段运行:已有标注及送检失败保护、删除/送检并发和重启恢复;
|
||||
隔离 Mongo 的旧数据/指纹/读取不补验等价检查;advisory 模式的完整工作台及返工浏览器流程。
|
||||
使用本机合成媒体和独立数据库,不使用公司数据。已知旧失败仍交由既有 Go/前端基线门禁比较。
|
||||
必需用例被跳过或未出现时失败;独立阶段在普通失败后继续,源码中途变化则停止并列出未验证阶段。
|
||||
CI 和发布策略均调用同一个入口,不能以只跑单元测试或 Pod Ready 代替业务验收。
|
||||
|
||||
每次运行在 `local-data/test-results/` 保存摘要和私有脱敏日志;浏览器在唯一运行目录保留失败截图、
|
||||
trace 和异常,不覆盖前一次失败。trace 仅用于合成夹具,不能对客户页面照搬。
|
||||
发布检查的显式证据复用范围限纯检查,失效和失败必须重跑,不能复用浏览器、数据库或性能结果。
|
||||
|
||||
## 源准入影响盘点
|
||||
|
||||
管理员可读取本组织清单,超级管理员才能读取部署范围清单;组织身份来自服务端会话。
|
||||
`projectId` 只能缩小组织查询,部署盘点不接受项目过滤。接口只返回按组织、阶段、状态分组的数量,
|
||||
不返回正文,不创建任务或补验作业,每个进程同时最多一份盘点,Mongo 执行上限 8 秒。
|
||||
超时/取消返回不完整错误,不能把部分结果作为可启用依据。
|
||||
|
||||
严格规则与现有规则变化时,应先将候选以 advisory 发布,再盘点候选规则会阻断的存量数据。
|
||||
发布工具拒绝旧构建、局部范围、错误数据库、过期、被修改或仍含阻断条目的清单。
|
||||
盘点不提供事务级数据冻结:仍需在发布安全审查中确定受控切换窗口;无法保证时保持 advisory。
|
||||
它不伪造证书,也不意味着源内容已通过完整检查。
|
||||
|
||||
## 查询与写锁观察
|
||||
|
||||
复用服务端生成的 request ID,记录固定阶段的耗时、Mongo 命令成功/失败次数、累计驱动耗时和
|
||||
受限错误分类,以及写锁获取耗时。只对已有请求收集,不追加查询或自动执行 explain。
|
||||
日志使用路由模板,不记录查询词、任务 ID、正文、意见、凭证、Mongo 命令或响应体。
|
||||
|
||||
驱动耗时包含网络等待,不能当作 Mongo 服务端执行时间;多个命令累计时间也不等于请求墙钟耗时。
|
||||
连接池等待、扫描量、服务端执行计划和 EIP 带宽没有因此被测量,仍需单独只读诊断。
|
||||
此修改未优化查询或增加业务后台计算,不构成容量达标结论。
|
||||
|
||||
## 构建、发布与恢复边界
|
||||
|
||||
泛宇构建固定 Node/Go 构建阶段及运行依赖摘要,使用同一权威配方并记录来源。
|
||||
镜像和唯一允许的源模式可在同一审查计划顺序切换,其他配置、组件和数据仍另列授权。
|
||||
处理器失败时停止更新 API,记录部分完成状态,不自动重试、回退或清除证据。
|
||||
|
||||
本批未启用 BOS 版本控制、Mongo PITR 或备份策略,也未进行恢复演练。保留送检快照和失败输入的
|
||||
已有应用级保护不能代替存储备份;备份保留期、成本、权限与隔离恢复演练需单独形成可批准方案。
|
||||
生产写验收、完整并发容量和故障恢复演练均不包含在本地门禁的通过结论中。
|
||||
@ -16,9 +16,11 @@
|
||||
再用 `python3 scripts/release.py status <id>` 查看缺失证据与下一步。
|
||||
历史聊天和“Pod 已 Ready”不代表业务验收完成。使用命令更新记录,保留失败和未验证项。
|
||||
|
||||
首版只支持 **Fanyu 现有 API 与切片处理器的同一不可变镜像更新**,并同步容器的
|
||||
`DROBOTICFLOW_BUILD_COMMIT`。保留 ConfigMap、Secret、PVC、Service、资源和副本数。
|
||||
需要更改配置、其他组件或数据库时,单独设计并审查变更计划;首版不提供绕过参数。
|
||||
支持 **Fanyu 现有 API 与切片处理器的同一不可变镜像更新**,并同步容器的
|
||||
`DROBOTICFLOW_BUILD_COMMIT`。可通过 `prepare --source-mode advisory|strict` 将唯一指定的
|
||||
`ANNOTATION_SOURCE_VALIDATION_MODE` 一起纳入同一计划和审批;处理器先更新并就绪,再更新 API。
|
||||
保留 ConfigMap、Secret、PVC、Service、其他配置、资源和副本数。
|
||||
需要更改其他配置、组件或数据库时,单独设计并审查变更计划,不提供绕过参数。
|
||||
|
||||
2026-10-17 已部署的源索引基线也可做上述镜像更新:处理器必须精确匹配
|
||||
fanyu-policy.json 引用的容量文件(2 副本、每副本 8 修订槽、4 GiB limit、
|
||||
@ -70,6 +72,11 @@ metadata 读取 digest,输出 `IMAGE=...@sha256:...` 供 `--image` 使用。
|
||||
目标 server、context、namespace UID 从已核实的集群身份取得,不能从一次错误连接中直接认可。
|
||||
工具检查 namespace UID 及 API server,namespace 名字相同也不能蒙混过关。
|
||||
|
||||
构建使用权威 `Dockerfile.api` 和 [固定运行依赖](api-runtime.json),不再临时生成包装 Dockerfile。
|
||||
输出旁的 `.provenance.json` 记录源码、运行基础摘要和配方摘要。当前固定基础为既有已验证镜像,
|
||||
复制新产物前移除旧应用;`Dockerfile.api-runtime` 提供后续专用基础的构建配方,发布它仍需单独授权。
|
||||
本地/其他环境直接构建保留原有 Debian 安装路径;泛宇发布脚本只接受不可变运行基础摘要。
|
||||
|
||||
```sh
|
||||
python3 scripts/release.py prepare --id <release-id> \
|
||||
--image <registry/repository@sha256:digest> \
|
||||
@ -95,7 +102,14 @@ python3 scripts/release.py check <release-id> --cluster
|
||||
不再把已替换的旧页面布局当成当前入口。返工浏览器夹具运行真实本地校验处理器,不能绕过源数据准入。
|
||||
统一候选涉及的额外组件与待确认发布项见 [组件发布计划](recent-integration-components.md)。首版运行流程工具测试、
|
||||
前端类型/构建及切片恢复定向回归,不冒充全套产品测试通过。任何失败阻断,不支持忽略开关。
|
||||
命令执行结果及输出摘要写入记录,不保存或回显原始输出;排查失败时单独重跑对应命令。
|
||||
命令执行结果、输入身份和日志摘要写入记录。本地检查的成功及失败输出先脱敏、限长,再以私有文件保存,
|
||||
不会记录集群对象原文;日志路径写入记录。浏览器失败截图及 trace 使用唯一运行目录,仅限隔离合成夹具。
|
||||
`check --local --reuse-successful` 仅可复用一小时内输入、安装依赖、工具、环境和日志摘要均未变的
|
||||
白名单纯检查。默认重跑全部;浏览器、Mongo、性能与集群检查始终重新执行。
|
||||
|
||||
新增强制 `test:work-availability`,复用已有送检保护、删除/送检并发、重启恢复、权限、旧数据
|
||||
Mongo 等价和 advisory 真实工作台回归。缺用例、跳过、失败、源码中途变化均不能算通过。
|
||||
CI 独立作业需要 Ubuntu 24.04 runner、Mongo 8、FFmpeg 和 Playwright;远端 CI 未执行时须明确待验证。
|
||||
|
||||
2026-10-20 起,`build:web` 后强制运行 `test:production:boot`,实际启动生产 dist 并验证登录页、非空 root 和零运行时错误。该门禁使用本机桩 API,不连接公司;须安装 Playwright 浏览器或通过 `BROWSER_EXECUTABLE` 指定已有 Chromium。构建依赖必须按当前 lockfile 独立安装;共享依赖版本清单或带 alias 的组件夹具不能代替实际产物启动验证。
|
||||
|
||||
@ -115,6 +129,16 @@ python3 scripts/release.py plan <release-id>
|
||||
显示旧/新镜像、构建标记、目标、带宽观察值、限制与回退说明,并生成计划摘要。
|
||||
核对两个部署的修改范围。计划不更新 ConfigMap、Secret、存储或网络。
|
||||
|
||||
新发布记录必须明确两个组件的源校验模式;未配置时禁止依赖代码默认严格模式。
|
||||
现有历史记录仍按原摘要核验。严格模式切换或严格准入关键规则变化须先以 advisory 部署候选,
|
||||
再由超级管理员获取 `GET /api/plugins/annotation-qa/source-readiness?scope=deployment` 的只读影响清单,
|
||||
用 `check --cluster --source-readiness-file <本地私有清单>` 绑定计划。
|
||||
清单必须覆盖同一数据库、命名空间、线上构建及旧数据截止配置,一小时内完整生成,所有条目均可准入;
|
||||
组织/项目局部清单、本地夹具、缺项、阻断条目、过期或内容变更均不允许启用严格模式。
|
||||
清单不补验、不排队、不改标注,也不是源数据校验证书。它是时间点盘点,不会冻结业务数据;
|
||||
`change-safety` 必须说明盘点至切换期间的新任务和在途变更如何受控,不满足时保持 advisory。
|
||||
当前模式不因获取清单而变化。详细边界见[可靠性防护](../operations/reliability-guardrails.md)。
|
||||
|
||||
部署前需要两份人工审查证据:
|
||||
|
||||
- `image-provenance`:构建记录将本次源码 commit 与镜像 digest 对应,候选镜像的验证情况明确。
|
||||
|
||||
5
docs/release/api-runtime.json
Normal file
5
docs/release/api-runtime.json
Normal file
@ -0,0 +1,5 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/fanyu/droboticflow-api@sha256:6a060aa2865ebe0527377a9f22a0f1feb6b23d1f099985a99cce25f6981743d2",
|
||||
"purpose": "Compatibility bootstrap: reuse approved production runtime; remove old application outputs. Replace with dedicated runtime only through an explicitly reviewed image publication."
|
||||
}
|
||||
@ -34,6 +34,7 @@
|
||||
["npm", "run", "test:browser:annotation-persistence"],
|
||||
["go", "test", "./internal/deliveryindex", "-count=1"],
|
||||
["go", "test", "./internal/worker", "-run", "TestCaptureCutover|Test.*V4Capture|TestDiscoverCaptureGroupsForWorkflow|TestClassificationAudit|TestInternalAuditFailure", "-count=1"],
|
||||
["go", "test", "./internal/api", "-run", "TestDurableAnnotationOutbox|TestSliceRecovery|TestReviewCannotMutateAnnotation|TestReviewDecision|Test.*ReviewDraft|TestReviewConfirmations|TestRouteSnapshot|Test.*Dictionary|TestProjectSliceNames|TestLegacyMaterializationWithoutPartScopedOriginalSelection|TestOptionalOriginalSource|TestAnnotationOriginalSource|TestClassificationBatch|TestWorkflowVersionActivation", "-count=1"]
|
||||
["go", "test", "./internal/api", "-run", "TestDurableAnnotationOutbox|TestSliceRecovery|TestReviewCannotMutateAnnotation|TestReviewDecision|Test.*ReviewDraft|TestReviewConfirmations|TestRouteSnapshot|Test.*Dictionary|TestProjectSliceNames|TestLegacyMaterializationWithoutPartScopedOriginalSelection|TestOptionalOriginalSource|TestAnnotationOriginalSource|TestClassificationBatch|TestWorkflowVersionActivation", "-count=1"],
|
||||
["python3", "scripts/test_work_availability.py"]
|
||||
]
|
||||
}
|
||||
|
||||
@ -2,12 +2,15 @@ package api
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
@ -30,6 +33,8 @@ func diagnosticTrace(next http.Handler) http.Handler {
|
||||
return
|
||||
}
|
||||
id := hex.EncodeToString(entropy[:])
|
||||
ctx, observation := store.WithReadObservation(r.Context(), id)
|
||||
r = r.WithContext(ctx)
|
||||
w.Header().Set("X-Request-ID", id)
|
||||
started := time.Now()
|
||||
wrapped := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
|
||||
@ -41,10 +46,19 @@ func diagnosticTrace(next http.Handler) http.Handler {
|
||||
} else if r.Header.Get("X-Diagnostic-Request") == "1" {
|
||||
kind = "diagnostic"
|
||||
}
|
||||
metrics, _ := json.Marshal(observation.Snapshot())
|
||||
if kind != "" {
|
||||
log.Printf("request_trace id=%s kind=%s status=%d elapsed_ms=%d", id, kind, wrapped.Status(), elapsed.Milliseconds())
|
||||
} else if elapsed >= slowRequestLogThreshold {
|
||||
log.Printf("request_trace id=%s kind=slow method=%s path=%s status=%d elapsed_ms=%d", id, r.Method, r.URL.Path, wrapped.Status(), elapsed.Milliseconds())
|
||||
log.Printf("request_trace id=%s kind=%s status=%d elapsed_ms=%d observations=%s", id, kind, wrapped.Status(), elapsed.Milliseconds(), metrics)
|
||||
} else if elapsed >= slowRequestLogThreshold || wrapped.Status() >= 500 {
|
||||
route := "/api/<unmatched>"
|
||||
if rc := chi.RouteContext(r.Context()); rc != nil && rc.RoutePattern() != "" {
|
||||
route = rc.RoutePattern()
|
||||
}
|
||||
reason := "slow"
|
||||
if elapsed < slowRequestLogThreshold {
|
||||
reason = "failed"
|
||||
}
|
||||
log.Printf("request_trace id=%s kind=%s method=%s path=%s status=%d elapsed_ms=%d observations=%s", id, reason, r.Method, route, wrapped.Status(), elapsed.Milliseconds(), metrics)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@ -2,7 +2,10 @@ package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@ -100,3 +103,36 @@ func TestDiagnosticTaskSummaryIsScopedAndReadOnly(t *testing.T) {
|
||||
t.Fatalf("anonymous: %d", anonymous.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiagnosticTraceCorrelatesReadPhasesWithoutQueryOrResourceIDs(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
old := log.Writer()
|
||||
log.SetOutput(&logs)
|
||||
defer log.SetOutput(old)
|
||||
router := chi.NewRouter()
|
||||
router.Use(diagnosticTrace)
|
||||
router.Get("/api/tasks/{id}", func(w http.ResponseWriter, r *http.Request) {
|
||||
if store.ReadRequestID(r.Context()) == "" {
|
||||
t.Fatal("missing request context")
|
||||
}
|
||||
finish := store.ObserveReadPhase(r.Context(), "candidate_query")
|
||||
finish()
|
||||
store.ReadCommandMonitor().Failed(r.Context(), &event.CommandFailedEvent{Failure: "i/o timeout private-body"})
|
||||
w.WriteHeader(500)
|
||||
})
|
||||
req := httptest.NewRequest("GET", "/api/tasks/private-id?search=secret-annotation", nil)
|
||||
req.Header.Set("X-Request-ID", "caller-id")
|
||||
resp := httptest.NewRecorder()
|
||||
router.ServeHTTP(resp, req)
|
||||
text := logs.String()
|
||||
for _, value := range []string{"private-id", "secret-annotation", "private-body", "caller-id"} {
|
||||
if strings.Contains(text, value) {
|
||||
t.Fatal("sensitive request information leaked")
|
||||
}
|
||||
}
|
||||
for _, value := range []string{"/api/tasks/{id}", "candidate_query", "mongo_failures", "client_or_network_deadline", resp.Header().Get("X-Request-ID")} {
|
||||
if !strings.Contains(text, value) {
|
||||
t.Fatalf("missing diagnostic: %s", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -410,7 +410,9 @@ func (server *server) prepareHumanPoolRead(writer http.ResponseWriter, request *
|
||||
}
|
||||
|
||||
func (server *server) readHumanItemsList(writer http.ResponseWriter, request *http.Request) (*humanItemListPage, *http.Request) {
|
||||
finishPrepare := store.ObserveReadPhase(request.Context(), "prepare")
|
||||
pool, query, limit, request, ok := server.prepareHumanPoolRead(writer, request)
|
||||
finishPrepare()
|
||||
if !ok {
|
||||
return nil, request
|
||||
}
|
||||
@ -429,12 +431,16 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
query.Limit = fetchSize
|
||||
candidateQuery := query
|
||||
candidateQuery.PoolCandidateProjection = true
|
||||
finishQuery := store.ObserveReadPhase(request.Context(), "candidate_query")
|
||||
items, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), candidateQuery)
|
||||
finishQuery()
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return nil, request
|
||||
}
|
||||
finishContext := store.ObserveReadPhase(request.Context(), "list_context")
|
||||
listCtx, listErr := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, items)
|
||||
finishContext()
|
||||
if listErr != nil {
|
||||
renderError(writer, http.StatusInternalServerError, listErr)
|
||||
return nil, request
|
||||
@ -446,7 +452,9 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
return nil, request
|
||||
}
|
||||
}
|
||||
finishHydrate := store.ObserveReadPhase(request.Context(), "hydrate")
|
||||
items, err = server.hydrateHumanPoolPage(request, items, pool, query, limit-len(filtered), batchByID)
|
||||
finishHydrate()
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return nil, request
|
||||
|
||||
@ -144,6 +144,8 @@ func (s *server) handleHumanPoolSummaries(w http.ResponseWriter, r *http.Request
|
||||
if page == nil {
|
||||
return
|
||||
}
|
||||
finishResponse := store.ObserveReadPhase(r.Context(), "response")
|
||||
defer finishResponse()
|
||||
summaries := make([]humanPoolSummary, 0, len(page.Items))
|
||||
for _, item := range page.Items {
|
||||
summaries = append(summaries, summarizeHumanPoolItem(item))
|
||||
|
||||
@ -6,6 +6,7 @@ import "github.com/go-chi/chi/v5"
|
||||
// plugin's templates and tasks (plus their deprecated /api/annotation-*
|
||||
// aliases), and the annotation label dictionaries.
|
||||
func (server *server) mountAnnotationRoutes(router chi.Router) {
|
||||
router.Get("/api/plugins/annotation-qa/source-readiness", server.handleSourceAdmissionReadiness)
|
||||
router.Get("/api/data-collection-tasks", server.handleCollectionTasksList)
|
||||
router.Get("/api/data-collection-tasks/{id}", server.handleCollectionTaskGet)
|
||||
router.Get("/api/annotation-templates", deprecatedAnnotationEndpoint(server.handleAnnotationTemplatesList))
|
||||
|
||||
@ -24,6 +24,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
@ -49,6 +50,7 @@ type validationIssue struct {
|
||||
}
|
||||
|
||||
type server struct {
|
||||
sourceReadinessRunning atomic.Bool
|
||||
outputOnce sync.Once
|
||||
outputReader *outputsnapshot.Reader
|
||||
humanEvidenceIndexes humanEvidenceCache
|
||||
@ -1437,11 +1439,16 @@ func invalidConfigurationHandler(err error) http.Handler {
|
||||
}
|
||||
|
||||
func (server *server) serviceMetadata() map[string]any {
|
||||
mode := "strict"
|
||||
if store.AnnotationSourceAuditAdvisory() {
|
||||
mode = "advisory"
|
||||
}
|
||||
return map[string]any{
|
||||
"service": "droboticflow-api",
|
||||
"buildCommit": server.buildCommit,
|
||||
"buildTime": server.buildTime,
|
||||
"image": server.image,
|
||||
"sourceAdmissionMode": mode,
|
||||
"service": "droboticflow-api",
|
||||
"buildCommit": server.buildCommit,
|
||||
"buildTime": server.buildTime,
|
||||
"image": server.image,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
45
internal/api/source_readiness.go
Normal file
45
internal/api/source_readiness.go
Normal file
@ -0,0 +1,45 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func (s *server) handleSourceAdmissionReadiness(w http.ResponseWriter, r *http.Request) {
|
||||
principal, allowed := s.productionAdmin(w, r)
|
||||
if !allowed {
|
||||
return
|
||||
}
|
||||
db, ok := s.store.(store.SourceReadinessStore)
|
||||
if !ok {
|
||||
renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source readiness reporting unavailable"})
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
org, project := s.requestOrganizationID(r), r.URL.Query().Get("projectId")
|
||||
if r.URL.Query().Get("scope") == "deployment" {
|
||||
if !principal.User.IsSuperAdmin || project != "" {
|
||||
renderJSON(w, http.StatusForbidden, map[string]string{"error": "deployment readiness requires super administrator and no project filter"})
|
||||
return
|
||||
}
|
||||
org = ""
|
||||
}
|
||||
if !s.sourceReadinessRunning.CompareAndSwap(false, true) {
|
||||
renderJSON(w, http.StatusTooManyRequests, map[string]string{"error": "source readiness report already running"})
|
||||
return
|
||||
}
|
||||
defer s.sourceReadinessRunning.Store(false)
|
||||
report, err := db.SourceAdmissionReadiness(ctx, org, project)
|
||||
if err != nil {
|
||||
renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source readiness report incomplete", "requestId": store.ReadRequestID(r.Context())})
|
||||
return
|
||||
}
|
||||
report.Namespace = s.k8sNamespace
|
||||
report.BuildCommit = s.buildCommit
|
||||
renderJSON(w, http.StatusOK, report)
|
||||
}
|
||||
68
internal/api/source_readiness_test.go
Normal file
68
internal/api/source_readiness_test.go
Normal file
@ -0,0 +1,68 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
|
||||
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "")
|
||||
f := newCombinedDraftFixture(t)
|
||||
missingTask := f.task
|
||||
missingTask.ID += "-missing-proof"
|
||||
if err := f.store.UpsertAnnotationTask(f.ctx, missingTask); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
missingItem := f.item
|
||||
missingItem.ID += "-missing-proof"
|
||||
missingItem.LegacyTaskID = missingTask.ID
|
||||
missingItem.ProcessingJob = nil
|
||||
if err := f.store.UpsertHumanItem(f.ctx, missingItem); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
var jobs store.AnnotationProcessingStore = f.store
|
||||
jobBefore, _ := jobs.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
path := "/api/plugins/annotation-qa/source-readiness"
|
||||
denied := f.request(http.MethodGet, path, f.annotator, nil)
|
||||
if denied.Code != 403 {
|
||||
t.Fatalf("non-admin readiness access: %d", denied.Code)
|
||||
}
|
||||
denied = f.request(http.MethodGet, path+"?scope=deployment", f.admin, nil)
|
||||
if denied.Code != 403 {
|
||||
t.Fatal("ordinary org admin got deployment-wide inventory")
|
||||
}
|
||||
r := f.request(http.MethodGet, path+"?organizationId=other", f.admin, nil)
|
||||
if r.Code != 200 {
|
||||
t.Fatalf("readiness: %d %s", r.Code, r.Body.String())
|
||||
}
|
||||
var report store.SourceReadiness
|
||||
if err := json.Unmarshal(r.Body.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !report.Complete || report.TargetMode != "strict" || report.Scope != "organization" || report.OrganizationID != f.task.OrganizationID || len(report.Rows) != 1 || report.Rows[0].Unverified != 1 {
|
||||
t.Fatalf("incorrect impact report: %+v", report)
|
||||
}
|
||||
super := f.admin
|
||||
super.IsSuperAdmin = true
|
||||
r = f.request(http.MethodGet, path+"?scope=deployment", super, nil)
|
||||
if r.Code != 200 {
|
||||
t.Fatalf("superadmin inventory: %d", r.Code)
|
||||
}
|
||||
if err := json.Unmarshal(r.Body.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.Scope != "deployment" || report.OrganizationID != "" {
|
||||
t.Fatal("wrong full deployment scope")
|
||||
}
|
||||
after, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
jobAfter, _ := jobs.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if !reflect.DeepEqual(before, after) || !reflect.DeepEqual(jobBefore, jobAfter) {
|
||||
t.Fatal("inventory changed task or audit queue")
|
||||
}
|
||||
}
|
||||
3
internal/api/testdata/routes.golden
vendored
3
internal/api/testdata/routes.golden
vendored
@ -1,7 +1,7 @@
|
||||
# Generated by TestRouteSnapshot; see routes_snapshot_test.go.
|
||||
# access: reach = endpoint reached; otherwise the status the middleware chain returned.
|
||||
global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache]
|
||||
routes=381
|
||||
routes=382
|
||||
DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
|
||||
DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
|
||||
DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
|
||||
@ -121,6 +121,7 @@ GET /api/plugins/annotation-qa/admin-pool | mw=[] | (*server).handleAdminAnnotat
|
||||
GET /api/plugins/annotation-qa/claim-count | mw=[] | (*server).handleAnnotationClaimCount-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/plugins/annotation-qa/claim-projects | mw=[] | (*server).handleAnnotationClaimProjects-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/plugins/annotation-qa/sidebar | mw=[] | (*server).handleAnnotationSidebar-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/plugins/annotation-qa/source-readiness | mw=[] | (*server).handleSourceAdmissionReadiness-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/plugins/annotation-qa/tasks | mw=[] | (*server).handleAnnotationTasksList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/plugins/annotation-qa/tasks/{id} | mw=[] | (*server).handleAnnotationTaskGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
GET /api/plugins/annotation-qa/tasks/{id}/frame-timeline | mw=[] | (*server).handleAnnotationTaskFrameTimelineGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
|
||||
|
||||
@ -61,6 +61,8 @@ func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, op
|
||||
// for callers without this proof), preserve uncertain-write lease semantics.
|
||||
func (s *MongoStore) withAccountMutationLockBeforeOperation(ctx context.Context, key string, operation func(context.Context) (any, error), operationStarted *bool) (result any, err error) {
|
||||
startedAt := time.Now()
|
||||
finishWait := ObserveReadPhase(ctx, "lock_acquire")
|
||||
defer finishWait()
|
||||
diagnose := func(event string) {
|
||||
digest := sha256.Sum256([]byte(key))
|
||||
log.Printf("[account-mutation-lock] event=%s resource=%s key_hash=%x elapsed_ms=%d", event, strings.SplitN(key, ":", 2)[0], digest[:6], time.Since(startedAt).Milliseconds())
|
||||
@ -111,6 +113,7 @@ func (s *MongoStore) withAccountMutationLockBeforeOperation(ctx context.Context,
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
finishWait()
|
||||
// Renew well inside the lease window (a fixed fraction of it) so normal
|
||||
// scheduling jitter never lets a live owner's lease lapse; only an
|
||||
// owner that stops renewing entirely — because it's gone — loses the
|
||||
|
||||
@ -124,6 +124,10 @@ func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain
|
||||
if AnnotationSourceAuditAdvisory() {
|
||||
return AnnotationSourceTaskMatches(task, item) && !AnnotationSourceUnavailable(item)
|
||||
}
|
||||
return annotationItemSourceJobsReadyStrict(task, item, timeline, source)
|
||||
}
|
||||
|
||||
func annotationItemSourceJobsReadyStrict(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool {
|
||||
if LegacyAnnotationSourceCompatible(task, item, timeline, source) {
|
||||
return true
|
||||
}
|
||||
@ -193,13 +197,17 @@ func AnnotationSourceAdmissionStatus(task domain.AnnotationTaskRecord, item doma
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return annotationSourceAdmissionStatusStrict(task, item, timeline, source)
|
||||
}
|
||||
|
||||
func annotationSourceAdmissionStatusStrict(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) string {
|
||||
if task.ID == "" {
|
||||
return "unverified"
|
||||
}
|
||||
if AnnotationSourceJobSuperseded(item, timeline) {
|
||||
source = nil
|
||||
}
|
||||
if AnnotationItemSourceJobsReady(task, item, timeline, source) {
|
||||
if annotationItemSourceJobsReadyStrict(task, item, timeline, source) {
|
||||
return ""
|
||||
}
|
||||
for _, job := range []*domain.AnnotationIndexJob{timeline, source} {
|
||||
@ -231,7 +239,11 @@ func AnnotationSourceAdmissionStatus(task domain.AnnotationTaskRecord, item doma
|
||||
// All joins use existing unique id indexes and project only admission fields.
|
||||
// Retain caller fields so administrative exception pages can display blocked rows.
|
||||
func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
|
||||
if AnnotationSourceAuditAdvisory() {
|
||||
return annotationSourceAdmissionStatusPipelineForMode(AnnotationSourceAuditAdvisory())
|
||||
}
|
||||
|
||||
func annotationSourceAdmissionStatusPipelineForMode(advisory bool) mongo.Pipeline {
|
||||
if advisory {
|
||||
blocked := bson.M{"$and": bson.A{
|
||||
bson.M{"$in": bson.A{"$currentStageId", bson.A{"annotation_combined", "semantic_annotation", "split_annotation"}}},
|
||||
bson.M{"$eq": bson.A{"$processingJob.status", "failed"}},
|
||||
|
||||
100
internal/store/read_observation.go
Normal file
100
internal/store/read_observation.go
Normal file
@ -0,0 +1,100 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
)
|
||||
|
||||
type readObservationKey struct{}
|
||||
|
||||
// ReadObservation contains only durations and counters, never commands, replies,
|
||||
// resource identifiers or credentials. It lives for one request, not in a cache.
|
||||
type ReadObservation struct {
|
||||
mu sync.Mutex
|
||||
id string
|
||||
phases map[string]time.Duration
|
||||
commands int
|
||||
failures int
|
||||
mongo time.Duration
|
||||
errorKinds map[string]int
|
||||
}
|
||||
|
||||
func WithReadObservation(ctx context.Context, id string) (context.Context, *ReadObservation) {
|
||||
o := &ReadObservation{id: id, phases: map[string]time.Duration{}, errorKinds: map[string]int{}}
|
||||
return context.WithValue(ctx, readObservationKey{}, o), o
|
||||
}
|
||||
|
||||
func ReadRequestID(ctx context.Context) string {
|
||||
if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok {
|
||||
return o.id
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func ObserveReadPhase(ctx context.Context, name string) func() {
|
||||
o, ok := ctx.Value(readObservationKey{}).(*ReadObservation)
|
||||
if !ok {
|
||||
return func() {}
|
||||
}
|
||||
// Fixed vocabulary prevents arbitrary customer values becoming log keys.
|
||||
switch name {
|
||||
case "prepare", "candidate_query", "list_context", "hydrate", "response", "lock_acquire":
|
||||
default:
|
||||
return func() {}
|
||||
}
|
||||
start := time.Now()
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { o.mu.Lock(); defer o.mu.Unlock(); o.phases[name] += time.Since(start) }) }
|
||||
}
|
||||
|
||||
func (o *ReadObservation) Snapshot() map[string]any {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
phases := map[string]int64{}
|
||||
for k, v := range o.phases {
|
||||
phases[k] = v.Milliseconds()
|
||||
}
|
||||
kinds := map[string]int{}
|
||||
for k, v := range o.errorKinds {
|
||||
kinds[k] = v
|
||||
}
|
||||
return map[string]any{"phases_ms": phases, "mongo_commands": o.commands, "mongo_failures": o.failures, "mongo_ms": o.mongo.Milliseconds(), "mongo_error_kinds": kinds}
|
||||
}
|
||||
|
||||
// Command duration includes driver/network time, not just server execution.
|
||||
// It is not connection checkout wait or docs/keys examined.
|
||||
// Those require independent evidence; never infer them from elapsed time.
|
||||
func ReadCommandMonitor() *event.CommandMonitor {
|
||||
record := func(ctx context.Context, duration int64, failed bool) {
|
||||
if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
o.commands++
|
||||
o.mongo += time.Duration(duration)
|
||||
if failed {
|
||||
o.failures++
|
||||
}
|
||||
}
|
||||
}
|
||||
return &event.CommandMonitor{
|
||||
Succeeded: func(ctx context.Context, e *event.CommandSucceededEvent) { record(ctx, e.DurationNanos, false) },
|
||||
Failed: func(ctx context.Context, e *event.CommandFailedEvent) {
|
||||
record(ctx, e.DurationNanos, true)
|
||||
if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok {
|
||||
kind := "driver_error"
|
||||
if strings.Contains(e.Failure, "MaxTimeMSExpired") {
|
||||
kind = "server_deadline"
|
||||
} else if strings.Contains(e.Failure, "deadline exceeded") || strings.Contains(e.Failure, "i/o timeout") {
|
||||
kind = "client_or_network_deadline"
|
||||
}
|
||||
o.mu.Lock()
|
||||
o.errorKinds[kind]++
|
||||
o.mu.Unlock()
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
67
internal/store/read_observation_test.go
Normal file
67
internal/store/read_observation_test.go
Normal file
@ -0,0 +1,67 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
)
|
||||
|
||||
func TestReadObservationIsPrivateConcurrentAndRequestScoped(t *testing.T) {
|
||||
ctx, o := WithReadObservation(context.Background(), "server-generated")
|
||||
monitor := ReadCommandMonitor()
|
||||
var wg sync.WaitGroup
|
||||
for range 20 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
end := ObserveReadPhase(ctx, "candidate_query")
|
||||
defer end()
|
||||
monitor.Succeeded(ctx, &event.CommandSucceededEvent{CommandFinishedEvent: event.CommandFinishedEvent{DurationNanos: int64(time.Millisecond), CommandName: "find"}})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
monitor.Failed(ctx, &event.CommandFailedEvent{CommandFinishedEvent: event.CommandFinishedEvent{DurationNanos: int64(time.Millisecond)}, Failure: "mongodb://password-secret@host private-customer-body"})
|
||||
ObserveReadPhase(ctx, "customer-should-not-be-key")()
|
||||
b, _ := json.Marshal(o.Snapshot())
|
||||
if strings.Contains(string(b), "secret") || strings.Contains(string(b), "customer") || strings.Contains(string(b), "find") {
|
||||
t.Fatal("command contents leaked")
|
||||
}
|
||||
if o.Snapshot()["mongo_commands"] != 21 || o.Snapshot()["mongo_failures"] != 1 {
|
||||
t.Fatal("counter mismatch")
|
||||
}
|
||||
if ReadRequestID(ctx) != "server-generated" || ReadRequestID(context.Background()) != "" {
|
||||
t.Fatal("request context leaked")
|
||||
}
|
||||
_, other := WithReadObservation(context.Background(), "other")
|
||||
if other.Snapshot()["mongo_commands"] != 0 {
|
||||
t.Fatal("cross request counters")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadObservationConcurrentFailureSnapshots(t *testing.T) {
|
||||
ctx, o := WithReadObservation(context.Background(), "generated")
|
||||
monitor := ReadCommandMonitor()
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for range 200 {
|
||||
monitor.Failed(ctx, &event.CommandFailedEvent{Failure: "context deadline exceeded"})
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for range 200 {
|
||||
_, _ = json.Marshal(o.Snapshot())
|
||||
}
|
||||
}()
|
||||
wg.Wait()
|
||||
if o.Snapshot()["mongo_failures"] != 200 {
|
||||
t.Fatal("missing failures")
|
||||
}
|
||||
}
|
||||
144
internal/store/source_readiness.go
Normal file
144
internal/store/source_readiness.go
Normal file
@ -0,0 +1,144 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
type SourceReadinessRow struct {
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
Stage string `json:"stage" bson:"stage"`
|
||||
Status string `json:"status" bson:"status"`
|
||||
Total int64 `json:"total" bson:"total"`
|
||||
Allowed int64 `json:"allowed" bson:"allowed"`
|
||||
Unverified int64 `json:"unverified" bson:"unverified"`
|
||||
Pending int64 `json:"pending" bson:"pending"`
|
||||
Failed int64 `json:"failed" bson:"failed"`
|
||||
}
|
||||
|
||||
type SourceReadiness struct {
|
||||
CurrentMode string `json:"currentMode"`
|
||||
Scope string `json:"scope"`
|
||||
Database string `json:"database"`
|
||||
Schema string `json:"schema"`
|
||||
OrganizationID string `json:"organizationId"`
|
||||
ProjectID string `json:"projectId"`
|
||||
Namespace string `json:"namespace"`
|
||||
BuildCommit string `json:"buildCommit"`
|
||||
GeneratedAt string `json:"generatedAt"`
|
||||
TargetMode string `json:"targetMode"`
|
||||
LegacySourceCutoff string `json:"legacySourceCutoff"`
|
||||
Complete bool `json:"complete"`
|
||||
Rows []SourceReadinessRow `json:"rows"`
|
||||
}
|
||||
|
||||
type SourceReadinessStore interface {
|
||||
SourceAdmissionReadiness(context.Context, string, string) (*SourceReadiness, error)
|
||||
}
|
||||
|
||||
func sourceReadiness(org, project string, rows []SourceReadinessRow) *SourceReadiness {
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
if rows[i].OrganizationID != rows[j].OrganizationID {
|
||||
return rows[i].OrganizationID < rows[j].OrganizationID
|
||||
}
|
||||
if rows[i].Stage != rows[j].Stage {
|
||||
return rows[i].Stage < rows[j].Stage
|
||||
}
|
||||
return rows[i].Status < rows[j].Status
|
||||
})
|
||||
mode := "strict"
|
||||
if AnnotationSourceAuditAdvisory() {
|
||||
mode = "advisory"
|
||||
}
|
||||
scope := "organization"
|
||||
if org == "" {
|
||||
scope = "deployment"
|
||||
}
|
||||
return &SourceReadiness{CurrentMode: mode, Scope: scope, Database: "memory", Schema: "droboticflow.source-readiness/v1", OrganizationID: org, ProjectID: project, GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), TargetMode: "strict", LegacySourceCutoff: os.Getenv("ANNOTATION_LEGACY_SOURCE_CUTOFF"), Complete: true, Rows: rows}
|
||||
}
|
||||
|
||||
// This is an explicit administrator inventory, never part of normal work reads.
|
||||
// Missing proof is reported, not repaired or enqueued. No task body is returned.
|
||||
func (s *MemoryStore) SourceAdmissionReadiness(ctx context.Context, org, project string) (*SourceReadiness, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
grouped := map[string]*SourceReadinessRow{}
|
||||
for _, item := range s.humanItems {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if org != "" && item.OrganizationID != org || item.PluginID != "annotation-qa" || item.LegacyTaskID == "" || project != "" && item.ProjectID != project {
|
||||
continue
|
||||
}
|
||||
key := item.OrganizationID + "\x00" + item.CurrentStageID + "\x00" + string(item.Status)
|
||||
row := grouped[key]
|
||||
if row == nil {
|
||||
row = &SourceReadinessRow{OrganizationID: item.OrganizationID, Stage: item.CurrentStageID, Status: string(item.Status)}
|
||||
grouped[key] = row
|
||||
}
|
||||
row.Total++
|
||||
task := s.annotationTasks[item.LegacyTaskID]
|
||||
t, tok := s.annotationIndexJobs[item.LegacyTaskID]
|
||||
j, jok := s.annotationIndexJobs[sourceJobForItem(item)]
|
||||
var tp, sp *domain.AnnotationIndexJob
|
||||
if tok {
|
||||
tp = &t
|
||||
}
|
||||
if jok {
|
||||
sp = &j
|
||||
}
|
||||
switch annotationSourceAdmissionStatusStrict(task, item, tp, sp) {
|
||||
case "":
|
||||
row.Allowed++
|
||||
case "pending":
|
||||
row.Pending++
|
||||
case "failed":
|
||||
row.Failed++
|
||||
default:
|
||||
row.Unverified++
|
||||
}
|
||||
}
|
||||
rows := []SourceReadinessRow{}
|
||||
for _, v := range grouped {
|
||||
rows = append(rows, *v)
|
||||
}
|
||||
return sourceReadiness(org, project, rows), nil
|
||||
}
|
||||
|
||||
func (s *MongoStore) SourceAdmissionReadiness(ctx context.Context, org, project string) (*SourceReadiness, error) {
|
||||
filter := bson.M{"pluginId": "annotation-qa", "legacyTaskId": bson.M{"$type": "string", "$ne": ""}}
|
||||
if org != "" {
|
||||
filter["organizationId"] = org
|
||||
}
|
||||
if project != "" {
|
||||
filter["projectId"] = project
|
||||
}
|
||||
pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: annotationSourceAdmissionProjection()}}}
|
||||
pipeline = append(pipeline, annotationSourceAdmissionStatusPipelineForMode(false)...)
|
||||
count := func(status string) bson.M {
|
||||
return bson.M{"$sum": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{"$_sourceAdmissionStatus", status}}, 1, 0}}}
|
||||
}
|
||||
pipeline = append(pipeline, bson.D{{Key: "$group", Value: bson.M{"_id": bson.M{"organizationId": "$organizationId", "stage": "$currentStageId", "status": "$status"}, "total": bson.M{"$sum": 1}, "allowed": count(""), "pending": count("pending"), "failed": count("failed"), "unverified": count("unverified")}}}, bson.D{{Key: "$project", Value: bson.M{"_id": 0, "organizationId": "$_id.organizationId", "stage": "$_id.stage", "status": "$_id.status", "total": 1, "allowed": 1, "pending": 1, "failed": 1, "unverified": 1}}})
|
||||
cursor, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetMaxTime(8*time.Second))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
rows := []SourceReadinessRow{}
|
||||
if err = cursor.All(ctx, &rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
report := sourceReadiness(org, project, rows)
|
||||
report.Database = s.db.Name()
|
||||
return report, nil
|
||||
}
|
||||
101
internal/store/source_readiness_test.go
Normal file
101
internal/store/source_readiness_test.go
Normal file
@ -0,0 +1,101 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func TestSourceReadinessMongoParityReadOnly(t *testing.T) {
|
||||
if os.Getenv("SOURCE_ADMISSION_LOCAL_MONGO") != "1" {
|
||||
t.Skip("isolated loopback Mongo opt-in")
|
||||
}
|
||||
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
|
||||
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "")
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Disconnect(context.Background())
|
||||
db := c.Database(fmt.Sprintf("source_readiness_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
ms := NewMongoStore(db)
|
||||
mem := NewMemoryStore()
|
||||
mem.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
|
||||
must := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before := map[string]*domain.HumanItemRecord{}
|
||||
for n, state := range []string{"", "queued", "failed", "completed", "foreign"} {
|
||||
id := fmt.Sprintf("i%d", n)
|
||||
org := "o"
|
||||
if state == "foreign" {
|
||||
org = "foreign"
|
||||
}
|
||||
task := domain.AnnotationTaskRecord{ID: "t" + id, OrganizationID: org, ProjectID: "p", RunID: id}
|
||||
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", RunID: id, PluginID: "annotation-qa", LegacyTaskID: task.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemClaimed, AssignedUserID: "owner", Version: 1, Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": map[string]any{"description": "retained"}}}}
|
||||
must(ms.UpsertAnnotationTask(ctx, task))
|
||||
must(mem.UpsertAnnotationTask(ctx, task))
|
||||
must(ms.UpsertHumanItem(ctx, item))
|
||||
must(mem.UpsertHumanItem(ctx, item))
|
||||
if state != "" && state != "foreign" {
|
||||
job := domain.AnnotationIndexJob{ID: task.ID, OrganizationID: org, ProjectID: "p", RunID: id, Status: state}
|
||||
if state == "completed" {
|
||||
job.SourceFingerprint = "fp"
|
||||
job.RevalidateSource = true
|
||||
}
|
||||
_, err = db.Collection("annotationIndexJobs").InsertOne(ctx, job)
|
||||
must(err)
|
||||
mem.annotationIndexJobs[task.ID] = job
|
||||
}
|
||||
before[id], err = ms.GetHumanItem(ctx, id)
|
||||
must(err)
|
||||
}
|
||||
for _, org := range []string{"o", ""} {
|
||||
a, err := ms.SourceAdmissionReadiness(ctx, org, "")
|
||||
must(err)
|
||||
b, err := mem.SourceAdmissionReadiness(ctx, org, "")
|
||||
must(err)
|
||||
a.GeneratedAt = ""
|
||||
b.GeneratedAt = ""
|
||||
a.Database = ""
|
||||
b.Database = ""
|
||||
if !reflect.DeepEqual(a, b) {
|
||||
t.Fatalf("Mongo/Go strict readiness mismatch: %+v / %+v", a, b)
|
||||
}
|
||||
total := int64(0)
|
||||
for _, r := range a.Rows {
|
||||
total += r.Total
|
||||
if r.Total != r.Allowed+r.Pending+r.Unverified+r.Failed {
|
||||
t.Fatal("counts do not reconcile")
|
||||
}
|
||||
}
|
||||
if org == "o" && total != 4 || org == "" && total != 5 {
|
||||
t.Fatal("readiness scope mismatch")
|
||||
}
|
||||
}
|
||||
for id, before := range before {
|
||||
after, err := ms.GetHumanItem(ctx, id)
|
||||
must(err)
|
||||
if !reflect.DeepEqual(before, after) {
|
||||
t.Fatal("inventory changed saved annotations or ownership")
|
||||
}
|
||||
}
|
||||
canceled, stop := context.WithCancel(ctx)
|
||||
stop()
|
||||
if r, err := ms.SourceAdmissionReadiness(canceled, "", ""); err == nil || r != nil {
|
||||
t.Fatal("canceled report claimed completeness")
|
||||
}
|
||||
}
|
||||
@ -104,7 +104,8 @@
|
||||
"predev:api": "npm run hooks:install",
|
||||
"predev:web": "npm run hooks:install",
|
||||
"predev:worker": "npm run hooks:install",
|
||||
"predev:api:replica": "npm run hooks:install"
|
||||
"predev:api:replica": "npm run hooks:install",
|
||||
"test:work-availability": "python3 scripts/test_work_availability.py"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tailwindcss/vite": "^4.1.14",
|
||||
|
||||
105
scripts/lib/release_checks.py
Normal file
105
scripts/lib/release_checks.py
Normal file
@ -0,0 +1,105 @@
|
||||
"""Private, bounded local-check evidence. Never log kubectl payloads here."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
SENSITIVE = '[A-Za-z0-9_-]{0,96}(?:password|passwd|secret|token|authorization|cookie|credential|access[_-]?key|secret[_-]?key|phone)[A-Za-z0-9_-]{0,96}|(?:BOS|BCE|AWS)[_-](?:AK|SK)|AK|SK'
|
||||
|
||||
def redact(text):
|
||||
"""Remove credential-bearing fields and URLs before persisting bounded local test output."""
|
||||
text = '\n'.join(line if len(line) <= 65536 else '<oversized-line-redacted>' for line in text.splitlines())
|
||||
text = re.sub('(?is)(?<![A-Za-z0-9_-])([\\"\\\']?(?:' + SENSITIVE + ')[\\"\\\']?\\s*[:=]\\s*)(\\"[^\\"]*\\"|\\\'[^\\\']*\\\'|[^\\s,}]+)', '\\1<redacted>', text)
|
||||
text = re.sub('(?i)\\bBearer\\s+\\S+', 'Bearer <redacted>', text)
|
||||
text = re.sub('(?i)\\b(?:https?|mongodb(?:\\+srv)?)://[^\\s\\"\\\'<>]+', '<redacted-url>', text)
|
||||
lines = ['<redacted-sensitive-line>' if re.search('(?i)\\b(?:' + SENSITIVE + ')\\b\\s*:', line) else line for line in text.splitlines()]
|
||||
text = '\n'.join(lines)
|
||||
limit = 2 * 1024 * 1024
|
||||
if len(text) > limit:
|
||||
text = text[:limit // 2] + '\n<output-truncated>\n' + text[-limit // 2:]
|
||||
return text
|
||||
|
||||
def write_log(folder, argv, output, result):
|
||||
folder = Path(folder)
|
||||
folder.mkdir(parents=True, exist_ok=True, mode=448)
|
||||
path = folder / ('check-' + uuid.uuid4().hex + '.log')
|
||||
data = redact(json.dumps({'argv': argv, 'result': result}) + '\n' + output) + '\n'
|
||||
with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 384), 'w') as f:
|
||||
f.write(data)
|
||||
return {'log': str(path.resolve()), 'logSHA256': hashlib.sha256(path.read_bytes()).hexdigest()}
|
||||
|
||||
def check_environment():
|
||||
"""Local synthetic checks inherit no company credentials or application/test switches."""
|
||||
return {k: v for k, v in os.environ.items() if not re.search('(?i)(password|secret|token|credential|access_key|secret_key|^BOS_|^BCE_|^AWS_|(^|_)MONGO(_|$)|(_URI|_URL)$|^KUBECONFIG$|^ANNOTATION_|^DROBOTICFLOW_|^DEPLOY_ENV$|^UPDATE_|^REGENERATE_|_TEST$|_TEST_|^TEST_.*MONGO)', k)}
|
||||
|
||||
def source_fingerprint(root):
|
||||
"""Hash the complete non-ignored source tree; unavailable Git identity is never reusable."""
|
||||
root = Path(root)
|
||||
h = hashlib.sha256()
|
||||
try:
|
||||
names = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=root, stderr=subprocess.DEVNULL).decode().split('\x00')
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
for name in sorted(set(filter(None, names))):
|
||||
path = root / name
|
||||
h.update(name.encode())
|
||||
if path.is_symlink():
|
||||
h.update(os.readlink(path).encode())
|
||||
elif path.is_file():
|
||||
h.update(path.read_bytes())
|
||||
else:
|
||||
h.update(b'<missing>')
|
||||
return h.hexdigest()
|
||||
|
||||
def identity(root, commit, policy_hash):
|
||||
"""Bind source, installed dependency contents, tool versions and a hash of the sanitized environment. Values and dependency contents are not persisted."""
|
||||
root = Path(root)
|
||||
h = hashlib.sha256()
|
||||
dependencies = root / 'node_modules'
|
||||
if dependencies.is_dir():
|
||||
visited = set()
|
||||
for base, dirs, files in os.walk(dependencies, followlinks=True):
|
||||
stat = Path(base).stat()
|
||||
key = (stat.st_dev, stat.st_ino)
|
||||
if key in visited:
|
||||
dirs[:] = []
|
||||
continue
|
||||
visited.add(key)
|
||||
h.update(str(Path(base).relative_to(dependencies)).encode())
|
||||
if Path(base).is_symlink():
|
||||
h.update(os.readlink(base).encode())
|
||||
dirs[:] = sorted((d for d in dirs if d not in ('.cache', '.vite', '.vite-temp')))
|
||||
for name in sorted(files):
|
||||
p = Path(base) / name
|
||||
h.update(str(p.relative_to(dependencies)).encode())
|
||||
if p.is_symlink():
|
||||
h.update(os.readlink(p).encode())
|
||||
else:
|
||||
with p.open('rb') as f:
|
||||
for block in iter(lambda: f.read(1024 * 1024), b''):
|
||||
h.update(block)
|
||||
else:
|
||||
h.update(b'no-installed-node-dependencies')
|
||||
versions = {}
|
||||
for name, args in [('python', ['python3', '--version']), ('node', ['node', '--version']), ('go', ['go', 'version']), ('ffmpeg', ['ffmpeg', '-version'])]:
|
||||
try:
|
||||
versions[name] = subprocess.check_output(args, cwd=root, stderr=subprocess.STDOUT, timeout=10).decode().splitlines()[0]
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
versions[name] = 'unavailable'
|
||||
env_hash = hashlib.sha256(json.dumps(check_environment(), sort_keys=True).encode()).hexdigest()
|
||||
source = source_fingerprint(root)
|
||||
complete = bool(source) and dependencies.is_dir() and all((v != 'unavailable' for v in versions.values()))
|
||||
return {'complete': complete, 'sourceTree': source, 'commit': commit, 'policyHash': policy_hash, 'dependencies': h.hexdigest(), 'tools': versions, 'environment': env_hash}
|
||||
|
||||
def reusable(entry, expected, argv):
|
||||
"""Only explicitly listed pure checks may reuse unchanged, unexpired and intact success evidence. Browser, Mongo and timing checks always rerun."""
|
||||
pure = argv[:3] == ['npm', 'run', 'lint:web'] or argv[:3] == ['npm', 'run', 'test:annotation:persistence'] or argv[:3] == ['npm', 'run', 'test:review:decisions'] or (argv[:3] == ['npm', 'run', 'test:review:drafts']) or (argv[:3] == ['python3', '-m', 'unittest'])
|
||||
if not expected.get('complete') or not pure or entry.get('identity') != expected or (entry.get('result') != 'passed'):
|
||||
return False
|
||||
if not 0 <= time.time() - entry.get('finishedUnix', 0) <= 3600:
|
||||
return False
|
||||
path = Path(entry.get('log', ''))
|
||||
return path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == entry.get('logSHA256')
|
||||
53
scripts/lib/source_readiness.py
Normal file
53
scripts/lib/source_readiness.py
Normal file
@ -0,0 +1,53 @@
|
||||
"""Validate an administrator's metadata-only strict-admission impact report."""
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
def validate_report(path, *, namespace, database, cutoff, live_commit, now=None):
|
||||
path = Path(path)
|
||||
if path.stat().st_size > 8 * 1024 * 1024:
|
||||
raise ValueError('Readiness report too large.')
|
||||
body = path.read_bytes()
|
||||
r = json.loads(body)
|
||||
if not isinstance(r, dict):
|
||||
raise ValueError('Invalid readiness object.')
|
||||
if not isinstance(live_commit, str) or not re.fullmatch('[0-9a-f]{40}', live_commit):
|
||||
raise ValueError('Unknown live source commit.')
|
||||
if r.get('scope') != 'deployment' or r.get('organizationId') != '' or r.get('database') != database:
|
||||
raise ValueError('Strict activation requires complete deployment-wide database scope.')
|
||||
if r.get('schema') != 'droboticflow.source-readiness/v1' or r.get('namespace') != namespace or r.get('targetMode') != 'strict' or (r.get('complete') is not True) or (r.get('projectId') != '') or (r.get('legacySourceCutoff', '') != (cutoff or '')) or (r.get('buildCommit') != live_commit):
|
||||
raise ValueError('Incomplete, scoped or mismatched strict readiness report.')
|
||||
if not isinstance(r.get('generatedAt'), str):
|
||||
raise ValueError('Invalid readiness timestamp.')
|
||||
at = dt.datetime.fromisoformat(r['generatedAt'].replace('Z', '+00:00'))
|
||||
if at.tzinfo is None:
|
||||
raise ValueError('Readiness timestamp must include timezone.')
|
||||
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
|
||||
if not 0 <= age <= 3600:
|
||||
raise ValueError('Readiness report is expired or in the future.')
|
||||
totals = {k: 0 for k in ('total', 'allowed', 'unverified', 'pending', 'failed')}
|
||||
rows = r['rows']
|
||||
if not isinstance(rows, list):
|
||||
raise ValueError('Invalid readiness rows.')
|
||||
seen = set()
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
raise ValueError('Invalid readiness row.')
|
||||
key = (row.get('organizationId'), row.get('stage'), row.get('status'))
|
||||
if not all((isinstance(x, str) for x in key)) or key in seen:
|
||||
raise ValueError('Duplicate or invalid readiness group.')
|
||||
seen.add(key)
|
||||
if not isinstance(row.get('stage'), str) or not isinstance(row.get('status'), str):
|
||||
raise ValueError('Missing readiness stage/status.')
|
||||
for k in totals:
|
||||
v = row.get(k)
|
||||
if type(v) is not int or v < 0:
|
||||
raise ValueError('Invalid readiness count.')
|
||||
totals[k] += v
|
||||
if row['total'] != sum((row[k] for k in ('allowed', 'unverified', 'pending', 'failed'))):
|
||||
raise ValueError('Readiness counts do not reconcile.')
|
||||
if totals['total'] != totals['allowed']:
|
||||
raise ValueError('Strict mode would block existing tasks; keep advisory and review the impact.')
|
||||
return {'path': str(path.resolve()), 'sha256': hashlib.sha256(body).hexdigest(), 'scope': 'deployment', 'generatedAt': r['generatedAt'], 'totals': totals}
|
||||
@ -27,8 +27,16 @@ commit="$(git -C "$ROOT_DIR" rev-parse HEAD)"
|
||||
build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
metadata="${2:-$(mktemp -t droboticflow-api-build.XXXXXX)}"
|
||||
|
||||
runtime_image="${API_RUNTIME_IMAGE:-$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["image"])' "$ROOT_DIR/docs/release/api-runtime.json")}"
|
||||
if [[ ! "$runtime_image" =~ ^[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]]; then
|
||||
echo "API_RUNTIME_IMAGE must be an immutable prepared runtime digest" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
"$DOCKER" buildx build \
|
||||
--platform "$PLATFORM" \
|
||||
--build-arg "API_RUNTIME_IMAGE=$runtime_image" \
|
||||
--build-arg "API_RUNTIME_PREPARED=1" \
|
||||
--build-arg "GOPROXY=${GOPROXY:-https://proxy.golang.org,direct}" \
|
||||
--build-arg "DROBOTICFLOW_BUILD_COMMIT=$commit" \
|
||||
--build-arg "DROBOTICFLOW_BUILD_TIME=$build_time" \
|
||||
@ -44,3 +52,9 @@ if [[ "$digest" != sha256:* ]]; then
|
||||
exit 1
|
||||
fi
|
||||
printf 'COMMIT=%s\nBUILD_TIME=%s\nMETADATA=%s\nIMAGE=%s@%s\n' "$commit" "$build_time" "$metadata" "$repository" "$digest"
|
||||
|
||||
python3 - "$metadata.provenance.json" "$ROOT_DIR/Dockerfile.api" "$commit" "$runtime_image" "$digest" "$build_time" <<'PYPROOF'
|
||||
import hashlib,json,pathlib,sys
|
||||
output,recipe,commit,runtime,digest,at=sys.argv[1:]
|
||||
pathlib.Path(output).write_text(json.dumps({"schemaVersion":1,"commit":commit,"runtimeImage":runtime,"imageDigest":digest,"buildTime":at,"recipeSHA256":hashlib.sha256(pathlib.Path(recipe).read_bytes()).hexdigest()},indent=2)+"\n")
|
||||
PYPROOF
|
||||
|
||||
@ -13,6 +13,11 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
import time
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from lib.release_checks import check_environment, identity, reusable, write_log
|
||||
from lib.source_readiness import validate_report
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RECORDS = ROOT / 'local-data/releases'
|
||||
@ -42,13 +47,18 @@ def digest(value):
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
||||
|
||||
|
||||
def run(argv, *, stdin=None, timeout=60):
|
||||
def run(argv, *, stdin=None, timeout=60, env=None):
|
||||
try:
|
||||
p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
raise Blocked('Command unavailable or timed out; outcome must be checked before retry.') from None
|
||||
# Do not echo kubectl output, URLs or credential-bearing errors.
|
||||
require(p.returncode == 0, 'Command failed: ' + argv[0] + ' (output withheld; no success recorded).')
|
||||
p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout, env=env)
|
||||
except (OSError, subprocess.TimeoutExpired) as e:
|
||||
error = Blocked('Command unavailable or timed out; outcome must be checked before retry.')
|
||||
error.check_output = str(getattr(e, 'stdout', '') or '') + str(getattr(e, 'stderr', '') or '')
|
||||
raise error from None
|
||||
# Only policy-local checks may persist the private redacted output.
|
||||
if p.returncode != 0:
|
||||
error = Blocked('Command failed: ' + argv[0] + ' (no success recorded).')
|
||||
error.check_output = p.stdout + '\n' + p.stderr
|
||||
raise error
|
||||
return p.stdout
|
||||
|
||||
|
||||
@ -175,7 +185,7 @@ def env_values(container, configs):
|
||||
return values
|
||||
|
||||
|
||||
def inspect_deployment(deployment, container_name, configs, p):
|
||||
def inspect_deployment(deployment, container_name, configs, p, *, ignore_source_mode=False):
|
||||
spec = deployment['spec']
|
||||
pod = spec['template']['spec']
|
||||
containers = pod['containers']
|
||||
@ -228,10 +238,10 @@ def inspect_deployment(deployment, container_name, configs, p):
|
||||
neutral = copy.deepcopy(spec)
|
||||
nc = neutral['template']['spec']['containers'][index]
|
||||
nc['image'] = '<release-image>'
|
||||
nc['env'] = [x for x in nc.get('env', []) if x['name'] != 'DROBOTICFLOW_BUILD_COMMIT']
|
||||
nc['env'] = [x for x in nc.get('env', []) if x['name'] not in (('DROBOTICFLOW_BUILD_COMMIT', 'ANNOTATION_SOURCE_VALIDATION_MODE') if ignore_source_mode else ('DROBOTICFLOW_BUILD_COMMIT',))]
|
||||
stamp = next((i for i, x in enumerate(c.get('env', [])) if x['name'] == 'DROBOTICFLOW_BUILD_COMMIT'), None)
|
||||
return {'uid': m['uid'], 'specHash': digest(spec), 'nonReleaseHash': digest(neutral),
|
||||
'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'],
|
||||
'sourceModeIndex': next((i for i,x in enumerate(c.get('env',[])) if x['name']=='ANNOTATION_SOURCE_VALIDATION_MODE'),None), 'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'],
|
||||
'resourceVersion': m['resourceVersion'], 'containerIndex': index,
|
||||
'ready': ready, 'env': {k: e.get(k) for k in SAFE_ENV}}
|
||||
|
||||
@ -251,7 +261,7 @@ def validate_legacy_source_cutoffs(envs):
|
||||
raise Blocked('Invalid legacy source admission cutoff.')
|
||||
|
||||
|
||||
def snapshot(r):
|
||||
def snapshot(r, *, allow_planned_transition=False):
|
||||
t, p = r['target'], policy()
|
||||
view = json.loads(kubectl(r, ['config', 'view', '--minify', '-o', 'json']))
|
||||
require(view['clusters'][0]['cluster']['server'] == t['server'], 'Kube API server mismatch.')
|
||||
@ -265,10 +275,17 @@ def snapshot(r):
|
||||
config_names.update(x['valueFrom']['configMapKeyRef']['name'] for x in c.get('env', [])
|
||||
if 'configMapKeyRef' in x.get('valueFrom', {}))
|
||||
configs = {n: get(r, 'configmap', n) for n in sorted(config_names)}
|
||||
deps = {n: inspect_deployment(d, p['deployments'][n], configs, p) for n, d in deployments.items()}
|
||||
deps = {n: inspect_deployment(d, p['deployments'][n], configs, p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in deployments.items()}
|
||||
envs = [d['env'] for d in deps.values()]
|
||||
require(len({e['ANNOTATION_WORKSPACE_CONNECTION_ID'] for e in envs}) == 1, 'API/processor storage connections differ.')
|
||||
validate_legacy_source_cutoffs(envs)
|
||||
if allow_planned_transition and r.get("sourceMode"):
|
||||
for e in envs: require(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in (r["sourceMode"], *(d["env"].get("ANNOTATION_SOURCE_VALIDATION_MODE") for d in r["baseline"]["deployments"].values())), "Unexpected source mode during rollout.")
|
||||
normalized = [dict(e, ANNOTATION_SOURCE_VALIDATION_MODE=r["sourceMode"]) for e in envs]
|
||||
validate_legacy_source_cutoffs(normalized)
|
||||
else:
|
||||
validate_legacy_source_cutoffs(envs)
|
||||
if r.get("sourceModeExplicit") and not r.get("sourceMode"):
|
||||
require(all(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in ("advisory", "strict") for e in envs), "New releases require an explicit source admission mode on both components.")
|
||||
pvc = get(r, 'pvc', p['workspacePVC'])
|
||||
require(pvc.get('status', {}).get('phase') == 'Bound', 'Workspace PVC is not Bound.')
|
||||
svc = get(r, 'service', 'droboticflow-api-public-eip')
|
||||
@ -287,7 +304,9 @@ def stable(s):
|
||||
|
||||
|
||||
def plan_data(r):
|
||||
return {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')}
|
||||
data = {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')}
|
||||
data.update({k:r[k] for k in ('sourceModeExplicit','sourceMode','sourceReadiness') if k in r})
|
||||
return data
|
||||
|
||||
|
||||
def manifest_platforms(image_ref):
|
||||
@ -317,11 +336,32 @@ def prepare(a):
|
||||
'target': {'kubeconfig': str(Path(a.kubeconfig).resolve()), 'context': a.context,
|
||||
'server': a.server.rstrip('/'), 'namespace': p['namespace'], 'namespaceUID': a.namespace_uid},
|
||||
'state': 'prepared', 'localChecks': [], 'baseline': None, 'planHash': None,
|
||||
'operations': [], 'evidence': {}, 'events': []}
|
||||
'operations': [], 'evidence': {}, 'events': [], 'sourceModeExplicit': True,
|
||||
'sourceMode': getattr(a, 'source_mode', None), 'sourceReadiness': None}
|
||||
event(r, 'prepared; no network access')
|
||||
print(json.dumps({'id': r['id'], 'state': r['state']}, ensure_ascii=False))
|
||||
|
||||
|
||||
def check_identity(r):
|
||||
return identity(ROOT, r['commit'], r['policyHash'])
|
||||
|
||||
|
||||
def run_local_check(r, argv, expected):
|
||||
entry = {'argv': argv, 'startedAt': now(), 'identity': expected}
|
||||
try:
|
||||
output = run(argv, timeout=1800, env=check_environment())
|
||||
except Blocked as e:
|
||||
entry.update(result='failed', finishedUnix=time.time())
|
||||
entry.update(write_log(record_path(r['id']).parent/'logs', argv, getattr(e, 'check_output', ''), 'failed'))
|
||||
r['localChecks'].append(entry)
|
||||
r['state'] = 'blocked'
|
||||
event(r, 'local check failed; private redacted log: ' + entry['log'])
|
||||
raise Blocked(str(e) + ' Log: ' + entry['log']) from None
|
||||
entry.update(result='passed', outputHash=digest(output), finishedUnix=time.time())
|
||||
entry.update(write_log(record_path(r['id']).parent/'logs', argv, output, 'passed'))
|
||||
return entry
|
||||
|
||||
|
||||
def check(r, a):
|
||||
fresh(r)
|
||||
require(not r['operations'], 'A deployment was attempted; check outcome with verify, never rewrite its plan.')
|
||||
@ -342,18 +382,18 @@ def check(r, a):
|
||||
r['state'] = 'blocked'
|
||||
event(r, 'local check failed')
|
||||
raise
|
||||
previous = {tuple(x['argv']): x for x in r['localChecks']}
|
||||
expected = check_identity(r)
|
||||
r['localChecks'] = []
|
||||
for argv in policy()['localChecks']:
|
||||
started = now()
|
||||
try:
|
||||
output = run(argv, timeout=1800)
|
||||
except Blocked:
|
||||
r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'failed'})
|
||||
r['state'] = 'blocked'
|
||||
event(r, 'local check failed')
|
||||
raise
|
||||
r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'passed', 'outputHash': digest(output)})
|
||||
old = previous.get(tuple(argv), {})
|
||||
if getattr(a, 'reuse_successful', False) and reusable(old, expected, argv):
|
||||
entry = {**old, 'reusedAt': now()}
|
||||
else:
|
||||
entry = run_local_check(r, argv, expected)
|
||||
r['localChecks'].append(entry)
|
||||
save(r)
|
||||
require(check_identity(r) == expected, 'Check environment/dependencies changed; evidence is not reusable.')
|
||||
fresh(r)
|
||||
if a.cluster:
|
||||
# Never preserve an earlier successful snapshot when a new read fails.
|
||||
@ -363,6 +403,15 @@ def check(r, a):
|
||||
s = snapshot(r)
|
||||
require(all(d['ready'] for d in s['deployments'].values()), 'Current deployments are not fully ready.')
|
||||
r['baseline'] = s
|
||||
if strict_transition(r):
|
||||
path = getattr(a, 'source_readiness_file', None)
|
||||
require(bool(path), 'Strict activation needs a fresh deployment-wide --source-readiness-file.')
|
||||
api = s['deployments']['droboticflow-api']
|
||||
try:
|
||||
r['sourceReadiness'] = validate_report(path, namespace=r['target']['namespace'], database=policy()['database'],
|
||||
cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit'])
|
||||
except (ValueError,KeyError,TypeError,AttributeError,OSError):
|
||||
raise Blocked('Strict readiness incomplete, blocked, expired or wrong scope; keep advisory.') from None
|
||||
except Blocked:
|
||||
r['state'] = 'blocked'
|
||||
event(r, 'cluster check failed')
|
||||
@ -371,20 +420,61 @@ def check(r, a):
|
||||
event(r, 'checks recorded; plan must still be generated')
|
||||
|
||||
|
||||
def target_source_mode(r, name):
|
||||
return r.get('sourceMode') or r['baseline']['deployments'][name]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE')
|
||||
|
||||
|
||||
def validate_planned_source_mode(r):
|
||||
if r.get("sourceModeExplicit"):
|
||||
require(all(target_source_mode(r,n) in ("advisory","strict") for n in r["baseline"]["deployments"]), "New releases require an explicit planned source mode.")
|
||||
|
||||
|
||||
def strict_transition(r):
|
||||
if not any(target_source_mode(r,n)=='strict' for n in r['baseline']['deployments']): return False
|
||||
if any(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE')!='strict' for d in r['baseline']['deployments'].values()): return True
|
||||
return strict_rules_changed(r)
|
||||
|
||||
|
||||
def strict_rules_changed(r):
|
||||
live=r['baseline']['deployments']['droboticflow-api']['buildCommit']
|
||||
if live==r['commit']: return False
|
||||
require(isinstance(live,str) and bool(re.fullmatch(r'[0-9a-f]{40}',live)), 'Unknown current source; strict activation requires advisory preparation first.')
|
||||
paths=['internal/store/annotation_source_admission.go','internal/store/annotation_availability.go',
|
||||
'internal/api/annotation_source_admission.go','internal/api/annotation_source_preparation.go',
|
||||
'internal/api/annotation_source_certificate.go','internal/api/annotation_source_availability.go']
|
||||
return bool(run(['git','diff','--name-only',live,r['commit'],'--',*paths]).strip())
|
||||
|
||||
|
||||
def validate_source_transition(r):
|
||||
if not strict_transition(r): return
|
||||
proof = r.get('sourceReadiness')
|
||||
require(bool(proof), 'Strict activation requires a complete deployment-wide source readiness report.')
|
||||
require(not strict_rules_changed(r), 'Strict admission rules changed; deploy advisory first, then inventory with the candidate rules.')
|
||||
api = r['baseline']['deployments']['droboticflow-api']
|
||||
try:
|
||||
current = validate_report(proof['path'], namespace=r['target']['namespace'], database=policy()['database'],
|
||||
cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit'])
|
||||
require(current['sha256'] == proof['sha256'], 'Source readiness report changed; recheck and approve a new plan.')
|
||||
except (ValueError, KeyError, TypeError, AttributeError, OSError) as e:
|
||||
raise Blocked('Source readiness report invalid: ' + type(e).__name__) from None
|
||||
|
||||
|
||||
def make_plan(r):
|
||||
fresh(r)
|
||||
require(not r['operations'], 'Deployment already attempted.')
|
||||
require([x['argv'] for x in r['localChecks']] == policy()['localChecks']
|
||||
and all(x['result'] == 'passed' for x in r['localChecks']), 'Required local checks incomplete.')
|
||||
require(r['baseline'] is not None, 'Read-only cluster check required.')
|
||||
validate_planned_source_mode(r)
|
||||
validate_source_transition(r)
|
||||
r['planHash'] = digest(plan_data(r))
|
||||
r['state'] = 'awaiting_approval'
|
||||
event(r, 'plan prepared; no authorization inferred')
|
||||
print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [
|
||||
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit']} for n in sorted(r['baseline']['deployments'])],
|
||||
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit'], 'sourceMode': {'from': r['baseline']['deployments'][n]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE'), 'to': target_source_mode(r,n)}} for n in sorted(r['baseline']['deployments'])],
|
||||
'sequence': 'Processor first; verify its complete rollout before patching API.',
|
||||
'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'],
|
||||
'warnings': ['Only image/build stamp changes; startup migrations require change-safety evidence.',
|
||||
'warnings': ['Only image/build stamp and the explicitly planned source mode may change; startup migrations require change-safety evidence.',
|
||||
'Image-to-source provenance requires independent evidence.',
|
||||
'Bandwidth adequacy and business acceptance remain manual evidence.'],
|
||||
'rollback': r['rollback']}, ensure_ascii=False, indent=2))
|
||||
@ -396,6 +486,8 @@ def deploy(r, a):
|
||||
require(r['planHash'] == a.confirm_plan == digest(plan_data(r)), 'Plan hash does not match.')
|
||||
require(bool(a.approval_ref.strip()), 'Record the explicit user approval reference for this exact plan.')
|
||||
require_mainline(r['commit'], refresh=True)
|
||||
validate_planned_source_mode(r)
|
||||
validate_source_transition(r)
|
||||
age = (dt.datetime.now(dt.timezone.utc) - dt.datetime.fromisoformat(r['baseline']['at'])).total_seconds()
|
||||
require(0 <= age <= 3600, 'Cluster check is older than one hour; recheck and obtain a new plan approval.')
|
||||
for key in ('image-provenance', 'change-safety'):
|
||||
@ -409,13 +501,13 @@ def deploy(r, a):
|
||||
# The processor must understand new durable payloads before API replicas can
|
||||
# enqueue them. The fixed Fanyu names sort processor before API.
|
||||
for name in sorted(current['deployments']):
|
||||
latest = snapshot(r)
|
||||
latest = snapshot(r, allow_planned_transition=True)
|
||||
require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.')
|
||||
for n, live in latest['deployments'].items():
|
||||
old = current['deployments'][n]
|
||||
done = any(o['deployment'] == n and o['result'] == 'acknowledged' for o in r['operations'])
|
||||
require(live['uid'] == old['uid'] and live['nonReleaseHash'] == old['nonReleaseHash'], 'Deployment changed during release.')
|
||||
require((live['image'] == r['image'] and live['buildCommit'] == r['commit']) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.')
|
||||
require((live['image'] == r['image'] and live['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or live['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n))) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.')
|
||||
dep = latest['deployments'][name]
|
||||
patch = [{'op': 'test', 'path': '/metadata/resourceVersion', 'value': dep['resourceVersion']},
|
||||
{'op': 'test', 'path': '/metadata/uid', 'value': dep['uid']},
|
||||
@ -425,6 +517,12 @@ def deploy(r, a):
|
||||
env_path = f"/spec/template/spec/containers/{dep['containerIndex']}/env/"
|
||||
patch.append({'op': 'add' if dep['buildCommitIndex'] is None else 'replace',
|
||||
'path': env_path + ('-' if dep['buildCommitIndex'] is None else str(dep['buildCommitIndex'])), 'value': stamp})
|
||||
if r.get('sourceMode'):
|
||||
mode_index = dep['sourceModeIndex']
|
||||
# If the stamp is appended first, a pre-existing mode keeps its index.
|
||||
patch.append({'op': 'add' if mode_index is None else 'replace',
|
||||
'path': env_path + ('-' if mode_index is None else str(mode_index)),
|
||||
'value': {'name': 'ANNOTATION_SOURCE_VALIDATION_MODE', 'value': r['sourceMode']}})
|
||||
op = {'deployment': name, 'before': dep['image'], 'desired': r['image'], 'at': now(), 'result': 'unknown'}
|
||||
r['operations'].append(op)
|
||||
save(r) # Persist intent before sending any mutation.
|
||||
@ -440,8 +538,8 @@ def deploy(r, a):
|
||||
event(r, 'processor patch acknowledged; waiting for rollout before API')
|
||||
try:
|
||||
kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330)
|
||||
ready = snapshot(r)['deployments'][name]
|
||||
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'],
|
||||
ready = snapshot(r, allow_planned_transition=True)['deployments'][name]
|
||||
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or ready['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,name)),
|
||||
'Processor rollout has not converged to the candidate.')
|
||||
except Blocked:
|
||||
r['state'] = 'deployed_pending_verification'
|
||||
@ -493,6 +591,7 @@ def verify(r):
|
||||
for n, d in s['deployments'].items():
|
||||
require(d['uid'] == expected['deployments'][n]['uid'], 'Deployment was replaced.')
|
||||
require(d['nonReleaseHash'] == expected['deployments'][n]['nonReleaseHash'], 'Deployment settings drifted beyond the approved image/build stamp.')
|
||||
if r.get('sourceModeExplicit'): require(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n), 'Source mode differs from approved plan.')
|
||||
require(d['image'] == r['image'] and d['buildCommit'] == r['commit'] and d['ready'], 'Target image rollout incomplete; outcome remains pending.')
|
||||
r['state'] = 'deployed_pending_acceptance'
|
||||
missing = [k for k in policy()['requiredEvidence'] if k not in r['evidence'] or not evidence_valid(r['evidence'][k], r)]
|
||||
@ -546,6 +645,8 @@ def main():
|
||||
elif cmd == 'check':
|
||||
c.add_argument('--local', action='store_true')
|
||||
c.add_argument('--cluster', action='store_true')
|
||||
c.add_argument('--source-readiness-file')
|
||||
c.add_argument('--reuse-successful', action='store_true', help='Reuse only unchanged successful pure local checks; database/browser checks always rerun.')
|
||||
elif cmd == 'deploy':
|
||||
c.add_argument('--confirm-plan', required=True)
|
||||
c.add_argument('--approval-ref', required=True)
|
||||
@ -554,6 +655,7 @@ def main():
|
||||
c.add_argument('--result', choices=['passed', 'failed', 'unverified'], required=True)
|
||||
c.add_argument('--file', required=True)
|
||||
c.add_argument('--note', required=True)
|
||||
prep.add_argument('--source-mode', choices=['strict','advisory'], help='Optional reviewed source-mode change in the same image plan.')
|
||||
a = parser.parse_args()
|
||||
try:
|
||||
if a.command == 'prepare':
|
||||
|
||||
85
scripts/test_work_availability.py
Normal file
85
scripts/test_work_availability.py
Normal file
@ -0,0 +1,85 @@
|
||||
"""Mandatory synthetic work/recovery gate; loopback Mongo only, no customer writes."""
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from lib.release_checks import check_environment, write_log, source_fingerprint
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CORE = {'TestSourceAdvisoryCompleteWorkAndReviewFlow', 'TestSourceAdvisoryPublishedReviewDoesNotRequireFullAudit', 'TestSourceAdvisoryPreparesIndexDespiteCadenceAuditFailure', 'TestSourceAdvisoryTaskScopeCannotBeBypassed', 'TestSourceAdvisoryIsolatedReworkDoesNotTrapRandomClaim', 'TestSourceAdvisoryOwnedFailureCanRetryTimeline', 'TestSubmissionLossGuardArchiveFailureBlocksReplacement', 'TestSubmissionLossGuardArchiveSurvivesRestartAndIsIdempotent', 'TestSubmissionLossGuardArchivesFailedInputBeforeReplacement', 'TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter', 'TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker', 'TestCombinedSubmissionFinalCASFailureDoesNotPublishQualityTask', 'TestDeletedAnnotatorSubmissionWinsCASRace', 'TestDeletedAnnotatorRestartFinishesResetOnce', 'TestReviewCannotMutateAnnotation', 'TestSourceAdvisoryDoesNotGateOnFullAudit', 'TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate', 'TestReadObservationIsPrivateConcurrentAndRequestScoped'}
|
||||
MONGO = {'TestSourceReadinessMongoParityReadOnly', 'TestSourceAdvisoryMongoEquivalenceAndBuffer', 'TestSourceAdmissionMongoClaimFence', 'TestSourceAdmissionMongoBatchMatchesLegacyPipeline', 'TestSourceAdmissionCurrentOutboxProofMongo', 'TestSourceAdmissionExistingTaskUpdateDoesNotEnqueueMongo', 'TestSourceValidationLegacyDetailsGETDoesNotEnqueueMongo', 'TestSubmissionRecoveryEventMongo'}
|
||||
|
||||
def verdict(stdout, code, required):
|
||||
events = []
|
||||
for line in stdout.splitlines():
|
||||
try:
|
||||
events.append(json.loads(line))
|
||||
except ValueError:
|
||||
continue
|
||||
passed = {e.get('Test') for e in events if e.get('Action') == 'pass'}
|
||||
skipped = sorted((e['Test'] for e in events if e.get('Action') == 'skip' and e.get('Test')))
|
||||
missing = sorted(required - passed)
|
||||
return {'passed': code == 0 and (not skipped) and (not missing), 'missingOrFailed': missing, 'skipped': skipped}
|
||||
|
||||
def phases():
|
||||
return [('core', CORE, {}), ('mongo', MONGO, {'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'SUBMISSION_RECOVERY_LOCAL_MONGO': '1', 'POOL_LOCAL_MONGO_URI': 'mongodb://127.0.0.1:27018'}), ('advisory-workbench', {'TestSourceAdvisoryWorkbenchBrowser'}, {'SOURCE_ADVISORY_BROWSER_TEST': '1'}), ('advisory-recovery', {'TestReworkBoundaryBrowserReplica'}, {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'REWORK_BOUNDARY_BROWSER_TEST': '1'})]
|
||||
|
||||
def run_phase(argv, env):
|
||||
# Terminate only this synthetic phase's process group on timeout, including
|
||||
# Go-launched Node/browser children. Never terminate a shared local service.
|
||||
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
text=True, start_new_session=True)
|
||||
try:
|
||||
stdout, stderr = child.communicate(timeout=360)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(child.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
stdout, stderr = child.communicate(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
stdout, stderr = child.communicate()
|
||||
return subprocess.CompletedProcess(argv, 124, stdout, stderr + '\nphase timed out')
|
||||
return subprocess.CompletedProcess(argv, child.returncode, stdout, stderr)
|
||||
|
||||
def main():
|
||||
folder = ROOT / 'local-data/test-results' / ('work-availability-' + uuid.uuid4().hex)
|
||||
folder.mkdir(parents=True, mode=448)
|
||||
before = source_fingerprint(ROOT)
|
||||
report = {'inputDigest': before, 'scope': 'synthetic local fixtures; no production acceptance/capacity claim', 'checks': [], 'passed': False}
|
||||
for name, required, switches in phases():
|
||||
argv = ['go', 'test', '-json', './internal/api', './internal/store', '-run', '^(' + '|'.join(sorted(required)) + ')$', '-count=1', '-timeout=5m']
|
||||
env = {**check_environment(), **switches, 'VERIFICATION_RUN_ID': uuid.uuid4().hex}
|
||||
try:
|
||||
r = run_phase(argv, env)
|
||||
row = verdict(r.stdout, r.returncode, required)
|
||||
output = r.stdout + '\n' + r.stderr
|
||||
except (OSError, subprocess.TimeoutExpired) as e:
|
||||
row = {'passed': False, 'error': 'runner failed or timed out'}
|
||||
output = str(e)
|
||||
report['inputsChangedDuringRun'] = source_fingerprint(ROOT) != before
|
||||
if report['inputsChangedDuringRun'] or not before:
|
||||
row.update(passed=False, error='Source identity changed or unavailable')
|
||||
row.update(phase=name, argv=argv, artifactRunId=env['VERIFICATION_RUN_ID'])
|
||||
row.update(write_log(folder, argv, output, 'passed' if row['passed'] else 'failed'))
|
||||
report['checks'].append(row)
|
||||
print(json.dumps(row), flush=True)
|
||||
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||
if report['inputsChangedDuringRun'] or not before:
|
||||
report['remainingUnverified'] = [p[0] for p in phases()[len(report['checks']):]]
|
||||
(folder/'report.json').write_text(json.dumps(report,indent=2)+'\n')
|
||||
return 2
|
||||
report['passed'] = all(row['passed'] for row in report['checks'])
|
||||
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||
print('Work availability gate ' + ('passed' if report['passed'] else 'failed') + '; production writes and capacity remain separate. Report: ' + str(folder / 'report.json'))
|
||||
return 0 if report['passed'] else 1
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -853,6 +853,36 @@
|
||||
"tests": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "reliability-guardrails",
|
||||
"paths": [
|
||||
"^scripts/(lib/(release_checks|source_readiness)\\.py|test_work_availability\\.py)$",
|
||||
"^tests/release/test_reliability_checks\\.py$",
|
||||
"^internal/(api|store)/(source_readiness|read_observation).*\\.go$",
|
||||
"^internal/api/diagnostic_trace.*\\.go$",
|
||||
"^docs/(operations/reliability-guardrails\\.md|release/api-runtime\\.json)$",
|
||||
"^Dockerfile\\.api-runtime$",
|
||||
"^cmd/api/main\\.go$"
|
||||
],
|
||||
"frontend": [],
|
||||
"go": {
|
||||
"./internal/api": ["^TestSourceReadiness", "^TestDiagnosticTrace"],
|
||||
"./internal/store": ["^TestReadObservation", "^TestSourceReadiness"]
|
||||
},
|
||||
"task": [
|
||||
{
|
||||
"argv": ["python3", "-m", "unittest", "discover", "-s", "tests/release", "-p", "test_*.py"],
|
||||
"tests": true
|
||||
},
|
||||
{
|
||||
"argv": ["npm", "run", "test:work-availability"],
|
||||
"tests": true
|
||||
}
|
||||
],
|
||||
"warnings": [
|
||||
"工作流门禁仅使用合成媒体和隔离 Mongo;严格模式清单不是校验证书,生产容量、备份恢复和部署需分别验证。"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { closeBrowserFixture } from "../../scripts/lib/browser-test-lifecycle.mjs";
|
||||
import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs";
|
||||
import fs from "node:fs";
|
||||
import assert from "node:assert/strict";
|
||||
@ -7,7 +8,11 @@ const { createServer } = require("vite"),
|
||||
{ chromium } = require("playwright");
|
||||
const api = process.env.REWORK_BOUNDARY_API;
|
||||
assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/);
|
||||
const root = new URL("../../local-data/verification/initial-slice-confirmation/", import.meta.url);
|
||||
const runId = process.env.VERIFICATION_RUN_ID || `${Date.now()}-${process.pid}`;
|
||||
assert.match(runId, /^[a-zA-Z0-9_-]+$/);
|
||||
const scenario = process.env.INITIAL_SLICE_CONFIRM_SCENARIO;
|
||||
assert.match(scenario, /^[a-z0-9-]+$/);
|
||||
const root = new URL(`../../local-data/verification/initial-slice-confirmation/${runId}/${scenario}/`, import.meta.url);
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
const report = { cases: [], errors: [], requests: [] };
|
||||
const vite = await createServer({
|
||||
@ -15,19 +20,22 @@ const vite = await createServer({
|
||||
configFile: "tests/browser/vite.platform-feedback.config.ts",
|
||||
server: { host: "127.0.0.1", port: 5204, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } },
|
||||
});
|
||||
let browser;
|
||||
let browser, owner, activeContext, activePage;
|
||||
try {
|
||||
await vite.listen();
|
||||
browser = await chromium.launch({
|
||||
owner = await chromium.launchServer({
|
||||
...browserExecutableOptions(),
|
||||
headless: true,
|
||||
args: ["--no-proxy-server"],
|
||||
});
|
||||
browser = await chromium.connect(owner.wsEndpoint());
|
||||
// A separate context for each case prevents autosaving one scenario into another.
|
||||
for (const status of [process.env.INITIAL_SLICE_CONFIRM_SCENARIO]) {
|
||||
await fetch(api + "/__boundary/reset?status=" + status);
|
||||
const context = await browser.newContext({ viewport: { width: 1600, height: 1100 } });
|
||||
const page = await context.newPage();
|
||||
activeContext = context;
|
||||
await context.tracing.start({ screenshots: true, snapshots: true, sources: false });
|
||||
const page = (activePage = await context.newPage());
|
||||
page.setDefaultTimeout(20000);
|
||||
page.on("pageerror", (e) => report.errors.push(e.message));
|
||||
page.on("response", async (r) => {
|
||||
@ -65,6 +73,7 @@ try {
|
||||
assert.equal(report.requests.length, 0, "unreadable content must not autosave or generate");
|
||||
await page.screenshot({ path: new URL(status + "-blocked.png", root).pathname });
|
||||
report.cases.push({ status, preserved: true });
|
||||
assert.deepEqual(report.errors, []);
|
||||
await context.close();
|
||||
continue;
|
||||
}
|
||||
@ -162,15 +171,20 @@ try {
|
||||
await bench.waitFor();
|
||||
await page.screenshot({ path: new URL(status + "-after.png", root).pathname });
|
||||
report.cases.push({ status, retainedAction: workspace.segments[0].id, sliceName: workspace.sliceSegments[0].name });
|
||||
assert.deepEqual(report.errors, []);
|
||||
await context.close();
|
||||
}
|
||||
assert.deepEqual(report.errors, []);
|
||||
} catch (error) {
|
||||
report.failure = { name: error.name, message: error.message, stack: error.stack };
|
||||
await activePage?.screenshot({ path: new URL("failure.png", root).pathname, fullPage: true }).catch(() => {});
|
||||
await activeContext?.tracing.stop({ path: new URL("failure-trace.zip", root).pathname }).catch(() => {});
|
||||
throw error;
|
||||
} finally {
|
||||
fs.writeFileSync(
|
||||
new URL(`report-${process.env.INITIAL_SLICE_CONFIRM_SCENARIO}.json`, root),
|
||||
JSON.stringify(report, null, 2),
|
||||
);
|
||||
await browser?.close();
|
||||
await vite.close();
|
||||
await closeBrowserFixture({ browser, owner, vite });
|
||||
}
|
||||
console.log(JSON.stringify(report));
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import { closeBrowserFixture } from "../../scripts/lib/browser-test-lifecycle.mjs";
|
||||
import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
@ -6,14 +7,18 @@ import { chromium } from "playwright";
|
||||
import { createServer } from "vite";
|
||||
const api = process.env.REWORK_BOUNDARY_API;
|
||||
assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/);
|
||||
const out = path.resolve("local-data/test-results/rework-boundary");
|
||||
const runId = process.env.VERIFICATION_RUN_ID || `${Date.now()}-${process.pid}`;
|
||||
assert.match(runId, /^[a-zA-Z0-9_-]+$/);
|
||||
const out = path.resolve("local-data/test-results/rework-boundary", runId);
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
const server = await createServer({
|
||||
cacheDir: "local-data/test-results/vite-rework-boundary-replica",
|
||||
server: { host: "127.0.0.1", port: 5202, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } },
|
||||
});
|
||||
let browser,
|
||||
let owner,
|
||||
browser,
|
||||
page,
|
||||
context,
|
||||
fault = "",
|
||||
releaseSave;
|
||||
const report = { cases: [], pageErrors: [], writes: [] };
|
||||
@ -29,13 +34,18 @@ const waitFor = async (check) => {
|
||||
};
|
||||
try {
|
||||
await server.listen();
|
||||
browser = await chromium.launch({
|
||||
owner = await chromium.launchServer({
|
||||
...browserExecutableOptions(),
|
||||
|
||||
headless: true,
|
||||
args: ["--no-proxy-server"],
|
||||
});
|
||||
const ctx = await browser.newContext({ viewport: { width: 1600, height: 1100 }, serviceWorkers: "block" });
|
||||
browser = await chromium.connect(owner.wsEndpoint());
|
||||
const ctx = (context = await browser.newContext({
|
||||
viewport: { width: 1600, height: 1100 },
|
||||
serviceWorkers: "block",
|
||||
}));
|
||||
await ctx.tracing.start({ screenshots: true, snapshots: true, sources: false });
|
||||
page = await ctx.newPage();
|
||||
page.setDefaultTimeout(12000);
|
||||
page.on("pageerror", (e) => report.pageErrors.push(e.message));
|
||||
@ -314,6 +324,8 @@ try {
|
||||
assert.deepEqual(report.pageErrors, []);
|
||||
console.log(JSON.stringify(report));
|
||||
} catch (e) {
|
||||
report.failure = { name: e.name, message: e.message, stack: e.stack };
|
||||
await context?.tracing.stop({ path: path.join(out, "failure-trace.zip") }).catch(() => {});
|
||||
if (page) {
|
||||
await page.screenshot({ path: path.join(out, "failure.png"), fullPage: true }).catch(() => {});
|
||||
fs.writeFileSync(
|
||||
@ -328,6 +340,5 @@ try {
|
||||
} finally {
|
||||
releaseSave?.();
|
||||
fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2));
|
||||
await browser?.close();
|
||||
await server.close();
|
||||
await closeBrowserFixture({ browser, owner, vite: server });
|
||||
}
|
||||
|
||||
@ -20,7 +20,7 @@ IMAGE = 'registry.example/api@sha256:' + 'a' * 64
|
||||
|
||||
|
||||
def fixture(name, container):
|
||||
config = {'data': {'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true',
|
||||
config = {'data': {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true',
|
||||
'ANNOTATION_PROCESSOR_MODE': 'external', 'ANNOTATION_WORKSPACE_BUCKET': 'drobodata-fanyu',
|
||||
'ANNOTATION_WORKSPACE_CONNECTION_ID': 'storage-example', 'WORKSPACE_MOUNT_PATH': '/workspace'},
|
||||
'metadata': {'uid': 'config-uid'}}
|
||||
@ -53,6 +53,7 @@ class ReleaseTests(unittest.TestCase):
|
||||
self.start_patch('require_mainline', lambda *args, **kwargs: None)
|
||||
self.start_patch('run', lambda *args, **kwargs: 'feature/release')
|
||||
self.start_patch('manifest_platforms', lambda image: ['linux/amd64'])
|
||||
self.start_patch('check_identity', lambda r: {'testEnvironment': 'stable', 'complete': True})
|
||||
self.ds = {}
|
||||
for n, c in self.p['deployments'].items():
|
||||
self.ds[n], config = fixture(n, c)
|
||||
@ -76,11 +77,11 @@ class ReleaseTests(unittest.TestCase):
|
||||
def load(self):
|
||||
return json.loads(m.record_path('test').read_text())
|
||||
|
||||
def snapshot(self, r):
|
||||
def snapshot(self, r, **kwargs):
|
||||
if self.before_snapshot:
|
||||
self.before_snapshot()
|
||||
return {'at': m.now(), 'deployments': {
|
||||
n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p) for n, d in self.ds.items()},
|
||||
n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in self.ds.items()},
|
||||
'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}},
|
||||
'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}}
|
||||
|
||||
@ -105,6 +106,9 @@ class ReleaseTests(unittest.TestCase):
|
||||
c['env'].append(ops[4]['value'])
|
||||
else:
|
||||
c['env'][int(ops[4]['path'].split('/')[-1])] = ops[4]['value']
|
||||
for extra in ops[5:]:
|
||||
if extra['op']=='add': c['env'].append(extra['value'])
|
||||
else: c['env'][int(extra['path'].split('/')[-1])] = extra['value']
|
||||
d['metadata']['resourceVersion'] = str(int(d['metadata']['resourceVersion']) + 1)
|
||||
return '{}'
|
||||
|
||||
@ -433,6 +437,15 @@ class ReleaseTests(unittest.TestCase):
|
||||
with patch.object(real, 'policy', return_value=self.p), patch.object(real, 'get', side_effect=fake_get), patch.object(real, 'kubectl', return_value=json.dumps({'clusters': [{'cluster': {'server': r['target']['server']}}]})):
|
||||
real.snapshot(r)
|
||||
self.assertNotIn('secret', reads)
|
||||
r['baseline'] = self.snapshot(r)
|
||||
r['sourceMode'] = 'strict'
|
||||
env = self.ds['droboticflow-annotation-processor']['spec']['template']['spec']['containers'][0]['env']
|
||||
env.append({'name':'ANNOTATION_SOURCE_VALIDATION_MODE','value':'strict'})
|
||||
with self.assertRaises(real.Blocked): real.snapshot(r)
|
||||
real.snapshot(r, allow_planned_transition=True)
|
||||
env[-1]['value'] = 'disabled'
|
||||
with self.assertRaises(real.Blocked): real.snapshot(r, allow_planned_transition=True)
|
||||
env.pop()
|
||||
replies['namespace']['metadata']['uid'] = 'wrong'
|
||||
with self.assertRaises(real.Blocked): real.snapshot(r)
|
||||
|
||||
@ -451,6 +464,104 @@ class ReleaseTests(unittest.TestCase):
|
||||
with self.assertRaises(m.Blocked):
|
||||
with m.locked('test'): pass
|
||||
|
||||
def test_combined_plan_changes_only_reviewed_mode_in_processor_api_order(self):
|
||||
self.configs['config']['data']['ANNOTATION_SOURCE_VALIDATION_MODE']='strict'
|
||||
self.r['sourceMode']='advisory'
|
||||
self.plan()
|
||||
approval=self.approve()
|
||||
m.deploy(self.r,approval)
|
||||
self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api'])
|
||||
self.assertEqual(self.rollout_waits[0][1],['droboticflow-annotation-processor'])
|
||||
for d in self.ds.values():
|
||||
env={e['name']:e.get('value') for e in d['spec']['template']['spec']['containers'][0]['env']}
|
||||
self.assertEqual(env['ANNOTATION_SOURCE_VALIDATION_MODE'],'advisory')
|
||||
m.verify(self.r)
|
||||
self.assertEqual(self.r['state'],'deployed_pending_acceptance')
|
||||
|
||||
def test_legacy_unset_mode_can_be_repaired_by_explicit_combined_plan(self):
|
||||
self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE')
|
||||
self.r['sourceMode']='advisory'
|
||||
self.plan();m.deploy(self.r,self.approve())
|
||||
self.assertEqual(len(self.writes),2)
|
||||
|
||||
def test_new_release_cannot_plan_an_implicit_strict_fallback(self):
|
||||
self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE')
|
||||
m.check(self.r,argparse.Namespace(local=True,cluster=True))
|
||||
with self.assertRaises(m.Blocked):m.make_plan(self.r)
|
||||
self.assertEqual(self.writes,[])
|
||||
|
||||
def test_mode_change_after_approval_invalidates_plan_without_writes(self):
|
||||
self.r['sourceMode']='advisory';self.plan();a=self.approve()
|
||||
self.r['sourceMode']='strict'
|
||||
with self.assertRaises(m.Blocked):m.deploy(self.r,a)
|
||||
self.assertEqual(self.writes,[])
|
||||
|
||||
def test_strict_activation_without_inventory_never_writes(self):
|
||||
self.r['sourceMode']='strict'
|
||||
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=True))
|
||||
self.assertEqual(self.writes,[])
|
||||
|
||||
def test_combined_mode_rollout_failure_leaves_api_unmodified(self):
|
||||
self.r['sourceMode']='advisory';self.plan();self.fail_rollout=True
|
||||
with self.assertRaises(m.Blocked):m.deploy(self.r,self.approve())
|
||||
self.assertEqual(self.writes,['droboticflow-annotation-processor'])
|
||||
self.assertEqual(self.r['state'],'deployed_pending_verification')
|
||||
|
||||
def test_failed_check_log_is_private_and_does_not_discard_failure(self):
|
||||
failure=m.Blocked('failed');failure.check_output='password: canary-secret\nTraceback: canary-failure'
|
||||
with patch.object(m,'run',side_effect=failure):
|
||||
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False))
|
||||
e=self.load()['localChecks'][-1];text=Path(e['log']).read_text()
|
||||
self.assertNotIn('canary-secret',text);self.assertIn('canary-failure',text)
|
||||
self.assertEqual(Path(e['log']).stat().st_mode&0o777,0o600)
|
||||
|
||||
def test_explicit_reuse_keeps_only_verified_successful_pure_checks(self):
|
||||
self.p['localChecks']=[['npm','run','lint:web']]
|
||||
self.r['policyHash']=m.digest(self.p)
|
||||
m.check(self.r,argparse.Namespace(local=True,cluster=False))
|
||||
with patch.object(m,'run',side_effect=m.Blocked('must not rerun pure check')):
|
||||
m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True))
|
||||
self.assertIn('reusedAt',self.r['localChecks'][0])
|
||||
Path(self.r['localChecks'][0]['log']).write_text('tampered')
|
||||
with patch.object(m,'run',side_effect=m.Blocked('must rerun tampered evidence')):
|
||||
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True))
|
||||
|
||||
def test_current_complete_deployment_report_binds_strict_mode_and_detects_tampering(self):
|
||||
for d in self.ds.values():
|
||||
d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']})
|
||||
self.r['sourceMode']='strict'
|
||||
report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'],
|
||||
'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True,
|
||||
'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(),
|
||||
'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]}
|
||||
path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report))
|
||||
m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path)))
|
||||
m.make_plan(self.r);a=self.approve()
|
||||
report['rows'][0]['allowed']=0;report['rows'][0]['failed']=1;path.write_text(json.dumps(report))
|
||||
with self.assertRaises(m.Blocked):m.deploy(self.r,a)
|
||||
self.assertEqual(self.writes,[])
|
||||
|
||||
def test_complete_report_allows_reviewed_strict_activation(self):
|
||||
for d in self.ds.values():
|
||||
d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']})
|
||||
self.r['sourceMode']='strict'
|
||||
report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'],
|
||||
'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True,
|
||||
'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(),
|
||||
'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]}
|
||||
path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report))
|
||||
m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path)))
|
||||
m.make_plan(self.r);m.deploy(self.r,self.approve());m.verify(self.r)
|
||||
self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api'])
|
||||
self.assertEqual(self.r['state'],'deployed_pending_acceptance')
|
||||
|
||||
def test_changed_strict_rule_needs_advisory_preparation_not_old_report(self):
|
||||
self.r['sourceMode']='strict';self.r['baseline']=self.snapshot(self.r)
|
||||
self.r['baseline']['deployments']['droboticflow-api']['buildCommit']='b'*40
|
||||
self.r['sourceReadiness']={'path':'not-read-yet'}
|
||||
with self.assertRaisesRegex(m.Blocked,'deploy advisory first'):m.validate_source_transition(self.r)
|
||||
|
||||
|
||||
|
||||
class EntrypointTests(unittest.TestCase):
|
||||
def test_cli_persists_and_resumes_offline(self):
|
||||
@ -459,8 +570,9 @@ class EntrypointTests(unittest.TestCase):
|
||||
(root / 'scripts').mkdir()
|
||||
(root / 'docs/release').mkdir(parents=True)
|
||||
shutil.copy(ROOT / 'scripts/release.py', root / 'scripts/release.py')
|
||||
shutil.copytree(ROOT / 'scripts/lib', root / 'scripts/lib', ignore=shutil.ignore_patterns('__pycache__','*.pyc'))
|
||||
shutil.copy(ROOT / 'docs/release/fanyu-policy.json', root / 'docs/release/fanyu-policy.json')
|
||||
(root / '.gitignore').write_text('local-data/\n')
|
||||
(root / '.gitignore').write_text('local-data/\n__pycache__/\n')
|
||||
for args in [ ['git', 'init', '-q'], ['git', 'add', '.'],
|
||||
['git', '-c', 'user.name=ReleaseTest', '-c', 'user.email=test@example.invalid',
|
||||
'commit', '-qm', 'fixture'] ]:
|
||||
|
||||
@ -29,6 +29,8 @@ class ReleaseBuildApiImageTest(unittest.TestCase):
|
||||
(self.repo / "scripts").mkdir(parents=True)
|
||||
shutil.copy(SCRIPT, self.repo / "scripts" / SCRIPT.name)
|
||||
(self.repo / "Dockerfile.api").write_text("FROM scratch\n")
|
||||
(self.repo / "docs/release").mkdir(parents=True)
|
||||
shutil.copy(ROOT / "docs/release/api-runtime.json", self.repo / "docs/release/api-runtime.json")
|
||||
git = ["git", "-C", str(self.repo), "-c", "user.name=t", "-c", "user.email=t@example.com"]
|
||||
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
|
||||
subprocess.run([*git, "add", "."], check=True)
|
||||
@ -52,6 +54,11 @@ class ReleaseBuildApiImageTest(unittest.TestCase):
|
||||
self.assertTrue(any(a.startswith("DROBOTICFLOW_BUILD_TIME=20") and a.endswith("Z") for a in args), args)
|
||||
self.assertIn(f"registry.example.com/team/droboticflow-api:{self.commit[:8]}", args)
|
||||
self.assertIn("--push", args)
|
||||
self.assertIn("API_RUNTIME_PREPARED=1", args)
|
||||
self.assertTrue(any(a.startswith("API_RUNTIME_IMAGE=") and "@sha256:" in a for a in args))
|
||||
proof=json.loads(pathlib.Path(str(metadata)+".provenance.json").read_text())
|
||||
self.assertEqual(proof["commit"],self.commit)
|
||||
self.assertIn("@sha256:",proof["runtimeImage"])
|
||||
self.assertIn(f"IMAGE=registry.example.com/team/droboticflow-api@sha256:{7:064d}", result.stdout)
|
||||
self.assertEqual(json.loads(metadata.read_text())["containerimage.digest"], f"sha256:{7:064d}")
|
||||
|
||||
|
||||
124
tests/release/test_reliability_checks.py
Normal file
124
tests/release/test_reliability_checks.py
Normal file
@ -0,0 +1,124 @@
|
||||
import datetime as dt
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import sys
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
|
||||
def module(name, path):
|
||||
s = importlib.util.spec_from_file_location(name, ROOT / path)
|
||||
m = importlib.util.module_from_spec(s)
|
||||
s.loader.exec_module(m)
|
||||
return m
|
||||
m = module('release_checks', 'scripts/lib/release_checks.py')
|
||||
g = module('work_availability', 'scripts/test_work_availability.py')
|
||||
ready = module('source_readiness', 'scripts/lib/source_readiness.py')
|
||||
|
||||
class CheckEvidenceTests(unittest.TestCase):
|
||||
|
||||
def test_private_bounded_redaction_including_multiline_secrets(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
result = m.write_log(d, ['fake-check'], 'password:\n"canary-pass"\nAuthorization: Bearer canary-token\nmongodb://user:canary-db@host/db\nhttps://host/x?signature=canary-url\nAWS_ACCESS_KEY_ID=canary-aws\nBOS_SK=canary-bos\nTraceback: visible failure', 'failed')
|
||||
p = Path(result['log'])
|
||||
text = p.read_text()
|
||||
for value in ['canary-pass', 'canary-token', 'canary-db', 'canary-url','canary-aws','canary-bos']:
|
||||
self.assertNotIn(value, text)
|
||||
self.assertIn('visible failure', text)
|
||||
self.assertEqual(p.stat().st_mode & 511, 384)
|
||||
|
||||
|
||||
def test_large_unstructured_output_is_redacted_with_bounded_runtime(self):
|
||||
code = "import sys;sys.path.insert(0," + repr(str(ROOT/'scripts')) + ");from lib.release_checks import redact;assert len(redact('a'*3000000))<1000"
|
||||
subprocess.run([sys.executable,'-c',code],check=True,capture_output=True,timeout=5)
|
||||
|
||||
def test_tampering_expiry_identity_and_external_checks_never_reuse(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
e = {'result': 'passed', 'identity': {'commit': 'x', 'complete': True}, 'finishedUnix': time.time(), **m.write_log(d, ['test'], 'ok', 'passed')}
|
||||
self.assertTrue(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
|
||||
self.assertFalse(m.reusable(e, {'commit': 'y', 'complete': True}, ['npm', 'run', 'lint:web']))
|
||||
for args in [['npm', 'run', 'test:browser:annotation-persistence'], ['python3', 'scripts/test_work_availability.py']]:
|
||||
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, args))
|
||||
self.assertFalse(m.reusable({**e, 'finishedUnix': 0}, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
|
||||
Path(e['log']).write_text('tampered')
|
||||
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
|
||||
|
||||
def test_production_credentials_and_inherited_switches_removed(self):
|
||||
with patch.dict(os.environ, {'POOL_LOCAL_MONGO_URI': 'production', 'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'MONGO_URI': 'production', 'BOS_SECRET_KEY': 'secret', 'SOURCE_ADVISORY_BROWSER_TEST': '1', 'ANNOTATION_SOURCE_VALIDATION_MODE': 'strict'}):
|
||||
e = m.check_environment()
|
||||
for key in ['POOL_LOCAL_MONGO_URI', 'SOURCE_ADMISSION_LOCAL_MONGO', 'MONGO_URI', 'BOS_SECRET_KEY', 'SOURCE_ADVISORY_BROWSER_TEST', 'ANNOTATION_SOURCE_VALIDATION_MODE']:
|
||||
self.assertNotIn(key, e)
|
||||
|
||||
def test_installed_dependency_and_symlink_changes_invalidate_identity(self):
|
||||
with tempfile.TemporaryDirectory() as d, patch.object(m.subprocess, 'check_output', return_value=b'version'):
|
||||
root = Path(d)
|
||||
pkg = root / 'node_modules/p'
|
||||
pkg.mkdir(parents=True)
|
||||
(pkg / 'index.js').write_text('a')
|
||||
a = m.identity(root, 'x', 'p')
|
||||
(pkg / 'index.js').write_text('b')
|
||||
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
|
||||
linked = root / 'linked'
|
||||
linked.mkdir()
|
||||
(linked / 'index.js').write_text('c')
|
||||
(root / 'node_modules/link').symlink_to(linked, target_is_directory=True)
|
||||
a = m.identity(root, 'x', 'p')
|
||||
(linked / 'index.js').write_text('d')
|
||||
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
|
||||
|
||||
def test_required_case_skip_missing_and_failure_are_not_passes(self):
|
||||
out = '\n'.join((json.dumps(e) for e in [{'Action': 'pass', 'Test': 'Required'}, {'Action': 'skip', 'Test': 'Required/child'}]))
|
||||
self.assertFalse(g.verdict(out, 0, {'Required'})['passed'])
|
||||
self.assertFalse(g.verdict('', 0, {'Required'})['passed'])
|
||||
self.assertFalse(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 1, {'Required'})['passed'])
|
||||
self.assertTrue(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 0, {'Required'})['passed'])
|
||||
|
||||
def test_existing_checks_are_wired_not_reimplemented(self):
|
||||
policy = json.loads((ROOT / 'docs/release/fanyu-policy.json').read_text())
|
||||
self.assertIn(['python3', 'scripts/test_work_availability.py'], policy['localChecks'])
|
||||
self.assertIn('npm run test:work-availability', (ROOT / '.gitea/workflows/web-ci.yml').read_text())
|
||||
self.assertIn('TestSourceAdvisoryCompleteWorkAndReviewFlow', g.CORE)
|
||||
self.assertIn('TestSubmissionRecoveryEventMongo', g.MONGO)
|
||||
|
||||
def test_independent_checks_continue_but_mixed_sources_stop(self):
|
||||
checks = [('first', {'Required'}, {}), ('second', {'Required'}, {})]
|
||||
output = json.dumps({'Action': 'pass', 'Test': 'Required'})
|
||||
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', return_value='stable'), patch.object(g, 'run_phase', side_effect=[subprocess.CompletedProcess([], 1, '', 'first failed'), subprocess.CompletedProcess([], 0, output, '')]) as run:
|
||||
self.assertEqual(g.main(), 1)
|
||||
self.assertEqual(run.call_count, 2)
|
||||
report = json.loads(next(Path(d).rglob('report.json')).read_text())
|
||||
self.assertFalse(report['passed'])
|
||||
self.assertTrue(report['checks'][1]['passed'])
|
||||
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', side_effect=['before', 'after']), patch.object(g, 'run_phase', return_value=subprocess.CompletedProcess([], 0, output, '')) as run:
|
||||
self.assertEqual(g.main(), 2)
|
||||
self.assertEqual(run.call_count, 1)
|
||||
report = json.loads(next(Path(d).rglob('report.json')).read_text())
|
||||
self.assertEqual(report['remainingUnverified'], ['second'])
|
||||
|
||||
class StrictReadinessTests(unittest.TestCase):
|
||||
|
||||
def report(self):
|
||||
return {'schema': 'droboticflow.source-readiness/v1', 'namespace': 'n', 'database': 'db', 'scope': 'deployment', 'organizationId': '', 'projectId': '', 'targetMode': 'strict', 'complete': True, 'legacySourceCutoff': '', 'buildCommit': 'c' * 40, 'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'rows': [{'organizationId': 'o', 'stage': 'quality_review', 'status': 'ready', 'total': 2, 'allowed': 2, 'unverified': 0, 'pending': 0, 'failed': 0}]}
|
||||
|
||||
def validate(self, r):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = Path(d) / 'report.json'
|
||||
p.write_text(json.dumps(r))
|
||||
return ready.validate_report(p, namespace='n', database='db', cutoff=None, live_commit='c' * 40)
|
||||
|
||||
def test_only_complete_current_deployment_inventory_can_activate(self):
|
||||
self.assertEqual(self.validate(self.report())['totals']['allowed'], 2)
|
||||
for patcher in [lambda r: r.update(scope='organization', organizationId='o'), lambda r: r.update(projectId='p'), lambda r: r.update(complete=False), lambda r: r.update(database='local-synthetic'), lambda r: r.update(namespace='other'), lambda r: r.update(buildCommit='a' * 40), lambda r: r.update(generatedAt='2000-01-01T00:00:00Z'), lambda r: r.update(legacySourceCutoff='2026-11-12T00:00:00Z'), lambda r: r['rows'][0].update(total=3), lambda r: r['rows'][0].update(allowed=True), lambda r: r['rows'][0].update(allowed=1, pending=1), lambda r: r['rows'].append(copy.deepcopy(r['rows'][0]))]:
|
||||
r = self.report()
|
||||
patcher(r)
|
||||
with self.subTest(r=r), self.assertRaises(ValueError):
|
||||
self.validate(r)
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user