From a35e84163a9ad9168d0bebcf1cf3abda38da98c0 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 11 Nov 2026 22:28:17 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Guard=20work=20availabi?= =?UTF-8?q?lity=20and=20preserve=20release=20verification=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitea/workflows/web-ci.yml | 40 ++++- .scratch/reliability-guardrails/work.md | 20 +++ Dockerfile.api | 35 ++-- Dockerfile.api-runtime | 9 + cmd/api/main.go | 2 +- docs/operations/reliability-guardrails.md | 48 ++++++ docs/release/README.md | 32 +++- docs/release/api-runtime.json | 5 + docs/release/fanyu-policy.json | 3 +- internal/api/diagnostic_trace.go | 20 ++- internal/api/diagnostic_trace_test.go | 36 ++++ internal/api/human_loop.go | 8 + internal/api/human_pool_reads.go | 2 + internal/api/routes_annotation.go | 1 + internal/api/server.go | 15 +- internal/api/source_readiness.go | 45 +++++ internal/api/source_readiness_test.go | 68 ++++++++ internal/api/testdata/routes.golden | 3 +- internal/store/account_mutation_lock.go | 3 + internal/store/annotation_source_admission.go | 16 +- internal/store/read_observation.go | 100 +++++++++++ internal/store/read_observation_test.go | 67 ++++++++ internal/store/source_readiness.go | 144 ++++++++++++++++ internal/store/source_readiness_test.go | 101 +++++++++++ package.json | 3 +- scripts/lib/release_checks.py | 105 ++++++++++++ scripts/lib/source_readiness.py | 53 ++++++ scripts/release-build-api-image.sh | 14 ++ scripts/release.py | 160 ++++++++++++++---- scripts/test_work_availability.py | 85 ++++++++++ scripts/verification-scopes.json | 30 ++++ tests/browser/initial-slice-confirmation.mjs | 26 ++- tests/browser/rework-boundary-replica.mjs | 23 ++- tests/release/test_release.py | 120 ++++++++++++- tests/release/test_release_build_api_image.py | 7 + tests/release/test_reliability_checks.py | 124 ++++++++++++++ 36 files changed, 1496 insertions(+), 77 deletions(-) create mode 100644 .scratch/reliability-guardrails/work.md create mode 100644 Dockerfile.api-runtime create mode 100644 docs/operations/reliability-guardrails.md create mode 100644 docs/release/api-runtime.json create mode 100644 internal/api/source_readiness.go create mode 100644 internal/api/source_readiness_test.go create mode 100644 internal/store/read_observation.go create mode 100644 internal/store/read_observation_test.go create mode 100644 internal/store/source_readiness.go create mode 100644 internal/store/source_readiness_test.go create mode 100644 scripts/lib/release_checks.py create mode 100644 scripts/lib/source_readiness.py create mode 100644 scripts/test_work_availability.py create mode 100644 tests/release/test_reliability_checks.py diff --git a/.gitea/workflows/web-ci.yml b/.gitea/workflows/web-ci.yml index 819e627a..ec2a4beb 100644 --- a/.gitea/workflows/web-ci.yml +++ b/.gitea/workflows/web-ci.yml @@ -7,7 +7,7 @@ name: web-ci # didn't boot the dev server. on: push: - branches: ['**'] + branches: ["**"] pull_request: jobs: @@ -41,7 +41,7 @@ jobs: - run: go test ./internal/api -run '^TestReviewPoolSearchBrowserReplica$' -count=1 -timeout=3m -v if: ${{ !cancelled() }} env: - REVIEW_POOL_BROWSER_TEST: '1' + REVIEW_POOL_BROWSER_TEST: "1" - uses: actions/upload-artifact@v4 if: always() with: @@ -69,3 +69,39 @@ jobs: - run: go test ./internal/store -run '^TestHumanPoolSearchMongo' -count=1 -timeout=3m -v env: POOL_LOCAL_MONGO_URI: mongodb://127.0.0.1:27018 + + work-availability: + # Current source/media paths require FFmpeg >=5.1 (-fps_mode). Keep the + # existing Ubuntu 22 checks intact and run this mandatory matrix on 24.04. + runs-on: ubuntu-24.04 + needs: pool-search-mongo + if: ${{ !cancelled() }} + services: + mongo: + image: mongo:8 + ports: + - 27018:27017 + options: >- + --health-cmd "mongosh --quiet --eval 'db.adminCommand({ping:1})'" + --health-interval 5s --health-timeout 5s --health-retries 12 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: sudo apt-get update && sudo apt-get install -y ffmpeg + - run: npx playwright install --with-deps chromium + - run: npm run test:work-availability + - uses: actions/upload-artifact@v4 + if: always() + with: + name: work-availability-logs + path: local-data/ + if-no-files-found: error + retention-days: 14 diff --git a/.scratch/reliability-guardrails/work.md b/.scratch/reliability-guardrails/work.md new file mode 100644 index 00000000..237aff8f --- /dev/null +++ b/.scratch/reliability-guardrails/work.md @@ -0,0 +1,20 @@ +# Reliability guardrails + +Base: dev 8d4b6726. User authorized local implementation of retrospective improvements. + +Scope: mandatory source-admission and annotation-preservation checks wired to CI/release; strict-mode readiness and explicit source-mode plans; private redacted failure logs and immutable check evidence; bounded request/DB/lock observations; reproducible runtime builds and reviewed combined image/config plans. + +Compatibility: keep current advisory policy, permissions, optimistic versions, annotations, legacy reads and historical release records. No production configuration/data changes, backfill, index/backup writes, push or deployment in this task. Existing unrelated admin pool test edits are protected. + +Verification: synthetic media and isolated local Mongo, existing baseline gates, release state-machine/CLI and security regression tests. No test may silently skip critical cases. Keep failures and unverified coverage visible. + +Acceptance: + +1. New releases require explicit effective source modes; a reviewed combined image/mode patch preserves all other configuration. Strict activation requires a complete, current deployment inventory under the candidate rules, without enqueueing validation or changing work. +2. Existing annotation preservation, submission/deletion races, restart recovery, permissions and legacy-data regressions become mandatory, with missing/skipped cases rejected. +3. The same gate exercises real local Mongo equivalence and advisory browser work/recovery in CI and release checks. +4. Local checks preserve bounded private redacted output and immutable evidence; browser failures retain unique artifacts. Reuse is explicit and limited to unchanged, unexpired successful pure checks. +5. Request-scoped numeric Mongo/query/lock observations reveal phase costs without customer data, extra queries or production explain. +6. Authoritative image builds pin runtime dependencies and record recipe provenance. Image/source-mode changes form one reviewable processor-before-API plan. Other production configuration, backups and data mutations remain outside this task. + +Limitations: inventory is point-in-time, not a data freeze or certificate; strict activation requires a controlled cutover window. Local tests do not prove production write acceptance, capacity or backup recovery. Remote CI awaits a separately authorized push. diff --git a/Dockerfile.api b/Dockerfile.api index ddebc818..eb120e9f 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -1,7 +1,8 @@ ARG DROBOTICFLOW_BUILD_COMMIT=unknown ARG DROBOTICFLOW_BUILD_TIME=unknown +ARG API_RUNTIME_IMAGE=debian:bookworm-slim -FROM node:22-alpine AS web-build +FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 AS web-build WORKDIR /app COPY package*.json ./ @@ -15,7 +16,7 @@ COPY src ./src COPY public ./public RUN npm run build:web -FROM golang:1.24-bookworm AS go-build +FROM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS go-build WORKDIR /src ARG GOPROXY=https://proxy.golang.org,direct @@ -26,23 +27,31 @@ COPY cmd ./cmd COPY internal ./internal RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-api ./cmd/api -FROM debian:bookworm-slim AS runtime +FROM ${API_RUNTIME_IMAGE} AS runtime +# Production release script supplies a pinned prepared runtime. Direct local/POC +# builds retain the original Debian setup unless explicitly opted in. +ARG API_RUNTIME_PREPARED=0 +USER root +WORKDIR /app +RUN if [ "$API_RUNTIME_PREPARED" = "1" ]; then \ + test -x /usr/bin/ffprobe && getent passwd 10001; \ + else \ + apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* && \ + groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow; \ + fi +# A compatibility runtime may be an older API image. Never retain its old app. +RUN rm -rf /app/dist /usr/local/bin/droboticflow-api +COPY --from=web-build --chown=10001:10001 /app/dist ./dist +COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api + +# Application identity must not invalidate the OS dependency layers. ARG DROBOTICFLOW_BUILD_COMMIT ARG DROBOTICFLOW_BUILD_TIME -WORKDIR /app -ENV PORT=3001 -ENV DIST_DIR=/app/dist +ENV PORT=3001 DIST_DIR=/app/dist ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT} ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME} LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT} LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME} - -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* -RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow - -COPY --from=web-build --chown=10001:10001 /app/dist ./dist -COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api - EXPOSE 3001 USER 10001:10001 CMD ["droboticflow-api"] diff --git a/Dockerfile.api-runtime b/Dockerfile.api-runtime new file mode 100644 index 00000000..f4531c89 --- /dev/null +++ b/Dockerfile.api-runtime @@ -0,0 +1,9 @@ +# Build/publish separately after approval; record this image's immutable digest +# in docs/release/api-runtime.json. No application or customer data is copied. +FROM docker.io/library/debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && \ + dpkg-query -W > /runtime-packages.txt && rm -rf /var/lib/apt/lists/* && \ + groupadd --system --gid 10001 droboticflow && \ + useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow +WORKDIR /app +USER 10001:10001 diff --git a/cmd/api/main.go b/cmd/api/main.go index 68b58bd1..457ae6ad 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -26,7 +26,7 @@ import ( func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) { return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) { - client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI)) + client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI).SetMonitor(store.ReadCommandMonitor())) if err != nil { return nil, err } diff --git a/docs/operations/reliability-guardrails.md b/docs/operations/reliability-guardrails.md new file mode 100644 index 00000000..754e8e80 --- /dev/null +++ b/docs/operations/reliability-guardrails.md @@ -0,0 +1,48 @@ +# 工作流程可靠性防护 + +本批修改把已存在的恢复测试和发布约束接入执行入口,不改变 advisory 的业务准入, +不改变权限、版本、领取、送检、审核或验收规则,不自动修改生产记录。 + +## 必跑回归与证据 + +`npm run test:work-availability` 分阶段运行:已有标注及送检失败保护、删除/送检并发和重启恢复; +隔离 Mongo 的旧数据/指纹/读取不补验等价检查;advisory 模式的完整工作台及返工浏览器流程。 +使用本机合成媒体和独立数据库,不使用公司数据。已知旧失败仍交由既有 Go/前端基线门禁比较。 +必需用例被跳过或未出现时失败;独立阶段在普通失败后继续,源码中途变化则停止并列出未验证阶段。 +CI 和发布策略均调用同一个入口,不能以只跑单元测试或 Pod Ready 代替业务验收。 + +每次运行在 `local-data/test-results/` 保存摘要和私有脱敏日志;浏览器在唯一运行目录保留失败截图、 +trace 和异常,不覆盖前一次失败。trace 仅用于合成夹具,不能对客户页面照搬。 +发布检查的显式证据复用范围限纯检查,失效和失败必须重跑,不能复用浏览器、数据库或性能结果。 + +## 源准入影响盘点 + +管理员可读取本组织清单,超级管理员才能读取部署范围清单;组织身份来自服务端会话。 +`projectId` 只能缩小组织查询,部署盘点不接受项目过滤。接口只返回按组织、阶段、状态分组的数量, +不返回正文,不创建任务或补验作业,每个进程同时最多一份盘点,Mongo 执行上限 8 秒。 +超时/取消返回不完整错误,不能把部分结果作为可启用依据。 + +严格规则与现有规则变化时,应先将候选以 advisory 发布,再盘点候选规则会阻断的存量数据。 +发布工具拒绝旧构建、局部范围、错误数据库、过期、被修改或仍含阻断条目的清单。 +盘点不提供事务级数据冻结:仍需在发布安全审查中确定受控切换窗口;无法保证时保持 advisory。 +它不伪造证书,也不意味着源内容已通过完整检查。 + +## 查询与写锁观察 + +复用服务端生成的 request ID,记录固定阶段的耗时、Mongo 命令成功/失败次数、累计驱动耗时和 +受限错误分类,以及写锁获取耗时。只对已有请求收集,不追加查询或自动执行 explain。 +日志使用路由模板,不记录查询词、任务 ID、正文、意见、凭证、Mongo 命令或响应体。 + +驱动耗时包含网络等待,不能当作 Mongo 服务端执行时间;多个命令累计时间也不等于请求墙钟耗时。 +连接池等待、扫描量、服务端执行计划和 EIP 带宽没有因此被测量,仍需单独只读诊断。 +此修改未优化查询或增加业务后台计算,不构成容量达标结论。 + +## 构建、发布与恢复边界 + +泛宇构建固定 Node/Go 构建阶段及运行依赖摘要,使用同一权威配方并记录来源。 +镜像和唯一允许的源模式可在同一审查计划顺序切换,其他配置、组件和数据仍另列授权。 +处理器失败时停止更新 API,记录部分完成状态,不自动重试、回退或清除证据。 + +本批未启用 BOS 版本控制、Mongo PITR 或备份策略,也未进行恢复演练。保留送检快照和失败输入的 +已有应用级保护不能代替存储备份;备份保留期、成本、权限与隔离恢复演练需单独形成可批准方案。 +生产写验收、完整并发容量和故障恢复演练均不包含在本地门禁的通过结论中。 diff --git a/docs/release/README.md b/docs/release/README.md index 470065cc..810fb1a7 100644 --- a/docs/release/README.md +++ b/docs/release/README.md @@ -16,9 +16,11 @@ 再用 `python3 scripts/release.py status ` 查看缺失证据与下一步。 历史聊天和“Pod 已 Ready”不代表业务验收完成。使用命令更新记录,保留失败和未验证项。 -首版只支持 **Fanyu 现有 API 与切片处理器的同一不可变镜像更新**,并同步容器的 -`DROBOTICFLOW_BUILD_COMMIT`。保留 ConfigMap、Secret、PVC、Service、资源和副本数。 -需要更改配置、其他组件或数据库时,单独设计并审查变更计划;首版不提供绕过参数。 +支持 **Fanyu 现有 API 与切片处理器的同一不可变镜像更新**,并同步容器的 +`DROBOTICFLOW_BUILD_COMMIT`。可通过 `prepare --source-mode advisory|strict` 将唯一指定的 +`ANNOTATION_SOURCE_VALIDATION_MODE` 一起纳入同一计划和审批;处理器先更新并就绪,再更新 API。 +保留 ConfigMap、Secret、PVC、Service、其他配置、资源和副本数。 +需要更改其他配置、组件或数据库时,单独设计并审查变更计划,不提供绕过参数。 2026-10-17 已部署的源索引基线也可做上述镜像更新:处理器必须精确匹配 fanyu-policy.json 引用的容量文件(2 副本、每副本 8 修订槽、4 GiB limit、 @@ -70,6 +72,11 @@ metadata 读取 digest,输出 `IMAGE=...@sha256:...` 供 `--image` 使用。 目标 server、context、namespace UID 从已核实的集群身份取得,不能从一次错误连接中直接认可。 工具检查 namespace UID 及 API server,namespace 名字相同也不能蒙混过关。 +构建使用权威 `Dockerfile.api` 和 [固定运行依赖](api-runtime.json),不再临时生成包装 Dockerfile。 +输出旁的 `.provenance.json` 记录源码、运行基础摘要和配方摘要。当前固定基础为既有已验证镜像, +复制新产物前移除旧应用;`Dockerfile.api-runtime` 提供后续专用基础的构建配方,发布它仍需单独授权。 +本地/其他环境直接构建保留原有 Debian 安装路径;泛宇发布脚本只接受不可变运行基础摘要。 + ```sh python3 scripts/release.py prepare --id \ --image \ @@ -95,7 +102,14 @@ python3 scripts/release.py check --cluster 不再把已替换的旧页面布局当成当前入口。返工浏览器夹具运行真实本地校验处理器,不能绕过源数据准入。 统一候选涉及的额外组件与待确认发布项见 [组件发布计划](recent-integration-components.md)。首版运行流程工具测试、 前端类型/构建及切片恢复定向回归,不冒充全套产品测试通过。任何失败阻断,不支持忽略开关。 -命令执行结果及输出摘要写入记录,不保存或回显原始输出;排查失败时单独重跑对应命令。 +命令执行结果、输入身份和日志摘要写入记录。本地检查的成功及失败输出先脱敏、限长,再以私有文件保存, +不会记录集群对象原文;日志路径写入记录。浏览器失败截图及 trace 使用唯一运行目录,仅限隔离合成夹具。 +`check --local --reuse-successful` 仅可复用一小时内输入、安装依赖、工具、环境和日志摘要均未变的 +白名单纯检查。默认重跑全部;浏览器、Mongo、性能与集群检查始终重新执行。 + +新增强制 `test:work-availability`,复用已有送检保护、删除/送检并发、重启恢复、权限、旧数据 +Mongo 等价和 advisory 真实工作台回归。缺用例、跳过、失败、源码中途变化均不能算通过。 +CI 独立作业需要 Ubuntu 24.04 runner、Mongo 8、FFmpeg 和 Playwright;远端 CI 未执行时须明确待验证。 2026-10-20 起,`build:web` 后强制运行 `test:production:boot`,实际启动生产 dist 并验证登录页、非空 root 和零运行时错误。该门禁使用本机桩 API,不连接公司;须安装 Playwright 浏览器或通过 `BROWSER_EXECUTABLE` 指定已有 Chromium。构建依赖必须按当前 lockfile 独立安装;共享依赖版本清单或带 alias 的组件夹具不能代替实际产物启动验证。 @@ -115,6 +129,16 @@ python3 scripts/release.py plan 显示旧/新镜像、构建标记、目标、带宽观察值、限制与回退说明,并生成计划摘要。 核对两个部署的修改范围。计划不更新 ConfigMap、Secret、存储或网络。 +新发布记录必须明确两个组件的源校验模式;未配置时禁止依赖代码默认严格模式。 +现有历史记录仍按原摘要核验。严格模式切换或严格准入关键规则变化须先以 advisory 部署候选, +再由超级管理员获取 `GET /api/plugins/annotation-qa/source-readiness?scope=deployment` 的只读影响清单, +用 `check --cluster --source-readiness-file <本地私有清单>` 绑定计划。 +清单必须覆盖同一数据库、命名空间、线上构建及旧数据截止配置,一小时内完整生成,所有条目均可准入; +组织/项目局部清单、本地夹具、缺项、阻断条目、过期或内容变更均不允许启用严格模式。 +清单不补验、不排队、不改标注,也不是源数据校验证书。它是时间点盘点,不会冻结业务数据; +`change-safety` 必须说明盘点至切换期间的新任务和在途变更如何受控,不满足时保持 advisory。 +当前模式不因获取清单而变化。详细边界见[可靠性防护](../operations/reliability-guardrails.md)。 + 部署前需要两份人工审查证据: - `image-provenance`:构建记录将本次源码 commit 与镜像 digest 对应,候选镜像的验证情况明确。 diff --git a/docs/release/api-runtime.json b/docs/release/api-runtime.json new file mode 100644 index 00000000..ad6255cf --- /dev/null +++ b/docs/release/api-runtime.json @@ -0,0 +1,5 @@ +{ + "schemaVersion": 1, + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/fanyu/droboticflow-api@sha256:6a060aa2865ebe0527377a9f22a0f1feb6b23d1f099985a99cce25f6981743d2", + "purpose": "Compatibility bootstrap: reuse approved production runtime; remove old application outputs. Replace with dedicated runtime only through an explicitly reviewed image publication." +} diff --git a/docs/release/fanyu-policy.json b/docs/release/fanyu-policy.json index 0c4e6857..289bd192 100644 --- a/docs/release/fanyu-policy.json +++ b/docs/release/fanyu-policy.json @@ -34,6 +34,7 @@ ["npm", "run", "test:browser:annotation-persistence"], ["go", "test", "./internal/deliveryindex", "-count=1"], ["go", "test", "./internal/worker", "-run", "TestCaptureCutover|Test.*V4Capture|TestDiscoverCaptureGroupsForWorkflow|TestClassificationAudit|TestInternalAuditFailure", "-count=1"], - ["go", "test", "./internal/api", "-run", "TestDurableAnnotationOutbox|TestSliceRecovery|TestReviewCannotMutateAnnotation|TestReviewDecision|Test.*ReviewDraft|TestReviewConfirmations|TestRouteSnapshot|Test.*Dictionary|TestProjectSliceNames|TestLegacyMaterializationWithoutPartScopedOriginalSelection|TestOptionalOriginalSource|TestAnnotationOriginalSource|TestClassificationBatch|TestWorkflowVersionActivation", "-count=1"] + ["go", "test", "./internal/api", "-run", "TestDurableAnnotationOutbox|TestSliceRecovery|TestReviewCannotMutateAnnotation|TestReviewDecision|Test.*ReviewDraft|TestReviewConfirmations|TestRouteSnapshot|Test.*Dictionary|TestProjectSliceNames|TestLegacyMaterializationWithoutPartScopedOriginalSelection|TestOptionalOriginalSource|TestAnnotationOriginalSource|TestClassificationBatch|TestWorkflowVersionActivation", "-count=1"], + ["python3", "scripts/test_work_availability.py"] ] } diff --git a/internal/api/diagnostic_trace.go b/internal/api/diagnostic_trace.go index 475bc308..af9a2383 100644 --- a/internal/api/diagnostic_trace.go +++ b/internal/api/diagnostic_trace.go @@ -2,12 +2,15 @@ package api import ( "crypto/rand" + "droboticflow/internal/store" "encoding/hex" + "encoding/json" "log" "net/http" "strings" "time" + "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" ) @@ -30,6 +33,8 @@ func diagnosticTrace(next http.Handler) http.Handler { return } id := hex.EncodeToString(entropy[:]) + ctx, observation := store.WithReadObservation(r.Context(), id) + r = r.WithContext(ctx) w.Header().Set("X-Request-ID", id) started := time.Now() wrapped := middleware.NewWrapResponseWriter(w, r.ProtoMajor) @@ -41,10 +46,19 @@ func diagnosticTrace(next http.Handler) http.Handler { } else if r.Header.Get("X-Diagnostic-Request") == "1" { kind = "diagnostic" } + metrics, _ := json.Marshal(observation.Snapshot()) if kind != "" { - log.Printf("request_trace id=%s kind=%s status=%d elapsed_ms=%d", id, kind, wrapped.Status(), elapsed.Milliseconds()) - } else if elapsed >= slowRequestLogThreshold { - log.Printf("request_trace id=%s kind=slow method=%s path=%s status=%d elapsed_ms=%d", id, r.Method, r.URL.Path, wrapped.Status(), elapsed.Milliseconds()) + log.Printf("request_trace id=%s kind=%s status=%d elapsed_ms=%d observations=%s", id, kind, wrapped.Status(), elapsed.Milliseconds(), metrics) + } else if elapsed >= slowRequestLogThreshold || wrapped.Status() >= 500 { + route := "/api/" + if rc := chi.RouteContext(r.Context()); rc != nil && rc.RoutePattern() != "" { + route = rc.RoutePattern() + } + reason := "slow" + if elapsed < slowRequestLogThreshold { + reason = "failed" + } + log.Printf("request_trace id=%s kind=%s method=%s path=%s status=%d elapsed_ms=%d observations=%s", id, reason, r.Method, route, wrapped.Status(), elapsed.Milliseconds(), metrics) } }) } diff --git a/internal/api/diagnostic_trace_test.go b/internal/api/diagnostic_trace_test.go index 4e2d88b4..3231ca34 100644 --- a/internal/api/diagnostic_trace_test.go +++ b/internal/api/diagnostic_trace_test.go @@ -2,7 +2,10 @@ package api import ( "bytes" + "droboticflow/internal/store" "encoding/json" + "github.com/go-chi/chi/v5" + "go.mongodb.org/mongo-driver/event" "log" "net/http" "net/http/httptest" @@ -100,3 +103,36 @@ func TestDiagnosticTaskSummaryIsScopedAndReadOnly(t *testing.T) { t.Fatalf("anonymous: %d", anonymous.Code) } } + +func TestDiagnosticTraceCorrelatesReadPhasesWithoutQueryOrResourceIDs(t *testing.T) { + var logs bytes.Buffer + old := log.Writer() + log.SetOutput(&logs) + defer log.SetOutput(old) + router := chi.NewRouter() + router.Use(diagnosticTrace) + router.Get("/api/tasks/{id}", func(w http.ResponseWriter, r *http.Request) { + if store.ReadRequestID(r.Context()) == "" { + t.Fatal("missing request context") + } + finish := store.ObserveReadPhase(r.Context(), "candidate_query") + finish() + store.ReadCommandMonitor().Failed(r.Context(), &event.CommandFailedEvent{Failure: "i/o timeout private-body"}) + w.WriteHeader(500) + }) + req := httptest.NewRequest("GET", "/api/tasks/private-id?search=secret-annotation", nil) + req.Header.Set("X-Request-ID", "caller-id") + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + text := logs.String() + for _, value := range []string{"private-id", "secret-annotation", "private-body", "caller-id"} { + if strings.Contains(text, value) { + t.Fatal("sensitive request information leaked") + } + } + for _, value := range []string{"/api/tasks/{id}", "candidate_query", "mongo_failures", "client_or_network_deadline", resp.Header().Get("X-Request-ID")} { + if !strings.Contains(text, value) { + t.Fatalf("missing diagnostic: %s", value) + } + } +} diff --git a/internal/api/human_loop.go b/internal/api/human_loop.go index 52c4f7a1..3f9a3157 100644 --- a/internal/api/human_loop.go +++ b/internal/api/human_loop.go @@ -410,7 +410,9 @@ func (server *server) prepareHumanPoolRead(writer http.ResponseWriter, request * } func (server *server) readHumanItemsList(writer http.ResponseWriter, request *http.Request) (*humanItemListPage, *http.Request) { + finishPrepare := store.ObserveReadPhase(request.Context(), "prepare") pool, query, limit, request, ok := server.prepareHumanPoolRead(writer, request) + finishPrepare() if !ok { return nil, request } @@ -429,12 +431,16 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht query.Limit = fetchSize candidateQuery := query candidateQuery.PoolCandidateProjection = true + finishQuery := store.ObserveReadPhase(request.Context(), "candidate_query") items, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), candidateQuery) + finishQuery() if err != nil { renderError(writer, http.StatusInternalServerError, err) return nil, request } + finishContext := store.ObserveReadPhase(request.Context(), "list_context") listCtx, listErr := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, items) + finishContext() if listErr != nil { renderError(writer, http.StatusInternalServerError, listErr) return nil, request @@ -446,7 +452,9 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht return nil, request } } + finishHydrate := store.ObserveReadPhase(request.Context(), "hydrate") items, err = server.hydrateHumanPoolPage(request, items, pool, query, limit-len(filtered), batchByID) + finishHydrate() if err != nil { renderError(writer, http.StatusInternalServerError, err) return nil, request diff --git a/internal/api/human_pool_reads.go b/internal/api/human_pool_reads.go index 99a205f3..a8a2c252 100644 --- a/internal/api/human_pool_reads.go +++ b/internal/api/human_pool_reads.go @@ -144,6 +144,8 @@ func (s *server) handleHumanPoolSummaries(w http.ResponseWriter, r *http.Request if page == nil { return } + finishResponse := store.ObserveReadPhase(r.Context(), "response") + defer finishResponse() summaries := make([]humanPoolSummary, 0, len(page.Items)) for _, item := range page.Items { summaries = append(summaries, summarizeHumanPoolItem(item)) diff --git a/internal/api/routes_annotation.go b/internal/api/routes_annotation.go index d3f003c3..fc431366 100644 --- a/internal/api/routes_annotation.go +++ b/internal/api/routes_annotation.go @@ -6,6 +6,7 @@ import "github.com/go-chi/chi/v5" // plugin's templates and tasks (plus their deprecated /api/annotation-* // aliases), and the annotation label dictionaries. func (server *server) mountAnnotationRoutes(router chi.Router) { + router.Get("/api/plugins/annotation-qa/source-readiness", server.handleSourceAdmissionReadiness) router.Get("/api/data-collection-tasks", server.handleCollectionTasksList) router.Get("/api/data-collection-tasks/{id}", server.handleCollectionTaskGet) router.Get("/api/annotation-templates", deprecatedAnnotationEndpoint(server.handleAnnotationTemplatesList)) diff --git a/internal/api/server.go b/internal/api/server.go index 595a3a5a..b33c5615 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -24,6 +24,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "unicode/utf8" @@ -49,6 +50,7 @@ type validationIssue struct { } type server struct { + sourceReadinessRunning atomic.Bool outputOnce sync.Once outputReader *outputsnapshot.Reader humanEvidenceIndexes humanEvidenceCache @@ -1437,11 +1439,16 @@ func invalidConfigurationHandler(err error) http.Handler { } func (server *server) serviceMetadata() map[string]any { + mode := "strict" + if store.AnnotationSourceAuditAdvisory() { + mode = "advisory" + } return map[string]any{ - "service": "droboticflow-api", - "buildCommit": server.buildCommit, - "buildTime": server.buildTime, - "image": server.image, + "sourceAdmissionMode": mode, + "service": "droboticflow-api", + "buildCommit": server.buildCommit, + "buildTime": server.buildTime, + "image": server.image, } } diff --git a/internal/api/source_readiness.go b/internal/api/source_readiness.go new file mode 100644 index 00000000..5fd2cdf1 --- /dev/null +++ b/internal/api/source_readiness.go @@ -0,0 +1,45 @@ +package api + +import ( + "context" + "net/http" + "time" + + "droboticflow/internal/store" +) + +func (s *server) handleSourceAdmissionReadiness(w http.ResponseWriter, r *http.Request) { + principal, allowed := s.productionAdmin(w, r) + if !allowed { + return + } + db, ok := s.store.(store.SourceReadinessStore) + if !ok { + renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source readiness reporting unavailable"}) + return + } + w.Header().Set("Cache-Control", "no-store") + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + org, project := s.requestOrganizationID(r), r.URL.Query().Get("projectId") + if r.URL.Query().Get("scope") == "deployment" { + if !principal.User.IsSuperAdmin || project != "" { + renderJSON(w, http.StatusForbidden, map[string]string{"error": "deployment readiness requires super administrator and no project filter"}) + return + } + org = "" + } + if !s.sourceReadinessRunning.CompareAndSwap(false, true) { + renderJSON(w, http.StatusTooManyRequests, map[string]string{"error": "source readiness report already running"}) + return + } + defer s.sourceReadinessRunning.Store(false) + report, err := db.SourceAdmissionReadiness(ctx, org, project) + if err != nil { + renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source readiness report incomplete", "requestId": store.ReadRequestID(r.Context())}) + return + } + report.Namespace = s.k8sNamespace + report.BuildCommit = s.buildCommit + renderJSON(w, http.StatusOK, report) +} diff --git a/internal/api/source_readiness_test.go b/internal/api/source_readiness_test.go new file mode 100644 index 00000000..e656224e --- /dev/null +++ b/internal/api/source_readiness_test.go @@ -0,0 +1,68 @@ +package api + +import ( + "encoding/json" + "net/http" + "reflect" + "testing" + + "droboticflow/internal/store" +) + +func TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate(t *testing.T) { + t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory") + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "") + f := newCombinedDraftFixture(t) + missingTask := f.task + missingTask.ID += "-missing-proof" + if err := f.store.UpsertAnnotationTask(f.ctx, missingTask); err != nil { + t.Fatal(err) + } + missingItem := f.item + missingItem.ID += "-missing-proof" + missingItem.LegacyTaskID = missingTask.ID + missingItem.ProcessingJob = nil + if err := f.store.UpsertHumanItem(f.ctx, missingItem); err != nil { + t.Fatal(err) + } + before, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + var jobs store.AnnotationProcessingStore = f.store + jobBefore, _ := jobs.GetAnnotationIndexJob(f.ctx, f.task.ID) + path := "/api/plugins/annotation-qa/source-readiness" + denied := f.request(http.MethodGet, path, f.annotator, nil) + if denied.Code != 403 { + t.Fatalf("non-admin readiness access: %d", denied.Code) + } + denied = f.request(http.MethodGet, path+"?scope=deployment", f.admin, nil) + if denied.Code != 403 { + t.Fatal("ordinary org admin got deployment-wide inventory") + } + r := f.request(http.MethodGet, path+"?organizationId=other", f.admin, nil) + if r.Code != 200 { + t.Fatalf("readiness: %d %s", r.Code, r.Body.String()) + } + var report store.SourceReadiness + if err := json.Unmarshal(r.Body.Bytes(), &report); err != nil { + t.Fatal(err) + } + if !report.Complete || report.TargetMode != "strict" || report.Scope != "organization" || report.OrganizationID != f.task.OrganizationID || len(report.Rows) != 1 || report.Rows[0].Unverified != 1 { + t.Fatalf("incorrect impact report: %+v", report) + } + super := f.admin + super.IsSuperAdmin = true + r = f.request(http.MethodGet, path+"?scope=deployment", super, nil) + if r.Code != 200 { + t.Fatalf("superadmin inventory: %d", r.Code) + } + if err := json.Unmarshal(r.Body.Bytes(), &report); err != nil { + t.Fatal(err) + } + if report.Scope != "deployment" || report.OrganizationID != "" { + t.Fatal("wrong full deployment scope") + } + after, _ := f.store.GetHumanItem(f.ctx, f.item.ID) + jobAfter, _ := jobs.GetAnnotationIndexJob(f.ctx, f.task.ID) + if !reflect.DeepEqual(before, after) || !reflect.DeepEqual(jobBefore, jobAfter) { + t.Fatal("inventory changed task or audit queue") + } +} diff --git a/internal/api/testdata/routes.golden b/internal/api/testdata/routes.golden index bc323526..62be43f4 100644 --- a/internal/api/testdata/routes.golden +++ b/internal/api/testdata/routes.golden @@ -1,7 +1,7 @@ # Generated by TestRouteSnapshot; see routes_snapshot_test.go. # access: reach = endpoint reached; otherwise the status the middleware chain returned. global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache] -routes=381 +routes=382 DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403 DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403 @@ -121,6 +121,7 @@ GET /api/plugins/annotation-qa/admin-pool | mw=[] | (*server).handleAdminAnnotat GET /api/plugins/annotation-qa/claim-count | mw=[] | (*server).handleAnnotationClaimCount-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/claim-projects | mw=[] | (*server).handleAnnotationClaimProjects-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/sidebar | mw=[] | (*server).handleAnnotationSidebar-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach +GET /api/plugins/annotation-qa/source-readiness | mw=[] | (*server).handleSourceAdmissionReadiness-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/tasks | mw=[] | (*server).handleAnnotationTasksList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/tasks/{id} | mw=[] | (*server).handleAnnotationTaskGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach GET /api/plugins/annotation-qa/tasks/{id}/frame-timeline | mw=[] | (*server).handleAnnotationTaskFrameTimelineGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach diff --git a/internal/store/account_mutation_lock.go b/internal/store/account_mutation_lock.go index 05d10756..3b664b8e 100644 --- a/internal/store/account_mutation_lock.go +++ b/internal/store/account_mutation_lock.go @@ -61,6 +61,8 @@ func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, op // for callers without this proof), preserve uncertain-write lease semantics. func (s *MongoStore) withAccountMutationLockBeforeOperation(ctx context.Context, key string, operation func(context.Context) (any, error), operationStarted *bool) (result any, err error) { startedAt := time.Now() + finishWait := ObserveReadPhase(ctx, "lock_acquire") + defer finishWait() diagnose := func(event string) { digest := sha256.Sum256([]byte(key)) log.Printf("[account-mutation-lock] event=%s resource=%s key_hash=%x elapsed_ms=%d", event, strings.SplitN(key, ":", 2)[0], digest[:6], time.Since(startedAt).Milliseconds()) @@ -111,6 +113,7 @@ func (s *MongoStore) withAccountMutationLockBeforeOperation(ctx context.Context, case <-time.After(25 * time.Millisecond): } } + finishWait() // Renew well inside the lease window (a fixed fraction of it) so normal // scheduling jitter never lets a live owner's lease lapse; only an // owner that stops renewing entirely — because it's gone — loses the diff --git a/internal/store/annotation_source_admission.go b/internal/store/annotation_source_admission.go index 81ebe4c0..8205e8c7 100644 --- a/internal/store/annotation_source_admission.go +++ b/internal/store/annotation_source_admission.go @@ -124,6 +124,10 @@ func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain if AnnotationSourceAuditAdvisory() { return AnnotationSourceTaskMatches(task, item) && !AnnotationSourceUnavailable(item) } + return annotationItemSourceJobsReadyStrict(task, item, timeline, source) +} + +func annotationItemSourceJobsReadyStrict(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool { if LegacyAnnotationSourceCompatible(task, item, timeline, source) { return true } @@ -193,13 +197,17 @@ func AnnotationSourceAdmissionStatus(task domain.AnnotationTaskRecord, item doma } return "" } + return annotationSourceAdmissionStatusStrict(task, item, timeline, source) +} + +func annotationSourceAdmissionStatusStrict(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) string { if task.ID == "" { return "unverified" } if AnnotationSourceJobSuperseded(item, timeline) { source = nil } - if AnnotationItemSourceJobsReady(task, item, timeline, source) { + if annotationItemSourceJobsReadyStrict(task, item, timeline, source) { return "" } for _, job := range []*domain.AnnotationIndexJob{timeline, source} { @@ -231,7 +239,11 @@ func AnnotationSourceAdmissionStatus(task domain.AnnotationTaskRecord, item doma // All joins use existing unique id indexes and project only admission fields. // Retain caller fields so administrative exception pages can display blocked rows. func annotationSourceAdmissionStatusPipeline() mongo.Pipeline { - if AnnotationSourceAuditAdvisory() { + return annotationSourceAdmissionStatusPipelineForMode(AnnotationSourceAuditAdvisory()) +} + +func annotationSourceAdmissionStatusPipelineForMode(advisory bool) mongo.Pipeline { + if advisory { blocked := bson.M{"$and": bson.A{ bson.M{"$in": bson.A{"$currentStageId", bson.A{"annotation_combined", "semantic_annotation", "split_annotation"}}}, bson.M{"$eq": bson.A{"$processingJob.status", "failed"}}, diff --git a/internal/store/read_observation.go b/internal/store/read_observation.go new file mode 100644 index 00000000..463da9bb --- /dev/null +++ b/internal/store/read_observation.go @@ -0,0 +1,100 @@ +package store + +import ( + "context" + "strings" + "sync" + "time" + + "go.mongodb.org/mongo-driver/event" +) + +type readObservationKey struct{} + +// ReadObservation contains only durations and counters, never commands, replies, +// resource identifiers or credentials. It lives for one request, not in a cache. +type ReadObservation struct { + mu sync.Mutex + id string + phases map[string]time.Duration + commands int + failures int + mongo time.Duration + errorKinds map[string]int +} + +func WithReadObservation(ctx context.Context, id string) (context.Context, *ReadObservation) { + o := &ReadObservation{id: id, phases: map[string]time.Duration{}, errorKinds: map[string]int{}} + return context.WithValue(ctx, readObservationKey{}, o), o +} + +func ReadRequestID(ctx context.Context) string { + if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok { + return o.id + } + return "" +} + +func ObserveReadPhase(ctx context.Context, name string) func() { + o, ok := ctx.Value(readObservationKey{}).(*ReadObservation) + if !ok { + return func() {} + } + // Fixed vocabulary prevents arbitrary customer values becoming log keys. + switch name { + case "prepare", "candidate_query", "list_context", "hydrate", "response", "lock_acquire": + default: + return func() {} + } + start := time.Now() + var once sync.Once + return func() { once.Do(func() { o.mu.Lock(); defer o.mu.Unlock(); o.phases[name] += time.Since(start) }) } +} + +func (o *ReadObservation) Snapshot() map[string]any { + o.mu.Lock() + defer o.mu.Unlock() + phases := map[string]int64{} + for k, v := range o.phases { + phases[k] = v.Milliseconds() + } + kinds := map[string]int{} + for k, v := range o.errorKinds { + kinds[k] = v + } + return map[string]any{"phases_ms": phases, "mongo_commands": o.commands, "mongo_failures": o.failures, "mongo_ms": o.mongo.Milliseconds(), "mongo_error_kinds": kinds} +} + +// Command duration includes driver/network time, not just server execution. +// It is not connection checkout wait or docs/keys examined. +// Those require independent evidence; never infer them from elapsed time. +func ReadCommandMonitor() *event.CommandMonitor { + record := func(ctx context.Context, duration int64, failed bool) { + if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok { + o.mu.Lock() + defer o.mu.Unlock() + o.commands++ + o.mongo += time.Duration(duration) + if failed { + o.failures++ + } + } + } + return &event.CommandMonitor{ + Succeeded: func(ctx context.Context, e *event.CommandSucceededEvent) { record(ctx, e.DurationNanos, false) }, + Failed: func(ctx context.Context, e *event.CommandFailedEvent) { + record(ctx, e.DurationNanos, true) + if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok { + kind := "driver_error" + if strings.Contains(e.Failure, "MaxTimeMSExpired") { + kind = "server_deadline" + } else if strings.Contains(e.Failure, "deadline exceeded") || strings.Contains(e.Failure, "i/o timeout") { + kind = "client_or_network_deadline" + } + o.mu.Lock() + o.errorKinds[kind]++ + o.mu.Unlock() + } + }, + } +} diff --git a/internal/store/read_observation_test.go b/internal/store/read_observation_test.go new file mode 100644 index 00000000..639b38b9 --- /dev/null +++ b/internal/store/read_observation_test.go @@ -0,0 +1,67 @@ +package store + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "go.mongodb.org/mongo-driver/event" +) + +func TestReadObservationIsPrivateConcurrentAndRequestScoped(t *testing.T) { + ctx, o := WithReadObservation(context.Background(), "server-generated") + monitor := ReadCommandMonitor() + var wg sync.WaitGroup + for range 20 { + wg.Add(1) + go func() { + defer wg.Done() + end := ObserveReadPhase(ctx, "candidate_query") + defer end() + monitor.Succeeded(ctx, &event.CommandSucceededEvent{CommandFinishedEvent: event.CommandFinishedEvent{DurationNanos: int64(time.Millisecond), CommandName: "find"}}) + }() + } + wg.Wait() + monitor.Failed(ctx, &event.CommandFailedEvent{CommandFinishedEvent: event.CommandFinishedEvent{DurationNanos: int64(time.Millisecond)}, Failure: "mongodb://password-secret@host private-customer-body"}) + ObserveReadPhase(ctx, "customer-should-not-be-key")() + b, _ := json.Marshal(o.Snapshot()) + if strings.Contains(string(b), "secret") || strings.Contains(string(b), "customer") || strings.Contains(string(b), "find") { + t.Fatal("command contents leaked") + } + if o.Snapshot()["mongo_commands"] != 21 || o.Snapshot()["mongo_failures"] != 1 { + t.Fatal("counter mismatch") + } + if ReadRequestID(ctx) != "server-generated" || ReadRequestID(context.Background()) != "" { + t.Fatal("request context leaked") + } + _, other := WithReadObservation(context.Background(), "other") + if other.Snapshot()["mongo_commands"] != 0 { + t.Fatal("cross request counters") + } +} + +func TestReadObservationConcurrentFailureSnapshots(t *testing.T) { + ctx, o := WithReadObservation(context.Background(), "generated") + monitor := ReadCommandMonitor() + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for range 200 { + monitor.Failed(ctx, &event.CommandFailedEvent{Failure: "context deadline exceeded"}) + } + }() + go func() { + defer wg.Done() + for range 200 { + _, _ = json.Marshal(o.Snapshot()) + } + }() + wg.Wait() + if o.Snapshot()["mongo_failures"] != 200 { + t.Fatal("missing failures") + } +} diff --git a/internal/store/source_readiness.go b/internal/store/source_readiness.go new file mode 100644 index 00000000..0dc7fb6a --- /dev/null +++ b/internal/store/source_readiness.go @@ -0,0 +1,144 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "os" + "sort" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type SourceReadinessRow struct { + OrganizationID string `json:"organizationId" bson:"organizationId"` + Stage string `json:"stage" bson:"stage"` + Status string `json:"status" bson:"status"` + Total int64 `json:"total" bson:"total"` + Allowed int64 `json:"allowed" bson:"allowed"` + Unverified int64 `json:"unverified" bson:"unverified"` + Pending int64 `json:"pending" bson:"pending"` + Failed int64 `json:"failed" bson:"failed"` +} + +type SourceReadiness struct { + CurrentMode string `json:"currentMode"` + Scope string `json:"scope"` + Database string `json:"database"` + Schema string `json:"schema"` + OrganizationID string `json:"organizationId"` + ProjectID string `json:"projectId"` + Namespace string `json:"namespace"` + BuildCommit string `json:"buildCommit"` + GeneratedAt string `json:"generatedAt"` + TargetMode string `json:"targetMode"` + LegacySourceCutoff string `json:"legacySourceCutoff"` + Complete bool `json:"complete"` + Rows []SourceReadinessRow `json:"rows"` +} + +type SourceReadinessStore interface { + SourceAdmissionReadiness(context.Context, string, string) (*SourceReadiness, error) +} + +func sourceReadiness(org, project string, rows []SourceReadinessRow) *SourceReadiness { + sort.Slice(rows, func(i, j int) bool { + if rows[i].OrganizationID != rows[j].OrganizationID { + return rows[i].OrganizationID < rows[j].OrganizationID + } + if rows[i].Stage != rows[j].Stage { + return rows[i].Stage < rows[j].Stage + } + return rows[i].Status < rows[j].Status + }) + mode := "strict" + if AnnotationSourceAuditAdvisory() { + mode = "advisory" + } + scope := "organization" + if org == "" { + scope = "deployment" + } + return &SourceReadiness{CurrentMode: mode, Scope: scope, Database: "memory", Schema: "droboticflow.source-readiness/v1", OrganizationID: org, ProjectID: project, GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), TargetMode: "strict", LegacySourceCutoff: os.Getenv("ANNOTATION_LEGACY_SOURCE_CUTOFF"), Complete: true, Rows: rows} +} + +// This is an explicit administrator inventory, never part of normal work reads. +// Missing proof is reported, not repaired or enqueued. No task body is returned. +func (s *MemoryStore) SourceAdmissionReadiness(ctx context.Context, org, project string) (*SourceReadiness, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + grouped := map[string]*SourceReadinessRow{} + for _, item := range s.humanItems { + if err := ctx.Err(); err != nil { + return nil, err + } + if org != "" && item.OrganizationID != org || item.PluginID != "annotation-qa" || item.LegacyTaskID == "" || project != "" && item.ProjectID != project { + continue + } + key := item.OrganizationID + "\x00" + item.CurrentStageID + "\x00" + string(item.Status) + row := grouped[key] + if row == nil { + row = &SourceReadinessRow{OrganizationID: item.OrganizationID, Stage: item.CurrentStageID, Status: string(item.Status)} + grouped[key] = row + } + row.Total++ + task := s.annotationTasks[item.LegacyTaskID] + t, tok := s.annotationIndexJobs[item.LegacyTaskID] + j, jok := s.annotationIndexJobs[sourceJobForItem(item)] + var tp, sp *domain.AnnotationIndexJob + if tok { + tp = &t + } + if jok { + sp = &j + } + switch annotationSourceAdmissionStatusStrict(task, item, tp, sp) { + case "": + row.Allowed++ + case "pending": + row.Pending++ + case "failed": + row.Failed++ + default: + row.Unverified++ + } + } + rows := []SourceReadinessRow{} + for _, v := range grouped { + rows = append(rows, *v) + } + return sourceReadiness(org, project, rows), nil +} + +func (s *MongoStore) SourceAdmissionReadiness(ctx context.Context, org, project string) (*SourceReadiness, error) { + filter := bson.M{"pluginId": "annotation-qa", "legacyTaskId": bson.M{"$type": "string", "$ne": ""}} + if org != "" { + filter["organizationId"] = org + } + if project != "" { + filter["projectId"] = project + } + pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: annotationSourceAdmissionProjection()}}} + pipeline = append(pipeline, annotationSourceAdmissionStatusPipelineForMode(false)...) + count := func(status string) bson.M { + return bson.M{"$sum": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{"$_sourceAdmissionStatus", status}}, 1, 0}}} + } + pipeline = append(pipeline, bson.D{{Key: "$group", Value: bson.M{"_id": bson.M{"organizationId": "$organizationId", "stage": "$currentStageId", "status": "$status"}, "total": bson.M{"$sum": 1}, "allowed": count(""), "pending": count("pending"), "failed": count("failed"), "unverified": count("unverified")}}}, bson.D{{Key: "$project", Value: bson.M{"_id": 0, "organizationId": "$_id.organizationId", "stage": "$_id.stage", "status": "$_id.status", "total": 1, "allowed": 1, "pending": 1, "failed": 1, "unverified": 1}}}) + cursor, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetMaxTime(8*time.Second)) + if err != nil { + return nil, err + } + defer cursor.Close(ctx) + rows := []SourceReadinessRow{} + if err = cursor.All(ctx, &rows); err != nil { + return nil, err + } + report := sourceReadiness(org, project, rows) + report.Database = s.db.Name() + return report, nil +} diff --git a/internal/store/source_readiness_test.go b/internal/store/source_readiness_test.go new file mode 100644 index 00000000..a118ddec --- /dev/null +++ b/internal/store/source_readiness_test.go @@ -0,0 +1,101 @@ +package store + +import ( + "context" + "fmt" + "os" + "reflect" + "testing" + "time" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func TestSourceReadinessMongoParityReadOnly(t *testing.T) { + if os.Getenv("SOURCE_ADMISSION_LOCAL_MONGO") != "1" { + t.Skip("isolated loopback Mongo opt-in") + } + t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory") + t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(3*time.Second)) + if err != nil { + t.Fatal(err) + } + defer c.Disconnect(context.Background()) + db := c.Database(fmt.Sprintf("source_readiness_%d", time.Now().UnixNano())) + defer db.Drop(context.Background()) + ms := NewMongoStore(db) + mem := NewMemoryStore() + mem.annotationIndexJobs = map[string]domain.AnnotationIndexJob{} + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + before := map[string]*domain.HumanItemRecord{} + for n, state := range []string{"", "queued", "failed", "completed", "foreign"} { + id := fmt.Sprintf("i%d", n) + org := "o" + if state == "foreign" { + org = "foreign" + } + task := domain.AnnotationTaskRecord{ID: "t" + id, OrganizationID: org, ProjectID: "p", RunID: id} + item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", RunID: id, PluginID: "annotation-qa", LegacyTaskID: task.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemClaimed, AssignedUserID: "owner", Version: 1, Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": map[string]any{"description": "retained"}}}} + must(ms.UpsertAnnotationTask(ctx, task)) + must(mem.UpsertAnnotationTask(ctx, task)) + must(ms.UpsertHumanItem(ctx, item)) + must(mem.UpsertHumanItem(ctx, item)) + if state != "" && state != "foreign" { + job := domain.AnnotationIndexJob{ID: task.ID, OrganizationID: org, ProjectID: "p", RunID: id, Status: state} + if state == "completed" { + job.SourceFingerprint = "fp" + job.RevalidateSource = true + } + _, err = db.Collection("annotationIndexJobs").InsertOne(ctx, job) + must(err) + mem.annotationIndexJobs[task.ID] = job + } + before[id], err = ms.GetHumanItem(ctx, id) + must(err) + } + for _, org := range []string{"o", ""} { + a, err := ms.SourceAdmissionReadiness(ctx, org, "") + must(err) + b, err := mem.SourceAdmissionReadiness(ctx, org, "") + must(err) + a.GeneratedAt = "" + b.GeneratedAt = "" + a.Database = "" + b.Database = "" + if !reflect.DeepEqual(a, b) { + t.Fatalf("Mongo/Go strict readiness mismatch: %+v / %+v", a, b) + } + total := int64(0) + for _, r := range a.Rows { + total += r.Total + if r.Total != r.Allowed+r.Pending+r.Unverified+r.Failed { + t.Fatal("counts do not reconcile") + } + } + if org == "o" && total != 4 || org == "" && total != 5 { + t.Fatal("readiness scope mismatch") + } + } + for id, before := range before { + after, err := ms.GetHumanItem(ctx, id) + must(err) + if !reflect.DeepEqual(before, after) { + t.Fatal("inventory changed saved annotations or ownership") + } + } + canceled, stop := context.WithCancel(ctx) + stop() + if r, err := ms.SourceAdmissionReadiness(canceled, "", ""); err == nil || r != nil { + t.Fatal("canceled report claimed completeness") + } +} diff --git a/package.json b/package.json index 4aabe311..150de3b9 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,8 @@ "predev:api": "npm run hooks:install", "predev:web": "npm run hooks:install", "predev:worker": "npm run hooks:install", - "predev:api:replica": "npm run hooks:install" + "predev:api:replica": "npm run hooks:install", + "test:work-availability": "python3 scripts/test_work_availability.py" }, "dependencies": { "@tailwindcss/vite": "^4.1.14", diff --git a/scripts/lib/release_checks.py b/scripts/lib/release_checks.py new file mode 100644 index 00000000..d3cf6263 --- /dev/null +++ b/scripts/lib/release_checks.py @@ -0,0 +1,105 @@ +"""Private, bounded local-check evidence. Never log kubectl payloads here.""" +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import time +import uuid +SENSITIVE = '[A-Za-z0-9_-]{0,96}(?:password|passwd|secret|token|authorization|cookie|credential|access[_-]?key|secret[_-]?key|phone)[A-Za-z0-9_-]{0,96}|(?:BOS|BCE|AWS)[_-](?:AK|SK)|AK|SK' + +def redact(text): + """Remove credential-bearing fields and URLs before persisting bounded local test output.""" + text = '\n'.join(line if len(line) <= 65536 else '' for line in text.splitlines()) + text = re.sub('(?is)(?', text) + text = re.sub('(?i)\\bBearer\\s+\\S+', 'Bearer ', text) + text = re.sub('(?i)\\b(?:https?|mongodb(?:\\+srv)?)://[^\\s\\"\\\'<>]+', '', text) + lines = ['' if re.search('(?i)\\b(?:' + SENSITIVE + ')\\b\\s*:', line) else line for line in text.splitlines()] + text = '\n'.join(lines) + limit = 2 * 1024 * 1024 + if len(text) > limit: + text = text[:limit // 2] + '\n\n' + text[-limit // 2:] + return text + +def write_log(folder, argv, output, result): + folder = Path(folder) + folder.mkdir(parents=True, exist_ok=True, mode=448) + path = folder / ('check-' + uuid.uuid4().hex + '.log') + data = redact(json.dumps({'argv': argv, 'result': result}) + '\n' + output) + '\n' + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 384), 'w') as f: + f.write(data) + return {'log': str(path.resolve()), 'logSHA256': hashlib.sha256(path.read_bytes()).hexdigest()} + +def check_environment(): + """Local synthetic checks inherit no company credentials or application/test switches.""" + return {k: v for k, v in os.environ.items() if not re.search('(?i)(password|secret|token|credential|access_key|secret_key|^BOS_|^BCE_|^AWS_|(^|_)MONGO(_|$)|(_URI|_URL)$|^KUBECONFIG$|^ANNOTATION_|^DROBOTICFLOW_|^DEPLOY_ENV$|^UPDATE_|^REGENERATE_|_TEST$|_TEST_|^TEST_.*MONGO)', k)} + +def source_fingerprint(root): + """Hash the complete non-ignored source tree; unavailable Git identity is never reusable.""" + root = Path(root) + h = hashlib.sha256() + try: + names = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=root, stderr=subprocess.DEVNULL).decode().split('\x00') + except (OSError, subprocess.SubprocessError): + return None + for name in sorted(set(filter(None, names))): + path = root / name + h.update(name.encode()) + if path.is_symlink(): + h.update(os.readlink(path).encode()) + elif path.is_file(): + h.update(path.read_bytes()) + else: + h.update(b'') + return h.hexdigest() + +def identity(root, commit, policy_hash): + """Bind source, installed dependency contents, tool versions and a hash of the sanitized environment. Values and dependency contents are not persisted.""" + root = Path(root) + h = hashlib.sha256() + dependencies = root / 'node_modules' + if dependencies.is_dir(): + visited = set() + for base, dirs, files in os.walk(dependencies, followlinks=True): + stat = Path(base).stat() + key = (stat.st_dev, stat.st_ino) + if key in visited: + dirs[:] = [] + continue + visited.add(key) + h.update(str(Path(base).relative_to(dependencies)).encode()) + if Path(base).is_symlink(): + h.update(os.readlink(base).encode()) + dirs[:] = sorted((d for d in dirs if d not in ('.cache', '.vite', '.vite-temp'))) + for name in sorted(files): + p = Path(base) / name + h.update(str(p.relative_to(dependencies)).encode()) + if p.is_symlink(): + h.update(os.readlink(p).encode()) + else: + with p.open('rb') as f: + for block in iter(lambda: f.read(1024 * 1024), b''): + h.update(block) + else: + h.update(b'no-installed-node-dependencies') + versions = {} + for name, args in [('python', ['python3', '--version']), ('node', ['node', '--version']), ('go', ['go', 'version']), ('ffmpeg', ['ffmpeg', '-version'])]: + try: + versions[name] = subprocess.check_output(args, cwd=root, stderr=subprocess.STDOUT, timeout=10).decode().splitlines()[0] + except (OSError, subprocess.SubprocessError): + versions[name] = 'unavailable' + env_hash = hashlib.sha256(json.dumps(check_environment(), sort_keys=True).encode()).hexdigest() + source = source_fingerprint(root) + complete = bool(source) and dependencies.is_dir() and all((v != 'unavailable' for v in versions.values())) + return {'complete': complete, 'sourceTree': source, 'commit': commit, 'policyHash': policy_hash, 'dependencies': h.hexdigest(), 'tools': versions, 'environment': env_hash} + +def reusable(entry, expected, argv): + """Only explicitly listed pure checks may reuse unchanged, unexpired and intact success evidence. Browser, Mongo and timing checks always rerun.""" + pure = argv[:3] == ['npm', 'run', 'lint:web'] or argv[:3] == ['npm', 'run', 'test:annotation:persistence'] or argv[:3] == ['npm', 'run', 'test:review:decisions'] or (argv[:3] == ['npm', 'run', 'test:review:drafts']) or (argv[:3] == ['python3', '-m', 'unittest']) + if not expected.get('complete') or not pure or entry.get('identity') != expected or (entry.get('result') != 'passed'): + return False + if not 0 <= time.time() - entry.get('finishedUnix', 0) <= 3600: + return False + path = Path(entry.get('log', '')) + return path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == entry.get('logSHA256') diff --git a/scripts/lib/source_readiness.py b/scripts/lib/source_readiness.py new file mode 100644 index 00000000..31f24c3c --- /dev/null +++ b/scripts/lib/source_readiness.py @@ -0,0 +1,53 @@ +"""Validate an administrator's metadata-only strict-admission impact report.""" +import datetime as dt +import hashlib +import json +import re +from pathlib import Path + +def validate_report(path, *, namespace, database, cutoff, live_commit, now=None): + path = Path(path) + if path.stat().st_size > 8 * 1024 * 1024: + raise ValueError('Readiness report too large.') + body = path.read_bytes() + r = json.loads(body) + if not isinstance(r, dict): + raise ValueError('Invalid readiness object.') + if not isinstance(live_commit, str) or not re.fullmatch('[0-9a-f]{40}', live_commit): + raise ValueError('Unknown live source commit.') + if r.get('scope') != 'deployment' or r.get('organizationId') != '' or r.get('database') != database: + raise ValueError('Strict activation requires complete deployment-wide database scope.') + if r.get('schema') != 'droboticflow.source-readiness/v1' or r.get('namespace') != namespace or r.get('targetMode') != 'strict' or (r.get('complete') is not True) or (r.get('projectId') != '') or (r.get('legacySourceCutoff', '') != (cutoff or '')) or (r.get('buildCommit') != live_commit): + raise ValueError('Incomplete, scoped or mismatched strict readiness report.') + if not isinstance(r.get('generatedAt'), str): + raise ValueError('Invalid readiness timestamp.') + at = dt.datetime.fromisoformat(r['generatedAt'].replace('Z', '+00:00')) + if at.tzinfo is None: + raise ValueError('Readiness timestamp must include timezone.') + age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds() + if not 0 <= age <= 3600: + raise ValueError('Readiness report is expired or in the future.') + totals = {k: 0 for k in ('total', 'allowed', 'unverified', 'pending', 'failed')} + rows = r['rows'] + if not isinstance(rows, list): + raise ValueError('Invalid readiness rows.') + seen = set() + for row in rows: + if not isinstance(row, dict): + raise ValueError('Invalid readiness row.') + key = (row.get('organizationId'), row.get('stage'), row.get('status')) + if not all((isinstance(x, str) for x in key)) or key in seen: + raise ValueError('Duplicate or invalid readiness group.') + seen.add(key) + if not isinstance(row.get('stage'), str) or not isinstance(row.get('status'), str): + raise ValueError('Missing readiness stage/status.') + for k in totals: + v = row.get(k) + if type(v) is not int or v < 0: + raise ValueError('Invalid readiness count.') + totals[k] += v + if row['total'] != sum((row[k] for k in ('allowed', 'unverified', 'pending', 'failed'))): + raise ValueError('Readiness counts do not reconcile.') + if totals['total'] != totals['allowed']: + raise ValueError('Strict mode would block existing tasks; keep advisory and review the impact.') + return {'path': str(path.resolve()), 'sha256': hashlib.sha256(body).hexdigest(), 'scope': 'deployment', 'generatedAt': r['generatedAt'], 'totals': totals} diff --git a/scripts/release-build-api-image.sh b/scripts/release-build-api-image.sh index 5d766381..603f818a 100755 --- a/scripts/release-build-api-image.sh +++ b/scripts/release-build-api-image.sh @@ -27,8 +27,16 @@ commit="$(git -C "$ROOT_DIR" rev-parse HEAD)" build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" metadata="${2:-$(mktemp -t droboticflow-api-build.XXXXXX)}" +runtime_image="${API_RUNTIME_IMAGE:-$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["image"])' "$ROOT_DIR/docs/release/api-runtime.json")}" +if [[ ! "$runtime_image" =~ ^[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]]; then + echo "API_RUNTIME_IMAGE must be an immutable prepared runtime digest" >&2 + exit 2 +fi + "$DOCKER" buildx build \ --platform "$PLATFORM" \ + --build-arg "API_RUNTIME_IMAGE=$runtime_image" \ + --build-arg "API_RUNTIME_PREPARED=1" \ --build-arg "GOPROXY=${GOPROXY:-https://proxy.golang.org,direct}" \ --build-arg "DROBOTICFLOW_BUILD_COMMIT=$commit" \ --build-arg "DROBOTICFLOW_BUILD_TIME=$build_time" \ @@ -44,3 +52,9 @@ if [[ "$digest" != sha256:* ]]; then exit 1 fi printf 'COMMIT=%s\nBUILD_TIME=%s\nMETADATA=%s\nIMAGE=%s@%s\n' "$commit" "$build_time" "$metadata" "$repository" "$digest" + +python3 - "$metadata.provenance.json" "$ROOT_DIR/Dockerfile.api" "$commit" "$runtime_image" "$digest" "$build_time" <<'PYPROOF' +import hashlib,json,pathlib,sys +output,recipe,commit,runtime,digest,at=sys.argv[1:] +pathlib.Path(output).write_text(json.dumps({"schemaVersion":1,"commit":commit,"runtimeImage":runtime,"imageDigest":digest,"buildTime":at,"recipeSHA256":hashlib.sha256(pathlib.Path(recipe).read_bytes()).hexdigest()},indent=2)+"\n") +PYPROOF diff --git a/scripts/release.py b/scripts/release.py index d00d9fe7..2daf0b86 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -13,6 +13,11 @@ import re import subprocess import sys import uuid +import time + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from lib.release_checks import check_environment, identity, reusable, write_log +from lib.source_readiness import validate_report ROOT = Path(__file__).resolve().parents[1] RECORDS = ROOT / 'local-data/releases' @@ -42,13 +47,18 @@ def digest(value): return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() -def run(argv, *, stdin=None, timeout=60): +def run(argv, *, stdin=None, timeout=60, env=None): try: - p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout) - except (OSError, subprocess.TimeoutExpired): - raise Blocked('Command unavailable or timed out; outcome must be checked before retry.') from None - # Do not echo kubectl output, URLs or credential-bearing errors. - require(p.returncode == 0, 'Command failed: ' + argv[0] + ' (output withheld; no success recorded).') + p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout, env=env) + except (OSError, subprocess.TimeoutExpired) as e: + error = Blocked('Command unavailable or timed out; outcome must be checked before retry.') + error.check_output = str(getattr(e, 'stdout', '') or '') + str(getattr(e, 'stderr', '') or '') + raise error from None + # Only policy-local checks may persist the private redacted output. + if p.returncode != 0: + error = Blocked('Command failed: ' + argv[0] + ' (no success recorded).') + error.check_output = p.stdout + '\n' + p.stderr + raise error return p.stdout @@ -175,7 +185,7 @@ def env_values(container, configs): return values -def inspect_deployment(deployment, container_name, configs, p): +def inspect_deployment(deployment, container_name, configs, p, *, ignore_source_mode=False): spec = deployment['spec'] pod = spec['template']['spec'] containers = pod['containers'] @@ -228,10 +238,10 @@ def inspect_deployment(deployment, container_name, configs, p): neutral = copy.deepcopy(spec) nc = neutral['template']['spec']['containers'][index] nc['image'] = '' - nc['env'] = [x for x in nc.get('env', []) if x['name'] != 'DROBOTICFLOW_BUILD_COMMIT'] + nc['env'] = [x for x in nc.get('env', []) if x['name'] not in (('DROBOTICFLOW_BUILD_COMMIT', 'ANNOTATION_SOURCE_VALIDATION_MODE') if ignore_source_mode else ('DROBOTICFLOW_BUILD_COMMIT',))] stamp = next((i for i, x in enumerate(c.get('env', [])) if x['name'] == 'DROBOTICFLOW_BUILD_COMMIT'), None) return {'uid': m['uid'], 'specHash': digest(spec), 'nonReleaseHash': digest(neutral), - 'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'], + 'sourceModeIndex': next((i for i,x in enumerate(c.get('env',[])) if x['name']=='ANNOTATION_SOURCE_VALIDATION_MODE'),None), 'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'], 'resourceVersion': m['resourceVersion'], 'containerIndex': index, 'ready': ready, 'env': {k: e.get(k) for k in SAFE_ENV}} @@ -251,7 +261,7 @@ def validate_legacy_source_cutoffs(envs): raise Blocked('Invalid legacy source admission cutoff.') -def snapshot(r): +def snapshot(r, *, allow_planned_transition=False): t, p = r['target'], policy() view = json.loads(kubectl(r, ['config', 'view', '--minify', '-o', 'json'])) require(view['clusters'][0]['cluster']['server'] == t['server'], 'Kube API server mismatch.') @@ -265,10 +275,17 @@ def snapshot(r): config_names.update(x['valueFrom']['configMapKeyRef']['name'] for x in c.get('env', []) if 'configMapKeyRef' in x.get('valueFrom', {})) configs = {n: get(r, 'configmap', n) for n in sorted(config_names)} - deps = {n: inspect_deployment(d, p['deployments'][n], configs, p) for n, d in deployments.items()} + deps = {n: inspect_deployment(d, p['deployments'][n], configs, p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in deployments.items()} envs = [d['env'] for d in deps.values()] require(len({e['ANNOTATION_WORKSPACE_CONNECTION_ID'] for e in envs}) == 1, 'API/processor storage connections differ.') - validate_legacy_source_cutoffs(envs) + if allow_planned_transition and r.get("sourceMode"): + for e in envs: require(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in (r["sourceMode"], *(d["env"].get("ANNOTATION_SOURCE_VALIDATION_MODE") for d in r["baseline"]["deployments"].values())), "Unexpected source mode during rollout.") + normalized = [dict(e, ANNOTATION_SOURCE_VALIDATION_MODE=r["sourceMode"]) for e in envs] + validate_legacy_source_cutoffs(normalized) + else: + validate_legacy_source_cutoffs(envs) + if r.get("sourceModeExplicit") and not r.get("sourceMode"): + require(all(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in ("advisory", "strict") for e in envs), "New releases require an explicit source admission mode on both components.") pvc = get(r, 'pvc', p['workspacePVC']) require(pvc.get('status', {}).get('phase') == 'Bound', 'Workspace PVC is not Bound.') svc = get(r, 'service', 'droboticflow-api-public-eip') @@ -287,7 +304,9 @@ def stable(s): def plan_data(r): - return {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')} + data = {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')} + data.update({k:r[k] for k in ('sourceModeExplicit','sourceMode','sourceReadiness') if k in r}) + return data def manifest_platforms(image_ref): @@ -317,11 +336,32 @@ def prepare(a): 'target': {'kubeconfig': str(Path(a.kubeconfig).resolve()), 'context': a.context, 'server': a.server.rstrip('/'), 'namespace': p['namespace'], 'namespaceUID': a.namespace_uid}, 'state': 'prepared', 'localChecks': [], 'baseline': None, 'planHash': None, - 'operations': [], 'evidence': {}, 'events': []} + 'operations': [], 'evidence': {}, 'events': [], 'sourceModeExplicit': True, + 'sourceMode': getattr(a, 'source_mode', None), 'sourceReadiness': None} event(r, 'prepared; no network access') print(json.dumps({'id': r['id'], 'state': r['state']}, ensure_ascii=False)) +def check_identity(r): + return identity(ROOT, r['commit'], r['policyHash']) + + +def run_local_check(r, argv, expected): + entry = {'argv': argv, 'startedAt': now(), 'identity': expected} + try: + output = run(argv, timeout=1800, env=check_environment()) + except Blocked as e: + entry.update(result='failed', finishedUnix=time.time()) + entry.update(write_log(record_path(r['id']).parent/'logs', argv, getattr(e, 'check_output', ''), 'failed')) + r['localChecks'].append(entry) + r['state'] = 'blocked' + event(r, 'local check failed; private redacted log: ' + entry['log']) + raise Blocked(str(e) + ' Log: ' + entry['log']) from None + entry.update(result='passed', outputHash=digest(output), finishedUnix=time.time()) + entry.update(write_log(record_path(r['id']).parent/'logs', argv, output, 'passed')) + return entry + + def check(r, a): fresh(r) require(not r['operations'], 'A deployment was attempted; check outcome with verify, never rewrite its plan.') @@ -342,18 +382,18 @@ def check(r, a): r['state'] = 'blocked' event(r, 'local check failed') raise + previous = {tuple(x['argv']): x for x in r['localChecks']} + expected = check_identity(r) r['localChecks'] = [] for argv in policy()['localChecks']: - started = now() - try: - output = run(argv, timeout=1800) - except Blocked: - r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'failed'}) - r['state'] = 'blocked' - event(r, 'local check failed') - raise - r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'passed', 'outputHash': digest(output)}) + old = previous.get(tuple(argv), {}) + if getattr(a, 'reuse_successful', False) and reusable(old, expected, argv): + entry = {**old, 'reusedAt': now()} + else: + entry = run_local_check(r, argv, expected) + r['localChecks'].append(entry) save(r) + require(check_identity(r) == expected, 'Check environment/dependencies changed; evidence is not reusable.') fresh(r) if a.cluster: # Never preserve an earlier successful snapshot when a new read fails. @@ -363,6 +403,15 @@ def check(r, a): s = snapshot(r) require(all(d['ready'] for d in s['deployments'].values()), 'Current deployments are not fully ready.') r['baseline'] = s + if strict_transition(r): + path = getattr(a, 'source_readiness_file', None) + require(bool(path), 'Strict activation needs a fresh deployment-wide --source-readiness-file.') + api = s['deployments']['droboticflow-api'] + try: + r['sourceReadiness'] = validate_report(path, namespace=r['target']['namespace'], database=policy()['database'], + cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit']) + except (ValueError,KeyError,TypeError,AttributeError,OSError): + raise Blocked('Strict readiness incomplete, blocked, expired or wrong scope; keep advisory.') from None except Blocked: r['state'] = 'blocked' event(r, 'cluster check failed') @@ -371,20 +420,61 @@ def check(r, a): event(r, 'checks recorded; plan must still be generated') +def target_source_mode(r, name): + return r.get('sourceMode') or r['baseline']['deployments'][name]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') + + +def validate_planned_source_mode(r): + if r.get("sourceModeExplicit"): + require(all(target_source_mode(r,n) in ("advisory","strict") for n in r["baseline"]["deployments"]), "New releases require an explicit planned source mode.") + + +def strict_transition(r): + if not any(target_source_mode(r,n)=='strict' for n in r['baseline']['deployments']): return False + if any(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE')!='strict' for d in r['baseline']['deployments'].values()): return True + return strict_rules_changed(r) + + +def strict_rules_changed(r): + live=r['baseline']['deployments']['droboticflow-api']['buildCommit'] + if live==r['commit']: return False + require(isinstance(live,str) and bool(re.fullmatch(r'[0-9a-f]{40}',live)), 'Unknown current source; strict activation requires advisory preparation first.') + paths=['internal/store/annotation_source_admission.go','internal/store/annotation_availability.go', + 'internal/api/annotation_source_admission.go','internal/api/annotation_source_preparation.go', + 'internal/api/annotation_source_certificate.go','internal/api/annotation_source_availability.go'] + return bool(run(['git','diff','--name-only',live,r['commit'],'--',*paths]).strip()) + + +def validate_source_transition(r): + if not strict_transition(r): return + proof = r.get('sourceReadiness') + require(bool(proof), 'Strict activation requires a complete deployment-wide source readiness report.') + require(not strict_rules_changed(r), 'Strict admission rules changed; deploy advisory first, then inventory with the candidate rules.') + api = r['baseline']['deployments']['droboticflow-api'] + try: + current = validate_report(proof['path'], namespace=r['target']['namespace'], database=policy()['database'], + cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit']) + require(current['sha256'] == proof['sha256'], 'Source readiness report changed; recheck and approve a new plan.') + except (ValueError, KeyError, TypeError, AttributeError, OSError) as e: + raise Blocked('Source readiness report invalid: ' + type(e).__name__) from None + + def make_plan(r): fresh(r) require(not r['operations'], 'Deployment already attempted.') require([x['argv'] for x in r['localChecks']] == policy()['localChecks'] and all(x['result'] == 'passed' for x in r['localChecks']), 'Required local checks incomplete.') require(r['baseline'] is not None, 'Read-only cluster check required.') + validate_planned_source_mode(r) + validate_source_transition(r) r['planHash'] = digest(plan_data(r)) r['state'] = 'awaiting_approval' event(r, 'plan prepared; no authorization inferred') print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [ - {'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit']} for n in sorted(r['baseline']['deployments'])], + {'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit'], 'sourceMode': {'from': r['baseline']['deployments'][n]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE'), 'to': target_source_mode(r,n)}} for n in sorted(r['baseline']['deployments'])], 'sequence': 'Processor first; verify its complete rollout before patching API.', 'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'], - 'warnings': ['Only image/build stamp changes; startup migrations require change-safety evidence.', + 'warnings': ['Only image/build stamp and the explicitly planned source mode may change; startup migrations require change-safety evidence.', 'Image-to-source provenance requires independent evidence.', 'Bandwidth adequacy and business acceptance remain manual evidence.'], 'rollback': r['rollback']}, ensure_ascii=False, indent=2)) @@ -396,6 +486,8 @@ def deploy(r, a): require(r['planHash'] == a.confirm_plan == digest(plan_data(r)), 'Plan hash does not match.') require(bool(a.approval_ref.strip()), 'Record the explicit user approval reference for this exact plan.') require_mainline(r['commit'], refresh=True) + validate_planned_source_mode(r) + validate_source_transition(r) age = (dt.datetime.now(dt.timezone.utc) - dt.datetime.fromisoformat(r['baseline']['at'])).total_seconds() require(0 <= age <= 3600, 'Cluster check is older than one hour; recheck and obtain a new plan approval.') for key in ('image-provenance', 'change-safety'): @@ -409,13 +501,13 @@ def deploy(r, a): # The processor must understand new durable payloads before API replicas can # enqueue them. The fixed Fanyu names sort processor before API. for name in sorted(current['deployments']): - latest = snapshot(r) + latest = snapshot(r, allow_planned_transition=True) require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.') for n, live in latest['deployments'].items(): old = current['deployments'][n] done = any(o['deployment'] == n and o['result'] == 'acknowledged' for o in r['operations']) require(live['uid'] == old['uid'] and live['nonReleaseHash'] == old['nonReleaseHash'], 'Deployment changed during release.') - require((live['image'] == r['image'] and live['buildCommit'] == r['commit']) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.') + require((live['image'] == r['image'] and live['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or live['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n))) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.') dep = latest['deployments'][name] patch = [{'op': 'test', 'path': '/metadata/resourceVersion', 'value': dep['resourceVersion']}, {'op': 'test', 'path': '/metadata/uid', 'value': dep['uid']}, @@ -425,6 +517,12 @@ def deploy(r, a): env_path = f"/spec/template/spec/containers/{dep['containerIndex']}/env/" patch.append({'op': 'add' if dep['buildCommitIndex'] is None else 'replace', 'path': env_path + ('-' if dep['buildCommitIndex'] is None else str(dep['buildCommitIndex'])), 'value': stamp}) + if r.get('sourceMode'): + mode_index = dep['sourceModeIndex'] + # If the stamp is appended first, a pre-existing mode keeps its index. + patch.append({'op': 'add' if mode_index is None else 'replace', + 'path': env_path + ('-' if mode_index is None else str(mode_index)), + 'value': {'name': 'ANNOTATION_SOURCE_VALIDATION_MODE', 'value': r['sourceMode']}}) op = {'deployment': name, 'before': dep['image'], 'desired': r['image'], 'at': now(), 'result': 'unknown'} r['operations'].append(op) save(r) # Persist intent before sending any mutation. @@ -440,8 +538,8 @@ def deploy(r, a): event(r, 'processor patch acknowledged; waiting for rollout before API') try: kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330) - ready = snapshot(r)['deployments'][name] - require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'], + ready = snapshot(r, allow_planned_transition=True)['deployments'][name] + require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or ready['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,name)), 'Processor rollout has not converged to the candidate.') except Blocked: r['state'] = 'deployed_pending_verification' @@ -493,6 +591,7 @@ def verify(r): for n, d in s['deployments'].items(): require(d['uid'] == expected['deployments'][n]['uid'], 'Deployment was replaced.') require(d['nonReleaseHash'] == expected['deployments'][n]['nonReleaseHash'], 'Deployment settings drifted beyond the approved image/build stamp.') + if r.get('sourceModeExplicit'): require(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n), 'Source mode differs from approved plan.') require(d['image'] == r['image'] and d['buildCommit'] == r['commit'] and d['ready'], 'Target image rollout incomplete; outcome remains pending.') r['state'] = 'deployed_pending_acceptance' missing = [k for k in policy()['requiredEvidence'] if k not in r['evidence'] or not evidence_valid(r['evidence'][k], r)] @@ -546,6 +645,8 @@ def main(): elif cmd == 'check': c.add_argument('--local', action='store_true') c.add_argument('--cluster', action='store_true') + c.add_argument('--source-readiness-file') + c.add_argument('--reuse-successful', action='store_true', help='Reuse only unchanged successful pure local checks; database/browser checks always rerun.') elif cmd == 'deploy': c.add_argument('--confirm-plan', required=True) c.add_argument('--approval-ref', required=True) @@ -554,6 +655,7 @@ def main(): c.add_argument('--result', choices=['passed', 'failed', 'unverified'], required=True) c.add_argument('--file', required=True) c.add_argument('--note', required=True) + prep.add_argument('--source-mode', choices=['strict','advisory'], help='Optional reviewed source-mode change in the same image plan.') a = parser.parse_args() try: if a.command == 'prepare': diff --git a/scripts/test_work_availability.py b/scripts/test_work_availability.py new file mode 100644 index 00000000..9e8f9daa --- /dev/null +++ b/scripts/test_work_availability.py @@ -0,0 +1,85 @@ +"""Mandatory synthetic work/recovery gate; loopback Mongo only, no customer writes.""" +import json +import os +import signal +from pathlib import Path +import subprocess +import sys +import time +import uuid +from lib.release_checks import check_environment, write_log, source_fingerprint +ROOT = Path(__file__).resolve().parents[1] +CORE = {'TestSourceAdvisoryCompleteWorkAndReviewFlow', 'TestSourceAdvisoryPublishedReviewDoesNotRequireFullAudit', 'TestSourceAdvisoryPreparesIndexDespiteCadenceAuditFailure', 'TestSourceAdvisoryTaskScopeCannotBeBypassed', 'TestSourceAdvisoryIsolatedReworkDoesNotTrapRandomClaim', 'TestSourceAdvisoryOwnedFailureCanRetryTimeline', 'TestSubmissionLossGuardArchiveFailureBlocksReplacement', 'TestSubmissionLossGuardArchiveSurvivesRestartAndIsIdempotent', 'TestSubmissionLossGuardArchivesFailedInputBeforeReplacement', 'TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter', 'TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker', 'TestCombinedSubmissionFinalCASFailureDoesNotPublishQualityTask', 'TestDeletedAnnotatorSubmissionWinsCASRace', 'TestDeletedAnnotatorRestartFinishesResetOnce', 'TestReviewCannotMutateAnnotation', 'TestSourceAdvisoryDoesNotGateOnFullAudit', 'TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate', 'TestReadObservationIsPrivateConcurrentAndRequestScoped'} +MONGO = {'TestSourceReadinessMongoParityReadOnly', 'TestSourceAdvisoryMongoEquivalenceAndBuffer', 'TestSourceAdmissionMongoClaimFence', 'TestSourceAdmissionMongoBatchMatchesLegacyPipeline', 'TestSourceAdmissionCurrentOutboxProofMongo', 'TestSourceAdmissionExistingTaskUpdateDoesNotEnqueueMongo', 'TestSourceValidationLegacyDetailsGETDoesNotEnqueueMongo', 'TestSubmissionRecoveryEventMongo'} + +def verdict(stdout, code, required): + events = [] + for line in stdout.splitlines(): + try: + events.append(json.loads(line)) + except ValueError: + continue + passed = {e.get('Test') for e in events if e.get('Action') == 'pass'} + skipped = sorted((e['Test'] for e in events if e.get('Action') == 'skip' and e.get('Test'))) + missing = sorted(required - passed) + return {'passed': code == 0 and (not skipped) and (not missing), 'missingOrFailed': missing, 'skipped': skipped} + +def phases(): + return [('core', CORE, {}), ('mongo', MONGO, {'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'SUBMISSION_RECOVERY_LOCAL_MONGO': '1', 'POOL_LOCAL_MONGO_URI': 'mongodb://127.0.0.1:27018'}), ('advisory-workbench', {'TestSourceAdvisoryWorkbenchBrowser'}, {'SOURCE_ADVISORY_BROWSER_TEST': '1'}), ('advisory-recovery', {'TestReworkBoundaryBrowserReplica'}, {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'REWORK_BOUNDARY_BROWSER_TEST': '1'})] + +def run_phase(argv, env): + # Terminate only this synthetic phase's process group on timeout, including + # Go-launched Node/browser children. Never terminate a shared local service. + child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True) + try: + stdout, stderr = child.communicate(timeout=360) + except subprocess.TimeoutExpired: + try: + os.killpg(child.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + stdout, stderr = child.communicate(timeout=5) + except subprocess.TimeoutExpired: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = child.communicate() + return subprocess.CompletedProcess(argv, 124, stdout, stderr + '\nphase timed out') + return subprocess.CompletedProcess(argv, child.returncode, stdout, stderr) + +def main(): + folder = ROOT / 'local-data/test-results' / ('work-availability-' + uuid.uuid4().hex) + folder.mkdir(parents=True, mode=448) + before = source_fingerprint(ROOT) + report = {'inputDigest': before, 'scope': 'synthetic local fixtures; no production acceptance/capacity claim', 'checks': [], 'passed': False} + for name, required, switches in phases(): + argv = ['go', 'test', '-json', './internal/api', './internal/store', '-run', '^(' + '|'.join(sorted(required)) + ')$', '-count=1', '-timeout=5m'] + env = {**check_environment(), **switches, 'VERIFICATION_RUN_ID': uuid.uuid4().hex} + try: + r = run_phase(argv, env) + row = verdict(r.stdout, r.returncode, required) + output = r.stdout + '\n' + r.stderr + except (OSError, subprocess.TimeoutExpired) as e: + row = {'passed': False, 'error': 'runner failed or timed out'} + output = str(e) + report['inputsChangedDuringRun'] = source_fingerprint(ROOT) != before + if report['inputsChangedDuringRun'] or not before: + row.update(passed=False, error='Source identity changed or unavailable') + row.update(phase=name, argv=argv, artifactRunId=env['VERIFICATION_RUN_ID']) + row.update(write_log(folder, argv, output, 'passed' if row['passed'] else 'failed')) + report['checks'].append(row) + print(json.dumps(row), flush=True) + (folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n') + if report['inputsChangedDuringRun'] or not before: + report['remainingUnverified'] = [p[0] for p in phases()[len(report['checks']):]] + (folder/'report.json').write_text(json.dumps(report,indent=2)+'\n') + return 2 + report['passed'] = all(row['passed'] for row in report['checks']) + (folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n') + print('Work availability gate ' + ('passed' if report['passed'] else 'failed') + '; production writes and capacity remain separate. Report: ' + str(folder / 'report.json')) + return 0 if report['passed'] else 1 +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 0a2e498b..57f6574e 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -853,6 +853,36 @@ "tests": true } ] + }, + { + "name": "reliability-guardrails", + "paths": [ + "^scripts/(lib/(release_checks|source_readiness)\\.py|test_work_availability\\.py)$", + "^tests/release/test_reliability_checks\\.py$", + "^internal/(api|store)/(source_readiness|read_observation).*\\.go$", + "^internal/api/diagnostic_trace.*\\.go$", + "^docs/(operations/reliability-guardrails\\.md|release/api-runtime\\.json)$", + "^Dockerfile\\.api-runtime$", + "^cmd/api/main\\.go$" + ], + "frontend": [], + "go": { + "./internal/api": ["^TestSourceReadiness", "^TestDiagnosticTrace"], + "./internal/store": ["^TestReadObservation", "^TestSourceReadiness"] + }, + "task": [ + { + "argv": ["python3", "-m", "unittest", "discover", "-s", "tests/release", "-p", "test_*.py"], + "tests": true + }, + { + "argv": ["npm", "run", "test:work-availability"], + "tests": true + } + ], + "warnings": [ + "工作流门禁仅使用合成媒体和隔离 Mongo;严格模式清单不是校验证书,生产容量、备份恢复和部署需分别验证。" + ] } ] } diff --git a/tests/browser/initial-slice-confirmation.mjs b/tests/browser/initial-slice-confirmation.mjs index dc78ce5b..4829da17 100644 --- a/tests/browser/initial-slice-confirmation.mjs +++ b/tests/browser/initial-slice-confirmation.mjs @@ -1,3 +1,4 @@ +import { closeBrowserFixture } from "../../scripts/lib/browser-test-lifecycle.mjs"; import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import fs from "node:fs"; import assert from "node:assert/strict"; @@ -7,7 +8,11 @@ const { createServer } = require("vite"), { chromium } = require("playwright"); const api = process.env.REWORK_BOUNDARY_API; assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); -const root = new URL("../../local-data/verification/initial-slice-confirmation/", import.meta.url); +const runId = process.env.VERIFICATION_RUN_ID || `${Date.now()}-${process.pid}`; +assert.match(runId, /^[a-zA-Z0-9_-]+$/); +const scenario = process.env.INITIAL_SLICE_CONFIRM_SCENARIO; +assert.match(scenario, /^[a-z0-9-]+$/); +const root = new URL(`../../local-data/verification/initial-slice-confirmation/${runId}/${scenario}/`, import.meta.url); fs.mkdirSync(root, { recursive: true }); const report = { cases: [], errors: [], requests: [] }; const vite = await createServer({ @@ -15,19 +20,22 @@ const vite = await createServer({ configFile: "tests/browser/vite.platform-feedback.config.ts", server: { host: "127.0.0.1", port: 5204, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, }); -let browser; +let browser, owner, activeContext, activePage; try { await vite.listen(); - browser = await chromium.launch({ + owner = await chromium.launchServer({ ...browserExecutableOptions(), headless: true, args: ["--no-proxy-server"], }); + browser = await chromium.connect(owner.wsEndpoint()); // A separate context for each case prevents autosaving one scenario into another. for (const status of [process.env.INITIAL_SLICE_CONFIRM_SCENARIO]) { await fetch(api + "/__boundary/reset?status=" + status); const context = await browser.newContext({ viewport: { width: 1600, height: 1100 } }); - const page = await context.newPage(); + activeContext = context; + await context.tracing.start({ screenshots: true, snapshots: true, sources: false }); + const page = (activePage = await context.newPage()); page.setDefaultTimeout(20000); page.on("pageerror", (e) => report.errors.push(e.message)); page.on("response", async (r) => { @@ -65,6 +73,7 @@ try { assert.equal(report.requests.length, 0, "unreadable content must not autosave or generate"); await page.screenshot({ path: new URL(status + "-blocked.png", root).pathname }); report.cases.push({ status, preserved: true }); + assert.deepEqual(report.errors, []); await context.close(); continue; } @@ -162,15 +171,20 @@ try { await bench.waitFor(); await page.screenshot({ path: new URL(status + "-after.png", root).pathname }); report.cases.push({ status, retainedAction: workspace.segments[0].id, sliceName: workspace.sliceSegments[0].name }); + assert.deepEqual(report.errors, []); await context.close(); } assert.deepEqual(report.errors, []); +} catch (error) { + report.failure = { name: error.name, message: error.message, stack: error.stack }; + await activePage?.screenshot({ path: new URL("failure.png", root).pathname, fullPage: true }).catch(() => {}); + await activeContext?.tracing.stop({ path: new URL("failure-trace.zip", root).pathname }).catch(() => {}); + throw error; } finally { fs.writeFileSync( new URL(`report-${process.env.INITIAL_SLICE_CONFIRM_SCENARIO}.json`, root), JSON.stringify(report, null, 2), ); - await browser?.close(); - await vite.close(); + await closeBrowserFixture({ browser, owner, vite }); } console.log(JSON.stringify(report)); diff --git a/tests/browser/rework-boundary-replica.mjs b/tests/browser/rework-boundary-replica.mjs index 2657e460..89b9f963 100644 --- a/tests/browser/rework-boundary-replica.mjs +++ b/tests/browser/rework-boundary-replica.mjs @@ -1,3 +1,4 @@ +import { closeBrowserFixture } from "../../scripts/lib/browser-test-lifecycle.mjs"; import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs"; import assert from "node:assert/strict"; import fs from "node:fs"; @@ -6,14 +7,18 @@ import { chromium } from "playwright"; import { createServer } from "vite"; const api = process.env.REWORK_BOUNDARY_API; assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/); -const out = path.resolve("local-data/test-results/rework-boundary"); +const runId = process.env.VERIFICATION_RUN_ID || `${Date.now()}-${process.pid}`; +assert.match(runId, /^[a-zA-Z0-9_-]+$/); +const out = path.resolve("local-data/test-results/rework-boundary", runId); fs.mkdirSync(out, { recursive: true }); const server = await createServer({ cacheDir: "local-data/test-results/vite-rework-boundary-replica", server: { host: "127.0.0.1", port: 5202, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } }, }); -let browser, +let owner, + browser, page, + context, fault = "", releaseSave; const report = { cases: [], pageErrors: [], writes: [] }; @@ -29,13 +34,18 @@ const waitFor = async (check) => { }; try { await server.listen(); - browser = await chromium.launch({ + owner = await chromium.launchServer({ ...browserExecutableOptions(), headless: true, args: ["--no-proxy-server"], }); - const ctx = await browser.newContext({ viewport: { width: 1600, height: 1100 }, serviceWorkers: "block" }); + browser = await chromium.connect(owner.wsEndpoint()); + const ctx = (context = await browser.newContext({ + viewport: { width: 1600, height: 1100 }, + serviceWorkers: "block", + })); + await ctx.tracing.start({ screenshots: true, snapshots: true, sources: false }); page = await ctx.newPage(); page.setDefaultTimeout(12000); page.on("pageerror", (e) => report.pageErrors.push(e.message)); @@ -314,6 +324,8 @@ try { assert.deepEqual(report.pageErrors, []); console.log(JSON.stringify(report)); } catch (e) { + report.failure = { name: e.name, message: e.message, stack: e.stack }; + await context?.tracing.stop({ path: path.join(out, "failure-trace.zip") }).catch(() => {}); if (page) { await page.screenshot({ path: path.join(out, "failure.png"), fullPage: true }).catch(() => {}); fs.writeFileSync( @@ -328,6 +340,5 @@ try { } finally { releaseSave?.(); fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2)); - await browser?.close(); - await server.close(); + await closeBrowserFixture({ browser, owner, vite: server }); } diff --git a/tests/release/test_release.py b/tests/release/test_release.py index fdb8be98..030a79df 100644 --- a/tests/release/test_release.py +++ b/tests/release/test_release.py @@ -20,7 +20,7 @@ IMAGE = 'registry.example/api@sha256:' + 'a' * 64 def fixture(name, container): - config = {'data': {'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true', + config = {'data': {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true', 'ANNOTATION_PROCESSOR_MODE': 'external', 'ANNOTATION_WORKSPACE_BUCKET': 'drobodata-fanyu', 'ANNOTATION_WORKSPACE_CONNECTION_ID': 'storage-example', 'WORKSPACE_MOUNT_PATH': '/workspace'}, 'metadata': {'uid': 'config-uid'}} @@ -53,6 +53,7 @@ class ReleaseTests(unittest.TestCase): self.start_patch('require_mainline', lambda *args, **kwargs: None) self.start_patch('run', lambda *args, **kwargs: 'feature/release') self.start_patch('manifest_platforms', lambda image: ['linux/amd64']) + self.start_patch('check_identity', lambda r: {'testEnvironment': 'stable', 'complete': True}) self.ds = {} for n, c in self.p['deployments'].items(): self.ds[n], config = fixture(n, c) @@ -76,11 +77,11 @@ class ReleaseTests(unittest.TestCase): def load(self): return json.loads(m.record_path('test').read_text()) - def snapshot(self, r): + def snapshot(self, r, **kwargs): if self.before_snapshot: self.before_snapshot() return {'at': m.now(), 'deployments': { - n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p) for n, d in self.ds.items()}, + n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in self.ds.items()}, 'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}}, 'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}} @@ -105,6 +106,9 @@ class ReleaseTests(unittest.TestCase): c['env'].append(ops[4]['value']) else: c['env'][int(ops[4]['path'].split('/')[-1])] = ops[4]['value'] + for extra in ops[5:]: + if extra['op']=='add': c['env'].append(extra['value']) + else: c['env'][int(extra['path'].split('/')[-1])] = extra['value'] d['metadata']['resourceVersion'] = str(int(d['metadata']['resourceVersion']) + 1) return '{}' @@ -433,6 +437,15 @@ class ReleaseTests(unittest.TestCase): with patch.object(real, 'policy', return_value=self.p), patch.object(real, 'get', side_effect=fake_get), patch.object(real, 'kubectl', return_value=json.dumps({'clusters': [{'cluster': {'server': r['target']['server']}}]})): real.snapshot(r) self.assertNotIn('secret', reads) + r['baseline'] = self.snapshot(r) + r['sourceMode'] = 'strict' + env = self.ds['droboticflow-annotation-processor']['spec']['template']['spec']['containers'][0]['env'] + env.append({'name':'ANNOTATION_SOURCE_VALIDATION_MODE','value':'strict'}) + with self.assertRaises(real.Blocked): real.snapshot(r) + real.snapshot(r, allow_planned_transition=True) + env[-1]['value'] = 'disabled' + with self.assertRaises(real.Blocked): real.snapshot(r, allow_planned_transition=True) + env.pop() replies['namespace']['metadata']['uid'] = 'wrong' with self.assertRaises(real.Blocked): real.snapshot(r) @@ -451,6 +464,104 @@ class ReleaseTests(unittest.TestCase): with self.assertRaises(m.Blocked): with m.locked('test'): pass + def test_combined_plan_changes_only_reviewed_mode_in_processor_api_order(self): + self.configs['config']['data']['ANNOTATION_SOURCE_VALIDATION_MODE']='strict' + self.r['sourceMode']='advisory' + self.plan() + approval=self.approve() + m.deploy(self.r,approval) + self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api']) + self.assertEqual(self.rollout_waits[0][1],['droboticflow-annotation-processor']) + for d in self.ds.values(): + env={e['name']:e.get('value') for e in d['spec']['template']['spec']['containers'][0]['env']} + self.assertEqual(env['ANNOTATION_SOURCE_VALIDATION_MODE'],'advisory') + m.verify(self.r) + self.assertEqual(self.r['state'],'deployed_pending_acceptance') + + def test_legacy_unset_mode_can_be_repaired_by_explicit_combined_plan(self): + self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE') + self.r['sourceMode']='advisory' + self.plan();m.deploy(self.r,self.approve()) + self.assertEqual(len(self.writes),2) + + def test_new_release_cannot_plan_an_implicit_strict_fallback(self): + self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE') + m.check(self.r,argparse.Namespace(local=True,cluster=True)) + with self.assertRaises(m.Blocked):m.make_plan(self.r) + self.assertEqual(self.writes,[]) + + def test_mode_change_after_approval_invalidates_plan_without_writes(self): + self.r['sourceMode']='advisory';self.plan();a=self.approve() + self.r['sourceMode']='strict' + with self.assertRaises(m.Blocked):m.deploy(self.r,a) + self.assertEqual(self.writes,[]) + + def test_strict_activation_without_inventory_never_writes(self): + self.r['sourceMode']='strict' + with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=True)) + self.assertEqual(self.writes,[]) + + def test_combined_mode_rollout_failure_leaves_api_unmodified(self): + self.r['sourceMode']='advisory';self.plan();self.fail_rollout=True + with self.assertRaises(m.Blocked):m.deploy(self.r,self.approve()) + self.assertEqual(self.writes,['droboticflow-annotation-processor']) + self.assertEqual(self.r['state'],'deployed_pending_verification') + + def test_failed_check_log_is_private_and_does_not_discard_failure(self): + failure=m.Blocked('failed');failure.check_output='password: canary-secret\nTraceback: canary-failure' + with patch.object(m,'run',side_effect=failure): + with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False)) + e=self.load()['localChecks'][-1];text=Path(e['log']).read_text() + self.assertNotIn('canary-secret',text);self.assertIn('canary-failure',text) + self.assertEqual(Path(e['log']).stat().st_mode&0o777,0o600) + + def test_explicit_reuse_keeps_only_verified_successful_pure_checks(self): + self.p['localChecks']=[['npm','run','lint:web']] + self.r['policyHash']=m.digest(self.p) + m.check(self.r,argparse.Namespace(local=True,cluster=False)) + with patch.object(m,'run',side_effect=m.Blocked('must not rerun pure check')): + m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True)) + self.assertIn('reusedAt',self.r['localChecks'][0]) + Path(self.r['localChecks'][0]['log']).write_text('tampered') + with patch.object(m,'run',side_effect=m.Blocked('must rerun tampered evidence')): + with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True)) + + def test_current_complete_deployment_report_binds_strict_mode_and_detects_tampering(self): + for d in self.ds.values(): + d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']}) + self.r['sourceMode']='strict' + report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'], + 'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True, + 'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(), + 'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]} + path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report)) + m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path))) + m.make_plan(self.r);a=self.approve() + report['rows'][0]['allowed']=0;report['rows'][0]['failed']=1;path.write_text(json.dumps(report)) + with self.assertRaises(m.Blocked):m.deploy(self.r,a) + self.assertEqual(self.writes,[]) + + def test_complete_report_allows_reviewed_strict_activation(self): + for d in self.ds.values(): + d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']}) + self.r['sourceMode']='strict' + report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'], + 'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True, + 'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(), + 'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]} + path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report)) + m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path))) + m.make_plan(self.r);m.deploy(self.r,self.approve());m.verify(self.r) + self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api']) + self.assertEqual(self.r['state'],'deployed_pending_acceptance') + + def test_changed_strict_rule_needs_advisory_preparation_not_old_report(self): + self.r['sourceMode']='strict';self.r['baseline']=self.snapshot(self.r) + self.r['baseline']['deployments']['droboticflow-api']['buildCommit']='b'*40 + self.r['sourceReadiness']={'path':'not-read-yet'} + with self.assertRaisesRegex(m.Blocked,'deploy advisory first'):m.validate_source_transition(self.r) + + class EntrypointTests(unittest.TestCase): def test_cli_persists_and_resumes_offline(self): @@ -459,8 +570,9 @@ class EntrypointTests(unittest.TestCase): (root / 'scripts').mkdir() (root / 'docs/release').mkdir(parents=True) shutil.copy(ROOT / 'scripts/release.py', root / 'scripts/release.py') + shutil.copytree(ROOT / 'scripts/lib', root / 'scripts/lib', ignore=shutil.ignore_patterns('__pycache__','*.pyc')) shutil.copy(ROOT / 'docs/release/fanyu-policy.json', root / 'docs/release/fanyu-policy.json') - (root / '.gitignore').write_text('local-data/\n') + (root / '.gitignore').write_text('local-data/\n__pycache__/\n') for args in [ ['git', 'init', '-q'], ['git', 'add', '.'], ['git', '-c', 'user.name=ReleaseTest', '-c', 'user.email=test@example.invalid', 'commit', '-qm', 'fixture'] ]: diff --git a/tests/release/test_release_build_api_image.py b/tests/release/test_release_build_api_image.py index 51833db6..78741c29 100644 --- a/tests/release/test_release_build_api_image.py +++ b/tests/release/test_release_build_api_image.py @@ -29,6 +29,8 @@ class ReleaseBuildApiImageTest(unittest.TestCase): (self.repo / "scripts").mkdir(parents=True) shutil.copy(SCRIPT, self.repo / "scripts" / SCRIPT.name) (self.repo / "Dockerfile.api").write_text("FROM scratch\n") + (self.repo / "docs/release").mkdir(parents=True) + shutil.copy(ROOT / "docs/release/api-runtime.json", self.repo / "docs/release/api-runtime.json") git = ["git", "-C", str(self.repo), "-c", "user.name=t", "-c", "user.email=t@example.com"] subprocess.run(["git", "init", "-q", str(self.repo)], check=True) subprocess.run([*git, "add", "."], check=True) @@ -52,6 +54,11 @@ class ReleaseBuildApiImageTest(unittest.TestCase): self.assertTrue(any(a.startswith("DROBOTICFLOW_BUILD_TIME=20") and a.endswith("Z") for a in args), args) self.assertIn(f"registry.example.com/team/droboticflow-api:{self.commit[:8]}", args) self.assertIn("--push", args) + self.assertIn("API_RUNTIME_PREPARED=1", args) + self.assertTrue(any(a.startswith("API_RUNTIME_IMAGE=") and "@sha256:" in a for a in args)) + proof=json.loads(pathlib.Path(str(metadata)+".provenance.json").read_text()) + self.assertEqual(proof["commit"],self.commit) + self.assertIn("@sha256:",proof["runtimeImage"]) self.assertIn(f"IMAGE=registry.example.com/team/droboticflow-api@sha256:{7:064d}", result.stdout) self.assertEqual(json.loads(metadata.read_text())["containerimage.digest"], f"sha256:{7:064d}") diff --git a/tests/release/test_reliability_checks.py b/tests/release/test_reliability_checks.py new file mode 100644 index 00000000..fdc0707c --- /dev/null +++ b/tests/release/test_reliability_checks.py @@ -0,0 +1,124 @@ +import datetime as dt +import copy +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import time +import sys +import subprocess +import unittest +from unittest.mock import patch +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) + +def module(name, path): + s = importlib.util.spec_from_file_location(name, ROOT / path) + m = importlib.util.module_from_spec(s) + s.loader.exec_module(m) + return m +m = module('release_checks', 'scripts/lib/release_checks.py') +g = module('work_availability', 'scripts/test_work_availability.py') +ready = module('source_readiness', 'scripts/lib/source_readiness.py') + +class CheckEvidenceTests(unittest.TestCase): + + def test_private_bounded_redaction_including_multiline_secrets(self): + with tempfile.TemporaryDirectory() as d: + result = m.write_log(d, ['fake-check'], 'password:\n"canary-pass"\nAuthorization: Bearer canary-token\nmongodb://user:canary-db@host/db\nhttps://host/x?signature=canary-url\nAWS_ACCESS_KEY_ID=canary-aws\nBOS_SK=canary-bos\nTraceback: visible failure', 'failed') + p = Path(result['log']) + text = p.read_text() + for value in ['canary-pass', 'canary-token', 'canary-db', 'canary-url','canary-aws','canary-bos']: + self.assertNotIn(value, text) + self.assertIn('visible failure', text) + self.assertEqual(p.stat().st_mode & 511, 384) + + + def test_large_unstructured_output_is_redacted_with_bounded_runtime(self): + code = "import sys;sys.path.insert(0," + repr(str(ROOT/'scripts')) + ");from lib.release_checks import redact;assert len(redact('a'*3000000))<1000" + subprocess.run([sys.executable,'-c',code],check=True,capture_output=True,timeout=5) + + def test_tampering_expiry_identity_and_external_checks_never_reuse(self): + with tempfile.TemporaryDirectory() as d: + e = {'result': 'passed', 'identity': {'commit': 'x', 'complete': True}, 'finishedUnix': time.time(), **m.write_log(d, ['test'], 'ok', 'passed')} + self.assertTrue(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web'])) + self.assertFalse(m.reusable(e, {'commit': 'y', 'complete': True}, ['npm', 'run', 'lint:web'])) + for args in [['npm', 'run', 'test:browser:annotation-persistence'], ['python3', 'scripts/test_work_availability.py']]: + self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, args)) + self.assertFalse(m.reusable({**e, 'finishedUnix': 0}, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web'])) + Path(e['log']).write_text('tampered') + self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web'])) + + def test_production_credentials_and_inherited_switches_removed(self): + with patch.dict(os.environ, {'POOL_LOCAL_MONGO_URI': 'production', 'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'MONGO_URI': 'production', 'BOS_SECRET_KEY': 'secret', 'SOURCE_ADVISORY_BROWSER_TEST': '1', 'ANNOTATION_SOURCE_VALIDATION_MODE': 'strict'}): + e = m.check_environment() + for key in ['POOL_LOCAL_MONGO_URI', 'SOURCE_ADMISSION_LOCAL_MONGO', 'MONGO_URI', 'BOS_SECRET_KEY', 'SOURCE_ADVISORY_BROWSER_TEST', 'ANNOTATION_SOURCE_VALIDATION_MODE']: + self.assertNotIn(key, e) + + def test_installed_dependency_and_symlink_changes_invalidate_identity(self): + with tempfile.TemporaryDirectory() as d, patch.object(m.subprocess, 'check_output', return_value=b'version'): + root = Path(d) + pkg = root / 'node_modules/p' + pkg.mkdir(parents=True) + (pkg / 'index.js').write_text('a') + a = m.identity(root, 'x', 'p') + (pkg / 'index.js').write_text('b') + self.assertNotEqual(a, m.identity(root, 'x', 'p')) + linked = root / 'linked' + linked.mkdir() + (linked / 'index.js').write_text('c') + (root / 'node_modules/link').symlink_to(linked, target_is_directory=True) + a = m.identity(root, 'x', 'p') + (linked / 'index.js').write_text('d') + self.assertNotEqual(a, m.identity(root, 'x', 'p')) + + def test_required_case_skip_missing_and_failure_are_not_passes(self): + out = '\n'.join((json.dumps(e) for e in [{'Action': 'pass', 'Test': 'Required'}, {'Action': 'skip', 'Test': 'Required/child'}])) + self.assertFalse(g.verdict(out, 0, {'Required'})['passed']) + self.assertFalse(g.verdict('', 0, {'Required'})['passed']) + self.assertFalse(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 1, {'Required'})['passed']) + self.assertTrue(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 0, {'Required'})['passed']) + + def test_existing_checks_are_wired_not_reimplemented(self): + policy = json.loads((ROOT / 'docs/release/fanyu-policy.json').read_text()) + self.assertIn(['python3', 'scripts/test_work_availability.py'], policy['localChecks']) + self.assertIn('npm run test:work-availability', (ROOT / '.gitea/workflows/web-ci.yml').read_text()) + self.assertIn('TestSourceAdvisoryCompleteWorkAndReviewFlow', g.CORE) + self.assertIn('TestSubmissionRecoveryEventMongo', g.MONGO) + + def test_independent_checks_continue_but_mixed_sources_stop(self): + checks = [('first', {'Required'}, {}), ('second', {'Required'}, {})] + output = json.dumps({'Action': 'pass', 'Test': 'Required'}) + with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', return_value='stable'), patch.object(g, 'run_phase', side_effect=[subprocess.CompletedProcess([], 1, '', 'first failed'), subprocess.CompletedProcess([], 0, output, '')]) as run: + self.assertEqual(g.main(), 1) + self.assertEqual(run.call_count, 2) + report = json.loads(next(Path(d).rglob('report.json')).read_text()) + self.assertFalse(report['passed']) + self.assertTrue(report['checks'][1]['passed']) + with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', side_effect=['before', 'after']), patch.object(g, 'run_phase', return_value=subprocess.CompletedProcess([], 0, output, '')) as run: + self.assertEqual(g.main(), 2) + self.assertEqual(run.call_count, 1) + report = json.loads(next(Path(d).rglob('report.json')).read_text()) + self.assertEqual(report['remainingUnverified'], ['second']) + +class StrictReadinessTests(unittest.TestCase): + + def report(self): + return {'schema': 'droboticflow.source-readiness/v1', 'namespace': 'n', 'database': 'db', 'scope': 'deployment', 'organizationId': '', 'projectId': '', 'targetMode': 'strict', 'complete': True, 'legacySourceCutoff': '', 'buildCommit': 'c' * 40, 'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'rows': [{'organizationId': 'o', 'stage': 'quality_review', 'status': 'ready', 'total': 2, 'allowed': 2, 'unverified': 0, 'pending': 0, 'failed': 0}]} + + def validate(self, r): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / 'report.json' + p.write_text(json.dumps(r)) + return ready.validate_report(p, namespace='n', database='db', cutoff=None, live_commit='c' * 40) + + def test_only_complete_current_deployment_inventory_can_activate(self): + self.assertEqual(self.validate(self.report())['totals']['allowed'], 2) + for patcher in [lambda r: r.update(scope='organization', organizationId='o'), lambda r: r.update(projectId='p'), lambda r: r.update(complete=False), lambda r: r.update(database='local-synthetic'), lambda r: r.update(namespace='other'), lambda r: r.update(buildCommit='a' * 40), lambda r: r.update(generatedAt='2000-01-01T00:00:00Z'), lambda r: r.update(legacySourceCutoff='2026-11-12T00:00:00Z'), lambda r: r['rows'][0].update(total=3), lambda r: r['rows'][0].update(allowed=True), lambda r: r['rows'][0].update(allowed=1, pending=1), lambda r: r['rows'].append(copy.deepcopy(r['rows'][0]))]: + r = self.report() + patcher(r) + with self.subTest(r=r), self.assertRaises(ValueError): + self.validate(r) +if __name__ == '__main__': + unittest.main()