🛡️ Guard work availability and preserve release verification evidence

This commit is contained in:
Codex 2026-11-11 22:28:17 -08:00
parent 8d4b672649
commit a35e84163a
36 changed files with 1496 additions and 77 deletions

View File

@ -7,7 +7,7 @@ name: web-ci
# didn't boot the dev server.
on:
push:
branches: ['**']
branches: ["**"]
pull_request:
jobs:
@ -41,7 +41,7 @@ jobs:
- run: go test ./internal/api -run '^TestReviewPoolSearchBrowserReplica$' -count=1 -timeout=3m -v
if: ${{ !cancelled() }}
env:
REVIEW_POOL_BROWSER_TEST: '1'
REVIEW_POOL_BROWSER_TEST: "1"
- uses: actions/upload-artifact@v4
if: always()
with:
@ -69,3 +69,39 @@ jobs:
- run: go test ./internal/store -run '^TestHumanPoolSearchMongo' -count=1 -timeout=3m -v
env:
POOL_LOCAL_MONGO_URI: mongodb://127.0.0.1:27018
work-availability:
# Current source/media paths require FFmpeg >=5.1 (-fps_mode). Keep the
# existing Ubuntu 22 checks intact and run this mandatory matrix on 24.04.
runs-on: ubuntu-24.04
needs: pool-search-mongo
if: ${{ !cancelled() }}
services:
mongo:
image: mongo:8
ports:
- 27018:27017
options: >-
--health-cmd "mongosh --quiet --eval 'db.adminCommand({ping:1})'"
--health-interval 5s --health-timeout 5s --health-retries 12
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm
- run: npm ci
- run: sudo apt-get update && sudo apt-get install -y ffmpeg
- run: npx playwright install --with-deps chromium
- run: npm run test:work-availability
- uses: actions/upload-artifact@v4
if: always()
with:
name: work-availability-logs
path: local-data/
if-no-files-found: error
retention-days: 14

View File

@ -0,0 +1,20 @@
# Reliability guardrails
Base: dev 8d4b6726. User authorized local implementation of retrospective improvements.
Scope: mandatory source-admission and annotation-preservation checks wired to CI/release; strict-mode readiness and explicit source-mode plans; private redacted failure logs and immutable check evidence; bounded request/DB/lock observations; reproducible runtime builds and reviewed combined image/config plans.
Compatibility: keep current advisory policy, permissions, optimistic versions, annotations, legacy reads and historical release records. No production configuration/data changes, backfill, index/backup writes, push or deployment in this task. Existing unrelated admin pool test edits are protected.
Verification: synthetic media and isolated local Mongo, existing baseline gates, release state-machine/CLI and security regression tests. No test may silently skip critical cases. Keep failures and unverified coverage visible.
Acceptance:
1. New releases require explicit effective source modes; a reviewed combined image/mode patch preserves all other configuration. Strict activation requires a complete, current deployment inventory under the candidate rules, without enqueueing validation or changing work.
2. Existing annotation preservation, submission/deletion races, restart recovery, permissions and legacy-data regressions become mandatory, with missing/skipped cases rejected.
3. The same gate exercises real local Mongo equivalence and advisory browser work/recovery in CI and release checks.
4. Local checks preserve bounded private redacted output and immutable evidence; browser failures retain unique artifacts. Reuse is explicit and limited to unchanged, unexpired successful pure checks.
5. Request-scoped numeric Mongo/query/lock observations reveal phase costs without customer data, extra queries or production explain.
6. Authoritative image builds pin runtime dependencies and record recipe provenance. Image/source-mode changes form one reviewable processor-before-API plan. Other production configuration, backups and data mutations remain outside this task.
Limitations: inventory is point-in-time, not a data freeze or certificate; strict activation requires a controlled cutover window. Local tests do not prove production write acceptance, capacity or backup recovery. Remote CI awaits a separately authorized push.

View File

@ -1,7 +1,8 @@
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
ARG DROBOTICFLOW_BUILD_TIME=unknown
ARG API_RUNTIME_IMAGE=debian:bookworm-slim
FROM node:22-alpine AS web-build
FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 AS web-build
WORKDIR /app
COPY package*.json ./
@ -15,7 +16,7 @@ COPY src ./src
COPY public ./public
RUN npm run build:web
FROM golang:1.24-bookworm AS go-build
FROM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS go-build
WORKDIR /src
ARG GOPROXY=https://proxy.golang.org,direct
@ -26,23 +27,31 @@ COPY cmd ./cmd
COPY internal ./internal
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/droboticflow-api ./cmd/api
FROM debian:bookworm-slim AS runtime
FROM ${API_RUNTIME_IMAGE} AS runtime
# Production release script supplies a pinned prepared runtime. Direct local/POC
# builds retain the original Debian setup unless explicitly opted in.
ARG API_RUNTIME_PREPARED=0
USER root
WORKDIR /app
RUN if [ "$API_RUNTIME_PREPARED" = "1" ]; then \
test -x /usr/bin/ffprobe && getent passwd 10001; \
else \
apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* && \
groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow; \
fi
# A compatibility runtime may be an older API image. Never retain its old app.
RUN rm -rf /app/dist /usr/local/bin/droboticflow-api
COPY --from=web-build --chown=10001:10001 /app/dist ./dist
COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api
# Application identity must not invalidate the OS dependency layers.
ARG DROBOTICFLOW_BUILD_COMMIT
ARG DROBOTICFLOW_BUILD_TIME
WORKDIR /app
ENV PORT=3001
ENV DIST_DIR=/app/dist
ENV PORT=3001 DIST_DIR=/app/dist
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow
COPY --from=web-build --chown=10001:10001 /app/dist ./dist
COPY --from=go-build /out/droboticflow-api /usr/local/bin/droboticflow-api
EXPOSE 3001
USER 10001:10001
CMD ["droboticflow-api"]

9
Dockerfile.api-runtime Normal file
View File

@ -0,0 +1,9 @@
# Build/publish separately after approval; record this image's immutable digest
# in docs/release/api-runtime.json. No application or customer data is copied.
FROM docker.io/library/debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates ffmpeg && \
dpkg-query -W > /runtime-packages.txt && rm -rf /var/lib/apt/lists/* && \
groupadd --system --gid 10001 droboticflow && \
useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow
WORKDIR /app
USER 10001:10001

View File

@ -26,7 +26,7 @@ import (
func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) {
return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) {
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI))
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI).SetMonitor(store.ReadCommandMonitor()))
if err != nil {
return nil, err
}

View File

@ -0,0 +1,48 @@
# 工作流程可靠性防护
本批修改把已存在的恢复测试和发布约束接入执行入口,不改变 advisory 的业务准入,
不改变权限、版本、领取、送检、审核或验收规则,不自动修改生产记录。
## 必跑回归与证据
`npm run test:work-availability` 分阶段运行:已有标注及送检失败保护、删除/送检并发和重启恢复;
隔离 Mongo 的旧数据/指纹/读取不补验等价检查;advisory 模式的完整工作台及返工浏览器流程。
使用本机合成媒体和独立数据库,不使用公司数据。已知旧失败仍交由既有 Go/前端基线门禁比较。
必需用例被跳过或未出现时失败;独立阶段在普通失败后继续,源码中途变化则停止并列出未验证阶段。
CI 和发布策略均调用同一个入口,不能以只跑单元测试或 Pod Ready 代替业务验收。
每次运行在 `local-data/test-results/` 保存摘要和私有脱敏日志;浏览器在唯一运行目录保留失败截图、
trace 和异常,不覆盖前一次失败。trace 仅用于合成夹具,不能对客户页面照搬。
发布检查的显式证据复用范围限纯检查,失效和失败必须重跑,不能复用浏览器、数据库或性能结果。
## 源准入影响盘点
管理员可读取本组织清单,超级管理员才能读取部署范围清单;组织身份来自服务端会话。
`projectId` 只能缩小组织查询,部署盘点不接受项目过滤。接口只返回按组织、阶段、状态分组的数量,
不返回正文,不创建任务或补验作业,每个进程同时最多一份盘点,Mongo 执行上限 8 秒。
超时/取消返回不完整错误,不能把部分结果作为可启用依据。
严格规则与现有规则变化时,应先将候选以 advisory 发布,再盘点候选规则会阻断的存量数据。
发布工具拒绝旧构建、局部范围、错误数据库、过期、被修改或仍含阻断条目的清单。
盘点不提供事务级数据冻结:仍需在发布安全审查中确定受控切换窗口;无法保证时保持 advisory。
它不伪造证书,也不意味着源内容已通过完整检查。
## 查询与写锁观察
复用服务端生成的 request ID,记录固定阶段的耗时、Mongo 命令成功/失败次数、累计驱动耗时和
受限错误分类,以及写锁获取耗时。只对已有请求收集,不追加查询或自动执行 explain。
日志使用路由模板,不记录查询词、任务 ID、正文、意见、凭证、Mongo 命令或响应体。
驱动耗时包含网络等待,不能当作 Mongo 服务端执行时间;多个命令累计时间也不等于请求墙钟耗时。
连接池等待、扫描量、服务端执行计划和 EIP 带宽没有因此被测量,仍需单独只读诊断。
此修改未优化查询或增加业务后台计算,不构成容量达标结论。
## 构建、发布与恢复边界
泛宇构建固定 Node/Go 构建阶段及运行依赖摘要,使用同一权威配方并记录来源。
镜像和唯一允许的源模式可在同一审查计划顺序切换,其他配置、组件和数据仍另列授权。
处理器失败时停止更新 API,记录部分完成状态,不自动重试、回退或清除证据。
本批未启用 BOS 版本控制、Mongo PITR 或备份策略,也未进行恢复演练。保留送检快照和失败输入的
已有应用级保护不能代替存储备份;备份保留期、成本、权限与隔离恢复演练需单独形成可批准方案。
生产写验收、完整并发容量和故障恢复演练均不包含在本地门禁的通过结论中。

View File

@ -16,9 +16,11 @@
再用 `python3 scripts/release.py status <id>` 查看缺失证据与下一步。
历史聊天和“Pod 已 Ready”不代表业务验收完成。使用命令更新记录,保留失败和未验证项。
首版只支持 **Fanyu 现有 API 与切片处理器的同一不可变镜像更新**,并同步容器的
`DROBOTICFLOW_BUILD_COMMIT`。保留 ConfigMap、Secret、PVC、Service、资源和副本数。
需要更改配置、其他组件或数据库时,单独设计并审查变更计划;首版不提供绕过参数。
支持 **Fanyu 现有 API 与切片处理器的同一不可变镜像更新**,并同步容器的
`DROBOTICFLOW_BUILD_COMMIT`。可通过 `prepare --source-mode advisory|strict` 将唯一指定的
`ANNOTATION_SOURCE_VALIDATION_MODE` 一起纳入同一计划和审批;处理器先更新并就绪,再更新 API。
保留 ConfigMap、Secret、PVC、Service、其他配置、资源和副本数。
需要更改其他配置、组件或数据库时,单独设计并审查变更计划,不提供绕过参数。
2026-10-17 已部署的源索引基线也可做上述镜像更新:处理器必须精确匹配
fanyu-policy.json 引用的容量文件(2 副本、每副本 8 修订槽、4 GiB limit、
@ -70,6 +72,11 @@ metadata 读取 digest,输出 `IMAGE=...@sha256:...` 供 `--image` 使用。
目标 server、context、namespace UID 从已核实的集群身份取得,不能从一次错误连接中直接认可。
工具检查 namespace UID 及 API server,namespace 名字相同也不能蒙混过关。
构建使用权威 `Dockerfile.api` 和 [固定运行依赖](api-runtime.json),不再临时生成包装 Dockerfile。
输出旁的 `.provenance.json` 记录源码、运行基础摘要和配方摘要。当前固定基础为既有已验证镜像,
复制新产物前移除旧应用;`Dockerfile.api-runtime` 提供后续专用基础的构建配方,发布它仍需单独授权。
本地/其他环境直接构建保留原有 Debian 安装路径;泛宇发布脚本只接受不可变运行基础摘要。
```sh
python3 scripts/release.py prepare --id <release-id> \
--image <registry/repository@sha256:digest> \
@ -95,7 +102,14 @@ python3 scripts/release.py check <release-id> --cluster
不再把已替换的旧页面布局当成当前入口。返工浏览器夹具运行真实本地校验处理器,不能绕过源数据准入。
统一候选涉及的额外组件与待确认发布项见 [组件发布计划](recent-integration-components.md)。首版运行流程工具测试、
前端类型/构建及切片恢复定向回归,不冒充全套产品测试通过。任何失败阻断,不支持忽略开关。
命令执行结果及输出摘要写入记录,不保存或回显原始输出;排查失败时单独重跑对应命令。
命令执行结果、输入身份和日志摘要写入记录。本地检查的成功及失败输出先脱敏、限长,再以私有文件保存,
不会记录集群对象原文;日志路径写入记录。浏览器失败截图及 trace 使用唯一运行目录,仅限隔离合成夹具。
`check --local --reuse-successful` 仅可复用一小时内输入、安装依赖、工具、环境和日志摘要均未变的
白名单纯检查。默认重跑全部;浏览器、Mongo、性能与集群检查始终重新执行。
新增强制 `test:work-availability`,复用已有送检保护、删除/送检并发、重启恢复、权限、旧数据
Mongo 等价和 advisory 真实工作台回归。缺用例、跳过、失败、源码中途变化均不能算通过。
CI 独立作业需要 Ubuntu 24.04 runner、Mongo 8、FFmpeg 和 Playwright;远端 CI 未执行时须明确待验证。
2026-10-20 起,`build:web` 后强制运行 `test:production:boot`,实际启动生产 dist 并验证登录页、非空 root 和零运行时错误。该门禁使用本机桩 API,不连接公司;须安装 Playwright 浏览器或通过 `BROWSER_EXECUTABLE` 指定已有 Chromium。构建依赖必须按当前 lockfile 独立安装;共享依赖版本清单或带 alias 的组件夹具不能代替实际产物启动验证。
@ -115,6 +129,16 @@ python3 scripts/release.py plan <release-id>
显示旧/新镜像、构建标记、目标、带宽观察值、限制与回退说明,并生成计划摘要。
核对两个部署的修改范围。计划不更新 ConfigMap、Secret、存储或网络。
新发布记录必须明确两个组件的源校验模式;未配置时禁止依赖代码默认严格模式。
现有历史记录仍按原摘要核验。严格模式切换或严格准入关键规则变化须先以 advisory 部署候选,
再由超级管理员获取 `GET /api/plugins/annotation-qa/source-readiness?scope=deployment` 的只读影响清单,
用 `check --cluster --source-readiness-file <本地私有清单>` 绑定计划。
清单必须覆盖同一数据库、命名空间、线上构建及旧数据截止配置,一小时内完整生成,所有条目均可准入;
组织/项目局部清单、本地夹具、缺项、阻断条目、过期或内容变更均不允许启用严格模式。
清单不补验、不排队、不改标注,也不是源数据校验证书。它是时间点盘点,不会冻结业务数据;
`change-safety` 必须说明盘点至切换期间的新任务和在途变更如何受控,不满足时保持 advisory。
当前模式不因获取清单而变化。详细边界见[可靠性防护](../operations/reliability-guardrails.md)。
部署前需要两份人工审查证据:
- `image-provenance`:构建记录将本次源码 commit 与镜像 digest 对应,候选镜像的验证情况明确。

View File

@ -0,0 +1,5 @@
{
"schemaVersion": 1,
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/fanyu/droboticflow-api@sha256:6a060aa2865ebe0527377a9f22a0f1feb6b23d1f099985a99cce25f6981743d2",
"purpose": "Compatibility bootstrap: reuse approved production runtime; remove old application outputs. Replace with dedicated runtime only through an explicitly reviewed image publication."
}

View File

@ -34,6 +34,7 @@
["npm", "run", "test:browser:annotation-persistence"],
["go", "test", "./internal/deliveryindex", "-count=1"],
["go", "test", "./internal/worker", "-run", "TestCaptureCutover|Test.*V4Capture|TestDiscoverCaptureGroupsForWorkflow|TestClassificationAudit|TestInternalAuditFailure", "-count=1"],
["go", "test", "./internal/api", "-run", "TestDurableAnnotationOutbox|TestSliceRecovery|TestReviewCannotMutateAnnotation|TestReviewDecision|Test.*ReviewDraft|TestReviewConfirmations|TestRouteSnapshot|Test.*Dictionary|TestProjectSliceNames|TestLegacyMaterializationWithoutPartScopedOriginalSelection|TestOptionalOriginalSource|TestAnnotationOriginalSource|TestClassificationBatch|TestWorkflowVersionActivation", "-count=1"]
["go", "test", "./internal/api", "-run", "TestDurableAnnotationOutbox|TestSliceRecovery|TestReviewCannotMutateAnnotation|TestReviewDecision|Test.*ReviewDraft|TestReviewConfirmations|TestRouteSnapshot|Test.*Dictionary|TestProjectSliceNames|TestLegacyMaterializationWithoutPartScopedOriginalSelection|TestOptionalOriginalSource|TestAnnotationOriginalSource|TestClassificationBatch|TestWorkflowVersionActivation", "-count=1"],
["python3", "scripts/test_work_availability.py"]
]
}

View File

@ -2,12 +2,15 @@ package api
import (
"crypto/rand"
"droboticflow/internal/store"
"encoding/hex"
"encoding/json"
"log"
"net/http"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
)
@ -30,6 +33,8 @@ func diagnosticTrace(next http.Handler) http.Handler {
return
}
id := hex.EncodeToString(entropy[:])
ctx, observation := store.WithReadObservation(r.Context(), id)
r = r.WithContext(ctx)
w.Header().Set("X-Request-ID", id)
started := time.Now()
wrapped := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
@ -41,10 +46,19 @@ func diagnosticTrace(next http.Handler) http.Handler {
} else if r.Header.Get("X-Diagnostic-Request") == "1" {
kind = "diagnostic"
}
metrics, _ := json.Marshal(observation.Snapshot())
if kind != "" {
log.Printf("request_trace id=%s kind=%s status=%d elapsed_ms=%d", id, kind, wrapped.Status(), elapsed.Milliseconds())
} else if elapsed >= slowRequestLogThreshold {
log.Printf("request_trace id=%s kind=slow method=%s path=%s status=%d elapsed_ms=%d", id, r.Method, r.URL.Path, wrapped.Status(), elapsed.Milliseconds())
log.Printf("request_trace id=%s kind=%s status=%d elapsed_ms=%d observations=%s", id, kind, wrapped.Status(), elapsed.Milliseconds(), metrics)
} else if elapsed >= slowRequestLogThreshold || wrapped.Status() >= 500 {
route := "/api/<unmatched>"
if rc := chi.RouteContext(r.Context()); rc != nil && rc.RoutePattern() != "" {
route = rc.RoutePattern()
}
reason := "slow"
if elapsed < slowRequestLogThreshold {
reason = "failed"
}
log.Printf("request_trace id=%s kind=%s method=%s path=%s status=%d elapsed_ms=%d observations=%s", id, reason, r.Method, route, wrapped.Status(), elapsed.Milliseconds(), metrics)
}
})
}

View File

@ -2,7 +2,10 @@ package api
import (
"bytes"
"droboticflow/internal/store"
"encoding/json"
"github.com/go-chi/chi/v5"
"go.mongodb.org/mongo-driver/event"
"log"
"net/http"
"net/http/httptest"
@ -100,3 +103,36 @@ func TestDiagnosticTaskSummaryIsScopedAndReadOnly(t *testing.T) {
t.Fatalf("anonymous: %d", anonymous.Code)
}
}
func TestDiagnosticTraceCorrelatesReadPhasesWithoutQueryOrResourceIDs(t *testing.T) {
var logs bytes.Buffer
old := log.Writer()
log.SetOutput(&logs)
defer log.SetOutput(old)
router := chi.NewRouter()
router.Use(diagnosticTrace)
router.Get("/api/tasks/{id}", func(w http.ResponseWriter, r *http.Request) {
if store.ReadRequestID(r.Context()) == "" {
t.Fatal("missing request context")
}
finish := store.ObserveReadPhase(r.Context(), "candidate_query")
finish()
store.ReadCommandMonitor().Failed(r.Context(), &event.CommandFailedEvent{Failure: "i/o timeout private-body"})
w.WriteHeader(500)
})
req := httptest.NewRequest("GET", "/api/tasks/private-id?search=secret-annotation", nil)
req.Header.Set("X-Request-ID", "caller-id")
resp := httptest.NewRecorder()
router.ServeHTTP(resp, req)
text := logs.String()
for _, value := range []string{"private-id", "secret-annotation", "private-body", "caller-id"} {
if strings.Contains(text, value) {
t.Fatal("sensitive request information leaked")
}
}
for _, value := range []string{"/api/tasks/{id}", "candidate_query", "mongo_failures", "client_or_network_deadline", resp.Header().Get("X-Request-ID")} {
if !strings.Contains(text, value) {
t.Fatalf("missing diagnostic: %s", value)
}
}
}

View File

@ -410,7 +410,9 @@ func (server *server) prepareHumanPoolRead(writer http.ResponseWriter, request *
}
func (server *server) readHumanItemsList(writer http.ResponseWriter, request *http.Request) (*humanItemListPage, *http.Request) {
finishPrepare := store.ObserveReadPhase(request.Context(), "prepare")
pool, query, limit, request, ok := server.prepareHumanPoolRead(writer, request)
finishPrepare()
if !ok {
return nil, request
}
@ -429,12 +431,16 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
query.Limit = fetchSize
candidateQuery := query
candidateQuery.PoolCandidateProjection = true
finishQuery := store.ObserveReadPhase(request.Context(), "candidate_query")
items, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), candidateQuery)
finishQuery()
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return nil, request
}
finishContext := store.ObserveReadPhase(request.Context(), "list_context")
listCtx, listErr := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, items)
finishContext()
if listErr != nil {
renderError(writer, http.StatusInternalServerError, listErr)
return nil, request
@ -446,7 +452,9 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
return nil, request
}
}
finishHydrate := store.ObserveReadPhase(request.Context(), "hydrate")
items, err = server.hydrateHumanPoolPage(request, items, pool, query, limit-len(filtered), batchByID)
finishHydrate()
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return nil, request

View File

@ -144,6 +144,8 @@ func (s *server) handleHumanPoolSummaries(w http.ResponseWriter, r *http.Request
if page == nil {
return
}
finishResponse := store.ObserveReadPhase(r.Context(), "response")
defer finishResponse()
summaries := make([]humanPoolSummary, 0, len(page.Items))
for _, item := range page.Items {
summaries = append(summaries, summarizeHumanPoolItem(item))

View File

@ -6,6 +6,7 @@ import "github.com/go-chi/chi/v5"
// plugin's templates and tasks (plus their deprecated /api/annotation-*
// aliases), and the annotation label dictionaries.
func (server *server) mountAnnotationRoutes(router chi.Router) {
router.Get("/api/plugins/annotation-qa/source-readiness", server.handleSourceAdmissionReadiness)
router.Get("/api/data-collection-tasks", server.handleCollectionTasksList)
router.Get("/api/data-collection-tasks/{id}", server.handleCollectionTaskGet)
router.Get("/api/annotation-templates", deprecatedAnnotationEndpoint(server.handleAnnotationTemplatesList))

View File

@ -24,6 +24,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf8"
@ -49,6 +50,7 @@ type validationIssue struct {
}
type server struct {
sourceReadinessRunning atomic.Bool
outputOnce sync.Once
outputReader *outputsnapshot.Reader
humanEvidenceIndexes humanEvidenceCache
@ -1437,11 +1439,16 @@ func invalidConfigurationHandler(err error) http.Handler {
}
func (server *server) serviceMetadata() map[string]any {
mode := "strict"
if store.AnnotationSourceAuditAdvisory() {
mode = "advisory"
}
return map[string]any{
"service": "droboticflow-api",
"buildCommit": server.buildCommit,
"buildTime": server.buildTime,
"image": server.image,
"sourceAdmissionMode": mode,
"service": "droboticflow-api",
"buildCommit": server.buildCommit,
"buildTime": server.buildTime,
"image": server.image,
}
}

View File

@ -0,0 +1,45 @@
package api
import (
"context"
"net/http"
"time"
"droboticflow/internal/store"
)
func (s *server) handleSourceAdmissionReadiness(w http.ResponseWriter, r *http.Request) {
principal, allowed := s.productionAdmin(w, r)
if !allowed {
return
}
db, ok := s.store.(store.SourceReadinessStore)
if !ok {
renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source readiness reporting unavailable"})
return
}
w.Header().Set("Cache-Control", "no-store")
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
org, project := s.requestOrganizationID(r), r.URL.Query().Get("projectId")
if r.URL.Query().Get("scope") == "deployment" {
if !principal.User.IsSuperAdmin || project != "" {
renderJSON(w, http.StatusForbidden, map[string]string{"error": "deployment readiness requires super administrator and no project filter"})
return
}
org = ""
}
if !s.sourceReadinessRunning.CompareAndSwap(false, true) {
renderJSON(w, http.StatusTooManyRequests, map[string]string{"error": "source readiness report already running"})
return
}
defer s.sourceReadinessRunning.Store(false)
report, err := db.SourceAdmissionReadiness(ctx, org, project)
if err != nil {
renderJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source readiness report incomplete", "requestId": store.ReadRequestID(r.Context())})
return
}
report.Namespace = s.k8sNamespace
report.BuildCommit = s.buildCommit
renderJSON(w, http.StatusOK, report)
}

View File

@ -0,0 +1,68 @@
package api
import (
"encoding/json"
"net/http"
"reflect"
"testing"
"droboticflow/internal/store"
)
func TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate(t *testing.T) {
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "")
f := newCombinedDraftFixture(t)
missingTask := f.task
missingTask.ID += "-missing-proof"
if err := f.store.UpsertAnnotationTask(f.ctx, missingTask); err != nil {
t.Fatal(err)
}
missingItem := f.item
missingItem.ID += "-missing-proof"
missingItem.LegacyTaskID = missingTask.ID
missingItem.ProcessingJob = nil
if err := f.store.UpsertHumanItem(f.ctx, missingItem); err != nil {
t.Fatal(err)
}
before, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
var jobs store.AnnotationProcessingStore = f.store
jobBefore, _ := jobs.GetAnnotationIndexJob(f.ctx, f.task.ID)
path := "/api/plugins/annotation-qa/source-readiness"
denied := f.request(http.MethodGet, path, f.annotator, nil)
if denied.Code != 403 {
t.Fatalf("non-admin readiness access: %d", denied.Code)
}
denied = f.request(http.MethodGet, path+"?scope=deployment", f.admin, nil)
if denied.Code != 403 {
t.Fatal("ordinary org admin got deployment-wide inventory")
}
r := f.request(http.MethodGet, path+"?organizationId=other", f.admin, nil)
if r.Code != 200 {
t.Fatalf("readiness: %d %s", r.Code, r.Body.String())
}
var report store.SourceReadiness
if err := json.Unmarshal(r.Body.Bytes(), &report); err != nil {
t.Fatal(err)
}
if !report.Complete || report.TargetMode != "strict" || report.Scope != "organization" || report.OrganizationID != f.task.OrganizationID || len(report.Rows) != 1 || report.Rows[0].Unverified != 1 {
t.Fatalf("incorrect impact report: %+v", report)
}
super := f.admin
super.IsSuperAdmin = true
r = f.request(http.MethodGet, path+"?scope=deployment", super, nil)
if r.Code != 200 {
t.Fatalf("superadmin inventory: %d", r.Code)
}
if err := json.Unmarshal(r.Body.Bytes(), &report); err != nil {
t.Fatal(err)
}
if report.Scope != "deployment" || report.OrganizationID != "" {
t.Fatal("wrong full deployment scope")
}
after, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
jobAfter, _ := jobs.GetAnnotationIndexJob(f.ctx, f.task.ID)
if !reflect.DeepEqual(before, after) || !reflect.DeepEqual(jobBefore, jobAfter) {
t.Fatal("inventory changed task or audit queue")
}
}

View File

@ -1,7 +1,7 @@
# Generated by TestRouteSnapshot; see routes_snapshot_test.go.
# access: reach = endpoint reached; otherwise the status the middleware chain returned.
global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache]
routes=381
routes=382
DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
@ -121,6 +121,7 @@ GET /api/plugins/annotation-qa/admin-pool | mw=[] | (*server).handleAdminAnnotat
GET /api/plugins/annotation-qa/claim-count | mw=[] | (*server).handleAnnotationClaimCount-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/plugins/annotation-qa/claim-projects | mw=[] | (*server).handleAnnotationClaimProjects-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/plugins/annotation-qa/sidebar | mw=[] | (*server).handleAnnotationSidebar-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/plugins/annotation-qa/source-readiness | mw=[] | (*server).handleSourceAdmissionReadiness-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/plugins/annotation-qa/tasks | mw=[] | (*server).handleAnnotationTasksList-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/plugins/annotation-qa/tasks/{id} | mw=[] | (*server).handleAnnotationTaskGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach
GET /api/plugins/annotation-qa/tasks/{id}/frame-timeline | mw=[] | (*server).handleAnnotationTaskFrameTimelineGet-fm | anon=401 token=401 super=403 admin=reach supplier=reach annotator=reach quality=reach acceptance=reach

View File

@ -61,6 +61,8 @@ func (s *MongoStore) withAccountMutationLock(ctx context.Context, key string, op
// for callers without this proof), preserve uncertain-write lease semantics.
func (s *MongoStore) withAccountMutationLockBeforeOperation(ctx context.Context, key string, operation func(context.Context) (any, error), operationStarted *bool) (result any, err error) {
startedAt := time.Now()
finishWait := ObserveReadPhase(ctx, "lock_acquire")
defer finishWait()
diagnose := func(event string) {
digest := sha256.Sum256([]byte(key))
log.Printf("[account-mutation-lock] event=%s resource=%s key_hash=%x elapsed_ms=%d", event, strings.SplitN(key, ":", 2)[0], digest[:6], time.Since(startedAt).Milliseconds())
@ -111,6 +113,7 @@ func (s *MongoStore) withAccountMutationLockBeforeOperation(ctx context.Context,
case <-time.After(25 * time.Millisecond):
}
}
finishWait()
// Renew well inside the lease window (a fixed fraction of it) so normal
// scheduling jitter never lets a live owner's lease lapse; only an
// owner that stops renewing entirely — because it's gone — loses the

View File

@ -124,6 +124,10 @@ func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain
if AnnotationSourceAuditAdvisory() {
return AnnotationSourceTaskMatches(task, item) && !AnnotationSourceUnavailable(item)
}
return annotationItemSourceJobsReadyStrict(task, item, timeline, source)
}
func annotationItemSourceJobsReadyStrict(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool {
if LegacyAnnotationSourceCompatible(task, item, timeline, source) {
return true
}
@ -193,13 +197,17 @@ func AnnotationSourceAdmissionStatus(task domain.AnnotationTaskRecord, item doma
}
return ""
}
return annotationSourceAdmissionStatusStrict(task, item, timeline, source)
}
func annotationSourceAdmissionStatusStrict(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) string {
if task.ID == "" {
return "unverified"
}
if AnnotationSourceJobSuperseded(item, timeline) {
source = nil
}
if AnnotationItemSourceJobsReady(task, item, timeline, source) {
if annotationItemSourceJobsReadyStrict(task, item, timeline, source) {
return ""
}
for _, job := range []*domain.AnnotationIndexJob{timeline, source} {
@ -231,7 +239,11 @@ func AnnotationSourceAdmissionStatus(task domain.AnnotationTaskRecord, item doma
// All joins use existing unique id indexes and project only admission fields.
// Retain caller fields so administrative exception pages can display blocked rows.
func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
if AnnotationSourceAuditAdvisory() {
return annotationSourceAdmissionStatusPipelineForMode(AnnotationSourceAuditAdvisory())
}
func annotationSourceAdmissionStatusPipelineForMode(advisory bool) mongo.Pipeline {
if advisory {
blocked := bson.M{"$and": bson.A{
bson.M{"$in": bson.A{"$currentStageId", bson.A{"annotation_combined", "semantic_annotation", "split_annotation"}}},
bson.M{"$eq": bson.A{"$processingJob.status", "failed"}},

View File

@ -0,0 +1,100 @@
package store
import (
"context"
"strings"
"sync"
"time"
"go.mongodb.org/mongo-driver/event"
)
type readObservationKey struct{}
// ReadObservation contains only durations and counters, never commands, replies,
// resource identifiers or credentials. It lives for one request, not in a cache.
type ReadObservation struct {
mu sync.Mutex
id string
phases map[string]time.Duration
commands int
failures int
mongo time.Duration
errorKinds map[string]int
}
func WithReadObservation(ctx context.Context, id string) (context.Context, *ReadObservation) {
o := &ReadObservation{id: id, phases: map[string]time.Duration{}, errorKinds: map[string]int{}}
return context.WithValue(ctx, readObservationKey{}, o), o
}
func ReadRequestID(ctx context.Context) string {
if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok {
return o.id
}
return ""
}
func ObserveReadPhase(ctx context.Context, name string) func() {
o, ok := ctx.Value(readObservationKey{}).(*ReadObservation)
if !ok {
return func() {}
}
// Fixed vocabulary prevents arbitrary customer values becoming log keys.
switch name {
case "prepare", "candidate_query", "list_context", "hydrate", "response", "lock_acquire":
default:
return func() {}
}
start := time.Now()
var once sync.Once
return func() { once.Do(func() { o.mu.Lock(); defer o.mu.Unlock(); o.phases[name] += time.Since(start) }) }
}
func (o *ReadObservation) Snapshot() map[string]any {
o.mu.Lock()
defer o.mu.Unlock()
phases := map[string]int64{}
for k, v := range o.phases {
phases[k] = v.Milliseconds()
}
kinds := map[string]int{}
for k, v := range o.errorKinds {
kinds[k] = v
}
return map[string]any{"phases_ms": phases, "mongo_commands": o.commands, "mongo_failures": o.failures, "mongo_ms": o.mongo.Milliseconds(), "mongo_error_kinds": kinds}
}
// Command duration includes driver/network time, not just server execution.
// It is not connection checkout wait or docs/keys examined.
// Those require independent evidence; never infer them from elapsed time.
func ReadCommandMonitor() *event.CommandMonitor {
record := func(ctx context.Context, duration int64, failed bool) {
if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok {
o.mu.Lock()
defer o.mu.Unlock()
o.commands++
o.mongo += time.Duration(duration)
if failed {
o.failures++
}
}
}
return &event.CommandMonitor{
Succeeded: func(ctx context.Context, e *event.CommandSucceededEvent) { record(ctx, e.DurationNanos, false) },
Failed: func(ctx context.Context, e *event.CommandFailedEvent) {
record(ctx, e.DurationNanos, true)
if o, ok := ctx.Value(readObservationKey{}).(*ReadObservation); ok {
kind := "driver_error"
if strings.Contains(e.Failure, "MaxTimeMSExpired") {
kind = "server_deadline"
} else if strings.Contains(e.Failure, "deadline exceeded") || strings.Contains(e.Failure, "i/o timeout") {
kind = "client_or_network_deadline"
}
o.mu.Lock()
o.errorKinds[kind]++
o.mu.Unlock()
}
},
}
}

View File

@ -0,0 +1,67 @@
package store
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"go.mongodb.org/mongo-driver/event"
)
func TestReadObservationIsPrivateConcurrentAndRequestScoped(t *testing.T) {
ctx, o := WithReadObservation(context.Background(), "server-generated")
monitor := ReadCommandMonitor()
var wg sync.WaitGroup
for range 20 {
wg.Add(1)
go func() {
defer wg.Done()
end := ObserveReadPhase(ctx, "candidate_query")
defer end()
monitor.Succeeded(ctx, &event.CommandSucceededEvent{CommandFinishedEvent: event.CommandFinishedEvent{DurationNanos: int64(time.Millisecond), CommandName: "find"}})
}()
}
wg.Wait()
monitor.Failed(ctx, &event.CommandFailedEvent{CommandFinishedEvent: event.CommandFinishedEvent{DurationNanos: int64(time.Millisecond)}, Failure: "mongodb://password-secret@host private-customer-body"})
ObserveReadPhase(ctx, "customer-should-not-be-key")()
b, _ := json.Marshal(o.Snapshot())
if strings.Contains(string(b), "secret") || strings.Contains(string(b), "customer") || strings.Contains(string(b), "find") {
t.Fatal("command contents leaked")
}
if o.Snapshot()["mongo_commands"] != 21 || o.Snapshot()["mongo_failures"] != 1 {
t.Fatal("counter mismatch")
}
if ReadRequestID(ctx) != "server-generated" || ReadRequestID(context.Background()) != "" {
t.Fatal("request context leaked")
}
_, other := WithReadObservation(context.Background(), "other")
if other.Snapshot()["mongo_commands"] != 0 {
t.Fatal("cross request counters")
}
}
func TestReadObservationConcurrentFailureSnapshots(t *testing.T) {
ctx, o := WithReadObservation(context.Background(), "generated")
monitor := ReadCommandMonitor()
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for range 200 {
monitor.Failed(ctx, &event.CommandFailedEvent{Failure: "context deadline exceeded"})
}
}()
go func() {
defer wg.Done()
for range 200 {
_, _ = json.Marshal(o.Snapshot())
}
}()
wg.Wait()
if o.Snapshot()["mongo_failures"] != 200 {
t.Fatal("missing failures")
}
}

View File

@ -0,0 +1,144 @@
package store
import (
"context"
"droboticflow/internal/domain"
"os"
"sort"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type SourceReadinessRow struct {
OrganizationID string `json:"organizationId" bson:"organizationId"`
Stage string `json:"stage" bson:"stage"`
Status string `json:"status" bson:"status"`
Total int64 `json:"total" bson:"total"`
Allowed int64 `json:"allowed" bson:"allowed"`
Unverified int64 `json:"unverified" bson:"unverified"`
Pending int64 `json:"pending" bson:"pending"`
Failed int64 `json:"failed" bson:"failed"`
}
type SourceReadiness struct {
CurrentMode string `json:"currentMode"`
Scope string `json:"scope"`
Database string `json:"database"`
Schema string `json:"schema"`
OrganizationID string `json:"organizationId"`
ProjectID string `json:"projectId"`
Namespace string `json:"namespace"`
BuildCommit string `json:"buildCommit"`
GeneratedAt string `json:"generatedAt"`
TargetMode string `json:"targetMode"`
LegacySourceCutoff string `json:"legacySourceCutoff"`
Complete bool `json:"complete"`
Rows []SourceReadinessRow `json:"rows"`
}
type SourceReadinessStore interface {
SourceAdmissionReadiness(context.Context, string, string) (*SourceReadiness, error)
}
func sourceReadiness(org, project string, rows []SourceReadinessRow) *SourceReadiness {
sort.Slice(rows, func(i, j int) bool {
if rows[i].OrganizationID != rows[j].OrganizationID {
return rows[i].OrganizationID < rows[j].OrganizationID
}
if rows[i].Stage != rows[j].Stage {
return rows[i].Stage < rows[j].Stage
}
return rows[i].Status < rows[j].Status
})
mode := "strict"
if AnnotationSourceAuditAdvisory() {
mode = "advisory"
}
scope := "organization"
if org == "" {
scope = "deployment"
}
return &SourceReadiness{CurrentMode: mode, Scope: scope, Database: "memory", Schema: "droboticflow.source-readiness/v1", OrganizationID: org, ProjectID: project, GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), TargetMode: "strict", LegacySourceCutoff: os.Getenv("ANNOTATION_LEGACY_SOURCE_CUTOFF"), Complete: true, Rows: rows}
}
// This is an explicit administrator inventory, never part of normal work reads.
// Missing proof is reported, not repaired or enqueued. No task body is returned.
func (s *MemoryStore) SourceAdmissionReadiness(ctx context.Context, org, project string) (*SourceReadiness, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
s.mu.Lock()
defer s.mu.Unlock()
grouped := map[string]*SourceReadinessRow{}
for _, item := range s.humanItems {
if err := ctx.Err(); err != nil {
return nil, err
}
if org != "" && item.OrganizationID != org || item.PluginID != "annotation-qa" || item.LegacyTaskID == "" || project != "" && item.ProjectID != project {
continue
}
key := item.OrganizationID + "\x00" + item.CurrentStageID + "\x00" + string(item.Status)
row := grouped[key]
if row == nil {
row = &SourceReadinessRow{OrganizationID: item.OrganizationID, Stage: item.CurrentStageID, Status: string(item.Status)}
grouped[key] = row
}
row.Total++
task := s.annotationTasks[item.LegacyTaskID]
t, tok := s.annotationIndexJobs[item.LegacyTaskID]
j, jok := s.annotationIndexJobs[sourceJobForItem(item)]
var tp, sp *domain.AnnotationIndexJob
if tok {
tp = &t
}
if jok {
sp = &j
}
switch annotationSourceAdmissionStatusStrict(task, item, tp, sp) {
case "":
row.Allowed++
case "pending":
row.Pending++
case "failed":
row.Failed++
default:
row.Unverified++
}
}
rows := []SourceReadinessRow{}
for _, v := range grouped {
rows = append(rows, *v)
}
return sourceReadiness(org, project, rows), nil
}
func (s *MongoStore) SourceAdmissionReadiness(ctx context.Context, org, project string) (*SourceReadiness, error) {
filter := bson.M{"pluginId": "annotation-qa", "legacyTaskId": bson.M{"$type": "string", "$ne": ""}}
if org != "" {
filter["organizationId"] = org
}
if project != "" {
filter["projectId"] = project
}
pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: annotationSourceAdmissionProjection()}}}
pipeline = append(pipeline, annotationSourceAdmissionStatusPipelineForMode(false)...)
count := func(status string) bson.M {
return bson.M{"$sum": bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{"$_sourceAdmissionStatus", status}}, 1, 0}}}
}
pipeline = append(pipeline, bson.D{{Key: "$group", Value: bson.M{"_id": bson.M{"organizationId": "$organizationId", "stage": "$currentStageId", "status": "$status"}, "total": bson.M{"$sum": 1}, "allowed": count(""), "pending": count("pending"), "failed": count("failed"), "unverified": count("unverified")}}}, bson.D{{Key: "$project", Value: bson.M{"_id": 0, "organizationId": "$_id.organizationId", "stage": "$_id.stage", "status": "$_id.status", "total": 1, "allowed": 1, "pending": 1, "failed": 1, "unverified": 1}}})
cursor, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetMaxTime(8*time.Second))
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
rows := []SourceReadinessRow{}
if err = cursor.All(ctx, &rows); err != nil {
return nil, err
}
report := sourceReadiness(org, project, rows)
report.Database = s.db.Name()
return report, nil
}

View File

@ -0,0 +1,101 @@
package store
import (
"context"
"fmt"
"os"
"reflect"
"testing"
"time"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func TestSourceReadinessMongoParityReadOnly(t *testing.T) {
if os.Getenv("SOURCE_ADMISSION_LOCAL_MONGO") != "1" {
t.Skip("isolated loopback Mongo opt-in")
}
t.Setenv("ANNOTATION_SOURCE_VALIDATION_MODE", "advisory")
t.Setenv("ANNOTATION_LEGACY_SOURCE_CUTOFF", "")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
c, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(3*time.Second))
if err != nil {
t.Fatal(err)
}
defer c.Disconnect(context.Background())
db := c.Database(fmt.Sprintf("source_readiness_%d", time.Now().UnixNano()))
defer db.Drop(context.Background())
ms := NewMongoStore(db)
mem := NewMemoryStore()
mem.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
must := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
before := map[string]*domain.HumanItemRecord{}
for n, state := range []string{"", "queued", "failed", "completed", "foreign"} {
id := fmt.Sprintf("i%d", n)
org := "o"
if state == "foreign" {
org = "foreign"
}
task := domain.AnnotationTaskRecord{ID: "t" + id, OrganizationID: org, ProjectID: "p", RunID: id}
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: "p", RunID: id, PluginID: "annotation-qa", LegacyTaskID: task.ID, CurrentStageID: "annotation_combined", Status: domain.HumanItemClaimed, AssignedUserID: "owner", Version: 1, Subject: domain.HumanSubjectRef{Payload: map[string]any{"annotationDraft": map[string]any{"description": "retained"}}}}
must(ms.UpsertAnnotationTask(ctx, task))
must(mem.UpsertAnnotationTask(ctx, task))
must(ms.UpsertHumanItem(ctx, item))
must(mem.UpsertHumanItem(ctx, item))
if state != "" && state != "foreign" {
job := domain.AnnotationIndexJob{ID: task.ID, OrganizationID: org, ProjectID: "p", RunID: id, Status: state}
if state == "completed" {
job.SourceFingerprint = "fp"
job.RevalidateSource = true
}
_, err = db.Collection("annotationIndexJobs").InsertOne(ctx, job)
must(err)
mem.annotationIndexJobs[task.ID] = job
}
before[id], err = ms.GetHumanItem(ctx, id)
must(err)
}
for _, org := range []string{"o", ""} {
a, err := ms.SourceAdmissionReadiness(ctx, org, "")
must(err)
b, err := mem.SourceAdmissionReadiness(ctx, org, "")
must(err)
a.GeneratedAt = ""
b.GeneratedAt = ""
a.Database = ""
b.Database = ""
if !reflect.DeepEqual(a, b) {
t.Fatalf("Mongo/Go strict readiness mismatch: %+v / %+v", a, b)
}
total := int64(0)
for _, r := range a.Rows {
total += r.Total
if r.Total != r.Allowed+r.Pending+r.Unverified+r.Failed {
t.Fatal("counts do not reconcile")
}
}
if org == "o" && total != 4 || org == "" && total != 5 {
t.Fatal("readiness scope mismatch")
}
}
for id, before := range before {
after, err := ms.GetHumanItem(ctx, id)
must(err)
if !reflect.DeepEqual(before, after) {
t.Fatal("inventory changed saved annotations or ownership")
}
}
canceled, stop := context.WithCancel(ctx)
stop()
if r, err := ms.SourceAdmissionReadiness(canceled, "", ""); err == nil || r != nil {
t.Fatal("canceled report claimed completeness")
}
}

View File

@ -104,7 +104,8 @@
"predev:api": "npm run hooks:install",
"predev:web": "npm run hooks:install",
"predev:worker": "npm run hooks:install",
"predev:api:replica": "npm run hooks:install"
"predev:api:replica": "npm run hooks:install",
"test:work-availability": "python3 scripts/test_work_availability.py"
},
"dependencies": {
"@tailwindcss/vite": "^4.1.14",

View File

@ -0,0 +1,105 @@
"""Private, bounded local-check evidence. Never log kubectl payloads here."""
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import time
import uuid
SENSITIVE = '[A-Za-z0-9_-]{0,96}(?:password|passwd|secret|token|authorization|cookie|credential|access[_-]?key|secret[_-]?key|phone)[A-Za-z0-9_-]{0,96}|(?:BOS|BCE|AWS)[_-](?:AK|SK)|AK|SK'
def redact(text):
"""Remove credential-bearing fields and URLs before persisting bounded local test output."""
text = '\n'.join(line if len(line) <= 65536 else '<oversized-line-redacted>' for line in text.splitlines())
text = re.sub('(?is)(?<![A-Za-z0-9_-])([\\"\\\']?(?:' + SENSITIVE + ')[\\"\\\']?\\s*[:=]\\s*)(\\"[^\\"]*\\"|\\\'[^\\\']*\\\'|[^\\s,}]+)', '\\1<redacted>', text)
text = re.sub('(?i)\\bBearer\\s+\\S+', 'Bearer <redacted>', text)
text = re.sub('(?i)\\b(?:https?|mongodb(?:\\+srv)?)://[^\\s\\"\\\'<>]+', '<redacted-url>', text)
lines = ['<redacted-sensitive-line>' if re.search('(?i)\\b(?:' + SENSITIVE + ')\\b\\s*:', line) else line for line in text.splitlines()]
text = '\n'.join(lines)
limit = 2 * 1024 * 1024
if len(text) > limit:
text = text[:limit // 2] + '\n<output-truncated>\n' + text[-limit // 2:]
return text
def write_log(folder, argv, output, result):
folder = Path(folder)
folder.mkdir(parents=True, exist_ok=True, mode=448)
path = folder / ('check-' + uuid.uuid4().hex + '.log')
data = redact(json.dumps({'argv': argv, 'result': result}) + '\n' + output) + '\n'
with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 384), 'w') as f:
f.write(data)
return {'log': str(path.resolve()), 'logSHA256': hashlib.sha256(path.read_bytes()).hexdigest()}
def check_environment():
"""Local synthetic checks inherit no company credentials or application/test switches."""
return {k: v for k, v in os.environ.items() if not re.search('(?i)(password|secret|token|credential|access_key|secret_key|^BOS_|^BCE_|^AWS_|(^|_)MONGO(_|$)|(_URI|_URL)$|^KUBECONFIG$|^ANNOTATION_|^DROBOTICFLOW_|^DEPLOY_ENV$|^UPDATE_|^REGENERATE_|_TEST$|_TEST_|^TEST_.*MONGO)', k)}
def source_fingerprint(root):
"""Hash the complete non-ignored source tree; unavailable Git identity is never reusable."""
root = Path(root)
h = hashlib.sha256()
try:
names = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=root, stderr=subprocess.DEVNULL).decode().split('\x00')
except (OSError, subprocess.SubprocessError):
return None
for name in sorted(set(filter(None, names))):
path = root / name
h.update(name.encode())
if path.is_symlink():
h.update(os.readlink(path).encode())
elif path.is_file():
h.update(path.read_bytes())
else:
h.update(b'<missing>')
return h.hexdigest()
def identity(root, commit, policy_hash):
"""Bind source, installed dependency contents, tool versions and a hash of the sanitized environment. Values and dependency contents are not persisted."""
root = Path(root)
h = hashlib.sha256()
dependencies = root / 'node_modules'
if dependencies.is_dir():
visited = set()
for base, dirs, files in os.walk(dependencies, followlinks=True):
stat = Path(base).stat()
key = (stat.st_dev, stat.st_ino)
if key in visited:
dirs[:] = []
continue
visited.add(key)
h.update(str(Path(base).relative_to(dependencies)).encode())
if Path(base).is_symlink():
h.update(os.readlink(base).encode())
dirs[:] = sorted((d for d in dirs if d not in ('.cache', '.vite', '.vite-temp')))
for name in sorted(files):
p = Path(base) / name
h.update(str(p.relative_to(dependencies)).encode())
if p.is_symlink():
h.update(os.readlink(p).encode())
else:
with p.open('rb') as f:
for block in iter(lambda: f.read(1024 * 1024), b''):
h.update(block)
else:
h.update(b'no-installed-node-dependencies')
versions = {}
for name, args in [('python', ['python3', '--version']), ('node', ['node', '--version']), ('go', ['go', 'version']), ('ffmpeg', ['ffmpeg', '-version'])]:
try:
versions[name] = subprocess.check_output(args, cwd=root, stderr=subprocess.STDOUT, timeout=10).decode().splitlines()[0]
except (OSError, subprocess.SubprocessError):
versions[name] = 'unavailable'
env_hash = hashlib.sha256(json.dumps(check_environment(), sort_keys=True).encode()).hexdigest()
source = source_fingerprint(root)
complete = bool(source) and dependencies.is_dir() and all((v != 'unavailable' for v in versions.values()))
return {'complete': complete, 'sourceTree': source, 'commit': commit, 'policyHash': policy_hash, 'dependencies': h.hexdigest(), 'tools': versions, 'environment': env_hash}
def reusable(entry, expected, argv):
"""Only explicitly listed pure checks may reuse unchanged, unexpired and intact success evidence. Browser, Mongo and timing checks always rerun."""
pure = argv[:3] == ['npm', 'run', 'lint:web'] or argv[:3] == ['npm', 'run', 'test:annotation:persistence'] or argv[:3] == ['npm', 'run', 'test:review:decisions'] or (argv[:3] == ['npm', 'run', 'test:review:drafts']) or (argv[:3] == ['python3', '-m', 'unittest'])
if not expected.get('complete') or not pure or entry.get('identity') != expected or (entry.get('result') != 'passed'):
return False
if not 0 <= time.time() - entry.get('finishedUnix', 0) <= 3600:
return False
path = Path(entry.get('log', ''))
return path.is_file() and hashlib.sha256(path.read_bytes()).hexdigest() == entry.get('logSHA256')

View File

@ -0,0 +1,53 @@
"""Validate an administrator's metadata-only strict-admission impact report."""
import datetime as dt
import hashlib
import json
import re
from pathlib import Path
def validate_report(path, *, namespace, database, cutoff, live_commit, now=None):
path = Path(path)
if path.stat().st_size > 8 * 1024 * 1024:
raise ValueError('Readiness report too large.')
body = path.read_bytes()
r = json.loads(body)
if not isinstance(r, dict):
raise ValueError('Invalid readiness object.')
if not isinstance(live_commit, str) or not re.fullmatch('[0-9a-f]{40}', live_commit):
raise ValueError('Unknown live source commit.')
if r.get('scope') != 'deployment' or r.get('organizationId') != '' or r.get('database') != database:
raise ValueError('Strict activation requires complete deployment-wide database scope.')
if r.get('schema') != 'droboticflow.source-readiness/v1' or r.get('namespace') != namespace or r.get('targetMode') != 'strict' or (r.get('complete') is not True) or (r.get('projectId') != '') or (r.get('legacySourceCutoff', '') != (cutoff or '')) or (r.get('buildCommit') != live_commit):
raise ValueError('Incomplete, scoped or mismatched strict readiness report.')
if not isinstance(r.get('generatedAt'), str):
raise ValueError('Invalid readiness timestamp.')
at = dt.datetime.fromisoformat(r['generatedAt'].replace('Z', '+00:00'))
if at.tzinfo is None:
raise ValueError('Readiness timestamp must include timezone.')
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
if not 0 <= age <= 3600:
raise ValueError('Readiness report is expired or in the future.')
totals = {k: 0 for k in ('total', 'allowed', 'unverified', 'pending', 'failed')}
rows = r['rows']
if not isinstance(rows, list):
raise ValueError('Invalid readiness rows.')
seen = set()
for row in rows:
if not isinstance(row, dict):
raise ValueError('Invalid readiness row.')
key = (row.get('organizationId'), row.get('stage'), row.get('status'))
if not all((isinstance(x, str) for x in key)) or key in seen:
raise ValueError('Duplicate or invalid readiness group.')
seen.add(key)
if not isinstance(row.get('stage'), str) or not isinstance(row.get('status'), str):
raise ValueError('Missing readiness stage/status.')
for k in totals:
v = row.get(k)
if type(v) is not int or v < 0:
raise ValueError('Invalid readiness count.')
totals[k] += v
if row['total'] != sum((row[k] for k in ('allowed', 'unverified', 'pending', 'failed'))):
raise ValueError('Readiness counts do not reconcile.')
if totals['total'] != totals['allowed']:
raise ValueError('Strict mode would block existing tasks; keep advisory and review the impact.')
return {'path': str(path.resolve()), 'sha256': hashlib.sha256(body).hexdigest(), 'scope': 'deployment', 'generatedAt': r['generatedAt'], 'totals': totals}

View File

@ -27,8 +27,16 @@ commit="$(git -C "$ROOT_DIR" rev-parse HEAD)"
build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
metadata="${2:-$(mktemp -t droboticflow-api-build.XXXXXX)}"
runtime_image="${API_RUNTIME_IMAGE:-$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["image"])' "$ROOT_DIR/docs/release/api-runtime.json")}"
if [[ ! "$runtime_image" =~ ^[A-Za-z0-9][A-Za-z0-9._:/-]*@sha256:[0-9a-f]{64}$ ]]; then
echo "API_RUNTIME_IMAGE must be an immutable prepared runtime digest" >&2
exit 2
fi
"$DOCKER" buildx build \
--platform "$PLATFORM" \
--build-arg "API_RUNTIME_IMAGE=$runtime_image" \
--build-arg "API_RUNTIME_PREPARED=1" \
--build-arg "GOPROXY=${GOPROXY:-https://proxy.golang.org,direct}" \
--build-arg "DROBOTICFLOW_BUILD_COMMIT=$commit" \
--build-arg "DROBOTICFLOW_BUILD_TIME=$build_time" \
@ -44,3 +52,9 @@ if [[ "$digest" != sha256:* ]]; then
exit 1
fi
printf 'COMMIT=%s\nBUILD_TIME=%s\nMETADATA=%s\nIMAGE=%s@%s\n' "$commit" "$build_time" "$metadata" "$repository" "$digest"
python3 - "$metadata.provenance.json" "$ROOT_DIR/Dockerfile.api" "$commit" "$runtime_image" "$digest" "$build_time" <<'PYPROOF'
import hashlib,json,pathlib,sys
output,recipe,commit,runtime,digest,at=sys.argv[1:]
pathlib.Path(output).write_text(json.dumps({"schemaVersion":1,"commit":commit,"runtimeImage":runtime,"imageDigest":digest,"buildTime":at,"recipeSHA256":hashlib.sha256(pathlib.Path(recipe).read_bytes()).hexdigest()},indent=2)+"\n")
PYPROOF

View File

@ -13,6 +13,11 @@ import re
import subprocess
import sys
import uuid
import time
sys.path.insert(0, str(Path(__file__).resolve().parent))
from lib.release_checks import check_environment, identity, reusable, write_log
from lib.source_readiness import validate_report
ROOT = Path(__file__).resolve().parents[1]
RECORDS = ROOT / 'local-data/releases'
@ -42,13 +47,18 @@ def digest(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
def run(argv, *, stdin=None, timeout=60):
def run(argv, *, stdin=None, timeout=60, env=None):
try:
p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout)
except (OSError, subprocess.TimeoutExpired):
raise Blocked('Command unavailable or timed out; outcome must be checked before retry.') from None
# Do not echo kubectl output, URLs or credential-bearing errors.
require(p.returncode == 0, 'Command failed: ' + argv[0] + ' (output withheld; no success recorded).')
p = subprocess.run(argv, cwd=ROOT, input=stdin, text=True, capture_output=True, timeout=timeout, env=env)
except (OSError, subprocess.TimeoutExpired) as e:
error = Blocked('Command unavailable or timed out; outcome must be checked before retry.')
error.check_output = str(getattr(e, 'stdout', '') or '') + str(getattr(e, 'stderr', '') or '')
raise error from None
# Only policy-local checks may persist the private redacted output.
if p.returncode != 0:
error = Blocked('Command failed: ' + argv[0] + ' (no success recorded).')
error.check_output = p.stdout + '\n' + p.stderr
raise error
return p.stdout
@ -175,7 +185,7 @@ def env_values(container, configs):
return values
def inspect_deployment(deployment, container_name, configs, p):
def inspect_deployment(deployment, container_name, configs, p, *, ignore_source_mode=False):
spec = deployment['spec']
pod = spec['template']['spec']
containers = pod['containers']
@ -228,10 +238,10 @@ def inspect_deployment(deployment, container_name, configs, p):
neutral = copy.deepcopy(spec)
nc = neutral['template']['spec']['containers'][index]
nc['image'] = '<release-image>'
nc['env'] = [x for x in nc.get('env', []) if x['name'] != 'DROBOTICFLOW_BUILD_COMMIT']
nc['env'] = [x for x in nc.get('env', []) if x['name'] not in (('DROBOTICFLOW_BUILD_COMMIT', 'ANNOTATION_SOURCE_VALIDATION_MODE') if ignore_source_mode else ('DROBOTICFLOW_BUILD_COMMIT',))]
stamp = next((i for i, x in enumerate(c.get('env', [])) if x['name'] == 'DROBOTICFLOW_BUILD_COMMIT'), None)
return {'uid': m['uid'], 'specHash': digest(spec), 'nonReleaseHash': digest(neutral),
'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'],
'sourceModeIndex': next((i for i,x in enumerate(c.get('env',[])) if x['name']=='ANNOTATION_SOURCE_VALIDATION_MODE'),None), 'buildCommitIndex': stamp, 'buildCommit': e.get('DROBOTICFLOW_BUILD_COMMIT'), 'image': c['image'],
'resourceVersion': m['resourceVersion'], 'containerIndex': index,
'ready': ready, 'env': {k: e.get(k) for k in SAFE_ENV}}
@ -251,7 +261,7 @@ def validate_legacy_source_cutoffs(envs):
raise Blocked('Invalid legacy source admission cutoff.')
def snapshot(r):
def snapshot(r, *, allow_planned_transition=False):
t, p = r['target'], policy()
view = json.loads(kubectl(r, ['config', 'view', '--minify', '-o', 'json']))
require(view['clusters'][0]['cluster']['server'] == t['server'], 'Kube API server mismatch.')
@ -265,10 +275,17 @@ def snapshot(r):
config_names.update(x['valueFrom']['configMapKeyRef']['name'] for x in c.get('env', [])
if 'configMapKeyRef' in x.get('valueFrom', {}))
configs = {n: get(r, 'configmap', n) for n in sorted(config_names)}
deps = {n: inspect_deployment(d, p['deployments'][n], configs, p) for n, d in deployments.items()}
deps = {n: inspect_deployment(d, p['deployments'][n], configs, p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in deployments.items()}
envs = [d['env'] for d in deps.values()]
require(len({e['ANNOTATION_WORKSPACE_CONNECTION_ID'] for e in envs}) == 1, 'API/processor storage connections differ.')
validate_legacy_source_cutoffs(envs)
if allow_planned_transition and r.get("sourceMode"):
for e in envs: require(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in (r["sourceMode"], *(d["env"].get("ANNOTATION_SOURCE_VALIDATION_MODE") for d in r["baseline"]["deployments"].values())), "Unexpected source mode during rollout.")
normalized = [dict(e, ANNOTATION_SOURCE_VALIDATION_MODE=r["sourceMode"]) for e in envs]
validate_legacy_source_cutoffs(normalized)
else:
validate_legacy_source_cutoffs(envs)
if r.get("sourceModeExplicit") and not r.get("sourceMode"):
require(all(e.get("ANNOTATION_SOURCE_VALIDATION_MODE") in ("advisory", "strict") for e in envs), "New releases require an explicit source admission mode on both components.")
pvc = get(r, 'pvc', p['workspacePVC'])
require(pvc.get('status', {}).get('phase') == 'Bound', 'Workspace PVC is not Bound.')
svc = get(r, 'service', 'droboticflow-api-public-eip')
@ -287,7 +304,9 @@ def stable(s):
def plan_data(r):
return {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')}
data = {k: r[k] for k in ('id', 'commit', 'image', 'policyHash', 'target', 'baseline', 'localChecks', 'purpose', 'rollback')}
data.update({k:r[k] for k in ('sourceModeExplicit','sourceMode','sourceReadiness') if k in r})
return data
def manifest_platforms(image_ref):
@ -317,11 +336,32 @@ def prepare(a):
'target': {'kubeconfig': str(Path(a.kubeconfig).resolve()), 'context': a.context,
'server': a.server.rstrip('/'), 'namespace': p['namespace'], 'namespaceUID': a.namespace_uid},
'state': 'prepared', 'localChecks': [], 'baseline': None, 'planHash': None,
'operations': [], 'evidence': {}, 'events': []}
'operations': [], 'evidence': {}, 'events': [], 'sourceModeExplicit': True,
'sourceMode': getattr(a, 'source_mode', None), 'sourceReadiness': None}
event(r, 'prepared; no network access')
print(json.dumps({'id': r['id'], 'state': r['state']}, ensure_ascii=False))
def check_identity(r):
return identity(ROOT, r['commit'], r['policyHash'])
def run_local_check(r, argv, expected):
entry = {'argv': argv, 'startedAt': now(), 'identity': expected}
try:
output = run(argv, timeout=1800, env=check_environment())
except Blocked as e:
entry.update(result='failed', finishedUnix=time.time())
entry.update(write_log(record_path(r['id']).parent/'logs', argv, getattr(e, 'check_output', ''), 'failed'))
r['localChecks'].append(entry)
r['state'] = 'blocked'
event(r, 'local check failed; private redacted log: ' + entry['log'])
raise Blocked(str(e) + ' Log: ' + entry['log']) from None
entry.update(result='passed', outputHash=digest(output), finishedUnix=time.time())
entry.update(write_log(record_path(r['id']).parent/'logs', argv, output, 'passed'))
return entry
def check(r, a):
fresh(r)
require(not r['operations'], 'A deployment was attempted; check outcome with verify, never rewrite its plan.')
@ -342,18 +382,18 @@ def check(r, a):
r['state'] = 'blocked'
event(r, 'local check failed')
raise
previous = {tuple(x['argv']): x for x in r['localChecks']}
expected = check_identity(r)
r['localChecks'] = []
for argv in policy()['localChecks']:
started = now()
try:
output = run(argv, timeout=1800)
except Blocked:
r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'failed'})
r['state'] = 'blocked'
event(r, 'local check failed')
raise
r['localChecks'].append({'argv': argv, 'startedAt': started, 'result': 'passed', 'outputHash': digest(output)})
old = previous.get(tuple(argv), {})
if getattr(a, 'reuse_successful', False) and reusable(old, expected, argv):
entry = {**old, 'reusedAt': now()}
else:
entry = run_local_check(r, argv, expected)
r['localChecks'].append(entry)
save(r)
require(check_identity(r) == expected, 'Check environment/dependencies changed; evidence is not reusable.')
fresh(r)
if a.cluster:
# Never preserve an earlier successful snapshot when a new read fails.
@ -363,6 +403,15 @@ def check(r, a):
s = snapshot(r)
require(all(d['ready'] for d in s['deployments'].values()), 'Current deployments are not fully ready.')
r['baseline'] = s
if strict_transition(r):
path = getattr(a, 'source_readiness_file', None)
require(bool(path), 'Strict activation needs a fresh deployment-wide --source-readiness-file.')
api = s['deployments']['droboticflow-api']
try:
r['sourceReadiness'] = validate_report(path, namespace=r['target']['namespace'], database=policy()['database'],
cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit'])
except (ValueError,KeyError,TypeError,AttributeError,OSError):
raise Blocked('Strict readiness incomplete, blocked, expired or wrong scope; keep advisory.') from None
except Blocked:
r['state'] = 'blocked'
event(r, 'cluster check failed')
@ -371,20 +420,61 @@ def check(r, a):
event(r, 'checks recorded; plan must still be generated')
def target_source_mode(r, name):
return r.get('sourceMode') or r['baseline']['deployments'][name]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE')
def validate_planned_source_mode(r):
if r.get("sourceModeExplicit"):
require(all(target_source_mode(r,n) in ("advisory","strict") for n in r["baseline"]["deployments"]), "New releases require an explicit planned source mode.")
def strict_transition(r):
if not any(target_source_mode(r,n)=='strict' for n in r['baseline']['deployments']): return False
if any(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE')!='strict' for d in r['baseline']['deployments'].values()): return True
return strict_rules_changed(r)
def strict_rules_changed(r):
live=r['baseline']['deployments']['droboticflow-api']['buildCommit']
if live==r['commit']: return False
require(isinstance(live,str) and bool(re.fullmatch(r'[0-9a-f]{40}',live)), 'Unknown current source; strict activation requires advisory preparation first.')
paths=['internal/store/annotation_source_admission.go','internal/store/annotation_availability.go',
'internal/api/annotation_source_admission.go','internal/api/annotation_source_preparation.go',
'internal/api/annotation_source_certificate.go','internal/api/annotation_source_availability.go']
return bool(run(['git','diff','--name-only',live,r['commit'],'--',*paths]).strip())
def validate_source_transition(r):
if not strict_transition(r): return
proof = r.get('sourceReadiness')
require(bool(proof), 'Strict activation requires a complete deployment-wide source readiness report.')
require(not strict_rules_changed(r), 'Strict admission rules changed; deploy advisory first, then inventory with the candidate rules.')
api = r['baseline']['deployments']['droboticflow-api']
try:
current = validate_report(proof['path'], namespace=r['target']['namespace'], database=policy()['database'],
cutoff=api['env'].get('ANNOTATION_LEGACY_SOURCE_CUTOFF'), live_commit=api['buildCommit'])
require(current['sha256'] == proof['sha256'], 'Source readiness report changed; recheck and approve a new plan.')
except (ValueError, KeyError, TypeError, AttributeError, OSError) as e:
raise Blocked('Source readiness report invalid: ' + type(e).__name__) from None
def make_plan(r):
fresh(r)
require(not r['operations'], 'Deployment already attempted.')
require([x['argv'] for x in r['localChecks']] == policy()['localChecks']
and all(x['result'] == 'passed' for x in r['localChecks']), 'Required local checks incomplete.')
require(r['baseline'] is not None, 'Read-only cluster check required.')
validate_planned_source_mode(r)
validate_source_transition(r)
r['planHash'] = digest(plan_data(r))
r['state'] = 'awaiting_approval'
event(r, 'plan prepared; no authorization inferred')
print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit']} for n in sorted(r['baseline']['deployments'])],
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit'], 'sourceMode': {'from': r['baseline']['deployments'][n]['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE'), 'to': target_source_mode(r,n)}} for n in sorted(r['baseline']['deployments'])],
'sequence': 'Processor first; verify its complete rollout before patching API.',
'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'],
'warnings': ['Only image/build stamp changes; startup migrations require change-safety evidence.',
'warnings': ['Only image/build stamp and the explicitly planned source mode may change; startup migrations require change-safety evidence.',
'Image-to-source provenance requires independent evidence.',
'Bandwidth adequacy and business acceptance remain manual evidence.'],
'rollback': r['rollback']}, ensure_ascii=False, indent=2))
@ -396,6 +486,8 @@ def deploy(r, a):
require(r['planHash'] == a.confirm_plan == digest(plan_data(r)), 'Plan hash does not match.')
require(bool(a.approval_ref.strip()), 'Record the explicit user approval reference for this exact plan.')
require_mainline(r['commit'], refresh=True)
validate_planned_source_mode(r)
validate_source_transition(r)
age = (dt.datetime.now(dt.timezone.utc) - dt.datetime.fromisoformat(r['baseline']['at'])).total_seconds()
require(0 <= age <= 3600, 'Cluster check is older than one hour; recheck and obtain a new plan approval.')
for key in ('image-provenance', 'change-safety'):
@ -409,13 +501,13 @@ def deploy(r, a):
# The processor must understand new durable payloads before API replicas can
# enqueue them. The fixed Fanyu names sort processor before API.
for name in sorted(current['deployments']):
latest = snapshot(r)
latest = snapshot(r, allow_planned_transition=True)
require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.')
for n, live in latest['deployments'].items():
old = current['deployments'][n]
done = any(o['deployment'] == n and o['result'] == 'acknowledged' for o in r['operations'])
require(live['uid'] == old['uid'] and live['nonReleaseHash'] == old['nonReleaseHash'], 'Deployment changed during release.')
require((live['image'] == r['image'] and live['buildCommit'] == r['commit']) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.')
require((live['image'] == r['image'] and live['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or live['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n))) if done else live['specHash'] == old['specHash'], 'Image/configuration drift during release.')
dep = latest['deployments'][name]
patch = [{'op': 'test', 'path': '/metadata/resourceVersion', 'value': dep['resourceVersion']},
{'op': 'test', 'path': '/metadata/uid', 'value': dep['uid']},
@ -425,6 +517,12 @@ def deploy(r, a):
env_path = f"/spec/template/spec/containers/{dep['containerIndex']}/env/"
patch.append({'op': 'add' if dep['buildCommitIndex'] is None else 'replace',
'path': env_path + ('-' if dep['buildCommitIndex'] is None else str(dep['buildCommitIndex'])), 'value': stamp})
if r.get('sourceMode'):
mode_index = dep['sourceModeIndex']
# If the stamp is appended first, a pre-existing mode keeps its index.
patch.append({'op': 'add' if mode_index is None else 'replace',
'path': env_path + ('-' if mode_index is None else str(mode_index)),
'value': {'name': 'ANNOTATION_SOURCE_VALIDATION_MODE', 'value': r['sourceMode']}})
op = {'deployment': name, 'before': dep['image'], 'desired': r['image'], 'at': now(), 'result': 'unknown'}
r['operations'].append(op)
save(r) # Persist intent before sending any mutation.
@ -440,8 +538,8 @@ def deploy(r, a):
event(r, 'processor patch acknowledged; waiting for rollout before API')
try:
kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330)
ready = snapshot(r)['deployments'][name]
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'],
ready = snapshot(r, allow_planned_transition=True)['deployments'][name]
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'] and (not r.get('sourceModeExplicit') or ready['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,name)),
'Processor rollout has not converged to the candidate.')
except Blocked:
r['state'] = 'deployed_pending_verification'
@ -493,6 +591,7 @@ def verify(r):
for n, d in s['deployments'].items():
require(d['uid'] == expected['deployments'][n]['uid'], 'Deployment was replaced.')
require(d['nonReleaseHash'] == expected['deployments'][n]['nonReleaseHash'], 'Deployment settings drifted beyond the approved image/build stamp.')
if r.get('sourceModeExplicit'): require(d['env'].get('ANNOTATION_SOURCE_VALIDATION_MODE') == target_source_mode(r,n), 'Source mode differs from approved plan.')
require(d['image'] == r['image'] and d['buildCommit'] == r['commit'] and d['ready'], 'Target image rollout incomplete; outcome remains pending.')
r['state'] = 'deployed_pending_acceptance'
missing = [k for k in policy()['requiredEvidence'] if k not in r['evidence'] or not evidence_valid(r['evidence'][k], r)]
@ -546,6 +645,8 @@ def main():
elif cmd == 'check':
c.add_argument('--local', action='store_true')
c.add_argument('--cluster', action='store_true')
c.add_argument('--source-readiness-file')
c.add_argument('--reuse-successful', action='store_true', help='Reuse only unchanged successful pure local checks; database/browser checks always rerun.')
elif cmd == 'deploy':
c.add_argument('--confirm-plan', required=True)
c.add_argument('--approval-ref', required=True)
@ -554,6 +655,7 @@ def main():
c.add_argument('--result', choices=['passed', 'failed', 'unverified'], required=True)
c.add_argument('--file', required=True)
c.add_argument('--note', required=True)
prep.add_argument('--source-mode', choices=['strict','advisory'], help='Optional reviewed source-mode change in the same image plan.')
a = parser.parse_args()
try:
if a.command == 'prepare':

View File

@ -0,0 +1,85 @@
"""Mandatory synthetic work/recovery gate; loopback Mongo only, no customer writes."""
import json
import os
import signal
from pathlib import Path
import subprocess
import sys
import time
import uuid
from lib.release_checks import check_environment, write_log, source_fingerprint
ROOT = Path(__file__).resolve().parents[1]
CORE = {'TestSourceAdvisoryCompleteWorkAndReviewFlow', 'TestSourceAdvisoryPublishedReviewDoesNotRequireFullAudit', 'TestSourceAdvisoryPreparesIndexDespiteCadenceAuditFailure', 'TestSourceAdvisoryTaskScopeCannotBeBypassed', 'TestSourceAdvisoryIsolatedReworkDoesNotTrapRandomClaim', 'TestSourceAdvisoryOwnedFailureCanRetryTimeline', 'TestSubmissionLossGuardArchiveFailureBlocksReplacement', 'TestSubmissionLossGuardArchiveSurvivesRestartAndIsIdempotent', 'TestSubmissionLossGuardArchivesFailedInputBeforeReplacement', 'TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter', 'TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker', 'TestCombinedSubmissionFinalCASFailureDoesNotPublishQualityTask', 'TestDeletedAnnotatorSubmissionWinsCASRace', 'TestDeletedAnnotatorRestartFinishesResetOnce', 'TestReviewCannotMutateAnnotation', 'TestSourceAdvisoryDoesNotGateOnFullAudit', 'TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate', 'TestReadObservationIsPrivateConcurrentAndRequestScoped'}
MONGO = {'TestSourceReadinessMongoParityReadOnly', 'TestSourceAdvisoryMongoEquivalenceAndBuffer', 'TestSourceAdmissionMongoClaimFence', 'TestSourceAdmissionMongoBatchMatchesLegacyPipeline', 'TestSourceAdmissionCurrentOutboxProofMongo', 'TestSourceAdmissionExistingTaskUpdateDoesNotEnqueueMongo', 'TestSourceValidationLegacyDetailsGETDoesNotEnqueueMongo', 'TestSubmissionRecoveryEventMongo'}
def verdict(stdout, code, required):
events = []
for line in stdout.splitlines():
try:
events.append(json.loads(line))
except ValueError:
continue
passed = {e.get('Test') for e in events if e.get('Action') == 'pass'}
skipped = sorted((e['Test'] for e in events if e.get('Action') == 'skip' and e.get('Test')))
missing = sorted(required - passed)
return {'passed': code == 0 and (not skipped) and (not missing), 'missingOrFailed': missing, 'skipped': skipped}
def phases():
return [('core', CORE, {}), ('mongo', MONGO, {'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'SUBMISSION_RECOVERY_LOCAL_MONGO': '1', 'POOL_LOCAL_MONGO_URI': 'mongodb://127.0.0.1:27018'}), ('advisory-workbench', {'TestSourceAdvisoryWorkbenchBrowser'}, {'SOURCE_ADVISORY_BROWSER_TEST': '1'}), ('advisory-recovery', {'TestReworkBoundaryBrowserReplica'}, {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'REWORK_BOUNDARY_BROWSER_TEST': '1'})]
def run_phase(argv, env):
# Terminate only this synthetic phase's process group on timeout, including
# Go-launched Node/browser children. Never terminate a shared local service.
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, start_new_session=True)
try:
stdout, stderr = child.communicate(timeout=360)
except subprocess.TimeoutExpired:
try:
os.killpg(child.pid, signal.SIGTERM)
except ProcessLookupError:
pass
try:
stdout, stderr = child.communicate(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
stdout, stderr = child.communicate()
return subprocess.CompletedProcess(argv, 124, stdout, stderr + '\nphase timed out')
return subprocess.CompletedProcess(argv, child.returncode, stdout, stderr)
def main():
folder = ROOT / 'local-data/test-results' / ('work-availability-' + uuid.uuid4().hex)
folder.mkdir(parents=True, mode=448)
before = source_fingerprint(ROOT)
report = {'inputDigest': before, 'scope': 'synthetic local fixtures; no production acceptance/capacity claim', 'checks': [], 'passed': False}
for name, required, switches in phases():
argv = ['go', 'test', '-json', './internal/api', './internal/store', '-run', '^(' + '|'.join(sorted(required)) + ')$', '-count=1', '-timeout=5m']
env = {**check_environment(), **switches, 'VERIFICATION_RUN_ID': uuid.uuid4().hex}
try:
r = run_phase(argv, env)
row = verdict(r.stdout, r.returncode, required)
output = r.stdout + '\n' + r.stderr
except (OSError, subprocess.TimeoutExpired) as e:
row = {'passed': False, 'error': 'runner failed or timed out'}
output = str(e)
report['inputsChangedDuringRun'] = source_fingerprint(ROOT) != before
if report['inputsChangedDuringRun'] or not before:
row.update(passed=False, error='Source identity changed or unavailable')
row.update(phase=name, argv=argv, artifactRunId=env['VERIFICATION_RUN_ID'])
row.update(write_log(folder, argv, output, 'passed' if row['passed'] else 'failed'))
report['checks'].append(row)
print(json.dumps(row), flush=True)
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
if report['inputsChangedDuringRun'] or not before:
report['remainingUnverified'] = [p[0] for p in phases()[len(report['checks']):]]
(folder/'report.json').write_text(json.dumps(report,indent=2)+'\n')
return 2
report['passed'] = all(row['passed'] for row in report['checks'])
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
print('Work availability gate ' + ('passed' if report['passed'] else 'failed') + '; production writes and capacity remain separate. Report: ' + str(folder / 'report.json'))
return 0 if report['passed'] else 1
if __name__ == '__main__':
sys.exit(main())

View File

@ -853,6 +853,36 @@
"tests": true
}
]
},
{
"name": "reliability-guardrails",
"paths": [
"^scripts/(lib/(release_checks|source_readiness)\\.py|test_work_availability\\.py)$",
"^tests/release/test_reliability_checks\\.py$",
"^internal/(api|store)/(source_readiness|read_observation).*\\.go$",
"^internal/api/diagnostic_trace.*\\.go$",
"^docs/(operations/reliability-guardrails\\.md|release/api-runtime\\.json)$",
"^Dockerfile\\.api-runtime$",
"^cmd/api/main\\.go$"
],
"frontend": [],
"go": {
"./internal/api": ["^TestSourceReadiness", "^TestDiagnosticTrace"],
"./internal/store": ["^TestReadObservation", "^TestSourceReadiness"]
},
"task": [
{
"argv": ["python3", "-m", "unittest", "discover", "-s", "tests/release", "-p", "test_*.py"],
"tests": true
},
{
"argv": ["npm", "run", "test:work-availability"],
"tests": true
}
],
"warnings": [
"工作流门禁仅使用合成媒体和隔离 Mongo;严格模式清单不是校验证书,生产容量、备份恢复和部署需分别验证。"
]
}
]
}

View File

@ -1,3 +1,4 @@
import { closeBrowserFixture } from "../../scripts/lib/browser-test-lifecycle.mjs";
import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs";
import fs from "node:fs";
import assert from "node:assert/strict";
@ -7,7 +8,11 @@ const { createServer } = require("vite"),
{ chromium } = require("playwright");
const api = process.env.REWORK_BOUNDARY_API;
assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/);
const root = new URL("../../local-data/verification/initial-slice-confirmation/", import.meta.url);
const runId = process.env.VERIFICATION_RUN_ID || `${Date.now()}-${process.pid}`;
assert.match(runId, /^[a-zA-Z0-9_-]+$/);
const scenario = process.env.INITIAL_SLICE_CONFIRM_SCENARIO;
assert.match(scenario, /^[a-z0-9-]+$/);
const root = new URL(`../../local-data/verification/initial-slice-confirmation/${runId}/${scenario}/`, import.meta.url);
fs.mkdirSync(root, { recursive: true });
const report = { cases: [], errors: [], requests: [] };
const vite = await createServer({
@ -15,19 +20,22 @@ const vite = await createServer({
configFile: "tests/browser/vite.platform-feedback.config.ts",
server: { host: "127.0.0.1", port: 5204, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } },
});
let browser;
let browser, owner, activeContext, activePage;
try {
await vite.listen();
browser = await chromium.launch({
owner = await chromium.launchServer({
...browserExecutableOptions(),
headless: true,
args: ["--no-proxy-server"],
});
browser = await chromium.connect(owner.wsEndpoint());
// A separate context for each case prevents autosaving one scenario into another.
for (const status of [process.env.INITIAL_SLICE_CONFIRM_SCENARIO]) {
await fetch(api + "/__boundary/reset?status=" + status);
const context = await browser.newContext({ viewport: { width: 1600, height: 1100 } });
const page = await context.newPage();
activeContext = context;
await context.tracing.start({ screenshots: true, snapshots: true, sources: false });
const page = (activePage = await context.newPage());
page.setDefaultTimeout(20000);
page.on("pageerror", (e) => report.errors.push(e.message));
page.on("response", async (r) => {
@ -65,6 +73,7 @@ try {
assert.equal(report.requests.length, 0, "unreadable content must not autosave or generate");
await page.screenshot({ path: new URL(status + "-blocked.png", root).pathname });
report.cases.push({ status, preserved: true });
assert.deepEqual(report.errors, []);
await context.close();
continue;
}
@ -162,15 +171,20 @@ try {
await bench.waitFor();
await page.screenshot({ path: new URL(status + "-after.png", root).pathname });
report.cases.push({ status, retainedAction: workspace.segments[0].id, sliceName: workspace.sliceSegments[0].name });
assert.deepEqual(report.errors, []);
await context.close();
}
assert.deepEqual(report.errors, []);
} catch (error) {
report.failure = { name: error.name, message: error.message, stack: error.stack };
await activePage?.screenshot({ path: new URL("failure.png", root).pathname, fullPage: true }).catch(() => {});
await activeContext?.tracing.stop({ path: new URL("failure-trace.zip", root).pathname }).catch(() => {});
throw error;
} finally {
fs.writeFileSync(
new URL(`report-${process.env.INITIAL_SLICE_CONFIRM_SCENARIO}.json`, root),
JSON.stringify(report, null, 2),
);
await browser?.close();
await vite.close();
await closeBrowserFixture({ browser, owner, vite });
}
console.log(JSON.stringify(report));

View File

@ -1,3 +1,4 @@
import { closeBrowserFixture } from "../../scripts/lib/browser-test-lifecycle.mjs";
import { browserExecutableOptions } from "../../scripts/lib/browser-executable.mjs";
import assert from "node:assert/strict";
import fs from "node:fs";
@ -6,14 +7,18 @@ import { chromium } from "playwright";
import { createServer } from "vite";
const api = process.env.REWORK_BOUNDARY_API;
assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/);
const out = path.resolve("local-data/test-results/rework-boundary");
const runId = process.env.VERIFICATION_RUN_ID || `${Date.now()}-${process.pid}`;
assert.match(runId, /^[a-zA-Z0-9_-]+$/);
const out = path.resolve("local-data/test-results/rework-boundary", runId);
fs.mkdirSync(out, { recursive: true });
const server = await createServer({
cacheDir: "local-data/test-results/vite-rework-boundary-replica",
server: { host: "127.0.0.1", port: 5202, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } },
});
let browser,
let owner,
browser,
page,
context,
fault = "",
releaseSave;
const report = { cases: [], pageErrors: [], writes: [] };
@ -29,13 +34,18 @@ const waitFor = async (check) => {
};
try {
await server.listen();
browser = await chromium.launch({
owner = await chromium.launchServer({
...browserExecutableOptions(),
headless: true,
args: ["--no-proxy-server"],
});
const ctx = await browser.newContext({ viewport: { width: 1600, height: 1100 }, serviceWorkers: "block" });
browser = await chromium.connect(owner.wsEndpoint());
const ctx = (context = await browser.newContext({
viewport: { width: 1600, height: 1100 },
serviceWorkers: "block",
}));
await ctx.tracing.start({ screenshots: true, snapshots: true, sources: false });
page = await ctx.newPage();
page.setDefaultTimeout(12000);
page.on("pageerror", (e) => report.pageErrors.push(e.message));
@ -314,6 +324,8 @@ try {
assert.deepEqual(report.pageErrors, []);
console.log(JSON.stringify(report));
} catch (e) {
report.failure = { name: e.name, message: e.message, stack: e.stack };
await context?.tracing.stop({ path: path.join(out, "failure-trace.zip") }).catch(() => {});
if (page) {
await page.screenshot({ path: path.join(out, "failure.png"), fullPage: true }).catch(() => {});
fs.writeFileSync(
@ -328,6 +340,5 @@ try {
} finally {
releaseSave?.();
fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2));
await browser?.close();
await server.close();
await closeBrowserFixture({ browser, owner, vite: server });
}

View File

@ -20,7 +20,7 @@ IMAGE = 'registry.example/api@sha256:' + 'a' * 64
def fixture(name, container):
config = {'data': {'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true',
config = {'data': {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'MONGO_DB': 'drobodata-fanyu', 'DROBOTICFLOW_REQUIRE_SESSION_AUTH': 'true',
'ANNOTATION_PROCESSOR_MODE': 'external', 'ANNOTATION_WORKSPACE_BUCKET': 'drobodata-fanyu',
'ANNOTATION_WORKSPACE_CONNECTION_ID': 'storage-example', 'WORKSPACE_MOUNT_PATH': '/workspace'},
'metadata': {'uid': 'config-uid'}}
@ -53,6 +53,7 @@ class ReleaseTests(unittest.TestCase):
self.start_patch('require_mainline', lambda *args, **kwargs: None)
self.start_patch('run', lambda *args, **kwargs: 'feature/release')
self.start_patch('manifest_platforms', lambda image: ['linux/amd64'])
self.start_patch('check_identity', lambda r: {'testEnvironment': 'stable', 'complete': True})
self.ds = {}
for n, c in self.p['deployments'].items():
self.ds[n], config = fixture(n, c)
@ -76,11 +77,11 @@ class ReleaseTests(unittest.TestCase):
def load(self):
return json.loads(m.record_path('test').read_text())
def snapshot(self, r):
def snapshot(self, r, **kwargs):
if self.before_snapshot:
self.before_snapshot()
return {'at': m.now(), 'deployments': {
n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p) for n, d in self.ds.items()},
n: m.inspect_deployment(d, self.p['deployments'][n], self.configs, self.p, ignore_source_mode=bool(r.get('sourceModeExplicit'))) for n, d in self.ds.items()},
'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}},
'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}}
@ -105,6 +106,9 @@ class ReleaseTests(unittest.TestCase):
c['env'].append(ops[4]['value'])
else:
c['env'][int(ops[4]['path'].split('/')[-1])] = ops[4]['value']
for extra in ops[5:]:
if extra['op']=='add': c['env'].append(extra['value'])
else: c['env'][int(extra['path'].split('/')[-1])] = extra['value']
d['metadata']['resourceVersion'] = str(int(d['metadata']['resourceVersion']) + 1)
return '{}'
@ -433,6 +437,15 @@ class ReleaseTests(unittest.TestCase):
with patch.object(real, 'policy', return_value=self.p), patch.object(real, 'get', side_effect=fake_get), patch.object(real, 'kubectl', return_value=json.dumps({'clusters': [{'cluster': {'server': r['target']['server']}}]})):
real.snapshot(r)
self.assertNotIn('secret', reads)
r['baseline'] = self.snapshot(r)
r['sourceMode'] = 'strict'
env = self.ds['droboticflow-annotation-processor']['spec']['template']['spec']['containers'][0]['env']
env.append({'name':'ANNOTATION_SOURCE_VALIDATION_MODE','value':'strict'})
with self.assertRaises(real.Blocked): real.snapshot(r)
real.snapshot(r, allow_planned_transition=True)
env[-1]['value'] = 'disabled'
with self.assertRaises(real.Blocked): real.snapshot(r, allow_planned_transition=True)
env.pop()
replies['namespace']['metadata']['uid'] = 'wrong'
with self.assertRaises(real.Blocked): real.snapshot(r)
@ -451,6 +464,104 @@ class ReleaseTests(unittest.TestCase):
with self.assertRaises(m.Blocked):
with m.locked('test'): pass
def test_combined_plan_changes_only_reviewed_mode_in_processor_api_order(self):
self.configs['config']['data']['ANNOTATION_SOURCE_VALIDATION_MODE']='strict'
self.r['sourceMode']='advisory'
self.plan()
approval=self.approve()
m.deploy(self.r,approval)
self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api'])
self.assertEqual(self.rollout_waits[0][1],['droboticflow-annotation-processor'])
for d in self.ds.values():
env={e['name']:e.get('value') for e in d['spec']['template']['spec']['containers'][0]['env']}
self.assertEqual(env['ANNOTATION_SOURCE_VALIDATION_MODE'],'advisory')
m.verify(self.r)
self.assertEqual(self.r['state'],'deployed_pending_acceptance')
def test_legacy_unset_mode_can_be_repaired_by_explicit_combined_plan(self):
self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE')
self.r['sourceMode']='advisory'
self.plan();m.deploy(self.r,self.approve())
self.assertEqual(len(self.writes),2)
def test_new_release_cannot_plan_an_implicit_strict_fallback(self):
self.configs['config']['data'].pop('ANNOTATION_SOURCE_VALIDATION_MODE')
m.check(self.r,argparse.Namespace(local=True,cluster=True))
with self.assertRaises(m.Blocked):m.make_plan(self.r)
self.assertEqual(self.writes,[])
def test_mode_change_after_approval_invalidates_plan_without_writes(self):
self.r['sourceMode']='advisory';self.plan();a=self.approve()
self.r['sourceMode']='strict'
with self.assertRaises(m.Blocked):m.deploy(self.r,a)
self.assertEqual(self.writes,[])
def test_strict_activation_without_inventory_never_writes(self):
self.r['sourceMode']='strict'
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=True))
self.assertEqual(self.writes,[])
def test_combined_mode_rollout_failure_leaves_api_unmodified(self):
self.r['sourceMode']='advisory';self.plan();self.fail_rollout=True
with self.assertRaises(m.Blocked):m.deploy(self.r,self.approve())
self.assertEqual(self.writes,['droboticflow-annotation-processor'])
self.assertEqual(self.r['state'],'deployed_pending_verification')
def test_failed_check_log_is_private_and_does_not_discard_failure(self):
failure=m.Blocked('failed');failure.check_output='password: canary-secret\nTraceback: canary-failure'
with patch.object(m,'run',side_effect=failure):
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False))
e=self.load()['localChecks'][-1];text=Path(e['log']).read_text()
self.assertNotIn('canary-secret',text);self.assertIn('canary-failure',text)
self.assertEqual(Path(e['log']).stat().st_mode&0o777,0o600)
def test_explicit_reuse_keeps_only_verified_successful_pure_checks(self):
self.p['localChecks']=[['npm','run','lint:web']]
self.r['policyHash']=m.digest(self.p)
m.check(self.r,argparse.Namespace(local=True,cluster=False))
with patch.object(m,'run',side_effect=m.Blocked('must not rerun pure check')):
m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True))
self.assertIn('reusedAt',self.r['localChecks'][0])
Path(self.r['localChecks'][0]['log']).write_text('tampered')
with patch.object(m,'run',side_effect=m.Blocked('must rerun tampered evidence')):
with self.assertRaises(m.Blocked):m.check(self.r,argparse.Namespace(local=True,cluster=False,reuse_successful=True))
def test_current_complete_deployment_report_binds_strict_mode_and_detects_tampering(self):
for d in self.ds.values():
d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']})
self.r['sourceMode']='strict'
report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'],
'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True,
'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(),
'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]}
path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report))
m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path)))
m.make_plan(self.r);a=self.approve()
report['rows'][0]['allowed']=0;report['rows'][0]['failed']=1;path.write_text(json.dumps(report))
with self.assertRaises(m.Blocked):m.deploy(self.r,a)
self.assertEqual(self.writes,[])
def test_complete_report_allows_reviewed_strict_activation(self):
for d in self.ds.values():
d['spec']['template']['spec']['containers'][0]['env'].append({'name':'DROBOTICFLOW_BUILD_COMMIT','value':self.r['commit']})
self.r['sourceMode']='strict'
report={'schema':'droboticflow.source-readiness/v1','namespace':self.p['namespace'],'database':self.p['database'],
'scope':'deployment','organizationId':'','projectId':'','targetMode':'strict','complete':True,
'legacySourceCutoff':'','buildCommit':self.r['commit'],'generatedAt':m.now(),
'rows':[{'organizationId':'o','stage':'quality_review','status':'ready','total':1,'allowed':1,'pending':0,'unverified':0,'failed':0}]}
path=Path(self.tmp.name)/'readiness.json';path.write_text(json.dumps(report))
m.check(self.r,argparse.Namespace(local=True,cluster=True,source_readiness_file=str(path)))
m.make_plan(self.r);m.deploy(self.r,self.approve());m.verify(self.r)
self.assertEqual(self.writes,['droboticflow-annotation-processor','droboticflow-api'])
self.assertEqual(self.r['state'],'deployed_pending_acceptance')
def test_changed_strict_rule_needs_advisory_preparation_not_old_report(self):
self.r['sourceMode']='strict';self.r['baseline']=self.snapshot(self.r)
self.r['baseline']['deployments']['droboticflow-api']['buildCommit']='b'*40
self.r['sourceReadiness']={'path':'not-read-yet'}
with self.assertRaisesRegex(m.Blocked,'deploy advisory first'):m.validate_source_transition(self.r)
class EntrypointTests(unittest.TestCase):
def test_cli_persists_and_resumes_offline(self):
@ -459,8 +570,9 @@ class EntrypointTests(unittest.TestCase):
(root / 'scripts').mkdir()
(root / 'docs/release').mkdir(parents=True)
shutil.copy(ROOT / 'scripts/release.py', root / 'scripts/release.py')
shutil.copytree(ROOT / 'scripts/lib', root / 'scripts/lib', ignore=shutil.ignore_patterns('__pycache__','*.pyc'))
shutil.copy(ROOT / 'docs/release/fanyu-policy.json', root / 'docs/release/fanyu-policy.json')
(root / '.gitignore').write_text('local-data/\n')
(root / '.gitignore').write_text('local-data/\n__pycache__/\n')
for args in [ ['git', 'init', '-q'], ['git', 'add', '.'],
['git', '-c', 'user.name=ReleaseTest', '-c', 'user.email=test@example.invalid',
'commit', '-qm', 'fixture'] ]:

View File

@ -29,6 +29,8 @@ class ReleaseBuildApiImageTest(unittest.TestCase):
(self.repo / "scripts").mkdir(parents=True)
shutil.copy(SCRIPT, self.repo / "scripts" / SCRIPT.name)
(self.repo / "Dockerfile.api").write_text("FROM scratch\n")
(self.repo / "docs/release").mkdir(parents=True)
shutil.copy(ROOT / "docs/release/api-runtime.json", self.repo / "docs/release/api-runtime.json")
git = ["git", "-C", str(self.repo), "-c", "user.name=t", "-c", "user.email=t@example.com"]
subprocess.run(["git", "init", "-q", str(self.repo)], check=True)
subprocess.run([*git, "add", "."], check=True)
@ -52,6 +54,11 @@ class ReleaseBuildApiImageTest(unittest.TestCase):
self.assertTrue(any(a.startswith("DROBOTICFLOW_BUILD_TIME=20") and a.endswith("Z") for a in args), args)
self.assertIn(f"registry.example.com/team/droboticflow-api:{self.commit[:8]}", args)
self.assertIn("--push", args)
self.assertIn("API_RUNTIME_PREPARED=1", args)
self.assertTrue(any(a.startswith("API_RUNTIME_IMAGE=") and "@sha256:" in a for a in args))
proof=json.loads(pathlib.Path(str(metadata)+".provenance.json").read_text())
self.assertEqual(proof["commit"],self.commit)
self.assertIn("@sha256:",proof["runtimeImage"])
self.assertIn(f"IMAGE=registry.example.com/team/droboticflow-api@sha256:{7:064d}", result.stdout)
self.assertEqual(json.loads(metadata.read_text())["containerimage.digest"], f"sha256:{7:064d}")

View File

@ -0,0 +1,124 @@
import datetime as dt
import copy
import importlib.util
import json
import os
from pathlib import Path
import tempfile
import time
import sys
import subprocess
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
def module(name, path):
s = importlib.util.spec_from_file_location(name, ROOT / path)
m = importlib.util.module_from_spec(s)
s.loader.exec_module(m)
return m
m = module('release_checks', 'scripts/lib/release_checks.py')
g = module('work_availability', 'scripts/test_work_availability.py')
ready = module('source_readiness', 'scripts/lib/source_readiness.py')
class CheckEvidenceTests(unittest.TestCase):
def test_private_bounded_redaction_including_multiline_secrets(self):
with tempfile.TemporaryDirectory() as d:
result = m.write_log(d, ['fake-check'], 'password:\n"canary-pass"\nAuthorization: Bearer canary-token\nmongodb://user:canary-db@host/db\nhttps://host/x?signature=canary-url\nAWS_ACCESS_KEY_ID=canary-aws\nBOS_SK=canary-bos\nTraceback: visible failure', 'failed')
p = Path(result['log'])
text = p.read_text()
for value in ['canary-pass', 'canary-token', 'canary-db', 'canary-url','canary-aws','canary-bos']:
self.assertNotIn(value, text)
self.assertIn('visible failure', text)
self.assertEqual(p.stat().st_mode & 511, 384)
def test_large_unstructured_output_is_redacted_with_bounded_runtime(self):
code = "import sys;sys.path.insert(0," + repr(str(ROOT/'scripts')) + ");from lib.release_checks import redact;assert len(redact('a'*3000000))<1000"
subprocess.run([sys.executable,'-c',code],check=True,capture_output=True,timeout=5)
def test_tampering_expiry_identity_and_external_checks_never_reuse(self):
with tempfile.TemporaryDirectory() as d:
e = {'result': 'passed', 'identity': {'commit': 'x', 'complete': True}, 'finishedUnix': time.time(), **m.write_log(d, ['test'], 'ok', 'passed')}
self.assertTrue(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
self.assertFalse(m.reusable(e, {'commit': 'y', 'complete': True}, ['npm', 'run', 'lint:web']))
for args in [['npm', 'run', 'test:browser:annotation-persistence'], ['python3', 'scripts/test_work_availability.py']]:
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, args))
self.assertFalse(m.reusable({**e, 'finishedUnix': 0}, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
Path(e['log']).write_text('tampered')
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
def test_production_credentials_and_inherited_switches_removed(self):
with patch.dict(os.environ, {'POOL_LOCAL_MONGO_URI': 'production', 'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'MONGO_URI': 'production', 'BOS_SECRET_KEY': 'secret', 'SOURCE_ADVISORY_BROWSER_TEST': '1', 'ANNOTATION_SOURCE_VALIDATION_MODE': 'strict'}):
e = m.check_environment()
for key in ['POOL_LOCAL_MONGO_URI', 'SOURCE_ADMISSION_LOCAL_MONGO', 'MONGO_URI', 'BOS_SECRET_KEY', 'SOURCE_ADVISORY_BROWSER_TEST', 'ANNOTATION_SOURCE_VALIDATION_MODE']:
self.assertNotIn(key, e)
def test_installed_dependency_and_symlink_changes_invalidate_identity(self):
with tempfile.TemporaryDirectory() as d, patch.object(m.subprocess, 'check_output', return_value=b'version'):
root = Path(d)
pkg = root / 'node_modules/p'
pkg.mkdir(parents=True)
(pkg / 'index.js').write_text('a')
a = m.identity(root, 'x', 'p')
(pkg / 'index.js').write_text('b')
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
linked = root / 'linked'
linked.mkdir()
(linked / 'index.js').write_text('c')
(root / 'node_modules/link').symlink_to(linked, target_is_directory=True)
a = m.identity(root, 'x', 'p')
(linked / 'index.js').write_text('d')
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
def test_required_case_skip_missing_and_failure_are_not_passes(self):
out = '\n'.join((json.dumps(e) for e in [{'Action': 'pass', 'Test': 'Required'}, {'Action': 'skip', 'Test': 'Required/child'}]))
self.assertFalse(g.verdict(out, 0, {'Required'})['passed'])
self.assertFalse(g.verdict('', 0, {'Required'})['passed'])
self.assertFalse(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 1, {'Required'})['passed'])
self.assertTrue(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 0, {'Required'})['passed'])
def test_existing_checks_are_wired_not_reimplemented(self):
policy = json.loads((ROOT / 'docs/release/fanyu-policy.json').read_text())
self.assertIn(['python3', 'scripts/test_work_availability.py'], policy['localChecks'])
self.assertIn('npm run test:work-availability', (ROOT / '.gitea/workflows/web-ci.yml').read_text())
self.assertIn('TestSourceAdvisoryCompleteWorkAndReviewFlow', g.CORE)
self.assertIn('TestSubmissionRecoveryEventMongo', g.MONGO)
def test_independent_checks_continue_but_mixed_sources_stop(self):
checks = [('first', {'Required'}, {}), ('second', {'Required'}, {})]
output = json.dumps({'Action': 'pass', 'Test': 'Required'})
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', return_value='stable'), patch.object(g, 'run_phase', side_effect=[subprocess.CompletedProcess([], 1, '', 'first failed'), subprocess.CompletedProcess([], 0, output, '')]) as run:
self.assertEqual(g.main(), 1)
self.assertEqual(run.call_count, 2)
report = json.loads(next(Path(d).rglob('report.json')).read_text())
self.assertFalse(report['passed'])
self.assertTrue(report['checks'][1]['passed'])
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', side_effect=['before', 'after']), patch.object(g, 'run_phase', return_value=subprocess.CompletedProcess([], 0, output, '')) as run:
self.assertEqual(g.main(), 2)
self.assertEqual(run.call_count, 1)
report = json.loads(next(Path(d).rglob('report.json')).read_text())
self.assertEqual(report['remainingUnverified'], ['second'])
class StrictReadinessTests(unittest.TestCase):
def report(self):
return {'schema': 'droboticflow.source-readiness/v1', 'namespace': 'n', 'database': 'db', 'scope': 'deployment', 'organizationId': '', 'projectId': '', 'targetMode': 'strict', 'complete': True, 'legacySourceCutoff': '', 'buildCommit': 'c' * 40, 'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'rows': [{'organizationId': 'o', 'stage': 'quality_review', 'status': 'ready', 'total': 2, 'allowed': 2, 'unverified': 0, 'pending': 0, 'failed': 0}]}
def validate(self, r):
with tempfile.TemporaryDirectory() as d:
p = Path(d) / 'report.json'
p.write_text(json.dumps(r))
return ready.validate_report(p, namespace='n', database='db', cutoff=None, live_commit='c' * 40)
def test_only_complete_current_deployment_inventory_can_activate(self):
self.assertEqual(self.validate(self.report())['totals']['allowed'], 2)
for patcher in [lambda r: r.update(scope='organization', organizationId='o'), lambda r: r.update(projectId='p'), lambda r: r.update(complete=False), lambda r: r.update(database='local-synthetic'), lambda r: r.update(namespace='other'), lambda r: r.update(buildCommit='a' * 40), lambda r: r.update(generatedAt='2000-01-01T00:00:00Z'), lambda r: r.update(legacySourceCutoff='2026-11-12T00:00:00Z'), lambda r: r['rows'][0].update(total=3), lambda r: r['rows'][0].update(allowed=True), lambda r: r['rows'][0].update(allowed=1, pending=1), lambda r: r['rows'].append(copy.deepcopy(r['rows'][0]))]:
r = self.report()
patcher(r)
with self.subTest(r=r), self.assertRaises(ValueError):
self.validate(r)
if __name__ == '__main__':
unittest.main()