From 322edf0c2ec6d0d6f75fc4ecce94704b5c121350 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Sun, 27 Sep 2026 02:45:58 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Harden=20test=20lane=20?= =?UTF-8?q?isolation=20and=20verify=20synthetic=20review=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../issues/01-environment.md | 8 +- .scratch/internal-test-lane/issues/02-seed.md | 8 +- .../internal-test-lane/issues/03-lifecycle.md | 8 +- .../issues/04-integration.md | 4 + .scratch/internal-test-lane/map.md | 6 + .scratch/internal-test-lane/verification.md | 35 ++ docs/internal-test-lane.md | 46 +- internal/testlane/apply.go | 2 +- internal/testlane/plan.go | 29 +- internal/testlane/plan_test.go | 1 + scripts/internal_test_lane.py | 8 + scripts/internal_test_lane/credentials.py | 15 +- scripts/internal_test_lane/environment.py | 6 +- scripts/internal_test_lane/lifecycle.py | 116 +++- tests/internal_test_lane/test_environment.py | 15 + tests/internal_test_lane/test_lifecycle.py | 76 +++ tests/internal_test_lane/test_verify_local.py | 98 ++++ tests/internal_test_lane/verify_local.py | 502 ++++++++++++++++++ 18 files changed, 934 insertions(+), 49 deletions(-) create mode 100644 .scratch/internal-test-lane/verification.md create mode 100644 tests/internal_test_lane/test_verify_local.py create mode 100644 tests/internal_test_lane/verify_local.py diff --git a/.scratch/internal-test-lane/issues/01-environment.md b/.scratch/internal-test-lane/issues/01-environment.md index 00ec400a..80f071bc 100644 --- a/.scratch/internal-test-lane/issues/01-environment.md +++ b/.scratch/internal-test-lane/issues/01-environment.md @@ -1,7 +1,7 @@ # Render the isolated environment Type: task -Status: claimed +Status: resolved Owner: test_environment (gpt-6-sol, high) Blocked by: none @@ -9,3 +9,9 @@ Implement spec component 1 using Python stdlib. Own environment renderer and its public-boundary tests. Do not touch existing production templates or access any cluster. Validate test identity, pinned images, separate authenticated DB/workspace, secret references, bounded processor and resource/network constraints. + +## Answer + +实现离线环境渲染、隔离拒绝规则、独立 Mongo 初始化和受限处理器。Mongo 非 root 官方镜像初始化在本地合成容器通过;公司 CNI/存储/入口未部署核验。 + +操作指南:[docs/internal-test-lane.md](../../../docs/internal-test-lane.md)。 diff --git a/.scratch/internal-test-lane/issues/02-seed.md b/.scratch/internal-test-lane/issues/02-seed.md index 217adb57..0085478f 100644 --- a/.scratch/internal-test-lane/issues/02-seed.md +++ b/.scratch/internal-test-lane/issues/02-seed.md @@ -1,7 +1,7 @@ # Seed independent task and role identities Type: task -Status: claimed +Status: resolved Owner: test_seed (gpt-6-sol, high) Blocked by: none @@ -9,3 +9,9 @@ Implement spec component 3. Own cmd/internal-test-seed and internal/testlane. Use normalized offline sample bundles, not production collection cloning. Add fresh-target guard, plan/confirm, real application-compatible identities/dictionaries, five private role credentials and scratch/snapshot support. Test synthetic inputs only. + +## Answer + +实现计划、摘要确认、专用 URI/工作区校验及五角色六任务初始化。使用新本地库的 readWrite 账号实际初始化成功;未读取客户素材。 + +操作指南:[docs/internal-test-lane.md](../../../docs/internal-test-lane.md)。 diff --git a/.scratch/internal-test-lane/issues/03-lifecycle.md b/.scratch/internal-test-lane/issues/03-lifecycle.md index 39924f44..bcc1ef23 100644 --- a/.scratch/internal-test-lane/issues/03-lifecycle.md +++ b/.scratch/internal-test-lane/issues/03-lifecycle.md @@ -1,7 +1,7 @@ # Import, prepare, archive and clean bounded test batches Type: task -Status: claimed +Status: resolved Owner: test_lifecycle (gpt-6-sol, high) Blocked by: none @@ -9,3 +9,9 @@ Implement spec component 2. Own CLI and lifecycle module/tests. Coordinate publi interfaces with tickets 01/02. No network or apply commands. Validate ownership, paths, digest and batch state; keep templates and latest three results; confirm old batch cleanup without following symlinks or touching unowned data. + +## Answer + +实现离线素材校验、准备、凭证、证据导入、归档和清理预览。真实合成导入/prepare/seeded/business-verified 状态转换和拒绝用例通过。 + +操作指南:[docs/internal-test-lane.md](../../../docs/internal-test-lane.md)。 diff --git a/.scratch/internal-test-lane/issues/04-integration.md b/.scratch/internal-test-lane/issues/04-integration.md index be163a1e..0f0792e0 100644 --- a/.scratch/internal-test-lane/issues/04-integration.md +++ b/.scratch/internal-test-lane/issues/04-integration.md @@ -9,3 +9,7 @@ Prepare independent synthetic assets and a new local Mongo target; verify comman and real business API flows after integration. Add usage/recovery/limitations guide, run targeted and full checks, obtain parallel standards/spec review from 17e7de7e, resolve findings and commit locally. No production data reads/writes or deployment. + +## Answer (core local verification complete) + +See [verification.md](../verification.md). Commands, isolation manifests and real local synthetic API flows have passed scoped verification and review. Standard product delivery packaging is an explicit remaining coverage gap; this ticket is not marked wholly resolved. Company deployment and customer sample copying have not been performed and still require specific approval. diff --git a/.scratch/internal-test-lane/map.md b/.scratch/internal-test-lane/map.md index f009e114..7605f8fc 100644 --- a/.scratch/internal-test-lane/map.md +++ b/.scratch/internal-test-lane/map.md @@ -122,3 +122,9 @@ Q1–Q3 均已采用推荐项。 ## Implementation — 2026-09-27 用户已要求本地实现,并确认从 `17e7de7e` 审查;核验受控命令、部署清单及独立本地合成业务流程。实际客户样本复制和集群部署仍需另行批准。实现票见 [issues](issues/),范围见 [spec.md](spec.md)。 + +## Implementation findings + +- [01 environment](issues/01-environment.md), [02 seed](issues/02-seed.md), [03 lifecycle](issues/03-lifecycle.md) implemented and locally verified. +- [04 integration](issues/04-integration.md): core five-role scratch/snapshot workflows verified through final acceptance; standard delivery-package worker/graph remains uncovered. See [verification and two-axis review](verification.md). +- No customer assets or production runtime changes. Company deployment, real sample copy and production publication remain separately authorized operations. diff --git a/.scratch/internal-test-lane/verification.md b/.scratch/internal-test-lane/verification.md new file mode 100644 index 00000000..f9bf9d97 --- /dev/null +++ b/.scratch/internal-test-lane/verification.md @@ -0,0 +1,35 @@ +# Internal test lane — local verification, 2026-09-27 + +Scope: no company cluster, customer database, BOS, customer account or customer asset was accessed or changed. No PR, push, or deployment. Independent local Mongo 27019, API 3019, synthetic H264/PTS/IMU only. Existing local 27017/27018 stores were not used. + +## Verified + +- Offline import of six generated samples (three per project), content/ownership checks and environment preparation. +- Fresh seed with five roles, two projects, six source tasks, using a Mongo user with only readWrite on the new test database. +- Five real session logins; random claim → asynchronous slicing → explicit blank action submission → quality approval → aggregate final acceptance: BLF and SL scratch samples both passed. +- BLF snapshot: original description and blank interval retained through slicing, submission, quality, and final acceptance. +- Quality snapshot: real materialization/submission restored it to quality_review at revision 2 without manufacturing review evidence. +- Unified frame timeline returned 8 frames with matching dual-camera boundaries. Both generated output videos have 8 H264 frames; annotation JSON retains a pending action with an empty description. +- CLI imported seeded and business-verified evidence for the actual local execution plan. These are explicitly operator-supplied reports with single-sample scope, not assertions of cluster or full release acceptance. +- Official cached Mongo image initialized nonroot uid 999 with dropped capabilities; renderer uses official MONGO_INITDB_ROOT_* keys. App-user creation and seed were separately verified in the fresh local Mongo. +- Python internal lane suite: 28 passed. Go cmd/internal-test-seed and internal/testlane passed. Web typecheck and production build passed. + +## Full-suite limits + +Frontend full suite: 606 passed, 13 failed (619 total). Full tracked Go cmd/internal suite: internal/api and internal/worker failed; 79 top-level / 110 leaf failure names, including missing local Python cv2 support and existing business expectation failures. Initial sandbox run additionally failed because httptest could not bind a local port; the unrestricted local rerun removed that obstacle. Existing application source and existing test files were not changed in this task. No claim of a globally green repository or exact equality with an older baseline failure count is made. + +Private raw reports/logs are under local-data/internal-test-lane/verification-20260927 and verification-final; credentials remain untracked and are not reproduced here. Placeholder image digests in local render configs were not deployed or verified as container images. + +## Standards + +Initial review against 17e7de7e found one ticket-recording rule issue and two possible duplicated-code smells. Completed tickets now include Answer records; workspace path validation and credential writes share helpers. No customer-safety or credential-commit violation was found. + +## Spec + +Initial review found four gaps. Retention quota now permits three retained batches plus one active batch and has a stable namespace-scoped name. Seed/business evidence is recorded with explicit scope. A quality snapshot can be replayed through real HTTP processing and stopped at its intended quality stage. + +One coverage gap remains: standard product delivery packaging. This version runs the human workflow through final acceptance and generates actual reviewed media/annotation JSON, but it has no independent workflow worker or package graph and does not verify the product delivery-package UI or exported package. Do not represent a homemade export or final acceptance alone as standard package acceptance. The next extension needs an isolated worker, test-only Job permissions, a bounded package graph, and separate acceptance. + +## Still requires explicit approval and verification + +Real six-source inventory/copy, company resource allocation, test-only TLS/ingress/PVC/CNI enforcement, deployment, browser acceptance on that environment, concurrency/capacity and rollback drills. Production promotion remains a separate explicitly approved action. Local test success cannot guarantee zero production impact. diff --git a/docs/internal-test-lane.md b/docs/internal-test-lane.md index 3f0afee4..cf6ad6f2 100644 --- a/docs/internal-test-lane.md +++ b/docs/internal-test-lane.md @@ -4,7 +4,7 @@ ## 离线准备 -使用仓库内的 `local-data/internal-test-lane/` 作为私有根目录。它被 Git 忽略,导入的原始模板和批次记录不能加入提交。一次批次的配置为 JSON:`schemaVersion: 1`、独立测试 `batchId` 和 `hostname`、生产入口列表 `productionOrigins`、以 SHA256 固定的 `image` 与 `mongoImage`、完整 `buildCommit`、`workspaceGi`、`mongoGi`,以及 API、processor、Mongo 各自的 CPU/内存 requests 和 limits;还需要 TLS Secret 名、镜像拉取 Secret 名和入口控制器 namespace。`verify-config` 会拒绝已知生产入口、IP 主机名、其他 namespace、非固定镜像及越界资源值。资源值和外部入口、存储类能力仍需获准只读盘点后确认。 +配置样例见 `scripts/internal_test_lane/config.example.json`;其中全零摘要和示例域名仅用于填写参考,不能作为已验证镜像部署。使用仓库内的 `local-data/internal-test-lane/` 作为私有根目录。它被 Git 忽略,导入的原始模板和批次记录不能加入提交。一次批次的配置为 JSON:`schemaVersion: 1`、独立测试 `batchId` 和 `hostname`、生产入口列表 `productionOrigins`、以 SHA256 固定的 `image` 与 `mongoImage`、完整 `buildCommit`、`workspaceGi`、`mongoGi`,以及 API、processor、Mongo 各自的 CPU/内存 requests 和 limits;还需要 TLS Secret 名、镜像拉取 Secret 名和入口控制器 namespace。`verify-config` 会拒绝已知生产入口、IP 主机名、其他 namespace、非固定镜像及越界资源值。资源值和外部入口、存储类能力仍需获准只读盘点后确认。 素材包 `bundle.json` 的 `schemaVersion` 为 `droboticflow.internal-test-seed/v1`。它含 `dictionaries`、六个 `samples` 和 `assets` 清单。BLF-data、SL-data 各三条完整 part,样本键使用小写字母、数字和连字符;至少各有一种 `scratch` 与 `snapshot` 模板。每个样本包含 `source.captureId`、正数 `durationSeconds` 和 `media` 中的双目视频、双目 PTS、IMU 路径;可额外含 `sourceManifest`。所有路径相对素材包目录,位于 `templates//`。`assets` 为每个引用文件列出相同路径、字节数和 SHA256。快照还需明确阶段、切片和动作;舍弃切片的原因必须存在于所属项目的舍弃原因字典快照中。字典和快照使用获准样本的真实内容,不能伪造已有审查或交付结果。导入前另行审查六个来源 ID/版本、文件清单与总大小。 @@ -26,7 +26,7 @@ python3 scripts/internal_test_lane.py credentials --config local-data/internal-t 以下是操作顺序,不是当前授权,也不是自动发布脚本。集群写入前须有该次部署的明确批准,并立即核对目标集群和 `droboticflow-internal-test` namespace;样本读取、测试存储写入和凭证处理分别遵循已批准范围。先核实 namespace、网络策略、配额、Ingress/TLS、存储类与节点可用资源。空间不足时保持暂停,不缩减生产资源。 -1. 审查 `environment.json`、`initializer.json`、私有 Secret 清单及 source/inventory 记录。环境清单建立独立 namespace、配额与网络策略、数据库和工作区 PVC;Mongo 目标副本为 1,API 与 processor 初始副本为 0。不要提前打开业务入口。测试 PVC 和数据库按批次独立,旧版本不能自动指向新批次迁移后的数据库。 +1. 审查 `environment.json`、`initializer.json`、私有 Secret 清单及 source/inventory 记录。环境清单建立独立 namespace、配额与网络策略、数据库和工作区 PVC;Mongo 目标副本为 1,API 与 processor 初始副本为 0。不要提前打开业务入口。测试 PVC 和数据库按批次独立,旧版本不能自动指向新批次迁移后的数据库。`internal-test-quota` 是整个测试 namespace 的固定名称配额,新批次清单会更新它;更新前人工核对现有 PVC、已用存储和运行工作负载,并确认新额度覆盖保留批次与新批次,且不低于当前实际占用。若此前部署过旧版按批次命名的 quota,应在单独审查和获准的远端清单中处理;旧对象不会因本地清单变化自动消失。 2. 在获准测试 namespace 创建清单与私有 Secret,等待独立 Mongo 就绪。只在隔离工作区准备临时 initializer Pod,上传**已复制并核验**的模板文件、规范化 `manifest.json`、匹配源码版本的 Linux seed 可执行文件及必要的私有账号/URI 文件。临时 Pod 不是长期服务;测试 API、processor 不带客户读取凭证。 3. 在 initializer 内对 `/workspace` 的文件重新运行 `internal-test-seed plan --manifest --assets-root /workspace --workspace-root /workspace --batch --out `。本地预览计划绑定的是本地导入路径,不能跨机器复用。核对新计划的六个样本、两项目、五账号、目标数据库和摘要后,以**同一** manifest、素材根目录与计划摘要运行 `internal-test-seed apply --plan --manifest --assets-root /workspace --accounts-file --mongo-uri-file --confirm `。该命令只接受专用测试 URI 和空数据库;若中途失败,保持停机并调查部分写入,不绕过拒绝检查。 4. 核实 seed 完成标记及素材路径/哈希,删除 initializer Pod 和暂存的私有文件,然后才将 API 与 processor 调至各 1 个副本。处理器切片并发为 1。确认入口明确标为测试、五角色登录和越权拒绝,再通过原业务界面完成领取、切片、标注、送检、质检、终审和测试交付。输出留在测试工作区,不进入客户出口。 @@ -50,3 +50,45 @@ python3 scripts/internal_test_lane.py clean --batch --confirm \ + --evidence local-data/internal-test-lane/inputs/-seeded-evidence.json \ + --plan local-data/internal-test-lane/inputs/.json \ + --result local-data/internal-test-lane/inputs/.json +python3 scripts/internal_test_lane.py record-evidence --batch \ + --evidence local-data/internal-test-lane/inputs/-business-evidence.json \ + --plan local-data/internal-test-lane/inputs/.json \ + --result local-data/internal-test-lane/inputs/.json +python3 scripts/internal_test_lane.py status +``` + +Seed 结果必须是 Go `apply` 成功返回的纯 JSON,包含同一数据库、`planSha256`、`created` 布尔值及 2 项目、6 样本、5 账号计数。HTTP 验证器的进度日志不能直接充当结果文件;保存其最终 JSON 对象后再计算文件摘要。该对象至少包含同一批次、样本键、该计划中的任务和 item ID、最终验收 item ID 与 `status: "accepted"`;若带 `checks`,每项都须为 `status: "passed"`。目前这只覆盖报告中的单条样本完整流程,状态里的 `scope` 会明确写为 `single-sample-http-workflow`。原始报告可能包含敏感信息,应留在私有本地目录;批次状态只保存摘要、目标、审查者和报告范围。 + + +## 本地合成流程复测 + +本轮使用全新 Mongo 端口 `27019` 和 API `3019`,没有复用已有 `27017/27018` 数据库。素材由 `tests/internal_test_lane/make_synthetic_bundle.py` 的 ffmpeg 测试画面生成。`tests/internal_test_lane/verify_local.py` 是针对这套本地验收夹具的脚本,固定读取 `local-data/internal-test-lane/verification-final/local-plan.json`、`credentials/accounts.json` 和 `workspace`;它不是集群验收脚本,也不会自动创建数据库或启动服务。 + +准备新的本地合成目录、通过本工具生成和应用计划、以该测试库的独立 readWrite 凭证启动 API/processor 后,可执行: + +```sh +python3 tests/internal_test_lane/verify_local.py --project SL-data --confirm --output local-data/internal-test-lane/verification-final/sl-result.json +python3 tests/internal_test_lane/verify_local.py --sample-key blf-2 --confirm --output local-data/internal-test-lane/verification-final/snapshot-result.json +python3 tests/internal_test_lane/verify_local.py --sample-key blf-3 --stop-at-quality-review --confirm --output local-data/internal-test-lane/verification-final/quality-snapshot-result.json +``` + +最后一条仅接受 manifest 中阶段为 `quality_review` 的快照,先重建真实媒体并保留动作,再停在质检阶段供复现;不会伪造质检证据。测试脚本会真实领取、送检、审核计划内任务,不能对同一份已完成任务无条件重复运行;应重新准备新批次。报告拒绝覆盖旧文件。实际样本键以自己的计划为准。 + + +## 标准交付打包的覆盖边界 + +当前首版没有启用工作流 worker,也没有复制客户的打包工作流。真实业务可到终审通过,并生成供审查的双目视频、PTS、IMU 和动作 JSON,但这不等于产品标准 delivery package 已经产出。终审后的 release partition 仍需 worker 处理;空测试工作流不能直接承担该步骤。不得将自制导出包或单纯终审成功当作标准交付验收。 + +若需要验收原产品的打包和交付界面,应补独立测试 worker、仅测试 namespace 的 Job 权限及有资源上限的打包工作流,然后另行验证。其部署与真实素材使用仍须具体授权;不能给现有测试身份追加生产权限来绕过隔离。 diff --git a/internal/testlane/apply.go b/internal/testlane/apply.go index 8d5c8a8c..a81e88fd 100644 --- a/internal/testlane/apply.go +++ b/internal/testlane/apply.go @@ -67,7 +67,7 @@ func ValidateMongoURI(raw, database, batch string) error { return errors.New("Mongo URI contains unsupported option") } } - if auth := u.Query().Get("authSource"); auth != "" && !((local && auth == "admin") || (cluster && auth == database)) { + if auth := u.Query().Get("authSource"); auth != "" && !(auth == database || (local && auth == "admin")) { return errors.New("Mongo URI authSource outside the internal test lane") } return nil diff --git a/internal/testlane/plan.go b/internal/testlane/plan.go index 03f862c0..b9afbe97 100644 --- a/internal/testlane/plan.go +++ b/internal/testlane/plan.go @@ -582,38 +582,13 @@ func ValidateWorkspaceRoot(root string) error { if !filepath.IsAbs(root) || filepath.Clean(root) != root || filepath.Base(root) != "workspace" { return errors.New("workspace root must be /workspace or a dedicated local workspace") } - wd, err := os.Getwd() + allowed, err := internalTestLocalRoot() if err != nil { return err } - for { - if _, err := os.Stat(filepath.Join(wd, "go.mod")); err == nil { - break - } - parent := filepath.Dir(wd) - if parent == wd { - return errors.New("repository root unavailable") - } - wd = parent - } - allowed := filepath.Join(wd, "local-data", "internal-test-lane") rel, err := filepath.Rel(allowed, root) if err != nil || rel == "." || strings.HasPrefix(rel, "..") || !strings.HasSuffix(rel, string(filepath.Separator)+"workspace") { return errors.New("workspace root outside dedicated internal-test-lane directory") } - if err := rejectLocalPathSymlinks(allowed, rel); err != nil { - return err - } - current := allowed - for _, part := range strings.Split(rel, string(filepath.Separator)) { - current = filepath.Join(current, part) - info, err := os.Lstat(current) - if err == nil && info.Mode()&os.ModeSymlink != 0 { - return errors.New("workspace path contains symlink") - } - if err != nil && !os.IsNotExist(err) { - return err - } - } - return nil + return rejectLocalPathSymlinks(allowed, rel) } diff --git a/internal/testlane/plan_test.go b/internal/testlane/plan_test.go index 584eefe5..a84a6e75 100644 --- a/internal/testlane/plan_test.go +++ b/internal/testlane/plan_test.go @@ -153,6 +153,7 @@ func TestMongoTargetGuard(t *testing.T) { db := "droboticflow_internal_test_batch_one" allowed := []string{ "mongodb://localhost:27019/" + db + "?authSource=admin", + "mongodb://local_app:private@localhost:27019/" + db + "?authSource=" + db, "mongodb://127.0.0.1:27019/" + db + "?authSource=admin", "mongodb://seed:private@internal-test-batch-one-mongo.droboticflow-internal-test.svc.cluster.local:27017/" + db + "?authSource=" + db, } diff --git a/scripts/internal_test_lane.py b/scripts/internal_test_lane.py index 953f641f..9ff4b5e8 100644 --- a/scripts/internal_test_lane.py +++ b/scripts/internal_test_lane.py @@ -43,6 +43,12 @@ def main(argv: list[str] | None = None) -> int: sub.add_parser("status", help="list local imports and batches without claiming remote success") + evidence = sub.add_parser("record-evidence", help="import a reviewed local seed or HTTP workflow report without querying its target") + evidence.add_argument("--batch", required=True) + evidence.add_argument("--evidence", type=Path, required=True, help="reviewer, target and version binding JSON") + evidence.add_argument("--plan", type=Path, required=True, help="actual approved execution plan JSON") + evidence.add_argument("--result", type=Path, required=True, help="Go apply or HTTP workflow result JSON") + archive = sub.add_parser("archive", help="record operator evidence that a batch is complete, stopped, and detached") archive.add_argument("--batch", required=True) archive.add_argument("--evidence", type=Path, required=True) @@ -76,6 +82,8 @@ def main(argv: list[str] | None = None) -> int: result = generate_credentials(_read_json(args.config), output) elif args.command == "status": result = lane.status() + elif args.command == "record-evidence": + result = lane.record_evidence(args.batch, args.evidence, args.plan, args.result) elif args.command == "archive": result = lane.archive(args.batch, args.evidence) elif args.command == "cleanup-plan": diff --git a/scripts/internal_test_lane/credentials.py b/scripts/internal_test_lane/credentials.py index 114a2a33..a549069b 100644 --- a/scripts/internal_test_lane/credentials.py +++ b/scripts/internal_test_lane/credentials.py @@ -9,11 +9,10 @@ from urllib.parse import quote from .environment import validate_config -def _write_private(path, value): +def _write_private(path, text): descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - json.dump(value, stream, ensure_ascii=False, indent=2) - stream.write("\n") + stream.write(text) stream.flush() os.fsync(stream.fileno()) @@ -58,13 +57,9 @@ def generate_credentials(config, outdir): for number, (role, name) in enumerate(roles, start=1) ]} secret_file, accounts_file = output / "secret.json", output / "accounts.json" - _write_private(secret_file, secret) - _write_private(accounts_file, accounts) + _write_private(secret_file, json.dumps(secret, ensure_ascii=False, indent=2) + "\n") + _write_private(accounts_file, json.dumps(accounts, ensure_ascii=False, indent=2) + "\n") uri_file = output / "mongo-uri.txt" - descriptor = os.open(uri_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - stream.write(values["MONGO_URI"] + "\n") - stream.flush() - os.fsync(stream.fileno()) + _write_private(uri_file, values["MONGO_URI"] + "\n") return {"secretManifest": str(secret_file), "accountsFile": str(accounts_file), "mongoUriFile": str(uri_file)} diff --git a/scripts/internal_test_lane/environment.py b/scripts/internal_test_lane/environment.py index 1b59cef6..5554d8b5 100644 --- a/scripts/internal_test_lane/environment.py +++ b/scripts/internal_test_lane/environment.py @@ -357,9 +357,9 @@ exit 1 if "ingressClassName" in c: ingress["spec"]["ingressClassName"] = c["ingressClassName"] items.append(ingress) - items.append(obj("v1", "ResourceQuota", prefix + "-quota", {"hard": { - "pods": "3", "persistentvolumeclaims": "6", - "requests.storage": f"{3 * (c['workspaceGi'] + c['mongoGi'])}Gi", + items.append(obj("v1", "ResourceQuota", "internal-test-quota", {"hard": { + "pods": "3", "persistentvolumeclaims": "8", + "requests.storage": f"{4 * (c['workspaceGi'] + c['mongoGi'])}Gi", "requests.cpu": _millicores(sum(_quantity(c["resources"][component]["requests"]["cpu"], "cpu", cpu=True) for component in ("api", "processor", "mongo"))), "limits.cpu": _millicores(sum(_quantity(c["resources"][component]["limits"]["cpu"], "cpu", cpu=True) for component in ("api", "processor", "mongo"))), "requests.memory": f"{sum(_quantity(c['resources'][component]['requests']['memory'], 'memory', cpu=False) for component in ('api', 'processor', 'mongo'))}Mi", diff --git a/scripts/internal_test_lane/lifecycle.py b/scripts/internal_test_lane/lifecycle.py index e3ec0e02..7aad60cc 100644 --- a/scripts/internal_test_lane/lifecycle.py +++ b/scripts/internal_test_lane/lifecycle.py @@ -59,6 +59,24 @@ def _read_json(path: Path) -> dict: return value +def _read_json_and_hash(path: Path) -> tuple[dict, str]: + path = Path(os.path.abspath(path)) + _check_no_symlink(path) + try: + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(fd, "rb") as file: + info = os.fstat(file.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > 10 * 1024 * 1024: + raise LaneError("evidence input must be a regular JSON file of at most 10 MiB") + contents = file.read() + data = json.loads(contents) + except (OSError, ValueError) as exc: + raise LaneError(f"cannot read evidence JSON at {path}: {exc}") from exc + if not isinstance(data, dict): + raise LaneError("evidence JSON must be an object") + return data, _digest(contents) + + def _atomic_json(path: Path, value: object) -> None: fd, name = tempfile.mkstemp(prefix=".writing-", dir=path.parent) try: @@ -102,6 +120,18 @@ def _valid_batch_id(batch_id: object) -> str: return batch_id +def _validate_observed_at(value: object) -> str: + if not isinstance(value, str): + raise LaneError("evidence observedAt must be an RFC3339 timestamp") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise LaneError("evidence observedAt must be an RFC3339 timestamp") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise LaneError("evidence observedAt must include a timezone") + return value + + def _hash_file(path: Path) -> tuple[str, int]: _check_no_symlink(path) mode = path.stat().st_mode @@ -394,6 +424,86 @@ class Lane: records.append(_read_json(child / "batch.json")) return {"owner": OWNER, "imports": import_ids, "batches": records, "unfinished": {"imports": pending_imports, "batches": pending}, "remoteState": "not-verified-by-this-tool"} + def record_evidence(self, batch_id: str, evidence_path: Path, plan_path: Path, result_path: Path) -> dict: + """Import a bounded operator report; never infer remote state ourselves.""" + _valid_batch_id(batch_id) + evidence = _read_json(evidence_path) + required = {"schemaVersion", "batchId", "stage", "reviewer", "observedAt", "target", "buildCommit", "image", "configSha256", "approvedExecutionPlanSha256", "resultSha256"} + if set(evidence) != required or evidence.get("schemaVersion") != 1 or evidence.get("batchId") != batch_id: + raise LaneError("evidence metadata fields/schema/batch do not match") + stage = evidence.get("stage") + if not isinstance(stage, str) or stage not in {"seeded", "business-verified"}: + raise LaneError("evidence stage must be seeded or business-verified") + reviewer = evidence.get("reviewer") + if not isinstance(reviewer, str) or not reviewer.strip() or len(reviewer) > 120 or any(ord(char) < 32 for char in reviewer): + raise LaneError("evidence reviewer must be a short nonblank name") + observed_at = _validate_observed_at(evidence.get("observedAt")) + for field in ("configSha256", "approvedExecutionPlanSha256", "resultSha256"): + if not isinstance(evidence.get(field), str) or not SHA_RE.fullmatch(evidence[field]): + raise LaneError(f"evidence {field} must be a SHA256 digest") + plan, plan_file_sha = _read_json_and_hash(plan_path) + result, result_sha = _read_json_and_hash(result_path) + if result_sha != evidence["resultSha256"] or plan.get("sha256") != evidence["approvedExecutionPlanSha256"]: + raise LaneError("execution plan or result hash differs from reviewed evidence") + with self.locked(): + directory = self.root / "batches" / batch_id + record = _read_json(directory / "batch.json") + expected_state = "prepared" if stage == "seeded" else "seeded" + if record.get("state") != expected_state: + raise LaneError(f"{stage} evidence requires batch state {expected_state}") + if evidence["buildCommit"] != record.get("buildCommit") or evidence["image"] != record.get("image") or evidence["configSha256"] != record.get("configSha256"): + raise LaneError("evidence source, image, or config differs from prepared batch") + config = _read_json(directory / "config.json") + target = evidence.get("target") + if not isinstance(target, dict) or target.get("databaseName") != config["databaseName"]: + raise LaneError("evidence target database differs from prepared batch") + if target.get("kind") == "local": + if set(target) != {"kind", "databaseName", "hostname", "namespace"} or not isinstance(target["hostname"], str) or target["hostname"] not in {"localhost", "127.0.0.1"} or target["namespace"] != "local": + raise LaneError("local evidence target must name a loopback host and local namespace") + if plan.get("assetsRoot") == "/workspace" or plan.get("workspaceRoot") == "/workspace": + raise LaneError("local evidence cannot use an in-cluster execution plan") + elif target.get("kind") == "cluster": + if set(target) != {"kind", "databaseName", "hostname", "namespace", "clusterContext"} or target["hostname"] != config["hostname"] or target["namespace"] != config["namespace"] or not isinstance(target["clusterContext"], str) or not re.fullmatch(r"[A-Za-z0-9._:/@-]{1,128}", target["clusterContext"]): + raise LaneError("cluster evidence target must match dedicated config and name a context") + if plan.get("assetsRoot") != "/workspace" or plan.get("workspaceRoot") != "/workspace": + raise LaneError("cluster evidence requires an in-cluster /workspace execution plan") + else: + raise LaneError("evidence target kind must be local or cluster") + preview = _read_json(directory / "seed-plan.json") + if plan.get("batchId") != batch_id or plan.get("database") != config["databaseName"] or plan.get("manifestSha256") != preview.get("manifestSha256") or plan.get("assets") != preview.get("assets"): + raise LaneError("execution plan source assets or database differ from prepared batch") + plan_rows, preview_rows = plan.get("samples"), preview.get("samples") + if not isinstance(plan_rows, list) or not isinstance(preview_rows, list) or len(plan_rows) != 6 or len(preview_rows) != 6 or any(not isinstance(item, dict) for item in plan_rows + preview_rows): + raise LaneError("execution plan must contain six sample mappings") + plan_samples = [(item.get("key"), item.get("project"), item.get("mode")) for item in plan_rows] + preview_samples = [(item.get("key"), item.get("project"), item.get("mode")) for item in preview_rows] + if plan_samples != preview_samples: + raise LaneError("execution plan sample mapping differs from prepared batch") + if stage == "seeded": + if result.get("database") != config["databaseName"] or result.get("planSha256") != plan["sha256"] or type(result.get("created")) is not bool or any(type(result.get(name)) is not int or result[name] != expected for name, expected in (("projects", 2), ("samples", 6), ("accounts", 5))): + raise LaneError("seed result is not a successful six-sample Go apply report") + scope = "six-sample-seed-report" + else: + prior = record.get("evidence", {}).get("seeded", {}) + if prior.get("approvedExecutionPlanSha256") != plan["sha256"] or prior.get("target") != target: + raise LaneError("business report must use the seeded execution plan and target") + sample_key = result.get("sampleKey") + sample = next((item for item in plan_rows if item.get("key") == sample_key), None) + acceptance_id = "human-item-fanyu-v5-prepackage-acceptance-" + str(sample.get("batchId")) if sample else None + if result.get("batchId") != batch_id or result.get("status") != "accepted" or sample is None or result.get("taskId") != sample.get("taskId") or result.get("itemId") != sample.get("itemId") or result.get("acceptanceItemId") != acceptance_id: + raise LaneError("HTTP result does not prove an accepted test workflow from this plan") + checks = result.get("checks") + if checks is not None and (not isinstance(checks, list) or not checks or any(not isinstance(check, dict) or check.get("status") != "passed" for check in checks)): + raise LaneError("HTTP report checks must all be passed") + scope = "single-sample-http-workflow" + summary = {"stage": stage, "reviewer": reviewer, "observedAt": observed_at, "target": target, "approvedExecutionPlanSha256": plan["sha256"], "executionPlanFileSha256": plan_file_sha, "resultSha256": result_sha, "importedAt": datetime.now(timezone.utc).isoformat(), "scope": scope, "verificationSource": "operator-supplied-report"} + record.setdefault("evidence", {})[stage] = summary + record["state"] = stage + record["remoteState"] = "not-verified-by-this-tool" + record["businessAcceptance"] = "operator-report-imported" if stage == "business-verified" else "not-recorded" + _atomic_json(directory / "batch.json", record) + return summary + def archive(self, batch_id: str, evidence_path: Path) -> dict: _valid_batch_id(batch_id) evidence = _read_json(evidence_path) @@ -405,10 +515,10 @@ class Lane: directory = self.root / "batches" / batch_id _check_no_symlink(directory) record = _read_json(directory / "batch.json") - if record.get("state") != "prepared": - raise LaneError("only a prepared batch can be archived") + if record.get("state") not in {"prepared", "seeded", "business-verified"}: + raise LaneError("only a prepared, seeded, or business-verified batch can be archived") _atomic_json(directory / "archive-evidence.json", evidence) - record.update({"state": "archived", "archivedAt": datetime.now(timezone.utc).isoformat(), "evidenceSha256": _digest(_canonical(evidence)), "remoteState": "operator-attested-stopped", "businessAcceptance": "not-inferred"}) + record.update({"state": "archived", "archivedAt": datetime.now(timezone.utc).isoformat(), "evidenceSha256": _digest(_canonical(evidence)), "remoteState": "operator-attested-stopped"}) _atomic_json(directory / "batch.json", record) return record diff --git a/tests/internal_test_lane/test_environment.py b/tests/internal_test_lane/test_environment.py index 25f139f2..f294c9f6 100644 --- a/tests/internal_test_lane/test_environment.py +++ b/tests/internal_test_lane/test_environment.py @@ -76,6 +76,21 @@ class EnvironmentTests(unittest.TestCase): with self.subTest(hostname=hostname), self.assertRaises(ValueError): validate_config(candidate) + def test_quota_allows_three_retained_batches_and_one_active(self): + c = config() + quota = next(x for x in render_environment(c)["items"] if x["kind"] == "ResourceQuota")["spec"]["hard"] + self.assertEqual(quota["persistentvolumeclaims"], "8") + self.assertEqual(quota["requests.storage"], f"{4 * (c['workspaceGi'] + c['mongoGi'])}Gi") + + def test_namespace_quota_is_replaced_across_batches(self): + first = next(x for x in render_environment(config())["items"] if x["kind"] == "ResourceQuota") + later = config() + later["batchId"] = "acceptance-2" + second = next(x for x in render_environment(later)["items"] if x["kind"] == "ResourceQuota") + self.assertEqual(first["metadata"]["name"], "internal-test-quota") + self.assertEqual(second["metadata"]["name"], first["metadata"]["name"]) + self.assertEqual(first["metadata"]["namespace"], second["metadata"]["namespace"]) + def test_runtime_has_only_batch_scoped_secret_refs_and_no_cluster_credential(self): items = render_environment(config())["items"] deployments = {x["metadata"]["name"]: x for x in items if x["kind"] == "Deployment"} diff --git a/tests/internal_test_lane/test_lifecycle.py b/tests/internal_test_lane/test_lifecycle.py index d0bf9c85..02f1ecba 100644 --- a/tests/internal_test_lane/test_lifecycle.py +++ b/tests/internal_test_lane/test_lifecycle.py @@ -171,6 +171,82 @@ class LifecycleTests(unittest.TestCase): with self.assertRaisesRegex(LaneError, "already exists"): self.lane.prepare("acceptance-1", imported["importId"], config_path, REPO) + def test_cli_records_seed_and_http_evidence_without_remote_claim(self): + from test_environment import config + + imported = self.lane.import_bundle(self.bundle) + cfg = config() + config_path = self.base / "config.json" + config_path.write_text(json.dumps(cfg)) + source = self.lane.root / "imports" / imported["importId"] + manifest = json.loads((source / "manifest.json").read_text()) + inventory = json.loads((source / "inventory.json").read_text()) + plan = { + "batchId": "acceptance-1", "database": "droboticflow_internal_test_acceptance_1", + "assetsRoot": str(source), "workspaceRoot": str(self.base / "workspace"), + "manifestSha256": "f" * 64, "sha256": "a" * 64, + "assets": [{"path": path, **entry} for path, entry in sorted(inventory.items())], + "samples": [{"key": sample["key"], "project": sample["project"], "mode": sample["mode"], + "batchId": "internal-test-batch-" + sample["key"], + "taskId": "internal-test-task-" + sample["key"], "itemId": "internal-test-item-" + sample["key"]} + for sample in manifest["samples"]], + } + + def fake_run(args, **kwargs): + Path(args[args.index("--out") + 1]).write_text(json.dumps(plan)) + return subprocess.CompletedProcess(args, 0, "", "") + + with mock.patch("scripts.internal_test_lane.lifecycle.subprocess.run", side_effect=fake_run): + prepared = self.lane.prepare("acceptance-1", imported["importId"], config_path, REPO) + plan_path = self.lane.root / "batches" / "acceptance-1" / "seed-plan.json" + target = {"kind": "local", "databaseName": plan["database"], "hostname": "127.0.0.1", "namespace": "local"} + + def record(stage, result, plan_sha="a" * 64): + result_path = self.base / f"{stage}-result.json" + result_path.write_text(json.dumps(result)) + metadata = {"schemaVersion": 1, "batchId": "acceptance-1", "stage": stage, + "reviewer": "internal tester", "observedAt": "2026-09-27T12:00:00Z", "target": target, + "buildCommit": cfg["buildCommit"], "image": cfg["image"], + "configSha256": prepared["configSha256"], + "approvedExecutionPlanSha256": plan_sha, + "resultSha256": hashlib.sha256(result_path.read_bytes()).hexdigest()} + metadata_path = self.base / f"{stage}-evidence.json" + metadata_path.write_text(json.dumps(metadata)) + return subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "record-evidence", + "--batch", "acceptance-1", "--evidence", str(metadata_path), + "--plan", str(plan_path), "--result", str(result_path)], + cwd=REPO, capture_output=True, text=True, check=False) + + seed_result = {"database": plan["database"], "planSha256": plan["sha256"], "created": True, + "projects": 2, "samples": 6, "accounts": 5} + refused = record("seeded", seed_result, plan_sha="b" * 64) + self.assertEqual(refused.returncode, 2) + self.assertIn("hash", refused.stderr) + sample = plan["samples"][0] + http_result = {"batchId": "acceptance-1", "sampleKey": sample["key"], "taskId": sample["taskId"], + "itemId": sample["itemId"], "acceptanceItemId": "human-item-fanyu-v5-prepackage-acceptance-" + sample["batchId"], + "status": "accepted", "checks": [{"name": "full-http-workflow", "status": "passed"}]} + before_seed = record("business-verified", http_result) + self.assertEqual(before_seed.returncode, 2) + self.assertIn("requires batch state seeded", before_seed.stderr) + self.assertEqual(record("seeded", seed_result).returncode, 0) + self.assertEqual(record("business-verified", http_result).returncode, 0) + completed = subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "status"], cwd=REPO, capture_output=True, text=True, check=False) + self.assertEqual(completed.returncode, 0) + row = json.loads(completed.stdout)["batches"][0] + self.assertEqual(row["state"], "business-verified") + self.assertEqual(row["remoteState"], "not-verified-by-this-tool") + self.assertEqual(row["evidence"]["business-verified"]["scope"], "single-sample-http-workflow") + self.assertEqual(row["businessAcceptance"], "operator-report-imported") + archive_path = self.base / "archive.json" + archive_path.write_text(json.dumps({"batchId": "acceptance-1", "tasksComplete": True, + "environmentStopped": True, "detached": True, "reviewer": "internal tester"})) + archived = subprocess.run([sys.executable, str(CLI), "--root", str(self.lane.root), "archive", + "--batch", "acceptance-1", "--evidence", str(archive_path)], + cwd=REPO, capture_output=True, text=True, check=False) + self.assertEqual(archived.returncode, 0, archived.stderr) + self.assertIn("business-verified", json.loads(archived.stdout)["evidence"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/internal_test_lane/test_verify_local.py b/tests/internal_test_lane/test_verify_local.py new file mode 100644 index 00000000..cf709d61 --- /dev/null +++ b/tests/internal_test_lane/test_verify_local.py @@ -0,0 +1,98 @@ +"""Offline refusal tests for the local HTTP acceptance runner.""" + +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + + +SCRIPT = Path(__file__).with_name("verify_local.py") +SPEC = importlib.util.spec_from_file_location("internal_test_verify_local", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +verify = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(verify) + + +class NoNetwork: + def open(self, *_args, **_kwargs): + raise AssertionError("test attempted a network request") + + +class VerifyLocalRefusalTests(unittest.TestCase): + def test_accepts_only_fixed_local_api_urls(self): + for url in ("http://127.0.0.1:3019", "http://localhost:3019"): + self.assertEqual(verify.LocalAPI(url).base_url, url) + for url in ( + "https://127.0.0.1:3019", + "http://127.0.0.1:3001", + "http://127.0.0.1:3019/api", + "http://127.0.0.1:3019@customer.example", + "http://internal-test.example:3019", + ): + with self.subTest(url=url), self.assertRaises(verify.CheckFailed): + verify.LocalAPI(url) + + def test_rejects_redirect_without_following_it(self): + with self.assertRaisesRegex(verify.CheckFailed, "redirect rejected"): + verify.RejectRedirects().redirect_request( + None, None, 302, "Found", {}, "http://customer.example/api/human-items" + ) + + def test_rejects_unreviewed_or_non_test_item_ids(self): + allowed = {"internal-test-item-abc123"} + self.assertEqual(verify.test_id("internal-test-item-abc123", allowed), "internal-test-item-abc123") + for item_id in ("customer-item", "internal-test-item-other", "../internal-test-item-abc123"): + with self.subTest(item_id=item_id), self.assertRaises(verify.CheckFailed): + verify.test_id(item_id, allowed) + + def test_rejects_unsafe_api_path_before_network(self): + api = verify.LocalAPI("http://127.0.0.1:3019") + api.opener = NoNetwork() + for path in ("http://customer.example/api/human-items", "/api/../admin", "/api//human-items"): + with self.subTest(path=path), self.assertRaises(verify.CheckFailed): + api.request("GET", path) + + def test_snapshot_action_check_requires_the_explicit_blank_record(self): + expected = [ + {"id": "internal-test-action-described", "start": 0, "end": 1, "description": "Turn", "status": "pending"}, + {"id": "internal-test-action-blank", "start": 1, "end": 2, "description": "", "status": "pending"}, + ] + verify.check_snapshot_actions({"segments": expected}, expected) + changed = [dict(row) for row in expected] + changed[1]["description"] = "Generated label" + with self.assertRaisesRegex(verify.CheckFailed, "meaning changed"): + verify.check_snapshot_actions({"segments": changed}, expected) + + def test_quality_stop_requires_manifest_requested_snapshot_stage(self): + snapshot = {"mode": "snapshot"} + quality = {"snapshot": {"stage": "quality_review"}} + verify.validate_quality_stop(snapshot, quality, True) + for selected, source in ( + (None, None), + ({"mode": "scratch"}, quality), + (snapshot, {"snapshot": {"stage": "annotation"}}), + ): + with self.subTest(selected=selected, source=source), self.assertRaises(verify.CheckFailed): + verify.validate_quality_stop(selected, source, True) + verify.validate_quality_stop(None, None, False) + + def test_report_is_new_private_file_inside_fixed_lane(self): + with tempfile.TemporaryDirectory() as directory: + lane = Path(directory) / "verification-final" + lane.mkdir() + report = lane / "report.json" + with patch.object(verify, "LANE", lane): + self.assertEqual(verify.write_report(str(report), {"status": "accepted"}), report) + self.assertEqual(json.loads(report.read_text()), {"status": "accepted"}) + self.assertEqual(os.stat(report).st_mode & 0o777, 0o600) + with self.assertRaisesRegex(verify.CheckFailed, "already exists"): + verify.write_report(str(report), {}) + with self.assertRaisesRegex(verify.CheckFailed, "under this local test lane"): + verify.write_report(str(lane.parent / "outside.json"), {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/internal_test_lane/verify_local.py b/tests/internal_test_lane/verify_local.py new file mode 100644 index 00000000..367f939e --- /dev/null +++ b/tests/internal_test_lane/verify_local.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +"""Exercise one seeded Capture through the real local HTTP workflow. + +This script writes only through the API at localhost:3019. It uses the reviewed +local seed plan and private test accounts; it never opens Mongo or customer data. +Random claim may assign snapshot samples before a scratch sample is selected. +Those test assignments remain in the test lane for later manual use. +""" + +from __future__ import annotations + +import argparse +import http.cookiejar +import json +import os +from pathlib import Path +import re +import sys +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit +from urllib.request import ( + HTTPCookieProcessor, + HTTPRedirectHandler, + OpenerDirector, + ProxyHandler, + Request, + build_opener, +) + + +REPO = Path(__file__).resolve().parents[2] +LANE = REPO / "local-data" / "internal-test-lane" / "verification-final" +PLAN = LANE / "local-plan.json" +ACCOUNTS = LANE / "credentials" / "accounts.json" +BASE_URLS = {"http://127.0.0.1:3019", "http://localhost:3019"} +ID_RE = re.compile(r"^internal-test-[a-z0-9-]+$") +SHA_RE = re.compile(r"^[a-f0-9]{64}$") + + +class CheckFailed(Exception): + pass + + +class RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, fp, code, msg, headers, newurl): # noqa: ANN001 + raise CheckFailed("API redirect rejected") + + +def under(path: Path, parent: Path) -> bool: + try: + path.relative_to(parent) + return True + except ValueError: + return False + + +def no_symlink(path: Path, boundary: Path) -> None: + current = path + while under(current, boundary): + if current.is_symlink(): + raise CheckFailed("test input path contains a symlink") + if current == boundary: + break + current = current.parent + + +def load_inputs(confirm: str) -> tuple[dict, dict, dict]: + if not SHA_RE.fullmatch(confirm): + raise CheckFailed("--confirm must be the reviewed plan SHA256") + for path in (LANE, PLAN, ACCOUNTS): + no_symlink(path, REPO) + if not PLAN.is_file() or not ACCOUNTS.is_file(): + raise CheckFailed("fixed local seed plan or private accounts file is missing") + if ACCOUNTS.stat().st_mode & 0o077: + raise CheckFailed("private accounts file must have mode 0600 or stricter") + plan = json.loads(PLAN.read_text(encoding="utf-8")) + if plan.get("sha256") != confirm: + raise CheckFailed("--confirm differs from fixed local plan") + batch = plan.get("batchId") + if not isinstance(batch, str) or not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", batch): + raise CheckFailed("plan batch ID invalid") + if plan.get("database") != "droboticflow_internal_test_" + batch.replace("-", "_"): + raise CheckFailed("plan database is outside batch scope") + assets_root = Path(str(plan.get("assetsRoot", ""))) + workspace_root = Path(str(plan.get("workspaceRoot", ""))) + if not under(assets_root, LANE / "lane" / "imports") or workspace_root != LANE / "workspace": + raise CheckFailed("plan paths are outside this local test lane") + for path in (assets_root, workspace_root): + no_symlink(path, REPO) + manifest_path = assets_root / "manifest.json" + no_symlink(manifest_path, REPO) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("schemaVersion") != "droboticflow.internal-test-seed/v1": + raise CheckFailed("normalized manifest schema invalid") + plan_samples = plan.get("samples") + manifest_samples = manifest.get("samples") + if not isinstance(plan_samples, list) or len(plan_samples) != 6 or not isinstance(manifest_samples, list) or len(manifest_samples) != 6: + raise CheckFailed("plan/manifest must contain exactly six test samples") + for planned, source in zip(plan_samples, manifest_samples, strict=True): + if (planned.get("key"), planned.get("project"), planned.get("mode")) != ( + source.get("key"), source.get("project"), source.get("mode") + ): + raise CheckFailed("plan/manifest sample mapping differs") + for field in ("workflowId", "runId", "batchId", "itemId", "taskId", "templateId"): + if not ID_RE.fullmatch(str(planned.get(field, ""))): + raise CheckFailed("plan contains an ID outside the test prefix") + for value in (plan.get("organizationId"), plan.get("supplierId"), *plan.get("projectIds", {}).values(), *plan.get("userIds", {}).values()): + if not ID_RE.fullmatch(str(value or "")): + raise CheckFailed("plan contains an ID outside the test prefix") + accounts = json.loads(ACCOUNTS.read_text(encoding="utf-8")) + rows = accounts.get("accounts") + if not isinstance(rows, list) or {row.get("role") for row in rows} != { + "admin", "supplier_manager", "annotator", "quality_reviewer", "acceptance_reviewer" + }: + raise CheckFailed("private account roles differ from five-role test lane") + return plan, manifest, {row["role"]: row for row in rows} + + +class LocalAPI: + def __init__(self, base_url: str): + if base_url not in BASE_URLS or urlsplit(base_url).path: + raise CheckFailed("API URL must be localhost:3019") + self.base_url = base_url + self.jar = http.cookiejar.CookieJar() + self.opener: OpenerDirector = build_opener(ProxyHandler({}), HTTPCookieProcessor(self.jar), RejectRedirects()) + + def request( + self, + method: str, + path: str, + body: dict | None = None, + *, + expected: tuple[int, ...] = (200,), + prefer_async: bool = False, + ) -> tuple[int, dict]: + if not path.startswith("/api/") or "//" in path or ".." in path or "?" in path and path.count("?") != 1: + raise CheckFailed("API path rejected") + data = None if body is None else json.dumps(body, ensure_ascii=False).encode("utf-8") + headers = {"Accept": "application/json"} + if data is not None: + headers["Content-Type"] = "application/json" + if prefer_async: + headers["Prefer"] = "respond-async" + if method not in ("GET", "HEAD") and path != "/api/auth/login": + csrf = next((cookie.value for cookie in self.jar if cookie.name == "droboticflow_csrf"), "") + if not csrf: + raise CheckFailed("authenticated session has no CSRF cookie") + headers["X-Droboticflow-CSRF-Token"] = csrf + request = Request(self.base_url + path, data=data, headers=headers, method=method) + try: + with self.opener.open(request, timeout=35) as response: + status, raw = response.status, response.read(2_000_000) + except HTTPError as error: + status, raw = error.code, error.read(2_000_000) + except URLError as error: + raise CheckFailed("local API connection failed") from error + try: + payload = json.loads(raw) if raw else {} + except json.JSONDecodeError as error: + raise CheckFailed(f"API returned non-JSON response (HTTP {status})") from error + if not isinstance(payload, dict): + raise CheckFailed(f"API returned unexpected response shape (HTTP {status})") + if status not in expected: + code = str(payload.get("code", ""))[:80] + raise CheckFailed(f"{method} {path.split('?')[0]} returned HTTP {status} code={code}") + return status, payload + + def login(self, row: dict, expected_user_id: str) -> None: + _, payload = self.request("POST", "/api/auth/login", {"phone": row["phone"], "password": row["password"]}) + user = payload.get("user", {}) + if user.get("id") != expected_user_id or user.get("role") != row["role"]: + raise CheckFailed("login identity differs from seed plan") + + +def test_id(value: str, allowed: set[str]) -> str: + if value not in allowed or not ID_RE.fullmatch(value): + raise CheckFailed("API returned an item outside reviewed seed plan") + return value + + +def item_get(api: LocalAPI, item_id: str, *, expected: tuple[int, ...] = (200,)) -> tuple[int, dict]: + status, payload = api.request("GET", f"/api/human-items/{item_id}", expected=expected) + if status == 200: + item = payload.get("item") + if not isinstance(item, dict) or item.get("id") != item_id: + raise CheckFailed("human item detail has wrong identity") + return status, item + return status, payload + + +def poll_item(api: LocalAPI, item_id: str, predicate, label: str, timeout_seconds: int) -> dict: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + _, item = item_get(api, item_id) + if predicate(item): + return item + workflow = item.get("subject", {}).get("payload", {}).get("sliceWorkflow", {}) + if workflow.get("status") == "failed": + raise CheckFailed(f"{label} failed; inspect local API/processor logs") + time.sleep(2) + raise CheckFailed(f"timed out waiting for {label}") + + +def latest_revision(api: LocalAPI, task_id: str) -> dict: + _, detail = api.request("GET", f"/api/plugins/annotation-qa/tasks/{task_id}") + revisions = detail.get("revisions") + if not isinstance(revisions, list) or not revisions: + raise CheckFailed("submitted annotation revision is missing") + latest = max(revisions, key=lambda row: int(row.get("revision", -1))) + if not isinstance(latest.get("segments"), list): + raise CheckFailed("submitted annotation revision has no action list") + return latest + + +def check_snapshot_actions(revision: dict, expected: list[dict]) -> None: + actual = revision["segments"] + if len(actual) != len(expected): + raise CheckFailed("snapshot action count changed after submission") + for original in expected: + matches = [row for row in actual if row.get("id") == original.get("id")] + if len(matches) != 1: + raise CheckFailed("snapshot action identity changed after submission") + row = matches[0] + if row.get("description", "") != original.get("description", "") or bool(row.get("isInvalid", False)) != bool(original.get("isInvalid", False)): + raise CheckFailed("snapshot action meaning changed after submission") + if abs(float(row["start"]) - float(original["start"])) > 1e-6 or abs(float(row["end"]) - float(original["end"])) > 1e-6: + raise CheckFailed("snapshot action interval changed after submission") + if not any(row.get("description", "") == "" and row.get("status") == "pending" and not row.get("isInvalid", False) for row in actual): + raise CheckFailed("explicit blank action record was lost") + + +def validate_quality_stop(selected: dict | None, source_sample: dict | None, enabled: bool) -> None: + if enabled and ( + selected is None or selected.get("mode") != "snapshot" + or source_sample is None or source_sample.get("snapshot", {}).get("stage") != "quality_review" + ): + raise CheckFailed("--stop-at-quality-review requires a snapshot requested at quality_review in the manifest") + + +def write_report(path_value: str, result: dict) -> Path: + output = Path(os.path.abspath(path_value)) + if not under(output, LANE) or output == LANE or output.suffix != ".json": + raise CheckFailed("report path must be a new JSON file under this local test lane") + if not output.parent.is_dir(): + raise CheckFailed("report parent directory does not exist") + no_symlink(output.parent, REPO) + no_symlink(output, REPO) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + descriptor = os.open(output, flags, 0o600) + except FileExistsError as error: + raise CheckFailed("report file already exists") from error + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(result, handle, ensure_ascii=False, indent=2) + handle.write("\n") + return output + + +def run(base_url: str, project_name: str, confirm: str, sample_key: str | None = None, + stop_at_quality_review: bool = False) -> dict: + plan, manifest, accounts = load_inputs(confirm) + selected = None + source_sample = None + if sample_key is not None: + pairs = [(planned, source) for planned, source in zip(plan["samples"], manifest["samples"], strict=True) if planned["key"] == sample_key] + if len(pairs) != 1: + raise CheckFailed("--sample-key is absent from reviewed plan") + selected, source_sample = pairs[0] + if selected["mode"] != "snapshot": + raise CheckFailed("--sample-key currently selects a snapshot sample") + if project_name and project_name != selected["project"]: + raise CheckFailed("--project and --sample-key refer to different projects") + project_name = selected["project"] + validate_quality_stop(selected, source_sample, stop_at_quality_review) + if not project_name: + project_name = "BLF-data" + project_id = plan["projectIds"].get(project_name) + if not isinstance(project_id, str) or not ID_RE.fullmatch(project_id): + raise CheckFailed("selected project is absent from reviewed plan") + project_samples = {row["itemId"]: row for row in plan["samples"] if row["project"] == project_name} + scratch = {row["itemId"]: row for row in project_samples.values() if row["mode"] == "scratch"} + if len(project_samples) != 3 or len(scratch) != 1: + raise CheckFailed("selected project must have three samples and one scratch sample") + dictionaries = manifest.get("dictionaries", {}).get(project_name, {}) + slice_names = dictionaries.get("sliceNames", []) + if not isinstance(slice_names, list) or not slice_names or not isinstance(slice_names[0], str): + raise CheckFailed("project slice dictionary is missing") + + sessions = {role: LocalAPI(base_url) for role in accounts} + for role, api in sessions.items(): + api.login(accounts[role], plan["userIds"][role]) + print(f"login {role}: ok") + + annotator = sessions["annotator"] + item = task = sample = None + if selected is not None: + sample = selected + item_id = test_id(sample["itemId"], set(project_samples)) + admin = sessions["admin"] + _, candidate = item_get(admin, item_id) + if candidate.get("currentStageId") != "annotation_combined": + raise CheckFailed("selected snapshot is no longer in annotation stage") + owner = candidate.get("assignedUserId", "") + annotator_id = plan["userIds"]["annotator"] + if owner == annotator_id and candidate.get("status") in ("claimed", "in_progress"): + item = candidate + print(f"snapshot {sample_key}: reusing test annotator assignment") + elif not owner and candidate.get("status") in ("ready", "rework"): + _, item = admin.request( + "PATCH", f"/api/human-items/{item_id}/assignment", + {"version": candidate["version"], "assignedUserId": annotator_id}, + ) + if item.get("id") != item_id or item.get("assignedUserId") != annotator_id: + raise CheckFailed("explicit test assignment did not bind the annotator") + print(f"snapshot {sample_key}: assigned by test administrator") + else: + raise CheckFailed("selected snapshot is already assigned or unavailable") + _, task_detail = annotator.request("GET", f"/api/plugins/annotation-qa/tasks/{sample['taskId']}") + task = task_detail.get("task") + if not isinstance(task, dict) or task.get("id") != sample["taskId"]: + raise CheckFailed("snapshot task differs from reviewed plan") + _, item = item_get(annotator, item_id) + else: + for attempt in range(3): + request_id = f"internal-test-http-claim-{plan['batchId']}-{attempt}-{int(time.time())}" + _, claimed = annotator.request( + "POST", "/api/plugins/annotation-qa/random-claims", + {"projectId": project_id, "requestId": request_id}, + ) + candidate = claimed.get("item") + if not isinstance(candidate, dict): + raise CheckFailed("random claim returned no test task") + sample_id = test_id(str(candidate.get("id", "")), set(project_samples)) + row = project_samples[sample_id] + returned_task = claimed.get("task") + if not isinstance(returned_task, dict) or returned_task.get("id") != row["taskId"]: + raise CheckFailed("claimed task differs from reviewed plan") + print(f"random claim {attempt+1}: {row['key']} ({row['mode']})") + if row["mode"] == "scratch": + item, task, sample = candidate, returned_task, row + break + if item is None or task is None or sample is None: + raise CheckFailed("three random claims did not yield the scratch sample") + + item_id = sample["itemId"] + task_id = sample["taskId"] + start, end = float(task["clipStart"]), float(task["clipEnd"]) + if start != 0 or not (0 < end <= 86_400): + raise CheckFailed("claimed scratch task has unexpected coverage") + snapshot_actions: list[dict] = [] + if sample["mode"] == "snapshot": + seeded = item.get("subject", {}).get("payload", {}).get("annotationWorkspace", {}) + proposed_slices = json.loads(json.dumps(seeded.get("sliceSegments", []))) + snapshot_actions = json.loads(json.dumps(seeded.get("segments", []))) + if not proposed_slices or not snapshot_actions or source_sample is None: + raise CheckFailed("snapshot workspace is incomplete") + expected_actions = source_sample.get("snapshot", {}).get("actions", []) + if len(snapshot_actions) != len(expected_actions): + raise CheckFailed("snapshot workspace differs from normalized manifest") + for actual_action, expected_action in zip(snapshot_actions, expected_actions, strict=True): + if actual_action.get("description", "") != expected_action.get("description", ""): + raise CheckFailed("snapshot description differs from normalized manifest") + expected_slice_ids = {row.get("id") for row in proposed_slices} + if len(expected_slice_ids) != len(proposed_slices) or not all(ID_RE.fullmatch(str(value)) for value in expected_slice_ids): + raise CheckFailed("snapshot slice IDs are outside test seed") + else: + slice_id = f"internal-test-http-slice-{sample['key']}" + proposed_slices = [{ + "id": slice_id, "name": slice_names[0], "start": start, "end": end, + "deliveryDisposition": "keep", "annotationStatus": "pending", + }] + _, confirmation = annotator.request( + "POST", f"/api/plugins/annotation-qa/tasks/{task_id}/slice-confirmations", + {"version": item["version"], "sliceSegments": proposed_slices}, expected=(202,), + ) + if confirmation.get("item", {}).get("id") != item_id: + raise CheckFailed("slice confirmation changed item identity") + print("slice confirmation: queued") + item = poll_item( + annotator, item_id, + lambda value: value.get("subject", {}).get("payload", {}).get("sliceWorkflow", {}).get("status") == "completed" + and value.get("subject", {}).get("payload", {}).get("sliceWorkflow", {}).get("phase") == "annotate", + "slice materialization", 240, + ) + workspace = item.get("subject", {}).get("payload", {}).get("annotationWorkspace", {}) + slices = workspace.get("sliceSegments") + if not isinstance(slices, list) or len(slices) != len(proposed_slices): + raise CheckFailed("materialized slice count changed") + for actual, proposed in zip(slices, proposed_slices, strict=True): + if actual.get("id") != proposed.get("id") or actual.get("name", "") != proposed.get("name", ""): + raise CheckFailed("materialized slice identity or name changed") + if sample["mode"] == "snapshot": + actions = snapshot_actions + else: + actual = slices[0] + actions = [{ + "id": f"internal-test-http-action-{sample['key']}", + "start": actual["start"], "end": actual["end"], + "status": "pending", "description": "", + }] + _, submitted = annotator.request( + "POST", f"/api/plugins/annotation-qa/tasks/{task_id}/combined-submissions", + {"version": item["version"], "sliceSegments": slices, "segments": actions, "requireSliceCompletion": True}, + expected=(201, 202), prefer_async=True, + ) + if submitted.get("item", submitted.get("humanItem", {})).get("id") not in (None, item_id): + raise CheckFailed("submission changed item identity") + print("action submission: accepted for processing") + item = poll_item( + annotator, item_id, + lambda value: value.get("currentStageId") == "quality_review" and value.get("currentRevision", 0) > 0 + and bool(value.get("subject", {}).get("payload", {}).get("materialization", {}).get("artifactId")), + "combined submission", 240, + ) + print(f"quality review ready: revision {item['currentRevision']}") + revision = latest_revision(annotator, task_id) + if sample["mode"] == "snapshot": + check_snapshot_actions(revision, snapshot_actions) + print("snapshot actions: preserved, including blank record") + elif not any(row.get("description", "") == "" and row.get("status") == "pending" for row in revision["segments"]): + raise CheckFailed("scratch blank action record was lost") + if stop_at_quality_review: + return {"batchId": plan["batchId"], "sampleKey": sample["key"], "mode": sample["mode"], + "taskId": task_id, "itemId": item_id, "status": "quality_review", + "currentRevision": item["currentRevision"]} + + quality = sessions["quality_reviewer"] + _, quality_item = item_get(quality, item_id) + _, claimed_quality = quality.request( + "POST", f"/api/human-items/{item_id}/claim", {"version": quality_item["version"]} + ) + if claimed_quality.get("id") != item_id or claimed_quality.get("assignedUserId") != plan["userIds"]["quality_reviewer"]: + raise CheckFailed("quality claim returned wrong assignment") + _, reviewed = quality.request( + "POST", f"/api/plugins/annotation-qa/tasks/{task_id}/reviews", + {"stage": "quality", "decision": "approved", "videoQuality": "good", + "annotationAccuracy": 100, "annotationCompleteness": 100, "overallGrade": "A"}, + expected=(201,), + ) + if reviewed.get("humanItem", {}).get("id") != item_id or reviewed.get("review", {}).get("decision") != "approved": + raise CheckFailed("quality approval response incomplete") + print("quality review: approved") + + acceptance_id = "human-item-fanyu-v5-prepackage-acceptance-" + sample["batchId"] + acceptance = sessions["acceptance_reviewer"] + deadline = time.monotonic() + 90 + aggregate = None + while time.monotonic() < deadline: + status, candidate = item_get(acceptance, acceptance_id, expected=(200, 404)) + if status == 200: + aggregate = candidate + break + time.sleep(2) + if aggregate is None: + raise CheckFailed("aggregate final acceptance item was not created") + if aggregate.get("currentStageId") != "prepackage_acceptance" or aggregate.get("status") not in ("ready", "submitted"): + raise CheckFailed("aggregate acceptance is not ready") + _, claimed_acceptance = acceptance.request( + "POST", f"/api/human-items/{acceptance_id}/claim", {"version": aggregate["version"]} + ) + if claimed_acceptance.get("id") != acceptance_id or claimed_acceptance.get("assignedUserId") != plan["userIds"]["acceptance_reviewer"]: + raise CheckFailed("acceptance claim returned wrong assignment") + _, decided = acceptance.request( + "POST", f"/api/human-items/{acceptance_id}/decisions", + {"version": claimed_acceptance["version"], "decision": "approved", "videoQuality": "good", + "annotationAccuracy": 100, "annotationCompleteness": 100, "overallGrade": "A"}, + ) + final_item = decided.get("item", decided) + if final_item.get("id") != acceptance_id or final_item.get("status") != "accepted": + raise CheckFailed("final acceptance did not reach accepted status") + print("final acceptance: approved") + return {"batchId": plan["batchId"], "sampleKey": sample["key"], "mode": sample["mode"], "taskId": task_id, + "itemId": item_id, "acceptanceItemId": acceptance_id, "status": "accepted"} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:3019", choices=sorted(BASE_URLS)) + parser.add_argument("--project", choices=("BLF-data", "SL-data"), help="scratch project; defaults to BLF-data") + parser.add_argument("--sample-key", help="reviewed snapshot key such as blf-2 or sl-2") + parser.add_argument("--stop-at-quality-review", action="store_true", + help="for a manifest quality_review snapshot, stop after slice confirmation and action submission") + parser.add_argument("--output", help="new 0600 JSON report path under the fixed local test lane") + parser.add_argument("--confirm", required=True, help="SHA256 from reviewed local-plan.json") + args = parser.parse_args() + try: + result = run(args.base_url, args.project, args.confirm, args.sample_key, args.stop_at_quality_review) + if args.output: + output = write_report(args.output, result) + print(f"report: {output}") + except (CheckFailed, KeyError, TypeError, ValueError, OSError) as error: + print(f"local HTTP verification stopped: {error}", file=sys.stderr) + return 1 + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())