diff --git a/.gitea/workflows/web-ci.yml b/.gitea/workflows/web-ci.yml index a2d4a8d4..819e627a 100644 --- a/.gitea/workflows/web-ci.yml +++ b/.gitea/workflows/web-ci.yml @@ -24,6 +24,7 @@ jobs: node-version: 22 cache: npm - run: npm ci + - run: npm run test:verification - run: npm run lint - run: npm run format:check - run: npm run test:go:gate @@ -33,6 +34,14 @@ jobs: if: ${{ !cancelled() }} - run: npm run test:browser:editing if: ${{ !cancelled() }} + - run: npm run test:browser:review-permissions + if: ${{ !cancelled() }} + - run: npm run test:browser:dashboard + if: ${{ !cancelled() }} + - run: go test ./internal/api -run '^TestReviewPoolSearchBrowserReplica$' -count=1 -timeout=3m -v + if: ${{ !cancelled() }} + env: + REVIEW_POOL_BROWSER_TEST: '1' - uses: actions/upload-artifact@v4 if: always() with: @@ -40,3 +49,23 @@ jobs: path: local-data/test-results/ if-no-files-found: warn retention-days: 14 + + pool-search-mongo: + runs-on: ubuntu-22.04 + services: + mongo: + image: mongo:8 + ports: + - 27018:27017 + options: >- + --health-cmd "mongosh --quiet --eval 'db.adminCommand({ping:1})'" + --health-interval 5s --health-timeout 5s --health-retries 12 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - run: go test ./internal/store -run '^TestHumanPoolSearchMongo' -count=1 -timeout=3m -v + env: + POOL_LOCAL_MONGO_URI: mongodb://127.0.0.1:27018 diff --git a/.scratch/acceptance-playback-rate/verification.md b/.scratch/acceptance-playback-rate/verification.md new file mode 100644 index 00000000..fffe6727 --- /dev/null +++ b/.scratch/acceptance-playback-rate/verification.md @@ -0,0 +1,40 @@ +# Acceptance playback-rate verification + +Base: `867d406c`. Synthetic local data only; no production writes. + +## Reproduction and fix + +The new browser regression failed against the unchanged base: after selecting 2x and advancing to slice two, the speed selector returned `1` instead of `2`. Physical acceptance slices intentionally change the media key and remount CaptureAnnotationWorkbench, whose rate was component-local state. + +FinalAcceptanceWorkspace now owns the rate and passes it to its media and pose workbenches. Callers that do not supply a rate retain the existing local-state behavior. No persistence across reloads is introduced. + +## Evidence + +- `npm run verify:task -- --base 867d406c --budget-seconds 600` ran type/architecture checks, changed-component formatting, verification tooling tests, affected frontend tests, the new browser regression, rework-boundary browser, review-draft browser and review-decision browser checks successfully. Its final permissions command could not launch because the default Playwright Chromium was absent. Original report: `local-data/test-results/verify-1793876380248594000/report.json` (retains that failure; not relabeled). +- Reran the permissions command with `BROWSER_EXECUTABLE=/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge`; passed with no errors, failures or unexpected requests. +- Strengthened and reran `tests/browser/acceptance-playback-rate.mjs`: physical clips must mount VIDEO and source-index clips must mount MOVI-PLAYER. Both paths passed three forward/backward transitions retaining 2x/1.5x, actual left/right element rates and play/pause. Report/screenshots: `local-data/test-results/acceptance-playback-rate/`. +- The source components did not change during these follow-ups. Unchanged editing, draft, decision and permission evidence is reused; only the strengthened synthetic fixture/browser test was rerun. +- Visually checked the acceptance page, both cameras, current-slice caption and retained 1.5x control. No customer-facing copy or layout changed. + +## Running the dedicated browser regression + +Requires ffmpeg, installed browser and local Vite on 127.0.0.1:5197 (`npm run dev -- --host 127.0.0.1 --port 5197 --strictPort`). The browser test intercepts only /api requests, permits GET/HEAD only, and supplies synthetic metadata and MP4s. It never claims, saves or submits a task. + +Run `BROWSER_PATH='/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge' node tests/browser/acceptance-playback-rate.mjs`. + +## Limits + +No production verification or deployment. No promise of retaining speed across page reloads. Broad release/capacity checks are not claimed by this local UI fix. The original task-gate report remains failed for its environment issue; the targeted follow-up above supplies the successful missing check. + +## Follow-up: shortcut navigation playback (base a341def2) + +- Built a real-browser keyboard loop (Ctrl+Arrow then immediate Space) using synthetic source-index media. Reproduced repeated failures with all bytes buffered: UI either remained in positioning or reverted to paused. Temporary probes and event traces are under `local-data/test-results/acceptance-shortcut-playback/`. +- Compared the pinned vendor implementation and native video properties: Movi derives seeking/readiness from its asynchronous state machine and paused from play intent. These diverge from the underlying media during rapid navigation; stale pause notifications could cancel a new play. The adapter now uses the current native engine's real paused/seeking/readyState; the software path retains vendor getters. No change to logical annotation times or seek accuracy. +- Extended the permanent browser regression to native clips, source-index clips with integer offsets, and source-index clips with fractional offsets. Each performs 15 transitions, checks both real media clocks advance after immediate keyboard play, and retains the playback rate. Passed twice with the fix (45 transitions per run). Screenshots/report: `local-data/test-results/shortcut-playback-fixed/`; task-run copy: `local-data/test-results/acceptance-playback-rate/`. +- Ran that same strengthened regression against the pre-fix MoviVideo from a341def2: failed waiting for playback after the shortcut (`local-data/test-results/shortcut-playback-baseline/browser-red.log`). Restored the fix automatically afterward. +- Task verification `verify-1793893323697578000/report.json`: type/architecture, format, and browser checks passed. Full frontend gate failed with one additional-to-known-baseline ID: `fanyu-comment-followup.test.mjs :: dashboard routes all workforce roles to the clean production quality panel`. Running the required gate with the pre-fix MoviVideo produces the same failure (`shortcut-playback-baseline/frontend-gate.log`); the tested dashboard sources are unchanged by this task. Known-failures was not expanded and the failed report remains failed. +- Actual acceptance screenshot checked with both cameras and 1.5x rate; no copy/layout changes. Production/client-specific verification remains pending; this is a local reproduction of the reported symptom, not proof of every customer's failure condition. + +- Media recovery across annotation, quality review and acceptance passed with forced software-engine reload, simulated failed loading, bounded retry, retained position, and annotation draft preservation: `REWORK_BOUNDARY_BROWSER_TEST=1 ANNOTATION_FEEDBACK_BROWSER_SCRIPT=local-data/test-results/acceptance-shortcut-playback/media-recovery.mjs go test -count=1 ./internal/api -run '^TestReworkBoundaryBrowserReplica$' -v` (35.16s). The original script failed before mounting due to a hardcoded missing `.vite/deps/react-dom_client.js`; the task-local copy changes only its Vite cache directory and React module import through a Vite-transformed module. No product/fixture assertions were removed. The original harness failure remains recorded, and no shared node_modules cache was repaired or overwritten. + +Local only. No push, PR, deployment, production data change or source-media change. Remaining release limitation: the pre-existing dashboard frontend assertion is still outside the known-failure baseline and blocks claiming the full task gate green. diff --git a/.scratch/acceptance-playback-rate/work.md b/.scratch/acceptance-playback-rate/work.md new file mode 100644 index 00000000..2c6fa2de --- /dev/null +++ b/.scratch/acceptance-playback-rate/work.md @@ -0,0 +1,16 @@ +# Preserve acceptance playback speed across slice changes + +Goal: Retain the selected playback rate when acceptance switches slices or media views, including both cameras. +Agent: Codex +Base: origin/dev @ 867d406c (contains live 6eee426e) +Scope: FinalAcceptanceWorkspace playback state, optional workbench rate ownership, isolated browser regression. +Out of scope: Persistence across browser reloads, global player defaults, task/permission changes, V4 migration, production writes or deployment. +Risk: Shared workbench must preserve its existing uncontrolled behavior for annotation and quality review; remounted acceptance media must receive the retained rate before playback. + +## Follow-up: intermittent shortcut navigation playback + +Follow-up base: `a341def2`. The user reports occasional inability to play after keyboard slice navigation. Reproduced locally with synthetic full-source clips and Ctrl+Arrow navigation followed immediately by Space; no production mutation. + +Scope: normalize Movi's HTMLMediaElement-shaped paused/seeking/readyState properties from its current native engine when present; preserve the existing software-engine getters, source/seek coordinates, media bytes, annotations and permissions. Restore original property descriptors on cleanup. Extend the browser regression to repeated keyboard navigation with integer and fractional source offsets. + +Compatibility: pinned vendored Movi 0.4.1 exposes the active native video via getHLSVideoElement(), including its direct native wrapper; the software engine returns null. Re-check that seam if the vendor is upgraded. No vendor bundle or network settings changed. diff --git a/.scratch/review-pool-search/index-plan.md b/.scratch/review-pool-search/index-plan.md new file mode 100644 index 00000000..40f95f7b --- /dev/null +++ b/.scratch/review-pool-search/index-plan.md @@ -0,0 +1,39 @@ +# Broad pool search: rejected metadata index experiment + +Base: c942a8ce on codex/review-pool-search; live source e4969acc. This is an opt-in follow-up, not a startup migration. + +## Evidence + +Production read-only phase counts on 2026-11-08 found about 11,900 quality/semantic candidates. The current source-generated wide search still exceeds the 8-second deadline. Root explain examined 11,903 full documents in 1,018 ms in one warm sample; the window/group prefix and foreign joins add more work. These samples are diagnostic, not an end-to-end p95 or a fixed dataset benchmark. Random claim uses a separate path. + +Hash batching and removing foreign lookup pipelines did not resolve the timeout and were not adopted. An initial synthetic index with an arbitrary prefix still fetched all documents. Ordering equality scope first is material: organizationId/pluginId/currentStageId, then createdAt/id, followed by projected metadata. + +`TestHumanPoolSearchMongoCoveringIndex` uses the actual aggregation with 1,000 synthetic 64 KiB payloads. With the candidate index, its root cursor examines zero documents, and matching results remain identical before creation, after creation, and after dropping the index. There is no hint in the tested aggregation. A separate 10,000-row projection experiment also examined zero documents with the correctly ordered index. It does not establish full-query latency or concurrency capacity. + +## Exact proposed production change + +- Target: Fanyu cluster, namespace droboticflow-fanyu; Mongo database drobodata-fanyu, collection humanItems. +- Create one non-unique, non-sparse index `pool_search_metadata_v1` using the ordered keys in `local-data/pool-v4-followup-20261108/rejected-index/index-spec.json`. +- No customer record updates, backfill, deletion, existing index removal, Pod/bandwidth changes, or deployment. +- The candidate API code's compact projection is still local; the live legacy projection may not gain the same coverage. After index creation, run the **local candidate's source-generated pipeline read-only** against the same filter/cursor and compare all matching IDs and count. Also check live API latency/errors because planner selection can change. +- Do not automatically add this index in MongoStore startup. Creation requires separate production-change authorization under AGENTS.md. + +## Cost, stop and rollback + +A 10,000-row synthetic 16 KiB-payload experiment measured about 1.8 MiB for the 16-field version and a 51 ms local build. The final spec adds supplier/author scope fields. These figures do not predict production build time, disk size, replication lag, CPU or write amplification; inspect actual capacity and active builds before creation. All indexed fields are scalar by contract; abort if historical array values make this a multikey/parallel-array index. + +Build adds CPU, storage I/O, disk usage and ongoing index maintenance. Monitor the existing service and cancel a build on sustained pressure rather than changing resources simultaneously. If the index worsens plans or gives no useful gain, drop **only this newly created index** with the same authorization. No customer-data rewrite is needed for rollback. + +Root coverage alone is not acceptance. If foreign lookups still dominate or total p95 does not meet the agreed threshold, retain pending status and profile the next phase. Do not widen permissions, omit legacy rows, change pagination, or increase the 8-second timeout. + +## Executed result and rollback + +The user explicitly authorized creation of this index and dropping only this index if ineffective. Preflight: humanItems had 25,802 records, about 250 MB uncompressed data, 62.5 MB storage, 19.3 MB existing indexes; Mongo cache used 3.54 / 7.86 GB. A paged read-only field check examined all 25,802 records and found no arrays in the indexed fields. The original one-shot type aggregate exceeded its 8-second diagnostic bound; the 52 bounded batches completed in 11.3 seconds. + +Creation took 4,597 ms; the new index occupied 15,835,136 bytes. The source-generated production root explain really used this index and examined zero documents, but the full **actual Go Store** candidate queries still took 8,001 ms and timed out for broad/all and available scopes. The acceptance example returned one match in 5,915 ms. These are scoped backend reads, not a customer browser/capacity pass. + +A direct old/new root projection comparison measured old index: 2,065 ms, 11,903 documents, 11,904 keys, 1 seek; candidate: 3,377 ms, zero documents, 11,904 keys, 9 seeks. Coverage did not translate to reduced overall cost. Hash batching and native equality batching also still timed out. Production activity and warm caches vary, so these do not establish a fixed-dataset percentage regression; they are sufficient to reject retaining an unproven extra index. + +The authorized rollback dropped only pool_search_metadata_v1 in 140 ms and verified it absent. Health/readiness returned HTTP 200 in 38/32 ms afterwards. No customer record, old index, resource or application image was changed. The experiment and local synthetic test are retained only under ignored local-data/, not installed by startup or made a permanent CI requirement. + +Next work remains: avoid performing full candidate-set sorting/grouping and foreign metadata joins before selective search. Candidate discovery must preserve direct IDs, capture/part combinations, historical payload/input names, categories, ordering and tenant/claim rules. A new implementation needs differential tests and realistic latency evidence before another production change. The broad-pool issue is **open**, not fixed by this experiment. diff --git a/.scratch/review-pool-search/nov5-fixes.md b/.scratch/review-pool-search/nov5-fixes.md new file mode 100644 index 00000000..fb6a7b62 --- /dev/null +++ b/.scratch/review-pool-search/nov5-fixes.md @@ -0,0 +1,58 @@ +# 旧任务 V4 交付及 11 月 5 日反馈核验 + +Base: 6d8b011c67ecadfb9354cd1871d20c21b58d1f87;在现有 codex/review-pool-search 上继续,不改其他工作树。 + +授权范围:本地修改、验证、提交,生产只读诊断;未推送、未建 PR、未部署,未改客户记录或 BOS 文件。飞书仅阅读。 + +## 相比现网的新改动 + +重新核对结果:当前工作树包含最新 origin/dev 与线上源码 e4969acc;本地已有三个尚未发布提交: + +- adc63af8:审核/验收按当前任务池独立请求,取消过期请求;批量关联搜索元数据,减少大载荷读取。已审核搜索不再被可领取池超时连带清空;**广范围搜索未完成性能验收**。 +- c942a8ce:修正工作树 hooks、验证入口与浏览器工具运行问题。 +- 6d8b011c:V4 历史包本地预览转换、可恢复批处理及查询索引实验记录。索引无端到端收益,已按原授权删除,没有保留生产索引。 +- 本批:旧任务 V4 打包快照与发布格式防护;鼠标进入批注时暂停并固定位置;舍弃切片批注目标;字幕上下位置偏好;Go 已知失败用官方基线门禁核对。 + +## 证据来源及日期 + +[数据平台测试](https://pcno45pc3jv8.feishu.cn/wiki/TImwwnFqyiIuC3kgIDHcIlHDnTc),读取修订 7543,11 月 5 日表格两条问题的解决栏均为空。 + +[平台调优群](https://applink.feishu.cn/client/chat/open?openChatId=oc_ff3c7f66ecfd607ffebd120f0113dbca)。CLI 时间为 America/Los_Angeles;北京时间 11 月 5 日主要反馈显示为本地 11 月 4 日。下面保留来源位置,不把当地 11 月 5 日晚的新反馈混称为北京时间同一天。群内“我看看/可以加设置”不算修复证据。 + +## 问题及处理状态 + +| 问题 | 核验与处理 | 状态 | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | +| 马迪:保存时间点意见后位置跳动(测试文档 11/5) | 下载原演示,画面是秒级位置变化;不能直接认定为源视频时间偏移。确认鼠标入口没有暂停/捕获当前帧,而 F 入口会暂停。补齐鼠标入口;既有编辑不追随播放头。旧钩子真实播放回归在暂停断言失败。 | 本地修复,三种播放器回归通过 | +| 马迪:舍弃切片无法批注(测试文档 11/5) | 服务端允许切片意见,拒绝舍弃片段的动作/时间点意见。验收未把舍弃状态传入工作台,前端仍提供被拒绝的目标。传递状态并回退到切片目标,不修改服务端业务规则。 | 本地修复,三种播放器保存回归通过 | +| [切片倍速重置](https://applink.feishu.cn/client/chat/open?openChatId=oc_ff3c7f66ecfd607ffebd120f0113dbca&position=452)、[快捷键切片后无法播放](https://applink.feishu.cn/client/chat/open?openChatId=oc_ff3c7f66ecfd607ffebd120f0113dbca&position=453) | 已在现网 e4969acc 修复,本批扩大原有验收真实播放器回归,防止回归。 | 现网已有修复;本批 45 次切片回归通过 | +| [当前意见卡显示“无”](https://applink.feishu.cn/client/chat/open?openChatId=oc_ff3c7f66ecfd607ffebd120f0113dbca&position=451) | 原截图当前为来源 1 / 切片 3,列表那条意见属于切片 2、01:37.152;不是同一个当前目标。不能据该截图证明批注丢失;保留当前目标卡与全部意见列表的不同范围。 | 原截图不构成同目标丢失证据;不宣称修复未知场景 | +| [审核搜题超时](https://applink.feishu.cn/client/chat/open?openChatId=oc_ff3c7f66ecfd607ffebd120f0113dbca&position=454) | 本地已有当前池请求隔离修复。此前与本轮生产只读实际 Store 对照:广范围 all / available 仍各 8001ms 超时,验收示例 5638ms、1 条。按来源聚集查询的实验也未达标,已撤回;不扩超时、不删候选、不保留无效索引。 | **未解决广范围慢查询** | +| [字幕遮挡手部](https://applink.feishu.cn/client/chat/open?openChatId=oc_ff3c7f66ecfd607ffebd120f0113dbca&position=456)(客户端当地 11/5 晚) | 加“字幕位置”上方/下方选择,保留隐藏字幕;保存本机偏好,切片、刷新后保持。仅移动展示层,不改视频、标注或快捷键。 | 本地修复,正常/只读/错误视觉核验通过 | +| [2x / 4x 视频卡顿](https://applink.feishu.cn/client/chat/open?openChatId=oc_ff3c7f66ecfd607ffebd120f0113dbca&position=440) | 用户提供的办公网报告:平台健康 107–148ms;同一 BOS 视频 Range 请求约 15s 超时,实收仅 0.02 / 0.22Mbps,另一次连接失败。这证明该次 BOS 到客户端传输异常,不能单靠通用测速判定播放链路正常;不足以判定 BOS 服务端或哪段网络的唯一责任。当前缺少新同条件链路复测,不改播放器掩盖传输故障。 | **未确认消除**;一般测速不等于视频链路测速 | + +## V4 根因与修复范围 + +V4 工作流实际已于 11 月 2 日建立并启用,镜像和参数正确。库存 35 个包均来自旧工作流的源任务;源父 Run 在切换前创建,11 月 6 日新产生的打包子 Run 仍用旧节点。这是“旧任务后续打包未纳入新格式要求”,不是 V4 开关未上线。 + +按用户最新“不得再新生成旧包”的要求,替代原 10 月 22 日在途任务继续旧协议的约定: + +1. 在同组织、项目、版本家族中选择唯一已发布且可见的 V4 配置,人工放行时冻结兼容打包节点、五项参数和摘要镜像;保留原任务及工作流归属。 +2. 执行、成功作业重启恢复和重试使用冻结节点。重试保留已验收基线及原始输入,更新交付分区关联;不重新进入人工关卡。 +3. 在任何目标写入前拒绝新的旧格式;V4 客户目录只能包含 part.json。审核历史是独立内部产物。 +4. 已完整交付旧包仍可逐成员校验后**只读**恢复状态;缺失、冲突或未完成旧包不续传,不原地转换。 +5. 缺少冻结原始选择、字典、分类或 PTS 的历史任务会明确失败,必须恢复可信证据后重发;不会用当前字典或猜测帧率补齐。已排队但没有新运行快照的旧子任务会被最终格式防护拦截,需要重新生成候选,不会自动改写已运行作业。 + +历史转换仍为 18/35 个已校验本地预览,另 17 个因证据缺口阻塞(10 个缺冻结切片字典、6 个缺固定 JSON、1 个全舍弃旧记录缺来源/字典)。6 个全舍弃预览存在历史标定版本证明限制,不能当成正式可发布。没有把预览上传或替换线上包。 + +## 验证 + +忽略目录 local-data/nov5-triage 保存脱敏诊断、旧版失败与候选验证;完整飞书文档捕获含账号表,不进入提交或对话输出。 + +- 旧行为复现:legacy-v4-baseline-red.log(旧格式成功新发、旧任务未收审计);note-baseline-red.log(鼠标意见入口不暂停);discard-baseline-red.log(舍弃状态仍提供时间点档)。 +- V4:v4-cutover-tests.log,隔离 MemoryStore / 假对象存储;新发防护、完整旧包零写入恢复、损坏/归属/多余成员、跨组织/项目/家族、固定镜像、重试祖先与分区恢复。 +- 最终任务门禁:local-data/test-results/verify-1794160542009742000/report.json,15 项全部通过,unmapped 为空,运行期间输入未改变。包括类型、格式、工具回归、前端相关回归、官方 Go 基线、V4 打包、三种验收播放器、返修边界、质检/验收草稿、决定竞态与只读权限。 +- 额外全量前端:frontend-baseline-gate-final.log,13 个失败均在既有 14 项基线内,无新增。首次沙箱运行的两个 EPERM 端口监听失败保留原始记录;允许本机测试服务后重跑通过,未改失败基线。 +- V4 Python 门禁 37 passed / 4 opt-in skipped;随后生成真实合成双目物化修订(real-materialization.log),补跑四项实际 V4 消费测试(real-v4-consumption.log)均通过,未把跳过直接算通过。 +- 真实验收截图与报告:local-data/test-results/acceptance-playback-rate/。native、movi、movi-fractional 各 15 次切片;鼠标写意见暂停真实媒体并固定保存位置;舍弃切片意见读回;字幕上下选择及刷新保持;加载失败、只读编辑锁均通过。已实际查看正常、只读及错误画面。 +- 最终候选未部署、未生产写验收、未容量验收;广范围搜索和 BOS 链路卡顿仍未完成。 diff --git a/.scratch/review-pool-search/work.md b/.scratch/review-pool-search/work.md new file mode 100644 index 00000000..faf31d26 --- /dev/null +++ b/.scratch/review-pool-search/work.md @@ -0,0 +1,51 @@ +# Reviewer pool search isolation and metadata reads + +Goal: Restore the reported reviewer history search without making it wait for unrelated pool queries. +Agent: Codex +Base: live e4969acc (codex/acceptance-playback-rate, PR #72), preserving its playback fixes. + +## Changes + +- Review/acceptance pages explicitly read the selected pool. Switching tabs cancels the old read; an available-pool timeout no longer discards a successful reviewed-pool search. Unread counts show an unknown value instead of zero. +- Keep action ownership scope unchanged when switching pool tabs; stale initial/page results are guarded separately by the active view. +- Separate page-read errors from action errors. A successful retry/refresh clears the read error without clearing an unresolved write error. +- Search sorts compact metadata, batches foreign lookups in groups of 64, uses one conservative text prefilter, and hydrates matching records in original order. Legacy recursive payload names are read in batches only where needed. +- Preserve dynamic legacy run-input reference names, tenant isolation, stable cursors, server eligibility, old response shapes and the 8-second deadline. No production data/index writes, migrations, or resource changes. + +## Verification + +Evidence directory: local-data/pool-search-verification/ (ignored). + +- final-freshness.log: contains current origin/dev and the live version. +- frontend-red.log: the new independent reviewed-search assertion fails against the exact baseline JavaScript (requests available instead of reviewed). +- baseline-regression.log: raw historical input-reference fixture fails against baseline Go source; current Mongo/memory parity passes. +- final-mongo-all.log: Mongo parity, 10,000 modern records, and a >100MB synthetic payload fixture pass, including cancellation, same-time cursor ties, cross-tenant links, category equivalence, payload-only fallback, candidate and identity responses. Find replies are included in traffic accounting. +- final-search-browser.log: selected-pool search, old history, slow/failed unrelated pool, cancellation, pagination retry and cleared read errors pass; screenshot inspected at local-data/test-results/review-pool-search/review-search.png. +- final-permissions-browser.log: review and acceptance authoring permissions pass using installed Edge. Initial default-browser attempt lacked a downloaded Playwright browser; rerun with Edge passed. +- final-lint.log: lint and lint-related frontend checks pass. +- remaining-checks-results.json and task-extra-*.log: source admission Mongo, source validation browser, review trace, rework boundaries, review drafts and decisions pass. The task plan was invoked but stopped at existing opt-in skips (RandomClaimLocalPreview and SourceValidationBrowserReplica); remaining checks were run separately with their prescribed environments. Do not describe the aggregate task gate as green. +- final-frontend-gate.log: the only remaining global frontend failure is fanyu-comment-followup.test.mjs :: dashboard routes all workforce roles to the clean production quality panel. baseline-frontend-gate.log reproduces the identical failure at e4969acc via the official gate. No failure baseline was expanded. + +## Production read-only evidence and limits + +- The reported part exists and is accepted; its current state was not changed. +- reviewer-scoped-readonly.log: the exact source-generated Mongo pipeline, restricted to each of the two actual reviewers' history IDs (79 and 98), finds one matching row in 583ms and 734ms. These are backend pipeline timings, not customer-browser end-to-end acceptance. No impersonation or customer writes. +- Wide administrator/all-history and broad available-pool search can still exceed 8 seconds. The selected-pool isolation fixes the reported history page being blocked by that failure; it does not prove all broad-pool latency or capacity has been resolved. +- No production write acceptance, 81-user load test, or release gates were performed for this task. Push, PR and deployment require new task-specific authorization. + +## Retrospective follow-up (base adc63af8) + +Scope: Repair worktree hook bootstrap, verification selection/dependencies/results, browser executable resolution and stale dashboard assertions. Wire the existing synthetic pool Mongo suite into CI. Reuse existing installed dependencies and isolated local services. +Risk: Local/CI tooling only; no business query, delivery contract, customer record, migration, resource or deployment changes in this follow-up. Keep the wide-pool performance limitation above open. + +Verification follow-up evidence: local-data/retro-guardrails/. The unchanged browser/product inputs passed all 23 task checks in verify-1793952872699139000/report.json (338.854s, no source drift). Earlier failures and the 600-second budget exhaustion remain recorded. Subsequent Docker COPY ordering and its regression affect only the dependency-install bootstrap: Docker-prepare regression and final quick checks are run separately; application/browser/Mongo inputs are unchanged. No known-failure baseline was expanded. CI wiring is validated locally; the remote CI has not run because this branch remains local. + +## 2026-11-08: wide-pool and all-delivery continuation (base c942a8ce) + +- Production freshness was confirmed against origin/dev and live e4969acc before investigation. No application deployment or customer-record mutation was performed. +- Full production delivery inventory: 35 records (27 accepted, 8 legacy no_deliverables). Added a local-only resumable V4 preview converter that reuses the native builder, preserves fixed revision checksums, refuses conflicting output, records every blocked item and never claims a complete audit/publication. All-discard conversion requires the fixed child-run confirmation to name the exact revision/artifact/hash. Historical source recovery requires the published Capture snapshot and an exact persisted Run captureUnit/objectKeys selection; it never uses the mutable current Capture association or current dictionaries. +- Converted and independently verified 18 previews: 12 published packages and 6 all-discard records. All 18 pass original left/right PTS coverage and calibration byte checks. The 6 all-discard cases have no historical calibration SHA: their checksums come from current bytes of the frozen BOS object selection, explicitly recorded. Historical object-byte version proof, complete internal audit, video decoding/downstream cuts and formal publication remain unverified. No online package/pointer was replaced. +- 17 records remain blocked: 10 missing frozen slice classification dictionaries; 6 unavailable/incomplete fixed revision JSONs (5 published + 1 all-discard); 1 older all-discard record lacks original-source and dictionary snapshots. Historical original selection was recovered for 9 of the 10 dictionary-blocked packages by cross-checking the published Capture snapshot with the original Run's explicit selection and the existing Go validator; this does not supply missing historical classification tags. A per-record Chinese follow-up list and historical-name confirmation list are included with the local evidence. +- Preview archive: local-data/delivery-v4-all-preview-20261108/V4交付包_18个已校验预览.zip. Full reports/evidence: local-data/pool-v4-followup-20261108/ and local-data/delivery-v4-all-preview-20261108/. Customer ZIP contains only 18 part.json files, README and public validation summary; no video/IMU/audit files. +- Broad pool remains open. The separately authorized metadata-index experiment really achieved zero root document fetches, but actual Go Store broad/all and available queries still timed out at 8,001 ms, acceptance example 5,915 ms. The index was explicitly rolled back and verified absent; health/readiness 200 afterwards. No unproven index or runtime query rewrite was retained. See index-plan.md for exact scope, cost, evidence and rollback. +- Earlier full task evidence (including synthetic covering-index, pool Mongo parity, permission and browser checks): verify-1794155814156557000/report.json passed. The rejected index experiment was then moved to ignored local-data rather than installed/added as a permanent CI requirement. Final converter/compatibility/tooling checks, including exact historical source recovery and all-discard acceptance guards: verify-1794157259610133000/report.json passed (8 converter cases plus native V4/real-materialization/legacy compatibility suites). These are scoped verification, not release or 81-user capacity acceptance. diff --git a/Dockerfile.api b/Dockerfile.api index 7d79037e..ddebc818 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -6,6 +6,7 @@ WORKDIR /app COPY package*.json ./ COPY scripts/check-git-hooks.mjs ./scripts/check-git-hooks.mjs +COPY scripts/lib/git-hooks.mjs ./scripts/lib/git-hooks.mjs RUN npm ci COPY index.html metadata.json package.json tsconfig.json vite.config.ts ./ diff --git a/docs/adr/0005-part-index-delivery-shape.md b/docs/adr/0005-part-index-delivery-shape.md index 0d3bbcae..c9eb08f0 100644 --- a/docs/adr/0005-part-index-delivery-shape.md +++ b/docs/adr/0005-part-index-delivery-shape.md @@ -59,3 +59,9 @@ status: accepted 下游需要仅凭总 JSON 和原始数据定位、切分、按类归组。新 v4 使用任务原始输入的 BOS 选择冻结八类源文件及 bucket/prefix,主文件不再依赖平台 workspace。动作同时明确切片内秒数、原始 part 秒数与原始硬件时间戳;用原始左右 PTS 各自解析硬件区间,避免用左视频秒数误切另一相机。输入快照指纹不是单个 BOS 对象内容版本的保证。实际切割和分类打包仍由下游执行。 缺少可靠来源、分类标签或时间证据时,选择拒绝新 v4 交付,而非“尽量输出”后让客户承受丢分类或错切的风险。发布同时验证消费契约、来源归属和文件存在。已交付结果不原地改写;历史只按证据补齐并产生新批次。这个要求适用于尚未发布的 v4,不改变 v1–v3;旧本地 v4 候选须重新生成。实现和本地消费验证见 `docs/delivery-classification-v4.zh-CN.md`。 + +## 2026-11-08 修订:旧任务完成后的 V4 格式约束 + +同一组织、项目、工作流版本家族已启用已发布且可见的 V4 工作流时,尚未交付的旧任务也必须生成 V4。人工任务、来源 Run、已验收修订和原始输入仍属于原工作流,不重绑、不重跑标注。放行子 Run 单独冻结兼容的打包节点、启用参数及摘要镜像;执行、重启恢复与重试使用该快照。冻结的节点仅替换打包运行配置,不修改原工作流。缺少原始输入选择、冻结分类字典或已验收证据时,明确失败并恢复证据,禁止输出旧格式兜底。 + +Worker 在写入交付目标前检查实际格式;V4 客户目录只允许 `part.json`,审核历史仍为独立内部产物。已完整发布的旧包继续支持逐成员归属、SHA-256、大小和完成清单校验后的**只读状态恢复**;不续传未完成旧包、不接管冲突目录、不改写旧完成清单。历史转换仍产生独立候选/新批次,生产转换与正式发布分别验收。 diff --git a/docs/agents/verification.md b/docs/agents/verification.md index a41c0533..8986c80e 100644 --- a/docs/agents/verification.md +++ b/docs/agents/verification.md @@ -34,6 +34,16 @@ - 退出码 0 仅表示所选检查通过;1 表示检查失败;2 表示超时、跳过、未映射、执行中源码变化或入口异常。报告中的 deferred/notCovered 始终保留,不把快速通过解释成完整通过。 - 遇到已有测试债务,使用既有 `test:frontend:gate` / `test:go:gate` 比较具体用例。不因失败而扩大 known-failures 基线,不手工忽略断言。 +## 检查选择与环境初始化 + +标准开发入口与本地验证先检查并补齐当前 worktree 的 Husky wrappers,复用依赖目录也执行该步骤。自定义 hooksPath 保持原样并报告冲突;CI、导出源码和显式 HUSKY=0 保留原有行为。 + +快速 Go 检查按包内实际用例展开范围规则,交互预览、Mongo、浏览器及容量用例由专项入口处理。任务映射显式声明所需 opt-in 环境;执行器清理继承的测试开关,避免误连外部服务。领取池 Mongo 等价和大载荷回归在任务级执行,也由 CI 的隔离 Mongo 服务执行。 + +独立检查在其他项失败或跳过后继续执行,报告保留每一项结果;依赖检查仅在前置检查通过后执行。例如构建失败时,生产产物启动检查标为未运行,避免读取旧 dist。预算耗尽仍停止后续执行,失败和未完成仍决定最终退出码。 + +浏览器门禁共用解析器:优先 BROWSER_EXECUTABLE,兼容 BROWSER_PATH;Mac 优先使用已安装的 Edge/Chrome,其余环境优先使用已安装的 Playwright Chromium,再查找本机 Chromium 浏览器。显式路径无效会报错。倍速回归默认创建并清理临时本地 Vite 服务,也支持显式指定本地测试 URL。 + ## 减少重复与环境准备 - 同一批同一输入只做一次成功验证;优先复用已有 npm 测试组、Go 编译/测试缓存和本地隔离 Mongo 服务,不为每次小改动重装依赖、重建镜像或重启整套副本。 diff --git a/docs/delivery-classification-v4.zh-CN.md b/docs/delivery-classification-v4.zh-CN.md index ee12a88b..99195c59 100644 --- a/docs/delivery-classification-v4.zh-CN.md +++ b/docs/delivery-classification-v4.zh-CN.md @@ -52,10 +52,16 @@ 回退旧 API 不是无损写入保证:新版本保存分类后,旧版本重建草稿数组可能丢弃字典快照。回退前应暂停相关写操作或准备能保留新字段的兼容回退版本;只保留恢复历史不等于能自动恢复所有后续修改。已交付不可变文件不因此原地迁移或覆盖。 -## 2026-10-22 启用决定 +## 2026-10-22 启用决定(在途任务范围已由 2026-11-08 修订替代) 用户随后确认启用 v4,范围仅为切换后新启动的工作流;在途任务、它们后续的审核/打包子运行和已交付文件继续原协议。此决定替代上节“继续保持关闭”的准备范围,不代表生产开关已经启用。实际发布仍需完成镜像、兼容性和目标核对,并审查具体部署及工作流切换计划。 用户明确确认当前无 tags 的启用词条以词条名称作为第一个独立 tags 标签。该初始化是一次有界字典变更:保留词条 ID、名称、用途、category、aliases 和既有历史,递增版本并记录新修订;已有非空 tags 不覆盖。历史任务中的冻结快照不因此重写,亦不反查新词条替代历史分类。后续 tags 仍由管理员维护。 新工作流使用新的定义版本及固定打包镜像。旧定义不原地改参数或镜像;切换必须保留已启动采集的原工作流归属,并阻止新监听器重新接入这些目录。新 v4 接入还需完整的原始文件选择,包含采集 meta;旧协议保留原有接入要求。准备与验收步骤见 [启用核对表](release/delivery-v4-activation.md)。 + +## 2026-11-08 旧任务交付修订 + +用户进一步要求不再新生成旧格式交付包。同一组织、项目和版本家族已启用 V4 时,旧任务后续放行也冻结兼容的 V4 打包配置;原工作流、人工任务归属和已验收内容不变。执行、重试与重启沿用冻结镜像及参数。无法证明原始来源、冻结字典、标签和时间证据时停在明确失败状态,先恢复证据,不回退旧格式。 + +已完整发布的旧文件仅允许校验完成清单、成员哈希和归属后只读恢复状态;缺失或冲突的旧上传不得继续补发。转换生成独立候选或新交付批次,不覆盖旧完成清单。线上替换、历史证据恢复和新批次正式发布需要分别验证;本地转换通过不代表全部历史包可正式交付。 diff --git a/internal/domain/types.go b/internal/domain/types.go index 36ccfe28..7f425499 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -790,6 +790,7 @@ type LightweightOperatorCodeManifest struct { } type RunMetadata struct { + CaptureDeliveryPackage *CaptureDeliveryPackageRuntime `json:"captureDeliveryPackage,omitempty" bson:"captureDeliveryPackage,omitempty"` LightweightOperatorCodeManifests map[string]LightweightOperatorCodeManifest `json:"lightweightOperatorCodeManifests,omitempty" bson:"lightweightOperatorCodeManifests,omitempty"` TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"` AnnotationTaskIDsByNode map[string][]string `json:"annotationTaskIdsByNode,omitempty" bson:"annotationTaskIdsByNode,omitempty"` @@ -797,6 +798,16 @@ type RunMetadata struct { HumanReleaseUniqueKey string `json:"humanReleaseUniqueKey,omitempty" bson:"humanReleaseUniqueKey,omitempty"` } +// A release child keeps its source workflow identity while freezing the newer +// package runtime. It never changes the source task or the stored workflow. +type CaptureDeliveryPackageRuntime struct { + SourceRunID string `json:"sourceRunId" bson:"sourceRunId"` + SourceWorkflowID string `json:"sourceWorkflowId" bson:"sourceWorkflowId"` + PolicyWorkflowID string `json:"policyWorkflowId" bson:"policyWorkflowId"` + PolicyVersion int `json:"policyVersion" bson:"policyVersion"` + Node WorkflowNode `json:"node" bson:"node"` +} + // RunFailure is the machine-readable counterpart to RunRecord.Error. Error is // intentionally retained for backwards compatibility and human-readable logs. type RunFailure struct { diff --git a/internal/store/human_pool_search.go b/internal/store/human_pool_search.go index 704f6d59..9b203412 100644 --- a/internal/store/human_pool_search.go +++ b/internal/store/human_pool_search.go @@ -4,7 +4,6 @@ import ( "context" "droboticflow/internal/domain" "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "regexp" "strings" @@ -94,19 +93,7 @@ func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q doma } filter = bson.M{"$and": bson.A{filter, visibility}} } - pipeline := mongo.Pipeline{ - {{Key: "$match", Value: filter}}, - {{Key: "$sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}}, - {{Key: "$project", Value: humanItemListProjection(domain.HumanItemQuery{ListProjection: true})}}, - adminPoolLookup("annotationTasks", "legacyTaskId", "poolTasks", org, bson.M{"_id": 0, "id": 1, "status": 1, "artifactId": 1, "sourceVideo": 1}), - } - if q.Search != "" || q.Capture != "" { - pipeline = append(pipeline, - adminPoolLookup("runs", "runId", "poolRuns", org, bson.M{"_id": 0, "id": 1, "taskId": 1, "projectName": 1, "captureDirectories": 1, "inputs.ref": 1, "nodeExecutions.artifacts.id": 1, "nodeExecutions.artifacts.name": 1, "nodeExecutions.artifacts.path": 1, "artifacts.id": 1, "artifacts.name": 1, "artifacts.path": 1}), - adminPoolLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org, bson.M{"_id": 0, "captureId": 1, "prefix": 1}), - adminPoolLookup("projects", "projectId", "poolProjects", org, bson.M{"_id": 0, "name": 1}), - ) - } + pipeline := humanPoolSearchPipeline(org, q, filter) for _, text := range []string{q.Search, q.Capture} { if text != "" { if searchFilter := humanPoolSearchPrefilter(text); searchFilter != nil { @@ -122,26 +109,36 @@ func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q doma return nil, err } defer cur.Close(ctx) - out := []domain.HumanItemRecord{} - for cur.Next(ctx) { - var row adminPoolRow - if err := cur.Decode(&row); err != nil { + ids := []string{} + for { + rows := []adminPoolRow{} + for len(rows) < 200 && cur.Next(ctx) { + var row adminPoolRow + if err := cur.Decode(&row); err != nil { + return nil, err + } + rows = append(rows, row) + } + if err := cur.Err(); err != nil { return nil, err } - if !humanPoolSearchRowMatches(row, q) { - continue + if len(rows) == 0 { + break } - item := row.HumanItemRecord - if q.PoolCandidateProjection { - item = item.PoolCandidate() + if err := s.humanPoolSearchLegacyPayloads(ctx, org, q, rows); err != nil { + return nil, err } - if q.IdentityProjection { - item = domain.HumanItemRecord{ID: item.ID} + for _, row := range rows { + if humanPoolSearchRowMatches(row, q) { + ids = append(ids, row.ID) + } + if q.Limit > 0 && len(ids) >= q.Limit { + break + } } - out = append(out, item) - if q.Limit > 0 && len(out) >= q.Limit { + if q.Limit > 0 && len(ids) >= q.Limit { break } } - return out, cur.Err() + return s.hydrateHumanPoolSearch(ctx, org, q, ids) } diff --git a/internal/store/human_pool_search_filter.go b/internal/store/human_pool_search_filter.go index 04abcb6f..3c4c5a57 100644 --- a/internal/store/human_pool_search_filter.go +++ b/internal/store/human_pool_search_filter.go @@ -16,10 +16,7 @@ func humanPoolSearchPrefilter(search string) bson.M { pattern := regexp.QuoteMeta(strings.ToLower(search)) pattern = strings.ReplaceAll(pattern, "/part", `(?:/|_|_camera_(?:left|right)_)part`) match := bson.A{} - for _, field := range []string{"id", "legacyTaskId", "subject.itemKey", "subject.artifactId", "poolTasks.artifactId", "poolTasks.sourceVideo", "poolCaptures.captureId", "poolCaptures.prefix", "poolProjects.name", "poolRuns.projectName", "poolRuns.captureDirectories", "poolRuns.inputs.ref.captureUnit", "poolRuns.inputs.ref.prefix", "poolRuns.inputs.ref.key", "poolRuns.inputs.ref.manifestKey", "poolRuns.artifacts.id", "poolRuns.artifacts.name", "poolRuns.artifacts.path", "poolRuns.nodeExecutions.artifacts.id", "poolRuns.nodeExecutions.artifacts.name", "poolRuns.nodeExecutions.artifacts.path"} { - match = append(match, bson.M{field: bson.M{"$regex": pattern, "$options": "i"}}) - match = append(match, bson.M{field: bson.M{"$regex": "%[0-9a-f]{2}", "$options": "i"}}) - } + match = append(match, bson.M{"$expr": bson.M{"$regexMatch": bson.M{"input": humanPoolSearchText(), "regex": pattern + "|%[0-9a-f]{2}", "options": "i"}}}) if split := strings.LastIndex(strings.ToLower(search), "/part"); split > 0 { captures := bson.A{} for _, field := range []string{"poolCaptures.captureId", "poolCaptures.prefix"} { diff --git a/internal/store/human_pool_search_join.go b/internal/store/human_pool_search_join.go new file mode 100644 index 00000000..07785b81 --- /dev/null +++ b/internal/store/human_pool_search_join.go @@ -0,0 +1,121 @@ +package store + +import ( + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +// Join fixed-size batches instead of sending four foreign queries per item. +// Batching does not limit candidates: every batch is searched, then the original +// item order is restored. Tenant filtering precedes the run -> capture link. +func humanPoolSearchPipeline(org string, q domain.HumanItemQuery, filter bson.M) mongo.Pipeline { + // Only search/category metadata participates in global sorts and joins. + // Hydrate matched IDs afterwards; the public record shape stays unchanged. + projection := bson.M{"_id": 0, "id": 1, "organizationId": 1, "pluginId": 1, "projectId": 1, "runId": 1, "legacyTaskId": 1, "createdAt": 1, "currentStageId": 1, "status": 1, "reviewRound": 1, "pendingCombinedSubmissionId": 1, "assignedUserId": 1, "subject.itemKey": 1, "subject.artifactId": 1, "subject.payload.sliceWorkflow.status": 1, "subject.payload.sliceWorkflow.phase": 1} + pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}, {{Key: "$project", Value: projection}}} + pipeline = append(pipeline, + // Number candidate batches; restore the stable createdAt/id order below. + bson.D{{Key: "$setWindowFields", Value: bson.M{"sortBy": bson.D{{Key: "createdAt", Value: -1}}, "output": bson.M{"_poolRow": bson.M{"$documentNumber": bson.M{}}}}}}, + bson.D{{Key: "$group", Value: bson.M{"_id": bson.M{"$floor": bson.M{"$divide": bson.A{bson.M{"$subtract": bson.A{"$_poolRow", 1}}, 64}}}, "poolItems": bson.M{"$push": "$$ROOT"}}}}, + humanPoolBatchLookup("annotationTasks", "poolItems.legacyTaskId", "poolTasks", org), + ) + fields := bson.M{"poolTasks": humanPoolTenantProjection("poolTasks", org, bson.M{ + "id": "$$row.id", "status": "$$row.status", "artifactId": "$$row.artifactId", "sourceVideo": "$$row.sourceVideo", + })} + if q.Search != "" || q.Capture != "" { + pipeline = append(pipeline, + humanPoolBatchLookup("runs", "poolItems.runId", "poolRuns", org), + humanPoolBatchLookup("projects", "poolItems.projectId", "poolProjects", org), + bson.D{{Key: "$set", Value: bson.M{"poolRuns": humanPoolTenantProjection("poolRuns", org, bson.M{ + "id": "$$row.id", "taskId": "$$row.taskId", "projectName": "$$row.projectName", "captureDirectories": "$$row.captureDirectories", + "inputs": humanPoolRunInputRefs(), "artifacts": humanPoolArtifactNames("$$row.artifacts"), + "nodeExecutions": bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{"$$row.nodeExecutions", bson.A{}}}, "as": "execution", "in": bson.M{"artifacts": humanPoolArtifactNames("$$execution.artifacts")}}}, + })}}}, + humanPoolBatchLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org), + ) + fields["poolCaptures"] = humanPoolTenantProjection("poolCaptures", org, bson.M{"id": "$$row.id", "captureId": "$$row.captureId", "prefix": "$$row.prefix"}) + fields["poolProjects"] = humanPoolTenantProjection("poolProjects", org, bson.M{"id": "$$row.id", "name": "$$row.name"}) + } + indexes := bson.M{} + for field := range fields { + indexes[field] = humanPoolJoinIndex(field) + } + // Runs were already projected before following their capture references. + if q.Search != "" || q.Capture != "" { + indexes["poolRuns"] = humanPoolJoinIndex("poolRuns") + } + pipeline = append(pipeline, bson.D{{Key: "$set", Value: fields}}, bson.D{{Key: "$set", Value: indexes}}) + pipeline = append(pipeline, bson.D{{Key: "$unwind", Value: "$poolItems"}}) + joined := bson.M{"poolTasks": humanPoolItemJoin("poolTasks", "$poolItems.legacyTaskId")} + if q.Search != "" || q.Capture != "" { + joined["poolRuns"] = humanPoolItemJoin("poolRuns", "$poolItems.runId") + joined["poolProjects"] = humanPoolItemJoin("poolProjects", "$poolItems.projectId") + joined["poolCaptures"] = bson.M{"$let": bson.M{"vars": bson.M{"run": humanPoolIndexedRecord("poolRuns", "$poolItems.runId")}, "in": humanPoolItemJoin("poolCaptures", "$$run.taskId")}} + + } + return append(pipeline, + bson.D{{Key: "$replaceWith", Value: bson.M{"$mergeObjects": bson.A{"$poolItems", joined}}}}, + bson.D{{Key: "$unset", Value: "_poolRow"}}, + bson.D{{Key: "$sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}}, + ) +} + +func humanPoolBatchLookup(from, local, as, org string) bson.D { + projection := bson.M{"_id": 0, "id": 1, "organizationId": 1} + fields := map[string][]string{ + "annotationTasks": {"status", "artifactId", "sourceVideo"}, + "runs": {"taskId", "projectName", "captureDirectories"}, + "captureRecords": {"captureId", "prefix"}, + "projects": {"name"}, + } + for _, field := range fields[from] { + projection[field] = 1 + } + if from == "runs" { + // CaptureDirectoriesFromRun gives materialized directories precedence. + // Only historical records need artifact and input-reference fallbacks. + hasDirectories := bson.M{"$gt": bson.A{bson.M{"$size": bson.M{"$ifNull": bson.A{"$captureDirectories", bson.A{}}}}, 0}} + projection["inputs"] = bson.M{"$cond": bson.A{hasDirectories, bson.M{}, bson.M{"$let": bson.M{"vars": bson.M{"row": "$$ROOT"}, "in": humanPoolRunInputRefs()}}}} + projection["artifacts"] = bson.M{"$cond": bson.A{hasDirectories, bson.A{}, humanPoolArtifactNames("$artifacts")}} + projection["nodeExecutions"] = bson.M{"$cond": bson.A{hasDirectories, bson.A{}, bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{"$nodeExecutions", bson.A{}}}, "as": "execution", "in": bson.M{"artifacts": humanPoolArtifactNames("$$execution.artifacts")}}}}} + } + + return bson.D{{Key: "$lookup", Value: bson.M{"from": from, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$match": bson.M{"organizationId": normalizedOrganizationID(org)}}, bson.M{"$project": projection}}, "as": as}}} +} + +func humanPoolTenantProjection(field, org string, projection bson.M) bson.M { + return bson.M{"$map": bson.M{ + "input": bson.M{"$filter": bson.M{"input": "$" + field, "as": "candidate", "cond": bson.M{"$eq": bson.A{"$$candidate.organizationId", normalizedOrganizationID(org)}}}}, + "as": "row", "in": projection, + }} +} + +func humanPoolArtifactNames(input string) bson.M { + return bson.M{"$map": bson.M{"input": bson.M{"$ifNull": bson.A{input, bson.A{}}}, "as": "artifact", "in": bson.M{"id": "$$artifact.id", "name": "$$artifact.name", "path": "$$artifact.path"}}} +} + +func humanPoolRunInputRefs() bson.M { + ref := bson.M{} + for _, field := range []string{"captureUnit", "prefix", "key", "manifestKey"} { + ref[field] = "$$input.v.ref." + field + } + return bson.M{"$arrayToObject": bson.M{"$map": bson.M{ + "input": bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$$row.inputs", bson.M{}}}}, + "as": "input", "in": bson.M{"k": "$$input.k", "v": bson.M{"ref": ref}}, + }}} +} + +func humanPoolIndexedRecord(field, id string) bson.M { + // indexOfArray keeps compatibility with Mongo 7.0; dynamic getField needs 7.2. + return bson.M{"$let": bson.M{"vars": bson.M{"position": bson.M{"$indexOfArray": bson.A{"$" + field + ".ids", bson.M{"$ifNull": bson.A{id, nil}}}}}, "in": bson.M{"$cond": bson.A{bson.M{"$gte": bson.A{"$$position", 0}}, bson.M{"$arrayElemAt": bson.A{"$" + field + ".rows", "$$position"}}, bson.M{}}}}} +} + +func humanPoolItemJoin(field, id string) bson.M { + return bson.M{"$let": bson.M{"vars": bson.M{"record": humanPoolIndexedRecord(field, id)}, "in": bson.M{"$cond": bson.A{bson.M{"$ne": bson.A{"$$record", bson.M{}}}, bson.A{"$$record"}, bson.A{}}}}} +} + +func humanPoolJoinIndex(field string) bson.M { + return bson.M{"$let": bson.M{"vars": bson.M{"rows": "$" + field}, "in": bson.M{"ids": "$$rows.id", "rows": "$$rows"}}} +} diff --git a/internal/store/human_pool_search_payload.go b/internal/store/human_pool_search_payload.go new file mode 100644 index 00000000..004bd8a5 --- /dev/null +++ b/internal/store/human_pool_search_payload.go @@ -0,0 +1,84 @@ +package store + +import ( + "context" + "droboticflow/internal/domain" + "time" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// Recursive names in old payloads remain searchable. Read those payloads in +// batches, after the globally sorted search metadata has left Mongo's buffer. +func (s *MongoStore) humanPoolSearchLegacyPayloads(ctx context.Context, org string, q domain.HumanItemQuery, rows []adminPoolRow) error { + if q.Search == "" && q.Capture == "" { + return nil + } + ids := []string{} + for _, row := range rows { + name := "" + if len(row.Captures) > 0 { + name = row.Captures[0].CaptureID + if name == "" { + name = domain.CaptureDirectoryFromDirectoryValue(row.Captures[0].Prefix) + } + } + if name == "" { + ids = append(ids, row.ID) + } + } + if len(ids) == 0 { + return nil + } + items, err := s.hydrateHumanPoolSearch(ctx, org, domain.HumanItemQuery{}, ids) + if err != nil { + return err + } + payloads := map[string]map[string]any{} + for _, item := range items { + payloads[item.ID] = item.Subject.Payload + } + for i := range rows { + if payload, ok := payloads[rows[i].ID]; ok { + rows[i].Subject.Payload = payload + } + } + return nil +} + +func (s *MongoStore) hydrateHumanPoolSearch(ctx context.Context, org string, q domain.HumanItemQuery, ids []string) ([]domain.HumanItemRecord, error) { + out := []domain.HumanItemRecord{} + if len(ids) == 0 { + return out, nil + } + projectionQuery := domain.HumanItemQuery{ListProjection: true} + if q.PoolCandidateProjection { + projectionQuery = domain.HumanItemQuery{PoolCandidateProjection: true} + } + if q.IdentityProjection { + projectionQuery = domain.HumanItemQuery{IdentityProjection: true} + } + cur, err := s.humanItems.Find(ctx, bson.M{"organizationId": normalizedOrganizationID(org), "id": bson.M{"$in": ids}}, options.Find().SetProjection(humanItemListProjection(projectionQuery)).SetMaxTime(8*time.Second).SetBatchSize(200)) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + byID := map[string]domain.HumanItemRecord{} + for cur.Next(ctx) { + var item domain.HumanItemRecord + if err := cur.Decode(&item); err != nil { + return nil, err + } + byID[item.ID] = item + } + if err := cur.Err(); err != nil { + return nil, err + } + for _, id := range ids { + if item, ok := byID[id]; ok { + out = append(out, item) + } + } + return out, nil +} diff --git a/internal/store/human_pool_search_test.go b/internal/store/human_pool_search_test.go index 906ab139..59d9a546 100644 --- a/internal/store/human_pool_search_test.go +++ b/internal/store/human_pool_search_test.go @@ -3,6 +3,7 @@ package store import ( "context" "droboticflow/internal/domain" + "errors" "fmt" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/event" @@ -11,15 +12,20 @@ import ( "net/url" "os" "reflect" + "strings" "sync/atomic" "testing" "time" ) -func TestHumanPoolSearchMongoParity(t *testing.T) { testHumanPoolSearchMongo(t, 963, false) } -func TestHumanPoolSearchMongoLargeModern(t *testing.T) { testHumanPoolSearchMongo(t, 10000, true) } +func TestHumanPoolSearchMongoParity(t *testing.T) { testHumanPoolSearchMongo(t, 963, false, false) } +func TestHumanPoolSearchMongoLargeModern(t *testing.T) { + testHumanPoolSearchMongo(t, 10000, true, false) +} -func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { +func TestHumanPoolSearchMongoLargePayload(t *testing.T) { testHumanPoolSearchMongo(t, 420, true, true) } + +func testHumanPoolSearchMongo(t *testing.T, count int, modern, heavy bool) { uri := os.Getenv("POOL_LOCAL_MONGO_URI") if uri == "" { t.Skip("isolated local Mongo opt-in") @@ -45,7 +51,7 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { }, Failed: func(_ context.Context, e *event.CommandFailedEvent) { t.Logf("mongo failed command=%s duration=%s maxTimeMS=%d failure=%s", e.CommandName, e.Duration, lastMaxTime.Load(), e.Failure) }, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { - if measuring.Load() && (e.CommandName == "aggregate" || e.CommandName == "getMore") { + if measuring.Load() && (e.CommandName == "aggregate" || e.CommandName == "getMore" || e.CommandName == "find") { replyBytes.Add(int64(len(e.Reply))) } }} @@ -71,6 +77,9 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { runs, captures := []any{}, []any{} for n := 0; n < count; n++ { i := domain.HumanItemRecord{ID: fmt.Sprintf("i-%04d", n), OrganizationID: org, PluginID: "annotation-qa", ProjectID: "p", RunID: "run", CurrentStageID: "quality_review", Status: domain.HumanItemReady, CreatedAt: time.Date(2026, 11, 1, 0, 0, n, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("FYLX_ITEM_%04d/part0002/camera_left.mp4", n), Payload: map[string]any{"annotationWorkspace": map[string]any{"secret": "not for lists"}}}} + if heavy && n >= 20 { + i.Subject.Payload["unusedCatalog"] = strings.Repeat("x", 256*1024) + } if modern { i.RunID = fmt.Sprintf("modern-run-%d", n) capture := domain.CaptureRecord{ID: fmt.Sprintf("modern-capture-%d", n), OrganizationID: org, CaptureID: fmt.Sprintf("FYLX_ITEM_%04d", n)} @@ -81,6 +90,9 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { _ = mem.UpsertCaptureRecord(ctx, capture) } + if n >= 63 && n <= 65 { + i.CreatedAt = "2026-11-01T00:01:03Z" + } if n >= 10 && n <= 18 { i.CurrentStageID = "annotation_combined" switch n { @@ -114,6 +126,23 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { if n == 3 { i.Subject.ItemKey = "special-[.*-capture" } + if n == 5 { + i.Subject.ItemKey = "historical-input-reference" + i.RunID = "legacy-input-run" + } + if n == 6 { + i.Subject.ItemKey = "cross-tenant-links" + i.RunID, i.ProjectID, i.LegacyTaskID = "foreign-run", "foreign-project", "foreign-task" + } + if n == 7 { + i.Subject.ItemKey = "cross-tenant-capture" + i.RunID = "foreign-capture-run" + } + if n == 8 { + i.Subject.ItemKey = "opaque-legacy-key" + i.RunID = "missing-historical-run" + i.Subject.Payload["importManifest"] = map[string]any{"sourceDirectory": "archive/payload-only-capture"} + } _ = mem.UpsertHumanItem(ctx, i) docs = append(docs, i) } @@ -131,6 +160,43 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { legacy := domain.RunRecord{ID: "legacy-run", OrganizationID: org, CaptureDirectories: []string{"历史目录"}} _ = mem.UpsertRun(ctx, legacy) _ = s.UpsertRun(ctx, legacy) + legacyInput := domain.RunRecord{ID: "legacy-input-run", OrganizationID: org, TaskID: "legacy-input-capture", Inputs: map[string]domain.ArtifactPayload{ + "sourceDirectory": {Mode: domain.ArtifactTransferModeRef, Ref: &domain.ArtifactReference{CaptureUnit: "FYLX_B07_AZER764008K_20260927_074440/part0002"}}, + }} + _ = mem.UpsertRun(ctx, legacyInput) + // Raw insert models records written before captureDirectories was materialized. + if _, err := s.runs.InsertOne(ctx, legacyInput); err != nil { + t.Fatal(err) + } + for _, capture := range []domain.CaptureRecord{ + {ID: "legacy-input-capture", OrganizationID: org, CaptureID: "other-display-name"}, + {ID: "same-org-poison", OrganizationID: org, CaptureID: "TENANT_LEAK"}, + {ID: "foreign-capture", OrganizationID: "other-org", CaptureID: "TENANT_LEAK"}, + } { + _ = mem.UpsertCaptureRecord(ctx, capture) + if _, err := s.captureRecords.InsertOne(ctx, capture); err != nil { + t.Fatal(err) + } + } + for _, run := range []domain.RunRecord{ + {ID: "foreign-run", OrganizationID: "other-org", TaskID: "same-org-poison", ProjectName: "TENANT_LEAK"}, + {ID: "foreign-capture-run", OrganizationID: org, TaskID: "foreign-capture"}, + } { + _ = mem.UpsertRun(ctx, run) + if _, err := s.runs.InsertOne(ctx, run); err != nil { + t.Fatal(err) + } + } + foreignProject := domain.ProjectRecord{ID: "foreign-project", OrganizationID: "other-org", Name: "TENANT_LEAK"} + _ = mem.UpsertProject(ctx, foreignProject) + if _, err := s.projects.InsertOne(ctx, foreignProject); err != nil { + t.Fatal(err) + } + foreignTask := domain.AnnotationTaskRecord{ID: "foreign-task", OrganizationID: "other-org", ArtifactID: "TENANT_LEAK"} + _ = mem.UpsertAnnotationTask(ctx, foreignTask) + if _, err := s.annotationTasks.InsertOne(ctx, foreignTask); err != nil { + t.Fatal(err) + } ids := func(t *testing.T, rows []domain.HumanItemRecord) []string { out := []string{} for _, i := range rows { @@ -141,9 +207,24 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { } return out } + t.Run("canceled-query", func(t *testing.T) { + canceled, stop := context.WithCancel(ctx) + stop() + if _, err := s.ListHumanItems(canceled, org, domain.HumanItemQuery{Search: "FYLX_ITEM"}); !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + }) for _, q := range []domain.HumanItemQuery{ + {Search: "FYLX_B07_AZER764008K_20260927_074440/part0002", Limit: 200}, + {Search: "FYLX_B07_AZER764008K_20260927_074440/part0002", Limit: 200, PoolCandidateProjection: true}, + {Search: "FYLX_ITEM", Limit: 200, IdentityProjection: true}, + {Search: "NO_SUCH_CAPTURE", Limit: 200, PoolCandidateProjection: true}, {Search: "FYLX_ITEM_0001/part0002", Limit: 200}, + {Search: "FYLX_ITEM", Limit: 10, Cursor: &domain.HumanItemCursor{CreatedAt: "2026-11-01T00:01:03Z", ID: "i-0064"}}, {Search: "NO_SUCH_CAPTURE", Limit: 200}, + {Search: "TENANT_LEAK", Limit: 200}, + {Search: "payload-only-capture", Limit: 200, PoolCandidateProjection: true}, + {Capture: "payload-only-capture", Limit: 200}, {Capture: "FYLX_ITEM_0001", Limit: 200}, {Capture: "历史目录", Limit: 200}, {Search: "FYLX_ITEM_0004/PART0002", Limit: 200}, @@ -156,6 +237,9 @@ func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) { {Category: "accepted", ReviewedPool: true, Limit: 200}, {Category: "rework_review", Limit: 200}, {Category: "pending_acceptance", Limit: 200}, {Category: "rework", Limit: 200}, {Category: "slicing", Limit: 200}, {Category: "pending_annotation", Limit: 200}, {Category: "pending_slice", Limit: 200}, {Category: "available", Limit: 200}, {Search: "FYLX_ITEM", IDs: []string{}, Limit: 200}, } { + if heavy && !q.PoolCandidateProjection && !q.IdentityProjection { + continue + } t.Run(fmt.Sprintf("search=%s_capture=%s_category=%s_reviewed=%t", q.Search, q.Capture, q.Category, q.ReviewedPool), func(t *testing.T) { q.ListProjection = true var clockBefore time.Time diff --git a/internal/store/human_pool_search_text.go b/internal/store/human_pool_search_text.go new file mode 100644 index 00000000..fd8d19a2 --- /dev/null +++ b/internal/store/human_pool_search_text.go @@ -0,0 +1,37 @@ +package store + +import "go.mongodb.org/mongo-driver/bson" + +// This is a conservative prefilter only. The existing Go matcher still makes +// the final decision. Joining metadata into one string avoids compiling and +// executing dozens of identical regex predicates for every candidate. +func humanPoolSearchText() bson.M { + artifacts := func(input string) bson.M { + return humanPoolTextArray(input, "artifact", humanPoolTextConcat("$$value", "$$artifact.id", "$$artifact.name", "$$artifact.path")) + } + refs := humanPoolTextArray(bson.M{"$objectToArray": bson.M{"$ifNull": bson.A{"$$run.inputs", bson.M{}}}}, "input", humanPoolTextConcat("$$value", "$$input.v.ref.captureUnit", "$$input.v.ref.prefix", "$$input.v.ref.key", "$$input.v.ref.manifestKey")) + runs := humanPoolTextArray("$poolRuns", "run", humanPoolTextConcat("$$value", "$$run.projectName", + humanPoolTextArray("$$run.captureDirectories", "directory", humanPoolTextConcat("$$value", "$$directory")), + refs, artifacts("$$run.artifacts"), + humanPoolTextArray("$$run.nodeExecutions", "execution", humanPoolTextConcat("$$value", artifacts("$$execution.artifacts"))), + )) + return humanPoolTextConcat("$id", "$legacyTaskId", "$subject.itemKey", "$subject.artifactId", runs, + humanPoolTextArray("$poolTasks", "task", humanPoolTextConcat("$$value", "$$task.artifactId", "$$task.sourceVideo")), + humanPoolTextArray("$poolCaptures", "capture", humanPoolTextConcat("$$value", "$$capture.captureId", "$$capture.prefix")), + humanPoolTextArray("$poolProjects", "project", humanPoolTextConcat("$$value", "$$project.name")), + ) +} + +func humanPoolTextConcat(values ...any) bson.M { + parts := bson.A{} + for _, value := range values { + parts = append(parts, " ", bson.M{"$ifNull": bson.A{value, ""}}) + } + return bson.M{"$concat": parts} +} + +func humanPoolTextArray(input any, name string, expression any) bson.M { + // reduce uses this/value; let gives the current element a distinct name + // so nested artifact/input reductions cannot shadow their parent record. + return bson.M{"$reduce": bson.M{"input": bson.M{"$ifNull": bson.A{input, bson.A{}}}, "initialValue": "", "in": bson.M{"$let": bson.M{"vars": bson.M{name: "$$this"}, "in": expression}}}} +} diff --git a/internal/worker/capture_cutover_test.go b/internal/worker/capture_cutover_test.go index d981c523..8f05d2d8 100644 --- a/internal/worker/capture_cutover_test.go +++ b/internal/worker/capture_cutover_test.go @@ -138,7 +138,19 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin t.Fatal(err) } } + publisher := &deliveryPublisher{} + f.runtime.artifactPublisher = publisher + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy bytes"), 0600); err != nil { + t.Fatal(err) + } + if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil { + t.Fatalf("old child cannot publish: %v", err) + } f.activate() + if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil { + t.Fatalf("completed legacy recovery failed: %v", err) + } for _, before := range []domain.CaptureRecord{active, done, launching} { after, _ := f.db.GetCaptureRecord(f.ctx, before.ID) if after == nil || after.WorkflowID != f.old.ID || after.RunID != before.RunID { @@ -149,15 +161,6 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin if err != nil || lookup == nil || lookup.ID != active.ID { t.Fatalf("old child lost Capture: %+v %v", lookup, err) } - publisher := &deliveryPublisher{} - f.runtime.artifactPublisher = publisher - root := t.TempDir() - if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy bytes"), 0600); err != nil { - t.Fatal(err) - } - if _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}); err != nil { - t.Fatalf("old child cannot publish: %v", err) - } child.Status = domain.RunStatusCompleted if err := f.db.UpsertRun(f.ctx, child); err != nil { t.Fatal(err) @@ -180,8 +183,8 @@ func TestCaptureCutoverPreservesStartedOwnershipAndParentChildDelivery(t *testin t.Fatalf("wrong new Run binding: %+v", newRun) } oldMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, child.WorkflowID) - if err != nil || oldMode { - t.Fatalf("old child selected v4: %v %v", oldMode, err) + if err != nil || !oldMode { + t.Fatalf("old child did not select v4 policy: %v %v", oldMode, err) } newMode, err := f.runtime.classificationDeliveryEnabled(f.ctx, newRun.WorkflowID) if err != nil || !newMode { diff --git a/internal/worker/capture_delivery.go b/internal/worker/capture_delivery.go index de152519..605ee0d8 100644 --- a/internal/worker/capture_delivery.go +++ b/internal/worker/capture_delivery.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -88,6 +89,7 @@ type deliveryTarget struct { stageRoot string marker string publish func(key string, body []byte) error + completed *domain.ExecutionArtifact } // prepareDeliveryTarget performs every check that must hold before the first @@ -96,6 +98,9 @@ type deliveryTarget struct { // ownership and the conflict rules of the shared destination prefix. func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (*deliveryTarget, error) { record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID) + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + record, err = runtime.frozenDeliveryCapture(ctx, run) + } if err != nil { return nil, err } @@ -139,6 +144,9 @@ func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.R if run.Trigger == domain.RunTriggerHumanRelease && run.ParentRunID != "" { ownerRunID = run.ParentRunID } + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + ownerRunID = run.Metadata.CaptureDeliveryPackage.SourceRunID + } owner := record.ID + ":" + ownerRunID // PutInternal is conditional-create. A failed invocation may leave staging // objects behind, so each retry needs its own private staging namespace. @@ -166,6 +174,13 @@ func (runtime *Runtime) prepareDeliveryTarget(ctx context.Context, run *domain.R return publisher.Delete(ctx, *connection, target.Bucket, stage) } if err := runtime.recheckV4DeliveryArtifact(ctx, run, artifact); err != nil { + if errors.Is(err, errLegacyDeliveryRequiresV4) { + completed, recoveryErr := completedLegacyDelivery(ctx, prepared, artifact) + if recoveryErr == nil { + prepared.completed = &completed + return prepared, nil + } + } return nil, err } known, err := publisher.Head(ctx, *connection, target.Bucket, prepared.marker) @@ -196,6 +211,9 @@ func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain. if err != nil { return artifact, err } + if prepared.completed != nil { + return *prepared.completed, nil + } target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner publishBytes := prepared.publish marker := prepared.marker @@ -399,6 +417,9 @@ func (runtime *Runtime) publishCaptureDeliveryFile(ctx context.Context, run *dom if err != nil { return artifact, err } + if prepared.completed != nil { + return *prepared.completed, nil + } target, connection, publisher, owner := prepared.target, prepared.connection, prepared.publisher, prepared.owner publishBytes := prepared.publish fileName := filepath.Base(routing.FileName) diff --git a/internal/worker/delivery_audit.go b/internal/worker/delivery_audit.go index cd234bc9..f5cf1aa1 100644 --- a/internal/worker/delivery_audit.go +++ b/internal/worker/delivery_audit.go @@ -76,7 +76,11 @@ func (runtime *Runtime) classificationDeliveryEnabled(ctx context.Context, workf if workflow == nil { return false, nil } - return workflowUsesV4CaptureDelivery(*workflow), nil + if workflowUsesV4CaptureDelivery(*workflow) { + return true, nil + } + policy, err := runtime.activeV4DeliveryPolicy(ctx, workflow) + return policy != nil, err } // Reuse normal internal-artifact persistence rather than exceeding the 64 KiB diff --git a/internal/worker/executor.go b/internal/worker/executor.go index 808922c6..eb228685 100644 --- a/internal/worker/executor.go +++ b/internal/worker/executor.go @@ -1094,6 +1094,18 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, if workflowErr != nil { workflow = nil } + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + if workflowErr != nil { + return executionResult{}, nil, workflowErr + } + workflow, err = deliveryRuntimeWorkflow(workflow, run) + if err != nil || workflow == nil { + if err == nil { + err = fmt.Errorf("missing delivery workflow") + } + return executionResult{}, nil, err + } + } settings, err := runtime.store.GetSettings(ctx) if err != nil { return executionResult{}, nil, err @@ -1152,7 +1164,10 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, return executionResult{}, nil, fmt.Errorf("project deletion in progress") } } - image, err := sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage) + image, frozen, err := frozenDeliveryPackageImage(run, node) + if err == nil && !frozen { + image, err = sourceIndexPackageImage(node, inputArtifacts, runtime.operatorRunnerImage) + } if err != nil { return executionResult{}, nil, err } diff --git a/internal/worker/human_release.go b/internal/worker/human_release.go index 5cdb04c9..27539464 100644 --- a/internal/worker/human_release.go +++ b/internal/worker/human_release.go @@ -157,6 +157,10 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch if gateExecution == nil { return fmt.Errorf("human release gate execution %s is missing", batch.GateNodeID) } + frozenPackage, err := runtime.freezeLegacyDeliveryRuntime(ctx, workflow, parent, target) + if err != nil { + return err + } artifacts, err := runtime.releasePartitionArtifacts(ctx, parent, *gateNode, *gateExecution, partition) if err != nil { return err @@ -189,6 +193,7 @@ func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch if child.Metadata == nil { child.Metadata = &domain.RunMetadata{} } + child.Metadata.CaptureDeliveryPackage = frozenPackage child.Metadata.HumanReleasePartitionID = partition.ID child.Metadata.HumanReleaseUniqueKey = partition.UniqueKey child.UpdatedAt = nowISO() diff --git a/internal/worker/legacy_delivery_recovery.go b/internal/worker/legacy_delivery_recovery.go new file mode 100644 index 00000000..200faebb --- /dev/null +++ b/internal/worker/legacy_delivery_recovery.go @@ -0,0 +1,144 @@ +package worker + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "droboticflow/internal/domain" +) + +// Recover only a byte-identical, fully completed immutable publication. This +// path cannot resume an incomplete legacy upload after a V4 family cutover. +func completedLegacyDelivery(ctx context.Context, target *deliveryTarget, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) { + info, err := os.Stat(artifact.Path) + if err != nil { + return artifact, err + } + paths := []string{artifact.Path} + kind := "part-index" + if info.IsDir() { + kind = "directory" + paths = nil + err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error { + if e != nil { + return e + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("legacy delivery symlink") + } + if !entry.IsDir() { + paths = append(paths, path) + } + return nil + }) + if err != nil { + return artifact, err + } + sort.Slice(paths, func(i, j int) bool { + a, b := filepath.Base(paths[i]) == "packages-manifest.json", filepath.Base(paths[j]) == "packages-manifest.json" + if a != b { + return !a + } + return paths[i] < paths[j] + }) + } + if len(paths) == 0 { + return artifact, fmt.Errorf("empty legacy delivery") + } + entries := make([]directoryManifestEntry, 0, len(paths)) + check := func(key, sha string, size int64) error { + meta, e := target.publisher.Head(ctx, *target.connection, target.target.Bucket, key) + if e != nil { + return e + } + if !deliveryObjectMatches(meta, target.owner, sha, size) { + return fmt.Errorf("legacy delivery incomplete or changed") + } + return nil + } + for _, path := range paths { + relative := filepath.Base(path) + if info.IsDir() { + relative, err = filepath.Rel(artifact.Path, path) + if err != nil { + return artifact, err + } + relative = filepath.ToSlash(relative) + } else { + routing, e := partIndexPayloadPeek(path) + if e != nil { + return artifact, e + } + relative = filepath.Base(routing.FileName) + } + if relative == "manifest.json" || relative == ".delivery-owner.json" || strings.HasPrefix(relative, "../") { + return artifact, fmt.Errorf("reserved legacy member") + } + f, e := os.Open(path) + if e != nil { + return artifact, e + } + hash := sha256.New() + size, e := io.Copy(hash, f) + closeErr := f.Close() + if e != nil { + return artifact, e + } + if closeErr != nil { + return artifact, closeErr + } + entry := directoryManifestEntry{Path: relative, Key: target.target.Prefix + "/" + relative, SizeBytes: size, SHA256: hex.EncodeToString(hash.Sum(nil))} + if e = check(entry.Key, entry.SHA256, size); e != nil { + return artifact, e + } + entries = append(entries, entry) + } + marker, _ := json.Marshal(map[string]string{"owner": target.owner}) + body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: kind, FileCount: len(entries), TotalBytes: sumManifestBytes(entries), Entries: entries}) + expected := map[string]bool{} + for _, entry := range entries { + expected[entry.Key] = true + } + manifestKey := target.target.Prefix + "/manifest.json" + for key, value := range map[string][]byte{target.marker: marker, manifestKey: body} { + hash := sha256.Sum256(value) + if e := check(key, hex.EncodeToString(hash[:]), int64(len(value))); e != nil { + return artifact, e + } + expected[key] = true + } + keys, err := target.lister.ListPrefix(ctx, *target.connection, target.target.Bucket, target.target.Prefix+"/") + if err != nil { + return artifact, err + } + if len(keys) != len(expected) { + return artifact, fmt.Errorf("legacy delivery member count differs") + } + for _, key := range keys { + if !expected[key] { + return artifact, fmt.Errorf("unexpected legacy delivery member") + } + } + artifact.Path = "bos://" + target.target.Bucket + "/" + target.target.Prefix + artifact.SizeBytes = sumManifestBytes(entries) + ref := &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.target.ConnectionID, Bucket: target.target.Bucket, Prefix: target.target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory} + if !info.IsDir() { + entry := entries[0] + artifact.Path = "bos://" + target.target.Bucket + "/" + entry.Key + ref.Prefix = "" + ref.Key = entry.Key + ref.SizeBytes = entry.SizeBytes + ref.SHA256 = entry.SHA256 + ref.Kind = domain.ArtifactRefKindObject + } + artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: ref} + return artifact, nil +} diff --git a/internal/worker/legacy_delivery_v4.go b/internal/worker/legacy_delivery_v4.go new file mode 100644 index 00000000..9f354fc5 --- /dev/null +++ b/internal/worker/legacy_delivery_v4.go @@ -0,0 +1,183 @@ +package worker + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "droboticflow/internal/domain" +) + +// The active released version is a delivery policy for its exact tenant, +// project and definition family. It does not rebind any in-flight source Run. +func (r *Runtime) activeV4DeliveryPolicy(ctx context.Context, source *domain.WorkflowRecord) (*domain.WorkflowRecord, error) { + if source == nil || source.ProjectID == "" || source.OrganizationID == "" || source.CaptureAutomation == nil { + return nil, nil + } + normalized := *source + domain.NormalizeWorkflowDefinitionMetadata(&normalized) + versions, err := r.store.ListWorkflowsByProject(ctx, source.ProjectID) + if err != nil { + return nil, err + } + var chosen *domain.WorkflowRecord + for _, candidate := range versions { + domain.NormalizeWorkflowDefinitionMetadata(&candidate) + if candidate.OrganizationID != source.OrganizationID || candidate.ProjectID != source.ProjectID || candidate.DefinitionFamilyID != normalized.DefinitionFamilyID || candidate.DefinitionState != domain.WorkflowDefinitionStateReleased || candidate.Hidden || !workflowUsesV4CaptureDelivery(candidate) { + continue + } + if chosen != nil { + return nil, fmt.Errorf("同一工作流版本家族有多个启用的 V4 交付配置,请先确认") + } + copy := candidate + chosen = © + } + return chosen, nil +} + +func (r *Runtime) freezeLegacyDeliveryRuntime(ctx context.Context, source *domain.WorkflowRecord, parent *domain.RunRecord, targetID string) (*domain.CaptureDeliveryPackageRuntime, error) { + if source == nil || parent == nil || workflowUsesV4CaptureDelivery(*source) || parent.Mode.Normalized() == domain.RunModeDebug { + return nil, nil + } + policy, err := r.activeV4DeliveryPolicy(ctx, source) + if err != nil || policy == nil { + return nil, err + } + old := domain.GetNodeByID(source.Nodes, targetID) + next := domain.GetNodeByID(policy.Nodes, targetID) + if old == nil || next == nil || old.Data.OperatorID != "operator-builtin-package" || next.Data.OperatorID != old.Data.OperatorID { + return nil, fmt.Errorf("旧任务缺少兼容的 V4 打包节点,请先核对交付配置") + } + image, err := sourceIndexPackageImage(*next, nil, r.operatorRunnerImage) + if err != nil { + return nil, err + } + expected := strings.TrimSpace(next.Data.OperatorImageID) + if expected == "" { + expected = strings.TrimSpace(next.Data.Image) + } + if image != expected { + return nil, fmt.Errorf("部署的 V4 打包镜像与已发布工作流不一致,请先核对") + } + var node domain.WorkflowNode + body, err := json.Marshal(old) + if err != nil { + return nil, err + } + if err = json.Unmarshal(body, &node); err != nil { + return nil, err + } + node.Data.Image, node.Data.OperatorImageID = image, image + node.Data.OperatorVersion = next.Data.OperatorVersion + node.Data.PublicationEvidence = next.Data.PublicationEvidence + node.Data.PublicationStatus = next.Data.PublicationStatus + node.Data.GenerationSource = next.Data.GenerationSource + node = applyNodeOverridesToNode(node, domain.NodeOverride{Hyperparams: []domain.HyperParam{ + {Key: "PIN_OPERATOR_IMAGE", Value: "true"}, {Key: "PART_INDEX_DELIVERY", Value: "true"}, {Key: "DELIVERY_JSON_V4", Value: "true"}, {Key: "DELIVERY_JSON_V3", Value: "false"}, {Key: "SOURCE_INDEX_DELIVERY", Value: "true"}, + }}) + audit := false + for _, port := range node.Data.Outputs { + if port.ID == "deliveryAudit" { + audit = true + } + } + if !audit { + node.Data.Outputs = append(node.Data.Outputs, domain.PortDefinition{ID: "deliveryAudit", Name: "deliveryAudit", Type: domain.PortTypeJSON}) + } + return &domain.CaptureDeliveryPackageRuntime{SourceRunID: parent.ID, SourceWorkflowID: source.ID, PolicyWorkflowID: policy.ID, PolicyVersion: policy.DefinitionVersion, Node: node}, nil +} + +func deliveryRuntimeWorkflow(workflow *domain.WorkflowRecord, run *domain.RunRecord) (*domain.WorkflowRecord, error) { + if workflow == nil || run == nil || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil { + return workflow, nil + } + frozen := run.Metadata.CaptureDeliveryPackage + if workflow.ID != run.WorkflowID || workflow.ID != frozen.SourceWorkflowID || workflow.OrganizationID != run.OrganizationID || workflow.ProjectID != run.ProjectID || frozen.SourceRunID == "" || frozen.PolicyWorkflowID == "" || frozen.Node.Data.OperatorID != "operator-builtin-package" || !workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{frozen.Node}}) || !sourcePackageDigestImage.MatchString(frozen.Node.Data.Image) || frozen.Node.Data.Image != frozen.Node.Data.OperatorImageID { + return nil, fmt.Errorf("V4 交付运行快照不完整或归属不符") + } + copy := *workflow + copy.Nodes = append([]domain.WorkflowNode(nil), workflow.Nodes...) + found := false + for i, node := range copy.Nodes { + if node.ID == frozen.Node.ID { + if node.Data.OperatorID != "operator-builtin-package" { + return nil, fmt.Errorf("V4 交付节点类型不符") + } + body, err := json.Marshal(frozen.Node) + if err != nil { + return nil, err + } + var replacement domain.WorkflowNode + if err = json.Unmarshal(body, &replacement); err != nil { + return nil, err + } + copy.Nodes[i] = replacement + found = true + } + } + if !found { + return nil, fmt.Errorf("V4 交付节点不存在") + } + return ©, nil +} + +func frozenDeliveryPackageImage(run *domain.RunRecord, node domain.WorkflowNode) (string, bool, error) { + if run == nil || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil || run.Metadata.CaptureDeliveryPackage.Node.ID != node.ID { + return "", false, nil + } + frozen := run.Metadata.CaptureDeliveryPackage.Node + if frozen.Data.OperatorID != "operator-builtin-package" || !workflowUsesV4CaptureDelivery(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{node}}) || !sourcePackageDigestImage.MatchString(frozen.Data.Image) || node.Data.Image != frozen.Data.Image { + return "", true, fmt.Errorf("V4 打包运行快照已改变,请先核对") + } + return frozen.Data.Image, true, nil +} + +// A retry with an explicit immutable delivery snapshot shares the accepted +// release attempt. Ordinary retry runs retain the Store's strict isolation. +func (r *Runtime) frozenDeliveryCapture(ctx context.Context, run *domain.RunRecord) (*domain.CaptureRecord, error) { + _, lineage, err := r.originalDeliveryRun(ctx, run) + if err != nil { + return nil, err + } + capture, err := r.store.GetCaptureRecord(ctx, run.TaskID) + if err != nil { + return nil, err + } + if capture == nil || capture.OrganizationID != run.OrganizationID || capture.ProjectID != run.ProjectID || capture.WorkflowID != run.WorkflowID || !lineage[capture.RunID] { + return nil, fmt.Errorf("frozen delivery capture scope mismatch") + } + return capture, nil +} + +// Persist the newest retry association before it launches. A restart repeats +// this safely; an unrelated or newer branch cannot be adopted or overwritten. +func (r *Runtime) attachFrozenReleaseRetry(ctx context.Context, run *domain.RunRecord) error { + if run.Trigger != domain.RunTriggerRetry || run.Metadata == nil || run.Metadata.CaptureDeliveryPackage == nil { + return nil + } + _, lineage, err := r.originalDeliveryRun(ctx, run) + if err != nil { + return err + } + partition, err := r.store.GetReleasePartitionByUniqueKey(ctx, run.OrganizationID, run.Metadata.HumanReleaseUniqueKey) + if err != nil { + return err + } + if partition == nil || partition.ID != run.Metadata.HumanReleasePartitionID || partition.UniqueKey != run.Metadata.HumanReleaseUniqueKey || !lineage[partition.ChildRunID] || partition.Status != domain.ReleasePartitionDispatched { + return fmt.Errorf("V4 retry release partition scope mismatch") + } + batch, err := r.store.GetHumanBatch(ctx, partition.BatchID) + if err != nil { + return err + } + if batch == nil || batch.OrganizationID != run.OrganizationID || batch.ProjectID != run.ProjectID || batch.WorkflowID != run.WorkflowID || batch.RunID != run.Metadata.CaptureDeliveryPackage.SourceRunID { + return fmt.Errorf("V4 retry release batch scope mismatch") + } + if partition.ChildRunID == run.ID { + return nil + } + partition.ChildRunID = run.ID + partition.UpdatedAt = nowISO() + return r.store.UpsertReleasePartition(ctx, *partition) +} diff --git a/internal/worker/legacy_delivery_v4_test.go b/internal/worker/legacy_delivery_v4_test.go new file mode 100644 index 00000000..35917b73 --- /dev/null +++ b/internal/worker/legacy_delivery_v4_test.go @@ -0,0 +1,348 @@ +package worker + +import ( + "context" + "droboticflow/internal/objectsync" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "droboticflow/internal/domain" +) + +func TestLegacyCaptureReleaseRequiresV4AfterFamilyCutover(t *testing.T) { + f := newCaptureCutoverFixture(t, "legacy-source") + record := f.seed("legacy-source", domain.CaptureRecordWaiting, "legacy-parent") + parent := domain.RunRecord{ID: record.RunID, OrganizationID: record.OrganizationID, ProjectID: record.ProjectID, WorkflowID: f.old.ID, TaskID: record.ID, Status: domain.RunStatusWaitingForHuman} + child := parent + child.ID = "legacy-child" + child.ParentRunID = parent.ID + child.Trigger = domain.RunTriggerHumanRelease + child.Status = domain.RunStatusRunning + for _, run := range []domain.RunRecord{parent, child} { + if err := f.db.UpsertRun(f.ctx, run); err != nil { + t.Fatal(err) + } + } + f.activate() + publisher := &deliveryPublisher{} + f.runtime.artifactPublisher = publisher + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "legacy.json"), []byte("legacy payload"), 0600); err != nil { + t.Fatal(err) + } + _, err := f.runtime.publishCaptureDelivery(f.ctx, &child, "package", domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root}) + if err == nil { + t.Fatal("a newly completed legacy task published an old-format customer package after V4 cutover") + } + if len(publisher.objects) != 0 { + t.Fatal("legacy rejection must precede owner markers and staging writes") + } + current, _ := f.db.GetCaptureRecord(f.ctx, record.ID) + if current.WorkflowID != f.old.ID { + t.Fatal("source ownership must not be rebound") + } +} + +func TestLegacyCaptureReleaseCollectsV4AuditAfterFamilyCutover(t *testing.T) { + f := newCaptureCutoverFixture(t, "legacy-audit") + f.activate() + enabled, err := f.runtime.classificationDeliveryEnabled(f.ctx, f.old.ID) + if err != nil || !enabled { + t.Fatalf("old task must collect the new customer format's internal audit after cutover: enabled=%v err=%v", enabled, err) + } +} + +func TestLegacyCaptureRuntimeIsFrozenAndScoped(t *testing.T) { + f := newCaptureCutoverFixture(t, "frozen") + f.activate() + parent := &domain.RunRecord{ID: "parent", OrganizationID: f.old.OrganizationID, ProjectID: f.old.ProjectID, WorkflowID: f.old.ID, TaskID: "capture"} + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", f.next.Nodes[0].Data.OperatorImageID) + frozen, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package") + if err != nil || frozen == nil { + t.Fatalf("freeze: %+v %v", frozen, err) + } + run := *parent + run.ID = "child" + run.ParentRunID = parent.ID + run.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: frozen} + effective, err := deliveryRuntimeWorkflow(&f.old, &run) + if err != nil { + t.Fatal(err) + } + if !workflowUsesV4CaptureDelivery(*effective) || workflowUsesV4CaptureDelivery(f.old) || run.WorkflowID != f.old.ID { + t.Fatal("execution must upgrade without mutating source ownership/configuration") + } + if len(effective.Nodes[0].Data.Outputs) != 1 || effective.Nodes[0].Data.Outputs[0].ID != "deliveryAudit" { + t.Fatal("missing internal audit output") + } + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "invalid-drift") + image, handled, err := frozenDeliveryPackageImage(&run, effective.Nodes[0]) + if err != nil || !handled || image != frozen.Node.Data.Image { + t.Fatalf("restart changed package image: %s %v", image, err) + } + run.OrganizationID = "foreign" + if _, err = deliveryRuntimeWorkflow(&f.old, &run); err == nil { + t.Fatal("foreign runtime accepted") + } + parent.Mode = domain.RunModeDebug + if frozen, err = f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package"); err != nil || frozen != nil { + t.Fatal("debug selection must be unchanged") + } +} + +func TestLegacyCaptureRuntimeRejectsIncompatiblePublishedImage(t *testing.T) { + f := newCaptureCutoverFixture(t, "image") + f.activate() + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", "registry.example/wrong@sha256:"+strings.Repeat("d", 64)) + if _, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, &domain.RunRecord{ID: "parent"}, "package"); err == nil { + t.Fatal("unexpected operator image accepted") + } +} + +func TestLegacyCapturePolicyNeverCrossesFamilyOrTenant(t *testing.T) { + for _, scope := range []string{"family", "organization", "project", "hidden"} { + t.Run(scope, func(t *testing.T) { + f := newCaptureCutoverFixture(t, "scope") + f.activate() + next := f.next + next.Hidden = false + switch scope { + case "family": + next.DefinitionFamilyID = "foreign" + case "organization": + next.OrganizationID = "foreign" + case "project": + next.ProjectID = "foreign" + case "hidden": + next.Hidden = true + } + if err := f.db.UpsertWorkflow(f.ctx, next); err != nil { + t.Fatal(err) + } + policy, err := f.runtime.activeV4DeliveryPolicy(f.ctx, &f.old) + if err != nil || policy != nil { + t.Fatalf("crossed %s: %+v %v", scope, policy, err) + } + }) + } +} + +func TestLegacyCaptureRetryPreservesPackageAndAcceptedBaseline(t *testing.T) { + f := newCaptureCutoverFixture(t, "retry") + f.activate() + t.Setenv("ANNOTATION_SOURCE_PACKAGE_IMAGE", f.next.Nodes[0].Data.OperatorImageID) + parent := &domain.RunRecord{ID: "source", OrganizationID: f.old.OrganizationID, ProjectID: f.old.ProjectID, WorkflowID: f.old.ID, TaskID: "capture"} + parent.Inputs = map[string]domain.ArtifactPayload{"sourceDirectory": {Mode: domain.ArtifactTransferModeRef, Type: domain.PortTypeFile, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: "cutover-storage", Bucket: "synthetic", Prefix: "incoming/retry", Kind: domain.ArtifactRefKindDirectory}}} + if err := f.db.UpsertRun(f.ctx, *parent); err != nil { + t.Fatal(err) + } + frozen, err := f.runtime.freezeLegacyDeliveryRuntime(f.ctx, &f.old, parent, "package") + if err != nil { + t.Fatal(err) + } + failed := *parent + failed.ID = "failed" + failed.ParentRunID = parent.ID + failed.RestartFromNodeID = "package" + failed.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: frozen, HumanReleasePartitionID: "partition", HumanReleaseUniqueKey: "key"} + failed.NodeExecutions = []domain.NodeExecutionRecord{{NodeID: "gate", Status: domain.NodeExecutionStatusCompleted, Artifacts: []domain.ExecutionArtifact{{Name: "accepted", Type: domain.PortTypeJSON, Data: map[string]any{"accepted": true}}}}, {NodeID: "package", Status: domain.NodeExecutionStatusFailed}} + if err := f.db.UpsertRun(f.ctx, failed); err != nil { + t.Fatal(err) + } + retry, err := f.runtime.enqueueRetryRun(f.ctx, &f.old, &failed) + if err != nil { + t.Fatal(err) + } + if retry.Metadata == nil || retry.Metadata.CaptureDeliveryPackage == nil || retry.Metadata.CaptureDeliveryPackage.SourceRunID != parent.ID || retry.Metadata.HumanReleasePartitionID != "partition" || retry.RestartFromNodeID != "package" || len(retry.NodeExecutions) != 1 || retry.NodeExecutions[0].NodeID != "gate" { + t.Fatalf("retry lost accepted delivery state: %+v", retry) + } + allowed := rerunAllowedNodeSet(domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{ID: "source"}, {ID: "gate"}, {ID: "package"}}, Edges: []domain.WorkflowEdge{{Source: "source", Target: "gate"}, {Source: "gate", Target: "package"}}}, retry) + if _, ok := allowed["package"]; !ok { + t.Fatal("retry lost package subtree") + } + if _, ok := allowed["gate"]; ok { + t.Fatal("retry reruns accepted human gate") + } + retry.Metadata.CaptureDeliveryPackage.Node.Data.Hyperparams[0].Value = "changed" + if frozen.Node.Data.Hyperparams[0].Value == "changed" { + t.Fatal("retry aliased previous immutable selection") + } +} + +// Every write method is counted; equal final objects alone would not prove a +// completed legacy recovery is read-only. +type readonlyLegacyPublisher struct { + deliveryPublisher + writes int +} + +func (p *readonlyLegacyPublisher) PutInternal(c context.Context, x domain.StorageConnectionRecord, b, k string, r io.Reader, n int64, t string, m map[string]string) (objectsync.ObjectMetadata, error) { + p.writes++ + return p.deliveryPublisher.PutInternal(c, x, b, k, r, n, t, m) +} +func (p *readonlyLegacyPublisher) PublishConditional(c context.Context, x domain.StorageConnectionRecord, b, s, k string, n int64, e, a, z string, m map[string]string) (objectsync.ObjectMetadata, error) { + p.writes++ + return p.deliveryPublisher.PublishConditional(c, x, b, s, k, n, e, a, z, m) +} +func (p *readonlyLegacyPublisher) Delete(c context.Context, x domain.StorageConnectionRecord, b, k string) error { + p.writes++ + return p.deliveryPublisher.Delete(c, x, b, k) +} + +func TestLegacyCaptureCompletedRecoveryNeverWrites(t *testing.T) { + for _, damage := range []string{"none", "missing", "modified", "extra", "owner", "file"} { + t.Run(damage, func(t *testing.T) { + f := newCaptureCutoverFixture(t, "completed") + record := f.seed("completed", domain.CaptureRecordWaiting, "parent") + run := domain.RunRecord{ID: "parent", OrganizationID: record.OrganizationID, ProjectID: record.ProjectID, WorkflowID: f.old.ID, TaskID: record.ID} + if err := f.db.UpsertRun(f.ctx, run); err != nil { + t.Fatal(err) + } + publisher := &readonlyLegacyPublisher{} + f.runtime.artifactPublisher = publisher + root := t.TempDir() + path := filepath.Join(root, "legacy.json") + if err := os.WriteFile(path, []byte("old"), 0600); err != nil { + t.Fatal(err) + } + artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Type: domain.PortTypeFile, Path: root} + publish := f.runtime.publishCaptureDelivery + if damage == "file" { + path = filepath.Join(root, "part.json") + if err := os.WriteFile(path, []byte(`{"schemaVersion":"droboticflow.delivery.part-index/v1","delivery":{"kind":"part_index","fileName":"part.json"}}`), 0600); err != nil { + t.Fatal(err) + } + artifact.Name = partIndexArtifactName + artifact.Path = path + publish = f.runtime.publishCaptureDeliveryFile + } + if _, err := publish(f.ctx, &run, "package", artifact); err != nil { + t.Fatal(err) + } + f.activate() + publisher.writes = 0 + key := record.Delivery.Bucket + "/" + record.Delivery.Prefix + "/legacy.json" + switch damage { + case "missing": + delete(publisher.objects, key) + case "modified": + publisher.metadata[key]["sha256"] = "changed" + case "extra": + publisher.objects[record.Delivery.Bucket+"/"+record.Delivery.Prefix+"/unexpected"] = []byte("x") + case "owner": + publisher.metadata[key]["delivery-owner"] = "foreign" + } + _, err := publish(f.ctx, &run, "package", artifact) + if damage == "none" || damage == "file" { + if err != nil { + t.Fatal(err) + } + } else if err == nil { + t.Fatal("damaged legacy publication was adopted") + } + if publisher.writes != 0 { + t.Fatalf("legacy recovery performed %d writes", publisher.writes) + } + }) + } +} + +func TestV4CustomerDirectoryRejectsInternalOrMediaMembers(t *testing.T) { + for _, name := range []string{"review-history.json", "left.mp4", "other.json"} { + t.Run(name, func(t *testing.T) { + r, pub, run, _, artifact := originalWorkerFixture(t) + if err := os.WriteFile(filepath.Join(artifact.Path, name), []byte("internal"), 0600); err != nil { + t.Fatal(err) + } + if _, err := r.publishCaptureDelivery(t.Context(), &run, "package", artifact); err == nil { + t.Fatal("V4 leaked extra member") + } + if len(pub.puts) > 0 || len(pub.published) > 0 { + t.Fatal("format guard ran after writes") + } + }) + } +} + +func TestFrozenV4RetryResolvesOnlyOriginalReleaseAncestry(t *testing.T) { + for _, scenario := range []string{"valid", "foreign", "cycle", "missing"} { + t.Run(scenario, func(t *testing.T) { + r, _, child, _, _ := originalWorkerFixture(t) + retry := child + retry.ID = "retry" + retry.ParentRunID = child.ID + retry.Trigger = domain.RunTriggerRetry + retry.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: &domain.CaptureDeliveryPackageRuntime{SourceRunID: child.ParentRunID}} + switch scenario { + case "foreign": + child.OrganizationID = "other" + case "cycle": + retry.Metadata.CaptureDeliveryPackage.SourceRunID = "unknown" + child.ParentRunID = retry.ID + case "missing": + child.ParentRunID = "unknown" + } + if err := r.store.UpsertRun(t.Context(), child); err != nil { + t.Fatal(err) + } + if err := r.store.UpsertRun(t.Context(), retry); err != nil { + t.Fatal(err) + } + _, err := r.originalDeliverySelection(t.Context(), &retry) + if (err == nil) != (scenario == "valid") { + t.Fatalf("scenario=%s err=%v", scenario, err) + } + }) + } +} + +func TestFrozenReleaseRetryRestoresPartitionAndCaptureAssociation(t *testing.T) { + r, _, child, _, _ := originalWorkerFixture(t) + retry := child + retry.ID = "retry" + retry.ParentRunID = child.ID + retry.Trigger = domain.RunTriggerRetry + retry.Metadata = &domain.RunMetadata{CaptureDeliveryPackage: &domain.CaptureDeliveryPackageRuntime{SourceRunID: child.ParentRunID}, HumanReleasePartitionID: "partition", HumanReleaseUniqueKey: "key"} + if err := r.store.UpsertRun(t.Context(), retry); err != nil { + t.Fatal(err) + } + batch := domain.HumanBatchRecord{ID: "batch", OrganizationID: child.OrganizationID, ProjectID: child.ProjectID, WorkflowID: child.WorkflowID, RunID: child.ParentRunID} + if err := r.store.UpsertHumanBatch(t.Context(), batch); err != nil { + t.Fatal(err) + } + partition := domain.ReleasePartitionRecord{ID: "partition", OrganizationID: child.OrganizationID, BatchID: batch.ID, ChildRunID: child.ID, UniqueKey: "key", Status: domain.ReleasePartitionDispatched} + if err := r.store.UpsertReleasePartition(t.Context(), partition); err != nil { + t.Fatal(err) + } + if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err != nil { + t.Fatal(err) + } + saved, err := r.store.GetReleasePartitionByUniqueKey(t.Context(), child.OrganizationID, "key") + if err != nil || saved.ChildRunID != retry.ID { + t.Fatalf("retry association: %+v %v", saved, err) + } + if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err != nil { + t.Fatal("restart did not preserve association", err) + } + capture, err := r.frozenDeliveryCapture(t.Context(), &retry) + if err != nil { + t.Fatal(err) + } + capture.RunID = retry.ID + if err := r.store.UpsertCaptureRecord(t.Context(), *capture); err != nil { + t.Fatal(err) + } + if _, err := r.originalDeliverySelection(t.Context(), &retry); err != nil { + t.Fatal("reconciled retry lost original source", err) + } + saved.ChildRunID = "unrelated" + if err := r.store.UpsertReleasePartition(t.Context(), *saved); err != nil { + t.Fatal(err) + } + if err := r.attachFrozenReleaseRetry(t.Context(), &retry); err == nil { + t.Fatal("unrelated branch was overwritten") + } +} diff --git a/internal/worker/original_source_recheck.go b/internal/worker/original_source_recheck.go index 95cf5923..d8a58ca7 100644 --- a/internal/worker/original_source_recheck.go +++ b/internal/worker/original_source_recheck.go @@ -3,6 +3,7 @@ package worker import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -14,28 +15,57 @@ import ( "droboticflow/internal/workspacescope" ) -// Release children carry accepted artifacts but no original Inputs. Resolve -// only the persisted human-release edge; retries are independent selections. -func (r *Runtime) originalDeliverySelection(ctx context.Context, run *domain.RunRecord) (*sourceidentity.Selection, error) { +// Follow only validated persisted release ancestry, including retries carrying +// the same frozen package selection. Never infer a source from current intake. +func (r *Runtime) originalDeliveryRun(ctx context.Context, run *domain.RunRecord) (*domain.RunRecord, map[string]bool, error) { if run == nil { - return nil, fmt.Errorf("missing original delivery run") + return nil, nil, fmt.Errorf("missing original delivery run") } original := run - if run.Trigger == domain.RunTriggerHumanRelease { + lineage := map[string]bool{run.ID: true} + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + wanted := run.Metadata.CaptureDeliveryPackage.SourceRunID + for original.ID != wanted { + if len(lineage) >= 32 || original.ParentRunID == "" { + return nil, nil, fmt.Errorf("original delivery lineage incomplete") + } + parent, err := r.store.GetRun(ctx, original.ParentRunID) + if err != nil { + return nil, nil, err + } + if parent == nil || lineage[parent.ID] || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != run.TaskID { + return nil, nil, fmt.Errorf("original delivery parent scope mismatch") + } + lineage[parent.ID] = true + original = parent + } + } else if run.Trigger == domain.RunTriggerHumanRelease { parent, err := r.store.GetRun(ctx, run.ParentRunID) if err != nil { - return nil, err + return nil, nil, err } if parent == nil || parent.ID == run.ID || parent.OrganizationID != run.OrganizationID || parent.ProjectID != run.ProjectID || parent.WorkflowID != run.WorkflowID || parent.TaskID != run.TaskID { - return nil, fmt.Errorf("original delivery parent scope mismatch") + return nil, nil, fmt.Errorf("original delivery parent scope mismatch") } original = parent + lineage[parent.ID] = true } - capture, err := r.store.GetCaptureRecordByRunID(ctx, original.ID) + return original, lineage, nil +} + +func (r *Runtime) originalDeliverySelection(ctx context.Context, run *domain.RunRecord) (*sourceidentity.Selection, error) { + original, lineage, err := r.originalDeliveryRun(ctx, run) if err != nil { return nil, err } - if capture == nil || capture.ID != original.TaskID || capture.RunID != original.ID && capture.RunID != run.ID { + capture, err := r.store.GetCaptureRecordByRunID(ctx, original.ID) + if run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil { + capture, err = r.frozenDeliveryCapture(ctx, run) + } + if err != nil { + return nil, err + } + if capture == nil || capture.ID != original.TaskID || !lineage[capture.RunID] { return nil, fmt.Errorf("original delivery capture lineage mismatch") } return sourceidentity.FromRun(original, capture) @@ -87,9 +117,25 @@ func (r *Runtime) preflightV4OriginalSource(ctx context.Context, run *domain.Run return r.recheckOriginalDeliverySelection(ctx, run, selection) } -// Detect the actual output schema, independent of mutable workflow switches. +var errLegacyDeliveryRequiresV4 = errors.New("该工作流已启用 V4,禁止新发布旧格式交付包;请使用已验收数据重新生成 V4") + +// Detect the actual output schema as well as the active format contract. // This guard runs before prepareDeliveryTarget writes even its owner marker. func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.RunRecord, artifact domain.ExecutionArtifact) error { + required := run != nil && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil + if !required && run != nil { + var err error + required, err = r.classificationDeliveryEnabled(ctx, run.WorkflowID) + if err != nil { + return err + } + } + legacy := func() error { + if required { + return errLegacyDeliveryRequiresV4 + } + return nil + } name := artifact.Path info, err := os.Stat(name) if err != nil { @@ -98,11 +144,11 @@ func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.Run if info.IsDir() { name = filepath.Join(name, "part.json") } else if filepath.Base(name) != "part.json" { - return nil + return legacy() } f, err := os.Open(name) if os.IsNotExist(err) { - return nil + return legacy() } // Legacy directory has no part.json. if err != nil { return err @@ -133,7 +179,16 @@ func (r *Runtime) recheckV4DeliveryArtifact(ctx context.Context, run *domain.Run return fmt.Errorf("invalid delivery part JSON") } if part.Schema != deliveryindex.ClassificationPartSchema { - return nil + return legacy() + } + if info.IsDir() { + members, e := os.ReadDir(artifact.Path) + if e != nil { + return e + } + if len(members) != 1 || members[0].Name() != "part.json" || !members[0].Type().IsRegular() { + return fmt.Errorf("V4 客户交付目录必须只有 part.json") + } } if len(raw) > 8<<20 { return fmt.Errorf("classification part exceeds 8 MiB") diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index 8e7c0a6f..5ea35648 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -1400,6 +1400,10 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run Suggestion: "重新从现有发布版本创建工作流后再运行", }) } + workflow, err = deliveryRuntimeWorkflow(workflow, run) + if err != nil { + return runtime.failReconciledRun(ctx, run, err.Error(), domain.RunFailure{Code: "invalid_delivery_runtime", Stage: "orchestration", Retryable: false}) + } node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID) if node == nil { return runtime.failReconciledRun(ctx, run, fmt.Sprintf("node %s was deleted while run %s was active", run.CurrentNodeID, run.ID), domain.RunFailure{ @@ -2288,7 +2292,15 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo if err != nil { return nil, err } - retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.Inputs, failedRun.TaskID) + inputs := failedRun.Inputs + if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil { + source, _, e := runtime.originalDeliveryRun(ctx, failedRun) + if e != nil { + return nil, e + } + inputs = source.Inputs + } + retryRun, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerRetry, inputs, failedRun.TaskID) if err != nil { return nil, err } @@ -2296,6 +2308,30 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo retryRun.ParentRunID = failedRun.ID retryRun.AvailableAt = nextRunAvailableAt(workflow) lightweight.MergeRunMetadata(retryRun, manifests) + if failedRun.Metadata != nil && failedRun.Metadata.CaptureDeliveryPackage != nil { + // Preserve the immutable package selection and accepted release baseline. + raw, e := json.Marshal(failedRun.Metadata.CaptureDeliveryPackage) + if e != nil { + return nil, e + } + var frozen domain.CaptureDeliveryPackageRuntime + if e = json.Unmarshal(raw, &frozen); e != nil { + return nil, e + } + if retryRun.Metadata == nil { + retryRun.Metadata = &domain.RunMetadata{} + } + retryRun.Metadata.CaptureDeliveryPackage = &frozen + retryRun.Metadata.HumanReleasePartitionID = failedRun.Metadata.HumanReleasePartitionID + retryRun.Metadata.HumanReleaseUniqueKey = failedRun.Metadata.HumanReleaseUniqueKey + retryRun.RestartFromNodeID = failedRun.RestartFromNodeID + for _, execution := range failedRun.NodeExecutions { + if execution.NodeID != frozen.Node.ID && execution.Status == domain.NodeExecutionStatusCompleted { + applyExecution(retryRun, execution) + } + } + } + retryRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *retryRun); err != nil { return nil, err @@ -2557,7 +2593,8 @@ func rerunAllowedNodeSet(workflow domain.WorkflowRecord, run *domain.RunRecord) strings.TrimSpace(run.DebugTarget.TargetNodeID): {}, } } - if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease) || strings.TrimSpace(run.RestartFromNodeID) == "" { + frozenDeliveryRetry := run.Trigger == domain.RunTriggerRetry && run.Metadata != nil && run.Metadata.CaptureDeliveryPackage != nil + if (run.Trigger != domain.RunTriggerRerunSubtree && run.Trigger != domain.RunTriggerHumanRelease && !frozenDeliveryRetry) || strings.TrimSpace(run.RestartFromNodeID) == "" { return nil } visited := map[string]struct{}{} @@ -2688,6 +2725,21 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor claimed.UpdatedAt = nowISO() return claimed, runtime.store.UpsertRun(ctx, *claimed) } + workflow, err = deliveryRuntimeWorkflow(workflow, claimed) + if err != nil { + claimed.Status = domain.RunStatusFailed + claimed.EndedAt = nowISO() + claimed.Error = err.Error() + claimed.UpdatedAt = nowISO() + return claimed, runtime.store.UpsertRun(ctx, *claimed) + } + if err = runtime.attachFrozenReleaseRetry(ctx, claimed); err != nil { + claimed.Status = domain.RunStatusFailed + claimed.EndedAt = nowISO() + claimed.Error = err.Error() + claimed.UpdatedAt = nowISO() + return claimed, runtime.store.UpsertRun(ctx, *claimed) + } if conflicts := domain.FindInputPortConflicts(workflow.Nodes, workflow.Edges); len(conflicts) > 0 { currentRun := claimed message := "工作流图非法:同一输入端口存在多条上游连线。请拆分输入,或为每个上传结果添加单独的上传算子。" diff --git a/package.json b/package.json index 593e3a80..4aabe311 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,13 @@ "verify:quick": "python3 scripts/verify-change.py --mode quick", "verify:task": "python3 scripts/verify-change.py --mode task", "verify:integration": "python3 scripts/verify-change.py --mode integration", - "test:verification": "python3 -m unittest discover -s tests/tooling -p test_verification.py" + "test:verification": "python3 -m unittest discover -s tests/tooling -p test_verification.py", + "predev": "npm run hooks:install", + "predev:full": "npm run hooks:install", + "predev:api": "npm run hooks:install", + "predev:web": "npm run hooks:install", + "predev:worker": "npm run hooks:install", + "predev:api:replica": "npm run hooks:install" }, "dependencies": { "@tailwindcss/vite": "^4.1.14", diff --git a/scripts/check-git-hooks.mjs b/scripts/check-git-hooks.mjs index d2c0afa2..1fc1854c 100644 --- a/scripts/check-git-hooks.mjs +++ b/scripts/check-git-hooks.mjs @@ -1,30 +1,11 @@ #!/usr/bin/env node -import { execFileSync } from "node:child_process"; -import { accessSync, constants, existsSync } from "node:fs"; +import { ensureGitHooks } from "./lib/git-hooks.mjs"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); try { - if (process.env.CI || process.env.HUSKY === "0" || !existsSync(resolve(root, ".git"))) { - console.log("Git hooks: skipped (CI, HUSKY=0, or exported source)"); - } else { - let configured = ""; - try { - configured = execFileSync("git", ["config", "--get", "core.hooksPath"], { cwd: root, encoding: "utf8" }).trim(); - } catch {} - if (process.argv.includes("--install")) { - if (configured && resolve(root, configured) !== resolve(root, ".husky/_")) - throw Error("Custom hooksPath detected; left unchanged"); - execFileSync(process.execPath, [resolve(root, "node_modules/husky/bin.js")], { cwd: root, stdio: "inherit" }); - configured = execFileSync("git", ["config", "--get", "core.hooksPath"], { cwd: root, encoding: "utf8" }).trim(); - } - if (resolve(root, configured) !== resolve(root, ".husky/_")) - throw Error("core.hooksPath must resolve to this checkout's .husky/_"); - for (const name of [".husky/_/pre-commit"]) accessSync(resolve(root, name), constants.X_OK); - accessSync(resolve(root, ".husky/_/h"), constants.R_OK); - accessSync(resolve(root, ".husky/pre-commit"), constants.R_OK); - console.log(`Git hooks: active in ${root}`); - } + const state = ensureGitHooks(root, { install: process.argv.includes("--install") }); + console.log(`Git hooks: ${state}${state === "active" ? ` in ${root}` : " (CI, HUSKY=0, or exported source)"}`); } catch (error) { console.error(`Git hooks unavailable: ${error.message}. Run npm run hooks:install in this checkout.`); process.exitCode = 1; diff --git a/scripts/delivery_v4_preview.py b/scripts/delivery_v4_preview.py new file mode 100644 index 00000000..444ba9c6 --- /dev/null +++ b/scripts/delivery_v4_preview.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Resume a local-only V4 conversion of inventoried, frozen delivery revisions. + +Input: inventory.json and packages//{old/,pinned-revision/,pinned-provenance.json}. +This produces customer JSON previews, never a complete audit or publishable run. +No network, current dictionaries, video downloads, or production writes are used. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from pathlib import Path +import tempfile + +from part_index_migration import load_operator_module, resolve_pose_policy, split_slices + + +def load(path): + return json.loads(path.read_text(encoding="utf8")) + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def convert(package, directory, output): + if package["status"] not in {"accepted", "no_deliverables"}: + raise ValueError("no published accepted manifest; frozen revision recovery required") + revision = directory / "pinned-revision" + manifest_bytes = (revision / "revision-manifest.json").read_bytes() + manifest = json.loads(manifest_bytes) + provenance = load(directory / "pinned-provenance.json") + delivery = load(directory / "old/packages-manifest.json") + lineage = delivery["lineage"] + if (provenance["sha256"] != sha(manifest_bytes) + or provenance["artifactId"] != lineage["acceptedSliceDirectoryArtifactId"] + or provenance["revision"] != manifest["revision"]): + raise ValueError("frozen revision provenance mismatch") + # Verify every input JSON against its accepted manifest before the builder reads it. + before = {} + for row in manifest["items"]: + relative = Path(row["directory"]) + if relative.is_absolute() or ".." in relative.parts: + raise ValueError("unsafe accepted slice directory") + verified = set() + files = row.get("files", []) + for entry in files.values() if isinstance(files, dict) else files: + if entry["name"] not in {"annotation.json", "synchronized-slice.json"}: + continue + path = revision / relative / entry["name"] + if not path.resolve().is_relative_to(revision.resolve()): + raise ValueError("accepted file escapes revision directory") + raw = path.read_bytes() + if not entry.get("sha256") or sha(raw) != entry["sha256"]: + raise ValueError("frozen annotation checksum mismatch") + before[path] = sha(raw) + verified.add(entry["name"]) + required = {"synchronized-slice.json"} + if row["deliveryDisposition"] == "keep": + required.add("annotation.json") + if not required <= verified: + raise ValueError("missing frozen JSON checksums") + source_basis = "accepted_revision_manifest" + recovery_path = directory / "source-selection-recovery.json" + if not manifest.get("originalSource") and recovery_path.exists(): + recovery = load(recovery_path) + raw_input = (directory / "source-selection-input.json").read_bytes() + evidence = json.loads(raw_input) + run, capture = evidence["run"], evidence["capture"] + frozen = delivery["capture"] + refs = [v["ref"] for v in run["inputs"].values()] + source = recovery["originalSource"] + if (recovery.get("basis") != "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys" + or recovery.get("inputSha256") != sha(raw_input) + or recovery.get("manifestSha256") != sha(manifest_bytes) + or run["id"] != lineage["sourceRunId"] or run["id"] != manifest["runId"] + or run["organizationId"] != package["organizationId"] + or capture["runId"] != run["id"] or capture["organizationId"] != run["organizationId"] + or capture["id"] != frozen["captureRecordId"] + or capture["captureId"] != frozen["captureId"] or capture["prefix"] != frozen["prefix"] + or len(refs) != 1): + raise ValueError("historical source recovery provenance mismatch") + ref = refs[0] + if (ref["provider"] != "BOS" or ref["kind"] != "directory" + or ref["captureUnit"] != frozen["captureId"] + or not frozen["prefix"].startswith(ref["prefix"] + "/") + or source["bucket"] != ref["bucket"] or source["prefix"] != ref["prefix"] + or source["partPrefix"] != frozen["prefix"] or source["captureUnit"] != frozen["captureId"] + or source["objectKeys"] != sorted(ref["objectKeys"]) + or source.get("snapshotFingerprint", "") != ref.get("versionId", "")): + raise ValueError("historical source selection mismatch") + manifest = copy.deepcopy(manifest) + manifest["originalSource"] = source + source_basis = "published_capture_snapshot_and_exact_persisted_run_captureUnit_objectKeys" + acceptance = load(directory / "old/acceptance-record.json") + kept, discarded = split_slices(manifest) + if package["status"] == "no_deliverables": + proof = load(directory / "empty-acceptance-proof.json") + accepted = proof.get("acceptedMaterialization", {}) + if (kept or not discarded or acceptance["finalAcceptance"].get("confirmed") is not True + or proof.get("packageId") != package["id"] or proof.get("confirmed") is not True + or proof.get("summaryArtifactId") not in package.get("artifactIds", []) + or accepted.get("artifactId") != lineage["acceptedSliceDirectoryArtifactId"] + or accepted.get("revision") != manifest["revision"] + or accepted.get("manifestSha256") != sha(manifest_bytes)): + raise ValueError("unconfirmed or mismatched all-discard acceptance evidence") + operator = load_operator_module() + final = acceptance["finalAcceptance"] + # Deliberately incomplete audit input: never export it or claim formal readiness. + preview_context = {"trace": [], "finalAcceptance": final} + bodies = [] + with tempfile.TemporaryDirectory(prefix="v4-preview-") as temp: + for attempt in range(2): + staging = Path(temp) / str(attempt) + staging.mkdir() + payload, _ = operator._build_part_index( + manifest, manifest["coverage"], kept, revision, + copy.deepcopy(delivery["calibration"]), resolve_pose_policy(acceptance), + {}, final, copy.deepcopy(delivery["capture"]), lineage, discarded, + classification_v4=True, + ) + def internal(*args, **kwargs): + return {"portId": "deliveryAudit", "previewOnly": True} + def virtual(run, node, name, path, kind, size, data=None, output=None): + return {"path": str(path), "portId": output["id"]} + artifacts = operator._emit_part_index_directory( + "local-preview", "package", staging, + {"outputs": [{"id": "deliveryAudit"}, {"id": "deliveryDirectory"}]}, + {"write_json_artifact": internal, "build_virtual_artifact": virtual}, + payload, preview_context, manifest, revision, + ) + folder = Path(artifacts[0]["path"]) + if {p.name for p in folder.iterdir()} != {"part.json"}: + raise ValueError("customer directory must contain only part.json") + bodies.append((folder / "part.json").read_bytes()) + if bodies[0] != bodies[1]: + raise ValueError("non-deterministic conversion") + for path, digest in before.items(): + if sha(path.read_bytes()) != digest: + raise ValueError("source changed during conversion") + if (revision / "revision-manifest.json").read_bytes() != manifest_bytes: + raise ValueError("source manifest changed during conversion") + calibration_proof = directory / "calibration-provenance.json" + calibration_basis = "published_manifest_checksums" + if calibration_proof.exists(): + evidence = load(calibration_proof) + if not all(entry.get("historicalStoredSHAAvailable") is True for entry in evidence): + calibration_basis = "original_BOS_selection_read_at_conversion; historical_checksum_unavailable" + document = json.loads(bodies[0]) + target = output / package["id"] / "part.json" + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != bodies[0]: + raise ValueError("refusing to replace an existing different preview") + target.write_bytes(bodies[0]) + return {"packageId": package["id"], "status": "converted-preview", "file": str(target), + "captureId": document["capture"]["captureId"], "partId": document["partId"], + "revision": document["revision"], "sha256": sha(bodies[0]), "bytes": len(bodies[0]), + "keptSlices": len(kept), "discardedSlices": len(discarded), + "manifestSha256": sha(manifest_bytes), "deterministic": True, + "calibrationEvidence": calibration_basis, + "originalSourceEvidence": source_basis, + "formalPublicationReady": False, "completeInternalAuditValidated": False, + "onlinePublished": False} + + +def run(root, output): + root, output = root.resolve(), output.resolve() + if output == root or root in output.parents: + raise ValueError("output must be outside the evidence directory") + inventory = load(root / "inventory.json") + ids = [p["id"] for p in inventory] + if len(ids) != len(set(ids)): + raise ValueError("duplicate inventory package IDs") + results = [] + output.mkdir(parents=True, exist_ok=True) + for package in inventory: + # Do not use an untrusted ID as a filesystem path. + if Path(package["id"]).name != package["id"] or package["id"] in {".", ".."}: + raise ValueError("unsafe package ID") + try: + result = convert(package, root / "packages" / package["id"], output) + except (ValueError, RuntimeError, KeyError, OSError, SystemExit) as error: + result = {"packageId": package["id"], "status": "blocked", "reason": str(error)} + results.append(result) + report = {"total": len(inventory), "processed": len(results), + "converted": sum(r["status"] == "converted-preview" for r in results), + "productionWrites": False, "formalPublicationReady": False, "packages": results} + (output / "conversion-report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + return report + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = run(args.evidence, args.out) + print(json.dumps({k: v for k, v in report.items() if k != "packages"}, ensure_ascii=False)) + raise SystemExit(0 if report["converted"] == report["total"] else 2) diff --git a/scripts/lib/browser-executable.mjs b/scripts/lib/browser-executable.mjs new file mode 100644 index 00000000..00506c87 --- /dev/null +++ b/scripts/lib/browser-executable.mjs @@ -0,0 +1,31 @@ +import { existsSync } from "node:fs"; +import { chromium } from "playwright"; + +export function browserExecutableOptions({ + env = process.env, + platform = process.platform, + exists = existsSync, + bundled = chromium.executablePath(), +} = {}) { + const explicit = env.BROWSER_EXECUTABLE || env.BROWSER_PATH; + if (explicit) { + if (!exists(explicit)) throw Error(`Configured browser does not exist: ${explicit}`); + return { executablePath: explicit }; + } + const candidates = + platform === "darwin" + ? [ + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + ] + : ["/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome"]; + const executablePath = + platform === "darwin" + ? candidates.find(exists) || (exists(bundled) ? bundled : undefined) + : exists(bundled) + ? bundled + : candidates.find(exists); + if (!executablePath) + throw Error("No Chromium browser found. Set BROWSER_EXECUTABLE or run npx playwright install chromium."); + return { executablePath }; +} diff --git a/scripts/lib/git-hooks.mjs b/scripts/lib/git-hooks.mjs new file mode 100644 index 00000000..81ecd397 --- /dev/null +++ b/scripts/lib/git-hooks.mjs @@ -0,0 +1,33 @@ +import { execFileSync } from "node:child_process"; +import { accessSync, constants, existsSync } from "node:fs"; +import { resolve } from "node:path"; + +export function ensureGitHooks(root, { install = false, env = process.env } = {}) { + if (env.CI || env.HUSKY === "0" || !existsSync(resolve(root, ".git"))) return "skipped"; + const git = (...args) => + execFileSync("git", args, { cwd: root, env, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim(); + let configured = ""; + try { + configured = git("config", "--get", "core.hooksPath"); + } catch {} + const expected = resolve(root, ".husky/_"); + if (configured && resolve(root, configured) !== expected) throw Error("Custom hooksPath detected; left unchanged"); + if (install && configured && resolve(root, configured) === expected) { + try { + accessSync(resolve(root, ".husky/_/pre-commit"), constants.X_OK); + accessSync(resolve(root, ".husky/_/h"), constants.R_OK); + accessSync(resolve(root, ".husky/pre-commit"), constants.R_OK); + return "active"; + } catch {} + } + if (install) { + execFileSync(process.execPath, [resolve(root, "node_modules/husky/bin.js")], { cwd: root, env, stdio: "pipe" }); + configured = git("config", "--get", "core.hooksPath"); + } + if (!configured || resolve(root, configured) !== expected) + throw Error("core.hooksPath must resolve to this checkout's .husky/_"); + accessSync(resolve(root, ".husky/_/pre-commit"), constants.X_OK); + accessSync(resolve(root, ".husky/_/h"), constants.R_OK); + accessSync(resolve(root, ".husky/pre-commit"), constants.R_OK); + return "active"; +} diff --git a/scripts/test-dashboard-presentation.mjs b/scripts/test-dashboard-presentation.mjs index 878f0ac1..6064c66d 100644 --- a/scripts/test-dashboard-presentation.mjs +++ b/scripts/test-dashboard-presentation.mjs @@ -3,7 +3,7 @@ import { createServer } from "node:net"; import { spawn } from "node:child_process"; import { once } from "node:events"; -import { mkdirSync, openSync, closeSync, existsSync } from "node:fs"; +import { mkdirSync, openSync, closeSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -25,10 +25,8 @@ const freePort = async () => { const apiPort = await freePort(); let webPort = await freePort(); while (webPort === apiPort) webPort = await freePort(); -const edge = "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"; const env = { ...process.env, - ...(process.env.BROWSER_EXECUTABLE ? {} : existsSync(edge) ? { BROWSER_EXECUTABLE: edge } : {}), VITE_API_BASE_URL: "", VITE_DEV_API_BASE_URL: `http://127.0.0.1:${apiPort}`, VITE_DEV_WORKER_BASE_URL: "http://127.0.0.1:1", @@ -86,7 +84,16 @@ try { ); const web = start( process.execPath, - ["node_modules/vite/bin/vite.js", "--host", "127.0.0.1", "--port", String(webPort), "--strictPort"], + [ + "node_modules/vite/bin/vite.js", + "--config", + "tests/browser/vite.platform-feedback.config.ts", + "--host", + "127.0.0.1", + "--port", + String(webPort), + "--strictPort", + ], "web.log", ); await Promise.all([ready(`http://127.0.0.1:${apiPort}/api/health`, api), ready(`http://127.0.0.1:${webPort}/`, web)]); diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 47d11f09..7ca83ae3 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1,6 +1,137 @@ { "version": 1, "scopes": [ + { + "name": "legacy-v4-release", + "paths": [ + "^internal/domain/types\\.go$", + "^internal/worker/(legacy_delivery_.*|capture_delivery|capture_cutover_test|delivery_audit|executor|human_release|original_source_recheck|runtime)\\.go$" + ], + "frontend": [], + "go": { + "./internal/worker": [ + "^Test(LegacyCapture|CaptureCutover|CaptureDelivery|V4|Original|DispatchHumanRelease|HumanRelease|ReleasePartition|SourceIndexPackage|RecoverSucceeded|Retry|Runtime)" + ] + }, + "task": [ + { + "argv": ["npm", "run", "test:go:gate"], + "tests": true + }, + { + "argv": [ + "python3", + "-m", + "pytest", + "tests/backend/test_delivery_v4_package.py", + "tests/backend/test_delivery_v4_real_materialization.py", + "tests/backend/test_legacy_package_compatibility.py", + "-q" + ], + "tests": true + } + ], + "warnings": [ + "V4 格式防护与旧任务运行快照只用隔离存储验证;缺少历史冻结事实的生产任务仍须单独恢复,不以旧格式回退。" + ] + }, + { + "name": "delivery-v4-preview", + "paths": ["^scripts/delivery_v4_preview\\.py$", "^tests/backend/test_delivery_v4_preview\\.py$"], + "frontend": [], + "go": {}, + "task": [ + { + "argv": [ + "python3", + "-m", + "pytest", + "tests/backend/test_delivery_v4_preview.py", + "tests/backend/test_delivery_v4_package.py", + "tests/backend/test_delivery_v4_real_materialization.py", + "tests/backend/test_legacy_package_compatibility.py", + "-q" + ], + "tests": true + } + ], + "warnings": ["仅验证本地 V4 转换预览;完整内部审计、真实视频切片与正式发布另行验收。"] + }, + { + "name": "verification-tooling", + "paths": [ + "^scripts/(check-git-hooks\\.mjs|lib/(git-hooks|browser-executable)\\.mjs|verify-change\\.py|verification-scopes\\.json)$", + "^tests/tooling/", + "^tests/frontend/(git-hooks|browser-executable|guardrails-ci)\\.test\\.mjs$", + "^\\.gitea/workflows/web-ci\\.yml$", + "^Dockerfile\\.api$" + ], + "frontend": [ + "tests/frontend/git-hooks.test.mjs", + "tests/frontend/browser-executable.test.mjs", + "tests/frontend/guardrails-ci.test.mjs", + "tests/frontend/dev-guardrails.test.mjs" + ], + "go": {}, + "task": [] + }, + { + "name": "verification-browser-harnesses", + "paths": [ + "^tests/browser/(review-drafts|dashboard-presentation-replica|run-editing-regression)\\.mjs$", + "^scripts/test-dashboard-presentation\\.mjs$" + ], + "frontend": ["tests/frontend/browser-executable.test.mjs"], + "go": {}, + "task": [ + { + "argv": ["npm", "run", "test:browser:editing"], + "tests": true + } + ] + }, + { + "name": "production-boot-harness", + "paths": ["^tests/browser/run-production-boot\\.mjs$"], + "frontend": ["tests/frontend/browser-executable.test.mjs"], + "go": {}, + "task": [ + { + "argv": ["npm", "run", "build:web"] + }, + { + "argv": ["npm", "run", "test:production:boot"], + "requires": [["npm", "run", "build:web"]], + "tests": true + } + ] + }, + { + "name": "acceptance-playback-and-feedback", + "paths": [ + "^src/components/FinalAcceptanceWorkspace\\.tsx$", + "^tests/browser/acceptance-playback-rate\\.mjs$", + "^tests/browser/fixtures/acceptance-playback-rate\\.tsx$", + "^src/components/(ReviewCaptions|review-viewing)\\.tsx?$", + "^src/hooks/useReviewDraft\\.ts$", + "^src/lib/(review-caption-preferences\\.js|review-navigation\\.js|review-opinions\\.ts)$", + "^src/styles/review-captions\\.css$" + ], + "frontend": [ + "tests/frontend/review-target-interaction.test.mjs", + "tests/frontend/review-caption-preferences.test.mjs", + "tests/frontend/review-captions.test.mjs" + ], + "go": {}, + "task": [ + { + "argv": ["node", "tests/browser/acceptance-playback-rate.mjs"], + "env": {}, + "tests": true + } + ], + "warnings": ["浏览器使用真实验收组件、三种播放器模式与隔离合成媒体/批注 API;不连接生产。"] + }, { "name": "review-navigation", "paths": [ @@ -8,13 +139,9 @@ "^internal/api/review_(continuation|trace)", "^tests/browser/review-trace\\.mjs$" ], - "frontend": [ - "tests/frontend/annotation-project-loading.test.mjs" - ], + "frontend": ["tests/frontend/annotation-project-loading.test.mjs"], "go": { - "./internal/api": [ - "^TestReviewContinuation" - ] + "./internal/api": ["^TestReviewContinuation"] }, "task": [ { @@ -54,9 +181,7 @@ "^TestWorkBufferRetains", "^TestRandomClaim" ], - "./internal/store": [ - "^TestSourceAdmissionMemoryClaimFence$" - ] + "./internal/store": ["^TestSourceAdmissionMemoryClaimFence$"] }, "task": [ { @@ -92,9 +217,7 @@ "tests": true } ], - "warnings": [ - "涉及准入/轮询:核对真实规模下查询次数、扫描量、超时与并发;局部回归不等于容量通过。" - ] + "warnings": ["涉及准入/轮询:核对真实规模下查询次数、扫描量、超时与并发;局部回归不等于容量通过。"] }, { "name": "pool-search", @@ -106,13 +229,11 @@ ], "frontend": [ "tests/frontend/annotation-pool-*.test.mjs", - "tests/frontend/review-pool-preview.test.mjs" + "tests/frontend/review-pool-preview.test.mjs", + "tests/frontend/review-pool-search.test.mjs" ], "go": { - "./internal/api": [ - "^TestHumanPoolSearch", - "^TestHumanPoolQCProjectScope" - ] + "./internal/api": ["^TestHumanPoolSearch", "^TestHumanPoolQCProjectScope"] }, "task": [ { @@ -130,6 +251,22 @@ "REVIEW_POOL_BROWSER_TEST": "1" }, "tests": true + }, + { + "argv": [ + "go", + "test", + "./internal/store", + "-run", + "^TestHumanPoolSearchMongo", + "-count=1", + "-timeout=3m", + "-v" + ], + "env": { + "POOL_LOCAL_MONGO_URI": "mongodb://127.0.0.1:27018" + }, + "tests": true } ] }, @@ -172,7 +309,8 @@ "^src/(pages/AnnotationOperations\\.tsx|components/CaptureAnnotationWorkbench\\.tsx)$", "^src/.*([Rr]eview[Dd]raft|review-draft|[Rr]eview[Dd]ecision|review-decision)", "^internal/api/review_(draft|decision|confirmation)", - "^tests/browser/review-(draft|decision|authoring)" + "^tests/browser/review-(draft|decision|authoring)", + "^tests/browser/review-draft-persistence\\.mjs$" ], "frontend": [ "tests/frontend/review-decision-controller.test.mjs", @@ -183,27 +321,15 @@ "go": {}, "task": [ { - "argv": [ - "npm", - "run", - "test:browser:review-drafts" - ], + "argv": ["npm", "run", "test:browser:review-drafts"], "tests": true }, { - "argv": [ - "npm", - "run", - "test:browser:review-decisions" - ], + "argv": ["npm", "run", "test:browser:review-decisions"], "tests": true }, { - "argv": [ - "npm", - "run", - "test:browser:review-permissions" - ], + "argv": ["npm", "run", "test:browser:review-permissions"], "tests": true } ] @@ -226,24 +352,15 @@ "tests/frontend/production-quality-dates.test.mjs" ], "go": { - "./internal/api": [ - "^TestProductionDashboardScopeIsolationAndFullCorpus$" - ] + "./internal/api": ["^TestProductionDashboardScopeIsolationAndFullCorpus$"] }, "task": [ { - "argv": [ - "npm", - "run", - "test:browser:dashboard" - ], + "argv": ["npm", "run", "test:browser:dashboard"], "tests": true }, { - "argv": [ - "python3", - "scripts/test_dashboard_stability.py" - ] + "argv": ["python3", "scripts/test_dashboard_stability.py"] }, { "argv": [ @@ -261,30 +378,20 @@ "tests": true } ], - "warnings": [ - "统计/查询改动须额外记录同数据规模的性能对照;完整 120 人容量另行验收。" - ] + "warnings": ["统计/查询改动须额外记录同数据规模的性能对照;完整 120 人容量另行验收。"] }, { "name": "production-statistics-model", - "paths": [ - "^internal/productionstats/model\\.go$" - ], + "paths": ["^internal/productionstats/model\\.go$"], "frontend": [], "go": {}, "task": [ { - "argv": [ - "npm", - "run", - "test:go:gate" - ], + "argv": ["npm", "run", "test:go:gate"], "tests": false } ], - "warnings": [ - "共享生产统计事实结构改动:使用既有 Go 基线门禁比较历史失败,不扩大失败基线。" - ] + "warnings": ["共享生产统计事实结构改动:使用既有 Go 基线门禁比较历史失败,不扩大失败基线。"] } ] } diff --git a/scripts/verify-change.py b/scripts/verify-change.py index 7ff8e4e8..66cc451f 100644 --- a/scripts/verify-change.py +++ b/scripts/verify-change.py @@ -28,7 +28,10 @@ def changed_files(root, base): def command(argv, env=None, tests=False): - return {'argv': argv, 'env': env or {}, 'tests': tests} + result = {'argv': argv, 'env': env or {}, 'tests': tests} + if argv == ['npm', 'run', 'test:production:boot']: + result['requires'] = [['npm', 'run', 'build:web']] + return result def tooling_package_change(root, base): @@ -39,12 +42,12 @@ def tooling_package_change(root, base): return False old_scripts, new_scripts = old.pop('scripts', {}), new.pop('scripts', {}) names = {name for name in set(old_scripts) | set(new_scripts) if old_scripts.get(name) != new_scripts.get(name)} - allowed = {'verify:plan', 'verify:quick', 'verify:task', 'verify:integration', 'test:verification'} + allowed = {'verify:plan', 'verify:quick', 'verify:task', 'verify:integration', 'test:verification', 'predev', 'predev:full', 'predev:api', 'predev:web', 'predev:worker', 'predev:api:replica'} return old == new and bool(names) and names <= allowed def plan_checks(root, files, policy, mode, base, tooling_package=False): - quick = [command(['git', 'diff', '--check', base, '--'])] + quick = [command(['git', 'diff', '--check', base, '--']), command(['node', 'scripts/check-git-hooks.mjs', '--install'])] task = [] warnings = [] unresolved = [] @@ -89,7 +92,7 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): fast = [] if test_file.exists(): names = re.findall(r'^func (Test\w+)\(', test_file.read_text(), re.M) - fast = [name for name in names if not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark', name)] + fast = [name for name in names if not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark|Preview', name)] if fast: go_patterns.setdefault(pkg, set()).update('^' + name + '$' for name in fast) if len(fast) != len(names) and f not in matched: @@ -100,6 +103,9 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): # Package-wide fallback is explicit at task level, not every edit. task.append(command(['go', 'test', pkg, '-timeout=5m', '-v'], tests=True)) continue + if f in ('scripts/verify-change.py', 'scripts/verification-scopes.json') or f.startswith('tests/tooling/'): + quick.append(command([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests/tooling', '-p', 'test_verification.py'], tests=True)) + continue if f in matched or f.endswith('.md') or f.startswith('docs/') and f.endswith(('.png', '.jpg', '.svg')): continue if f.startswith('src/') and f.endswith('.css'): @@ -120,9 +126,6 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): command(['npm', 'run', 'build:web']), command(['npm', 'run', 'test:production:boot'])]) warnings.append('依赖/公共配置变化:任务级扩大到集成检查,核对安装与锁文件。') continue - if f in ('scripts/verify-change.py', 'scripts/verification-scopes.json') or f.startswith('tests/tooling/'): - quick.append(command([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests/tooling', '-p', 'test_verification.py'], tests=True)) - continue if path.exists() and f.endswith('.py'): quick.append(command([sys.executable, 'scripts/verify-change.py', '--python-syntax', f])) elif path.exists() and f.endswith('.mjs'): @@ -131,9 +134,24 @@ def plan_checks(root, files, policy, mode, base, tooling_package=False): quick.append(command(['bash', '-n', f])) unresolved.append(f + ': 尚无行为验证映射;补充映射/专项证据,语法通过不代表功能通过。') if frontend: - quick.append(command(['node', '--test', *sorted(frontend)], tests=True)) + known = json.loads((root/'tests/known-failures.json').read_text()).get('frontend', []) + known_files = {item.split(' :: ', 1)[0] for item in known} + if any(Path(name).name in known_files for name in frontend): + quick.append(command(['npm', 'run', 'test:frontend:gate'])) + else: + quick.append(command(['node', '--test', *sorted(frontend)], tests=True)) for pkg, patterns in sorted(go_patterns.items()): - quick.append(command(['go', 'test', pkg, '-run', '|'.join(sorted(patterns)), '-timeout=2m', '-v'], tests=True)) + names = set() + for test_file in (root/pkg).glob('*_test.go'): + names.update(re.findall(r'^func (Test\w+)\(', test_file.read_text(), re.M)) + selected_names = sorted(name for name in names if any(re.search(p, name) for p in patterns) + and not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark|Preview', name)) + if selected_names: + known_go = json.loads((root/'tests/known-failures.json').read_text()).get('go', []) + if any(pkg.removeprefix('./') + ' :: ' + name in known_go for name in selected_names): + quick.append(command(['npm', 'run', 'test:go:gate'], tests=True)) + else: + quick.append(command(['go', 'test', pkg, '-run', '|'.join('^' + name + '$' for name in selected_names), '-timeout=2m', '-v'], tests=True)) integration = [command(['npm', 'run', x]) for x in ['lint:web', 'format:check', 'test:frontend:gate', 'test:go:gate', 'build:web', 'test:production:boot']] def unique(commands): @@ -216,6 +234,24 @@ def run_check(check, root, logfile, timeout): return {**check, 'status': status, 'exitCode': code, 'seconds': round(time.monotonic()-start, 3), 'log': str(logfile)} +def run_checks(checks, root, folder, budget): + results = [] + start = time.monotonic() + for index, check in enumerate(checks): + remaining = budget - (time.monotonic() - start) + if remaining <= 0: + results.append({**check, 'status': 'not-run', 'reason': 'budget exhausted'}) + continue + if any(not any(r['argv'] == required and r['status'] == 'passed' for r in results) + for required in check.get('requires', [])): + results.append({**check, 'status': 'not-run', 'reason': 'dependency did not pass'}) + continue + result = run_check(check, root, folder/f'{index:02}.log', remaining) + results.append(result) + print(f"{result['status']}: {' '.join(check['argv'])} ({result['seconds']}s)", flush=True) + return results + + def runtime_identity(root): versions = {'python': sys.version.split()[0]} for name, argv in [('node', ['node', '--version']), ('go', ['go', 'version'])]: @@ -259,17 +295,7 @@ def main(argv=None): 'environmentDigest': hashlib.sha256(json.dumps(dict(os.environ), sort_keys=True).encode()).hexdigest(), 'startedAt': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'results': []} start = time.monotonic() - for index, check in enumerate(plan['commands']): - remaining = budget - (time.monotonic()-start) - if remaining <= 0: - report['results'].append({**check, 'status': 'not-run', 'reason': 'budget exhausted'}) - continue - result = run_check(check, ROOT, folder/f'{index:02}.log', remaining) - report['results'].append(result) - print(f"{result['status']}: {' '.join(check['argv'])} ({result['seconds']}s)", flush=True) - if result['status'] != 'passed': - report['results'].extend({**c, 'status': 'not-run', 'reason': 'previous check did not pass'} for c in plan['commands'][index+1:]) - break + report['results'] = run_checks(plan['commands'], ROOT, folder, budget) report['inputsChangedDuringRun'] = fingerprint(ROOT) != before statuses = {r['status'] for r in report['results']} code = 1 if 'failed' in statuses else 2 if (statuses - {'passed'} or plan['unmapped'] or report['inputsChangedDuringRun']) else 0 diff --git a/src/components/CaptureAnnotationWorkbench.tsx b/src/components/CaptureAnnotationWorkbench.tsx index a84181e6..396f9ba1 100644 --- a/src/components/CaptureAnnotationWorkbench.tsx +++ b/src/components/CaptureAnnotationWorkbench.tsx @@ -16,6 +16,7 @@ import { userErrorMessage } from "../lib/user-error-message.js"; import { reviewCommandRequiresWrite, reviewShortcut, reviewShortcutHelp } from "../lib/review-shortcuts.js"; import type { ReviewViewingProps, ReviewViewingController, ReviewPosition, ReviewNavigation } from "./review-viewing"; import { ReviewCaptionDetails, ReviewVideoCaption, type ReviewCaption } from "./ReviewCaptions"; +import { readReviewCaptionPlacement, writeReviewCaptionPlacement } from "../lib/review-caption-preferences.js"; import { reviewCaptionAtTime } from "../lib/review-captions.js"; import { sliceEditActionsWithDrafts, workbenchDraftsFingerprint } from "../lib/annotation-autosave.js"; import { DiscardReasonFields, type DiscardReasonChoice } from "./DiscardReasonFields"; @@ -320,6 +321,8 @@ type Props = ReviewViewingProps & { browserReadyMedia?: boolean; seekRequest?: { time: number; revision: number } | null; cameraPreferenceKey?: string; + playbackRate?: number; + onPlaybackRateChange?: (rate: number) => void; continuousActions?: boolean; progressSaved?: boolean; onSaveProgress?: (segments: AnnotationSegment[]) => Promise; @@ -471,6 +474,8 @@ export function CaptureAnnotationWorkbench({ browserReadyMedia = false, seekRequest, cameraPreferenceKey = "", + playbackRate: preferredPlaybackRate, + onPlaybackRateChange, continuousActions = false, progressSaved = true, onSaveProgress, @@ -586,8 +591,10 @@ export function CaptureAnnotationWorkbench({ }, [selectedSliceId, onSelectedSliceChange]); const [currentTime, setCurrentTime] = useState(clipStart); const [captionsVisible, setCaptionsVisible] = useState(true); + const [captionPlacement, setCaptionPlacement] = useState(readReviewCaptionPlacement); const [sliceZoom, setSliceZoom] = useState(false); - const [playbackRate, setPlaybackRate] = useState(1); + const [localPlaybackRate, setLocalPlaybackRate] = useState(1); + const playbackRate = preferredPlaybackRate ?? localPlaybackRate; const [showLeft, setShowLeft] = useState(initialCameraVisibility?.left ?? true); // Ordinary layouts load the left stream only until the right checkbox is // explicitly selected. Focused camera preferences are applied separately. @@ -2759,6 +2766,7 @@ export function CaptureAnnotationWorkbench({ const reviewSelectedActionId = reviewNavigation?.selectedActionId || editingAnnotation?.id || null; const reviewPosition: ReviewPosition = { time: currentTime, + discarded: selectedSlice.deliveryDisposition === "discard" || providedReviewCaption?.kind === "discarded", timeBasis: mediaOnly ? "slice-seconds" : "capture-seconds", sliceId: selectedSlice.id, navigationSliceId: selectedSlice.id, @@ -2784,6 +2792,7 @@ export function CaptureAnnotationWorkbench({ currentTime, selectedSlice.id, reviewPosition.actionId, + reviewPosition.discarded, reviewSelectedActionId, timelineReady, onReviewPositionChange, @@ -2835,6 +2844,7 @@ export function CaptureAnnotationWorkbench({ } const position: ReviewPosition = { time, + discarded: owner.deliveryDisposition === "discard" || providedReviewCaption?.kind === "discarded", timeBasis: mediaOnly ? "slice-seconds" : "capture-seconds", sliceId: owner.id, navigationSliceId: owner.id, @@ -2958,7 +2968,8 @@ export function CaptureAnnotationWorkbench({ const sliceBoundaryIssue = timelineReady ? sliceTimelineIssue(slices, clipStart, clipEnd, frameBoundaries) : ""; const changePlaybackRate = (rate: number) => { if (saving || phase === "materializing" || !CAPTURE_PLAYBACK_RATES.includes(rate)) return; - setPlaybackRate(rate); + setLocalPlaybackRate(rate); + onPlaybackRateChange?.(rate); if (leftVideoRef.current) leftVideoRef.current.playbackRate = rate; if (rightVideoRef.current) rightVideoRef.current.playbackRate = rate; }; @@ -3988,6 +3999,23 @@ export function CaptureAnnotationWorkbench({ > {captionsVisible ? "隐藏字幕" : "显示字幕"} + {isReviewViewing && captionsVisible && ( + + )} {focusedLayout ? (