forked from longtao.wu/droboticflow
🛡️ Gate invalid source tasks and fix complete review pool search
This commit is contained in:
parent
99af829a42
commit
63714753a9
@ -14,3 +14,4 @@ Risk: eligibility, legacy source compatibility, source retry and draft preservat
|
||||
- Pool lane owns internal/api/human_loop.go list handler, pool search storage support, src/services/api/human-loop*, src/lib/annotation-project-loading*, pool hooks/UI in AnnotationOperations.tsx, and related tests/docs. Do not edit source validation/ingestion modules or human_loop.go claim helpers. Coordinate shared types before editing.
|
||||
- User's explicit authorization boundary overrides repository defaults: remain local; do not push/create PR/deploy or mutate company data.
|
||||
- Contract/meta lane owns worker/capture_layout.go raw-human default required files, authoritative backend/operators/builtin/orbbec_data_normalize/operator.py and generated mirror, related normalization tests/docs. Reuse documented segment metadata checks; reject malformed IMU while preserving legitimate part boundary gaps.
|
||||
- Pool lane reused only QC candidate project/supplier scope and its two regression files from existing local commit b2324c19; unrelated workforce/statistics changes are not included. Modern Mongo search prefilter preserves old metadata fallback, no new indexes. Frontend source retry/timeline edits remain root-owned.
|
||||
|
||||
@ -120,8 +120,10 @@ def _validate_transcoded_pts(video: pathlib.Path, pts: pathlib.Path, expected_fr
|
||||
return {"videoFrames": frames, "ptsRows": count, "sourceDurationSeconds": duration, "startTimestampUs": first, "endTimestampUs": previous, "validation": "count_and_timestamp_checks_passed", "timestamps": "original_unmodified"}
|
||||
|
||||
|
||||
def _validate_raw_imu(path: pathlib.Path, start: int, end: int) -> None:
|
||||
def _validate_raw_imu(path: pathlib.Path, start: int, end: int, *, require_coverage: bool = True) -> None:
|
||||
ranges = {}
|
||||
previous = None
|
||||
pair = set()
|
||||
with path.open(encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
if not {"timestamp_us", "x", "y", "z", "type"}.issubset(reader.fieldnames or []):
|
||||
@ -133,9 +135,27 @@ def _validate_raw_imu(path: pathlib.Path, start: int, end: int) -> None:
|
||||
raise ValueError("invalid value")
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行无效") from exc
|
||||
kind = row["type"].strip().lower()
|
||||
kind = (row.get("type") or "").strip().lower()
|
||||
if kind not in {"accel", "gyro"}:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行类型无效,只支持 accel/gyro")
|
||||
if previous is not None and timestamp < previous:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行时间戳未递增")
|
||||
if timestamp != previous:
|
||||
if previous is not None and pair != {"accel", "gyro"}:
|
||||
raise RuntimeError(f"原始 IMU 时间戳 {previous} 缺少 accel/gyro 配对")
|
||||
pair = set()
|
||||
if kind in pair:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行 {kind} 重复")
|
||||
pair.add(kind)
|
||||
previous = timestamp
|
||||
low, high = ranges.get(kind, (timestamp, timestamp))
|
||||
ranges[kind] = (min(low,timestamp), max(high,timestamp))
|
||||
if previous is None:
|
||||
raise RuntimeError("原始 IMU 没有有效数据")
|
||||
if pair != {"accel", "gyro"}:
|
||||
raise RuntimeError(f"原始 IMU 时间戳 {previous} 缺少 accel/gyro 配对")
|
||||
if not require_coverage:
|
||||
return
|
||||
for kind in ("accel", "gyro"):
|
||||
if kind not in ranges or ranges[kind][0] > start or ranges[kind][1] < end:
|
||||
raise RuntimeError(f"原始 IMU {kind} 未覆盖视频 PTS 时间范围")
|
||||
@ -218,9 +238,8 @@ def _validate_fallback_pair(path, paired, entries, source_path, output_dir, para
|
||||
if raw_human:
|
||||
if not paired.get("imu"):
|
||||
raise RuntimeError("原始数据人工版缺少 IMU")
|
||||
# Temporarily bypass the raw-human IMU content/coverage gate: source
|
||||
# sensor streams can cross part boundaries. Keep the paired IMU intact.
|
||||
# _validate_raw_imu(materialize(paired["imu"]), validation["startTimestampUs"], validation["endTimestampUs"])
|
||||
# Validate content without requiring the part's IMU to cover video endpoints.
|
||||
_validate_raw_imu(materialize(paired["imu"]), validation["startTimestampUs"], validation["endTimestampUs"], require_coverage=False)
|
||||
return validation
|
||||
|
||||
|
||||
|
||||
@ -48,3 +48,17 @@ npm run test:frontend:gate
|
||||
HTTP、Mongo、审核决定、交付 JSON 和后台任务协议均未修改,不需要数据迁移;回退前端会恢复原来的加载及分页行为。布局 agent 仍只接触展示配置,不接入任务可见性、领取、审核或交付判定。
|
||||
|
||||
审核会话、意见草稿和正式决定已经各有现有 hook。正式决定状态机及决定完成后的页面编排已在[第四阶段](review-decision-coordination.md)收敛;意见草稿、会话与连续审核保留各自职责。独立镜像发布、JSON v4 和实际 agent 尚未实现。
|
||||
|
||||
## 2026-11-02:审核池全范围搜索与分页
|
||||
|
||||
质检与验收工作区改用独立的 `review` 读取模式:可领取、我的任务、已审核分别保存服务端游标,每页最多 200 条;切换页签复用已加载页,加载更多只读取当前池。验收将原来的两个阶段合并为一次 `stageIds` 查询,不再通过 `listAllItems` 逐页拉完整池。任务关联仍按 50 个 ID、最多 4 个并发读取;前端显示“已加载”及未加载完的 `+`,不冒充实际总量。
|
||||
|
||||
`GET /api/human-items` 兼容新增 `search/capture/category`。搜索在分页之前执行,支持 Capture、part、项目名称及任务身份;`Capture/part0002` 也匹配旧式 `Capture_camera_left_part0002.mp4`。输入按普通文字匹配,不执行用户提供的正则表达式。正常排序仍为稳定的 `createdAt/id` 降序,旧任务新送检不会改写导入时间,但能够通过全范围搜索及继续分页找到;不以易变的更新时间作为分页键。
|
||||
|
||||
Mongo 对有权威 Capture 记录的数据先筛选名称和状态,仅返回匹配候选;缺少该来源的旧记录保留原有元数据回退。查询保持 8 秒超时、200 条传输批次及取消传播,不加载工作区草稿。质检可领取候选进一步使用本请求核对的项目和供应商必要范围,复用 `b2324c19` 中的范围收缩及等价测试;共享验收、已审核历史不套用这项供应商收缩。候选仍经过原服务端实时可见性与 `canClaim` 校验,没有权限缓存或新增索引。
|
||||
|
||||
已审核分类沿用原有展示口径:三角色流程中,质检通过的来源任务在最终验收未通过前仍显示“待验收”,最终验收通过后才显示“已通过”。筛选在分页前按这项有效状态判断;每批只批量补充验收状态,不逐条加载下游任务正文,也不改写来源任务。`TestReviewedHistoryCategoryPaginationUsesDownstreamStatus` 覆盖 205 条混合状态历史记录、连续分页、无重复/遗漏及返回详情完整性。
|
||||
|
||||
搜索、状态、项目和账号变化会清空旧游标并取消旧请求;迟到回包不能覆盖新列表,续页失败保留原游标供重试。新接口返回 `filtersApplied: true`。新前端在筛选时遇到旧服务未返回该标记会提示版本暂不支持,不显示未经筛选的数据;旧客户端忽略新增字段。发布时先升级服务端,再启用前端筛选。
|
||||
|
||||
本地证据:`TestHumanPoolSearch*` 覆盖前方有 962 条新任务的旧 part、三种池各 611 条连续分页、供应商隔离、自审禁止和普通文字匹配;`TestHumanPoolSearchMongo*` 对照 Memory/Mongo 与 1 万条数据;`TestReviewPoolSearchBrowserReplica` 使用真实 HTTP、合成账号与真实页面验证搜索、分页、失败重试和旧回包处理。完整容量仍须独立验收。
|
||||
|
||||
37
docs/architecture/source-validation-admission.md
Normal file
37
docs/architecture/source-validation-admission.md
Normal file
@ -0,0 +1,37 @@
|
||||
# 源数据校验与标注准入
|
||||
|
||||
2026-11-02:修复文件已接入、后台校验失败而任务仍能领取的问题。接入文件规则见[采集布局契约](../capture-layout-compatibility.md),固定源与媒体校验沿用 [ADR 0010](../adr/0010-source-backed-slice-revisions.md)。
|
||||
|
||||
## 状态与准入
|
||||
|
||||
- “文件齐全”只证明所选目录的必需对象存在、非空并稳定,不代表 PTS、IMU、视频或 meta 内容有效。
|
||||
- 现代完整视频任务(`sourceIsFullVideo && templateVersion >= 5 && artifactId`)必须有本任务的时间轴校验成功记录。记录缺失、排队、运行或失败时不进入可领取集合;既有后台队列负责实际校验,不在轮询或领取时重复解析整个源文件。
|
||||
- 本任务当前切片/送检 outbox 所引用的源证书作业如果排队或失败,同样阻断分配。新 outbox 明确保存私有 `sourceIndex` 标志;已物化的 `materialization.sourceIndex` 也构成必须有证书的证据。已知源索引任务缺少所需作业时不放行。
|
||||
- 历史物理切片 outbox 也带 `sourceFingerprint`,该字段单独存在不能证明它使用源证书协议。无源索引证据、无校验作业的旧格式任务保持旧路径,展示为未核验而非伪称校验成功;任何已存在的失败记录仍生效。
|
||||
- 随机候选列表、可领取数量与最终原子领取都核对同一条件,指定领取也在存储层核对。失败不会清空任务、解除已有归属或重写标注草稿。指定领取以持久化的当前源状态为准;即使 outbox 变化未改变任务版本,陈旧领取快照也不能覆盖新作业。管理员仍可查看异常任务和既有资料。
|
||||
|
||||
## 诊断与恢复
|
||||
|
||||
任务详情和状态返回兼容新增字段 `sourceValidation`:`status`、`code`、`message`、`required`、`retryAllowed`。状态为 `valid / pending / failed / unverified`。`frame-timeline` 被阻断时返回 422,稳定错误码为 `source_validation_pending / source_validation_failed / source_validation_unverified`,同一摘要位于 `details.sourceValidation`。
|
||||
|
||||
普通操作者只看到安全的具体事实,例如 IMU 行号、非法类型或视频与时间戳不一致;不透传凭证、路径、堆栈或整段后台日志。源数据提供方负责修复,管理员在确认后重新校验。工作区保留必要阻断提示与恢复操作,不增加常驻教程。
|
||||
|
||||
组织管理员调用 `POST /api/plugins/annotation-qa/tasks/{id}/source-validation-retries`,提交 `{ "sourceRepaired": true }`。服务端再次校验组织和任务权限。已有排队/运行中的校验拒绝重复重置;重新校验不等于已恢复,只有校验完成才能再次分配。
|
||||
|
||||
源索引恢复由任务 ID 的校验作业完整验证当前源版本并固定其证书;记录实际成功的 `sourceFingerprint` 和被替代的旧 outbox 指纹 `supersededSourceFingerprint`。旧 `source-v1:A` 作业仍保留失败事实,不以 B 的验证结果把 A 改成成功。该替代只适用于确切旧指纹 A;之后 C 的排队、缺证书或失败不能继承 B 的放行。
|
||||
|
||||
修复过程不改写已有切片/标注内容、不替换审核冻结版本、不自动重放旧失败送检。用户可按原流程继续编辑或确认处理;旧处理输入与当前源不一致时仍要求重新确认。
|
||||
|
||||
工作台若在已通过的任务中发现新的源校验错误,会按读取时的作业代次把该记录标为失败。新一轮重校验开始后,迟到的旧请求或旧校验器不能覆盖它。存储写入与领取使用既有项目写保护;不持锁解析视频。读取取消、超时或网络故障不作为内容错误撤销准入。
|
||||
|
||||
旧任务缺少校验记录时仍沿用兼容路径;但时间轴实际发现内容异常后,会原子插入失败记录并提供管理员重校验入口。插入不能覆盖并发开始或完成的新校验,不要求所有历史任务批量重验。只有历史 `materialization.sourceIndex` 标志、没有 outbox 指纹的任务,明确重校验成功且记录当前源指纹后才能恢复;后续出现新 outbox 指纹仍必须核对其校验状态。
|
||||
|
||||
源文件由责任方明确修复;平台不自动改写、舍弃或删除源文件。舍弃仍需原有明确业务操作。
|
||||
|
||||
## 查询成本与兼容
|
||||
|
||||
普通轮询只读取状态记录;领取列表和计数按候选身份使用现有 ID 索引,进入聚合前投影必要字段,不传输标注大载荷。没有新增长期权限缓存或生产索引。
|
||||
|
||||
本地 1,200 条任务、每条 64 KiB subject 的 Mongo 夹具,原始 subject 共 75 MiB,候选 JSON 为约 0.49 MiB。新增状态关联比原来不做准入判断的计数更贵;实际对照数值见[验证记录](../verification/source-validation-admission.md)。该测试不能替代 120 人完整作业验收。
|
||||
|
||||
接口新增字段可由旧客户端忽略。恢复作业需要理解 `revalidateSource` 的新版校验器;旧校验器仅完成时间轴而未提供源指纹证明时,新 API 保持未核验,不据此放行。发布须协调 API、校验器及接入算子版本;未执行生产回填、生产源修复或上线。
|
||||
@ -3,7 +3,7 @@
|
||||
- Flat legacy directories keep the existing filename requirements and one Run per directory. Nested files cannot satisfy root requirements. Multiple candidates for a required type block admission rather than selecting the first file.
|
||||
- A direct `part[0-9]+` child selects segmented layout. Each part is an independent unit, identified by workflow + parent prefix + part. Capture names are not restricted to FYLX or a date pattern.
|
||||
- Part video names must contain the actual Capture name, camera side and actual part name. Original `.mp4/.mov/.mkv` and suffixed variants such as `_silent_480p_wm.mp4` are recognized. Multiple variants block intake as ambiguous.
|
||||
- Part PTS and IMU names must match the Capture and part exactly. Video-specific `<video-stem>_pts.csv` takes precedence over raw part PTS when present. Only the parent's two `<capture>_calibration_camera.yaml` / `<capture>_calibration_imu.yaml` files are shared. Audio and meta are optional.
|
||||
- Part PTS and IMU names must match the Capture and part exactly. Video-specific `<video-stem>_pts.csv` takes precedence over raw part PTS when present. Only the parent's two `<capture>_calibration_camera.yaml` / `<capture>_calibration_imu.yaml` files are shared. Audio remains optional. Metadata requirements for raw-human and v4 intake are defined below; other legacy workflows keep meta optional.
|
||||
- Root sensor data plus part directories is a structural conflict. Missing/empty files block only their unit; missing shared calibration blocks all affected parts. Custom required patterns remain supported for flat layouts; for part layouts they add constraints within the selected unit, never permit cross-part matching.
|
||||
- A directory reference records the parent `prefix`, explicit `objectKeys` and display `captureUnit`. Both directory reading and normalization enforce the selection, including on retry. Missing selected files fail closed. Each part's fingerprint includes only its selected files and shared calibration/meta.
|
||||
- Existing admitted parent records retain their Run association and are not split/relaunched by scanning. Discarded or launching parents also prevent child admissions. No completion manifest is required; non-empty required files and the existing stability window still apply.
|
||||
@ -13,6 +13,16 @@
|
||||
|
||||
Filename compatibility does not prove frame correspondence. In particular, a transcoded `_silent_480p_wm.mp4` plus raw `_part0001_pts.csv` is recognized by intake but remains subject to the existing normalizer rejection when a matching transformed-video PTS/frame mapping is absent. This change must not invent timestamps or silently bypass that safety check.
|
||||
|
||||
### 2026-11-02: raw-human source admission
|
||||
|
||||
For newly launched workflows whose Orbbec normalizer uses `PROCESSING_PROFILE=raw_human`, the selected files must include exactly one nonempty `meta_<capture>.json` at the capture root. A same-name file inside a part or another capture does not satisfy this requirement. Discovery checks presence and size; launch repeats the frozen-selection check. V4's existing eight-file selection remains required independently. Already-created Runs keep their bindings and annotation history; this does not migrate or relaunch them.
|
||||
|
||||
The normalizer remains the authority for meta content: `streams.color_left.segments` and `streams.color_right.segments` provide the relevant camera's records; the selected record requires `index` matching the current part and a positive integer `frame_count` (legacy digit strings accepted). The count must match the actual decoded video frames and PTS rows. Optional `file` / `pts_file` identities must agree with capture, camera and part; matching must be unique. Legacy index/count-only records remain supported when that part has no explicit filename identities. No additional device, scene or business fields are invented as mandatory metadata. Existing admitted legacy inputs without meta retain their previous normalization path; present metadata is always checked.
|
||||
|
||||
Raw-human normalization checks IMU headers `timestamp_us,x,y,z,type`, positive timestamps, finite coordinates, monotonic timestamps, and exactly one `accel` / `gyro` pair per timestamp. Invalid types such as `gyr`, incomplete final pairs, duplicate samples and empty data fail before normalization publishes the task input. This restores content validation without the previous full-video endpoint coverage requirement: legal part boundaries can end a few milliseconds beyond the local IMU samples. Source bytes are never repaired, renamed or truncated by validation.
|
||||
|
||||
File completeness only permits normalization to start; it is not a content-validation certificate or permission to claim an annotation task. The source validation and claim gate must also pass. Ingestion changes require the worker and normalizer runtime image together; local tests do not update deployed operator image digests.
|
||||
|
||||
## Validation
|
||||
|
||||
Go tests cover flat/part grouping, independent identities/fingerprints, shared calibration, missing/empty/wrong-part files, duplicate videos, mixed layouts, historical-parent protection, Run binding/de-duplication and retry references. Python tests cover scope enforcement in both readers, parent calibration pairing and unchanged PTS safety.
|
||||
|
||||
51
docs/verification/review-pool-search.md
Normal file
51
docs/verification/review-pool-search.md
Normal file
@ -0,0 +1,51 @@
|
||||
# 审核池全范围搜索与分页验证(2026-11-02)
|
||||
|
||||
## 范围和接口
|
||||
|
||||
基线为 `origin/dev@5d1135a0`。原服务只返回首批候选,前端丢失游标并在本地搜索;红测试在目标之前放置 962 条更新任务,旧实现返回 200 条无关记录,目标无法搜索。
|
||||
|
||||
新服务兼容新增 `search/capture/category`,按普通文字处理;搜索与状态筛选先于分页。响应新增 `filtersApplied: true`。旧前端不受新增字段影响;新前端遇到不支持筛选的旧服务显示简短错误,不把未筛选结果当成正确搜索。每池每页 200 条,独立游标;界面区分已加载数量,不制造总量。仍由服务端判断可见性、自审和领取资格。
|
||||
|
||||
质检可领取候选的项目/供应商必要范围及等价测试来自已有本地提交 `b2324c19`;未合并该提交的其他修改。共享验收和历史已审核任务不套用该供应商预筛选。
|
||||
|
||||
## 已通过
|
||||
|
||||
- API 定向回归:`TestHumanPool*`、`TestHumanItemsList*`、`TestReviewedPool*`。覆盖 962 条之后的旧任务、三池各 611 条无重复分页、供应商隔离、自审不可领取、普通文字搜索。已审核分类根据当前下游状态过滤后再分页,避免把质检已通过、仍待验收的历史任务误归为最终通过。
|
||||
- `TestReviewListsBoundRepeatedPermissionReads` 通过,原读取次数上限未放宽;新增兼容字段导致的整段 JSON 断言改为验证非空任务变为空任务,仍验证权限撤销立即生效。
|
||||
- Store 定向回归:`TestHumanPoolReadScope*`、`TestHumanItem*`。本次默认运行中 Mongo opt-in 测试跳过;此前明确启用的范围 Memory/Mongo 对照通过。
|
||||
- 前端任务池套件 52 项、新搜索模块 7 项通过;最后追加的搜索/分页组合 24 项通过。
|
||||
- TypeScript 和本次涉及的 Prettier 检查通过,`git diff --check` 通过。
|
||||
- 真实浏览器+真实 Go HTTP+隔离 MemoryStore 的 613 条任务:7 个场景通过,约 12.26 秒,无页面异常和写请求。包括首批 200、续页 400、失败重试到 600、首批之外 part 搜索、迟到搜索保护、我的任务及已审核搜索。
|
||||
- 本地 Mongo 7(27017)临时隔离库:最终 963 条历史记录与 10000 条现代记录的全部搜索、Capture、状态分类(含两项 ReviewedPool 分类)、游标、空 ID 范围与 MemoryStore 结果一致,完整运行通过,约 11.53 秒。同客户端连续两遍查询的诊断对照也通过。
|
||||
- 本地 Mongo 8(27018)最终三轮中,第 1、3 轮全部场景通过;第 2 轮的两次失败均有下述时钟跳变证据。不能记为三轮持续稳定性通过。
|
||||
|
||||
日志位于本机 `/tmp/pool-*.log`;浏览器报告及截图位于忽略路径 `local-data/test-results/review-pool-search/`。全量前后端门禁由集成阶段执行,本报告不替代门禁。
|
||||
|
||||
## 读取测量与限制
|
||||
|
||||
现代 10000 条夹具使用权威 Capture 链接。Mongo 8 的成功样本中,找旧 part 和无命中查询约 1.4–1.9 秒,Mongo 命令回包分别约 1408 和 735 字节;无命中不会把全部现代候选传回 API。精确业务判断仍在 Go 层复核,缺少权威 Capture 的旧数据保留回退;963 条全为旧数据时,无命中仍传输约 519 KB 的窄字段元数据。含空白的跨字段搜索也保留宽候选回退。
|
||||
|
||||
这些是隔离单请求样本,不是 p95 或 120 人容量结论。未新增生产索引、资源或缓存;未写入生产数据。
|
||||
|
||||
## 提前超时已定位为本地 Docker 时钟跳变
|
||||
|
||||
已将复现缩减为无业务代码的 10000 条小记录聚合,分别排查原/新 Mongo 实例、复用/新建客户端、显式 session、不同 lookup、无 Go 驱动的 mongosh。最终在失败操作前后直接读取 `hello.localTime`,与本机单调计时比较:
|
||||
|
||||
| 场景 | 服务端时间变化 | 单调耗时 | 结果 |
|
||||
| --- | --- | --- | --- |
|
||||
| 最小复现 | `1793615107197 → 1793615143974`,前跳 36.777 秒 | 43.79 ms | `MaxTimeMSExpired` |
|
||||
| Mongo 7 最终矩阵第一次尝试 | 前跳 36.981 秒 | 327.42 ms | `MaxTimeMSExpired` |
|
||||
| Mongo 8 最终三轮中的失败搜索 | 前跳 36.989 秒 | 341.92 ms | `MaxTimeMSExpired` |
|
||||
| 同轮失败状态筛选 | 前跳 36.923 秒 | 78.06 ms | `MaxTimeMSExpired` |
|
||||
|
||||
失败命令实际发送 `maxTimeMS=8000`。Mongo 的操作上下文按服务端 FastClock 时间计算和比较截止时间,约 37 秒前跳会立即越过 8 秒截止点;这解释了服务端 profiler 只记录数百毫秒,却返回超时的现象。见 [MongoDB 8.0.32 官方实现](https://github.com/mongodb/mongo/blob/r8.0.32/src/mongo/db/operation_context.cpp#L115)。Mongo 7 也复现,因此不是已证实的 Mongo 8 版本特有缺陷,也不是列表业务查询或 Go 上下文复用造成的超时。
|
||||
|
||||
当前代码保留原有组织过滤、字段投影和 server/client 双 8 秒限制;没有延长超时、重试吞错或换成未验证的 lookup。诊断期间去掉 maxTimeMS 等实验均已撤回。测试按独立子例执行,单次环境错误不会跳过后续业务断言,失败仍正常上报。
|
||||
|
||||
证据保存在忽略目录 `local-data/verification/review-pool-search/`:
|
||||
|
||||
- `clock-jump-minimal.log` / `clock-jump-minimal.go`:最小复现与前后时钟。
|
||||
- `mongo7-final.log`:保留 Mongo 7 受跳时影响的失败;`mongo7-stable-window.log`:随后完整通过。
|
||||
- `mongo8-final-clock.log`:先前整组中断记录;`mongo8-final-subtests.log`:最终三轮的全部子例与时钟关联。
|
||||
|
||||
未发现已安装的本机原生 mongod;没有安装软件、修改 Docker 时间或配置。一次性诊断容器已删除,原 Mongo 设置未改动;profiler 只启用于随后删除的合成库。环境跳时的原因定位已完成,本轮没有修复 Docker 宿主时钟。生产 p95、并发容量和长时间稳定性仍需独立环境及真实负载验收,不能用本次功能对照代替。
|
||||
74
docs/verification/source-gate-pool-search.md
Normal file
74
docs/verification/source-gate-pool-search.md
Normal file
@ -0,0 +1,74 @@
|
||||
# 源数据准入与审核池修复:集成验证
|
||||
|
||||
基线为 `origin/dev@5d1135a0`,分支 `codex/source-gate-pool-search`。实施前只读核对线上 API / Processor 为 `005eb934`、Worker 为 `63512da7`,均在本次基线内。其他工作树及其未合并改动未纳入或覆盖。
|
||||
|
||||
## 修复范围
|
||||
|
||||
1. 接入文件齐全与内容校验通过分开判断。新 raw-human 接入检查 meta,恢复 IMU 内容验证;已知校验失败、等待或缺少必需证明的任务不可领取,已有归属和草稿保留。管理员确认修复后可重新校验,通过后恢复。
|
||||
2. 可领取、我的任务、已审核均使用服务端搜索和游标分页,每页最多 200 条;供应商、当前权限及自审规则保留。数量文案为已加载数量,不能当作全部任务总数。
|
||||
3. 工作台异常只显示必要错误及恢复操作;正常状态不增加说明性备注。管理员可筛选源数据异常任务。
|
||||
|
||||
接口、兼容性和分项证据:
|
||||
|
||||
- [源数据校验准入契约](../architecture/source-validation-admission.md)及[验证记录](source-validation-admission.md)
|
||||
- [接入布局契约](../capture-layout-compatibility.md)
|
||||
- [任务池加载契约](../architecture/annotation-pool-loading.md)及[搜索验证记录](review-pool-search.md)
|
||||
|
||||
## 本地真实页面验证
|
||||
|
||||
均使用合成数据和隔离 HTTP 服务,不使用生产任务或客户文件。
|
||||
|
||||
- 源校验页面:真实 Go API、后台校验器与浏览器,验证具体 IMU 错误、普通标注员无重试权限、管理员取消确认不写入、确认只提交一次、校验期间继续阻断、修复后时间轴恢复;草稿、版本和归属不变。最终通过,约 14.43 秒。
|
||||
- 审核池页面:613 条任务,验证首批 200、续页 400、失败后重试到 600、首批之外的 part 搜索、迟到请求保护、我的任务和已审核搜索。最终 7 个场景通过,无页面异常和写请求,约 12.82 秒。
|
||||
- 生产构建启动:使用已安装的 Edge 验证构建产物,登录页面可见、无页面异常、无意外写请求。首次运行缺少 Playwright 下载的 Chromium,改用现有浏览器后通过;没有改动产品代码规避启动检查。
|
||||
|
||||
本机证据位于忽略路径 `local-data/verification/source-gate-pool-search/`、`local-data/test-results/source-validation-browser/`、`local-data/test-results/review-pool-search/`。这些目录不提交客户数据或凭证。
|
||||
|
||||
浏览器夹具复跑命令(串行执行):
|
||||
|
||||
```sh
|
||||
SOURCE_VALIDATION_BROWSER_TEST=1 go test ./internal/api -run '^TestSourceValidationBrowserReplica$' -count=1 -timeout=4m -v
|
||||
REVIEW_POOL_BROWSER_TEST=1 go test ./internal/api -run '^TestReviewPoolSearchBrowserReplica$' -count=1 -timeout=5m -v
|
||||
```
|
||||
|
||||
最终审核池测试曾因错用 opt-in 变量而被跳过;保留该次日志,修正变量后完整重跑通过,跳过不计作通过。
|
||||
|
||||
## 门禁与基线说明
|
||||
|
||||
本次不改动 `tests/known-failures.json`;全量回归使用项目官方 Go / frontend gate 比较具体失败用例。
|
||||
|
||||
| 检查 | 结果 |
|
||||
| --- | --- |
|
||||
| `npm run test:frontend:gate` | 通过;13 项失败均在原有 14 项基线内,零新增 |
|
||||
| `npm run test:go:gate` | 通过;68 项失败均在原有 70 项基线内,零新增 |
|
||||
| `npm run lint:web` | TypeScript 与架构检查通过 |
|
||||
| `npm run lint:go`、`npm run build:go` | 格式检查及 API、Worker、协调器构建通过 |
|
||||
| `npm run lint:scripts`、`npm run operators:repos:check` | 通过;算子镜像源码与权威源码一致 |
|
||||
| `npm run format:check`、`git diff --check` | 通过 |
|
||||
| `npm run build:web`、`npm run test:production:boot` | 通过;构建仍有已有的大 chunk 提醒 |
|
||||
| raw-human 接入与归一化 Python 回归 | 33 项通过 |
|
||||
| Fanyu 标注算子 Python 回归 | 18 项通过 |
|
||||
|
||||
- 新增管理员重校验路由通过授权快照生成器更新;普通标注、质检、验收均不能调用。
|
||||
- 原委托对象存储时间轴用例改为先确认等待校验,再运行真实校验器,验证左右时间轴及只使用 HEAD/GET。
|
||||
- 原权限读数测试改为解析 JSON 后检查任务集合;新增兼容字段 `filtersApplied` 不再被当作权限错误。读数上限及撤权后空列表断言保留。
|
||||
- 补齐两个历史恢复边界:无旧校验记录的任务在实际发现内容异常后原子记录失败,不覆盖并发的新校验;只有历史 sourceIndex 标记、没有 outbox 指纹的记录可在本次明确重校验成功后恢复,后续新指纹仍须重新核对。真实 API、Memory/Mongo 对照通过。
|
||||
- 指定领取在存储边界核对当前持久化的源身份并执行 CAS,陈旧提议不能覆盖新的处理记录。该项属于一致性加固,未将合成的同版本源身份竞争报告为已发生的线上故障。
|
||||
- 两项旧前端断言仍匹配已经被替换的 `selectScope` 调用;沿用已有本地提交 `6969411b` 的精确断言修复,验证当前 `editSelected` 行为,未引入该分支其他修改。
|
||||
- 基线中 4 个前端文件未符合全局 Prettier 门禁,本次只格式化 `ReviewHistorySummary`、`WorkforceStatisticsPanel`、`useAnnotationOutputData`、`useReviewContinuation`,不改变行为。
|
||||
|
||||
## Mongo 环境限制
|
||||
|
||||
搜索矩阵包含 963 条历史任务和 10,000 条现代任务。Mongo 7 最终完整矩阵通过,Mongo 8 三轮中第 1、3 轮完整通过;第 2 轮的提前超时有直接时钟证据:服务端 `hello.localTime` 在一次查询期间前跳约 37 秒,本机单调计时仅经过几十到几百毫秒,触发 `maxTimeMS=8000`。Mongo 7 也捕获了相同现象,不能归因为 Mongo 8 特有查询缺陷。
|
||||
|
||||
业务查询保持客户端及服务端双 8 秒限制,没有采用删除超时、增加重试或吞掉错误的绕过。没有修改 Docker 或系统时间配置。每项业务等价场景均有完整通过证据,但这台 Docker 环境上的三轮连续稳定性未通过,不能代替生产版本的容量验收;具体矩阵与诊断见[搜索验证记录](review-pool-search.md)。
|
||||
|
||||
## 发布与未验收边界
|
||||
|
||||
- 当前仅本地修改与验证;生产源修复、写验收、推送、PR 和部署均未执行。
|
||||
- 新搜索客户端要求服务端确认筛选已应用;应先升级服务端。旧客户端继续兼容。恢复作业要求新版校验器理解源指纹证明。
|
||||
- 新增字段不要求数据迁移;回退到旧 API 会失去本次准入阻断,已知异常任务可能重新进入分配。生产回退方案必须保留这道阻断,不能通过删除校验记录恢复领取。
|
||||
- 完整接入修复同时涉及 API、校验器、Worker 与 normalizer 算子运行镜像,仅更新 API 不能覆盖新采集接入检查。
|
||||
- `GET /tasks/{id}/frame-timeline` 会在发现新的内容异常时持久化隔离状态,原有路径也可能持久化时间轴。**生产只读验证不能把这个 GET 当成无写入探针**;只读诊断使用任务 `?view=status` / 摘要,实际检测及重校验需要对应生产操作授权。
|
||||
- 未自动修改或删除源文件、回填历史记录、重放旧送检、创建生产索引或调整资源。新 meta 检查不等于对历史任务补做了 meta 完整校验。
|
||||
- 单请求 Mongo 结果及页面回归不代表 120 人完整作业、8 小时运行或故障接管已通过;容量仍待隔离环境验收。源准入关联使部分计数变贵,分项记录保留实际开销,不以减少任务或放宽权限换取性能。
|
||||
44
docs/verification/source-validation-admission.md
Normal file
44
docs/verification/source-validation-admission.md
Normal file
@ -0,0 +1,44 @@
|
||||
# 源校验准入本地验证
|
||||
|
||||
基线:`origin/dev` 的 `5d1135a0`。任务分支 `codex/source-gate-pool-search`。仅本地内存、临时媒体及独立命名 Mongo 测试库;未修改生产数据。
|
||||
|
||||
## 可复现缺陷
|
||||
|
||||
新增 `TestSourceValidationFailureBlocksRandomClaimWithoutLosingDraft` 与 `TestPendingSourceValidationIsNotAssigned`,经真实随机领取 HTTP handler 复现:失败或等待校验的 `random-item-0` 被分配。修复前两项均失败,修复后通过。
|
||||
|
||||
## 功能与恢复
|
||||
|
||||
- 校验失败/等待不领取、不计入可领取数量;已有草稿、版本、归属保留。
|
||||
- 管理员确认修复后可重校验,其他角色拒绝;重校验完成前继续阻断。
|
||||
- 已过期校验器、迟到的读取失败不能覆盖新的校验代次。
|
||||
- 未要求重验的旧任务实际发现非法 IMU 后会保存异常;无旧记录的失败插入不能覆盖并发新校验。旧任务修复、payload-only 源索引记录补证均经过真实校验器恢复;草稿与归属不变,跨组织读取和非管理员重验拒绝。
|
||||
- 直接领取的存储边界加固:同版本内源作业变更时,陈旧快照不能覆盖当前缺证、失败或有效的新作业。内存与 Mongo 使用同一回归;这不是已观察到的线上生产竞态。
|
||||
- 已完成时间轴仍受当前源证书失败约束;新源索引 outbox 缺证书拒绝,旧物理 outbox 不强制新增协议。
|
||||
- 原指纹 A 失败,修复 B 成功,仅替代 A;之后 C 缺证书/失败继续阻断。内存和真实 Mongo 覆盖等价。
|
||||
- 真实合成双目视频/IMU:末行 gyro 改成 gyr,真实解析拒绝;恢复文件并调用管理员重校验,真实后台校验器建立当前指纹证书,旧失败作业与已保存内容不变。
|
||||
- 原来校验成功的文件再次损坏,工作台检测后撤销准入并给出可执行重校验入口。取消或超时的读取请求不会把有效源数据误标为异常;新增取消请求回归先复现误标,再通过。
|
||||
- 现有随机领取、源索引、时间轴与源校验等待回归通过。委托只读对象存储用真实校验器完成验证后,左右时间轴正常读取,全程仅 HEAD/GET;未校验时明确返回等待状态。
|
||||
|
||||
可复跑:
|
||||
|
||||
```sh
|
||||
GOCACHE=/tmp/droboticflow-dashboard-go-cache go test ./internal/api -run 'Test(SourceIndex|Annotation.*Timeline|Annotation.*Source|HardwareIMU|SourceValidationWait|RandomClaim|SourceFailureDetected|SourceRepair|RequiredSource|SourceValidation|PendingSourceValidation|CurrentPreparedSourceFailure)' -count=1
|
||||
SOURCE_ADMISSION_LOCAL_MONGO=1 GOCACHE=/tmp/droboticflow-dashboard-go-cache go test ./internal/store -run '^TestSourceAdmission(Memory|Mongo)' -count=1 -v
|
||||
POOL_LOCAL_MONGO_URI=mongodb://127.0.0.1:27018 GOCACHE=/tmp/droboticflow-dashboard-go-cache go test ./internal/store -run '^TestClaimCountMongoSparseBacklogPlan$' -count=1 -v
|
||||
```
|
||||
|
||||
## 查询对照
|
||||
|
||||
同一本地 Mongo、1,200 条全部可用任务、每条 64 KiB subject(共 75 MiB),连续三轮:
|
||||
|
||||
| 指标 | 第 1 轮 | 第 2 轮 | 第 3 轮 |
|
||||
| ---------------------------- | --------- | --------- | --------- |
|
||||
| 原计数逻辑(不做源校验准入) | 7.43 ms | 6.17 ms | 4.24 ms |
|
||||
| 加准入过滤后的计数 | 143.78 ms | 133.14 ms | 139.71 ms |
|
||||
| 准入候选列表 | 134.21 ms | 139.77 ms | 144.18 ms |
|
||||
|
||||
每轮候选 JSON 均为 508,801 字节,任务数未减少。1200 条保留历史仅 1 条候选的稀疏场景,计数约 3–10 ms,关键读计划最大扫描量为 1。
|
||||
|
||||
计数的额外关联开销真实存在。一次条件关联改写在 Mongo 8 上反而变为约 750 ms,已撤回,保留实测更快的 ID 等值关联与必要字段投影。没有通过放宽准入或减少可见任务改善数字。
|
||||
|
||||
上述是本地单请求查询对照,不代表 120 人完整作业、生产媒体/数据库/网络负载或长期运行已达标。浏览器和全量门禁由本次整体验证记录汇总。
|
||||
@ -30,7 +30,7 @@ func (s *server) adminAnnotationPoolQuery(w http.ResponseWriter, r *http.Request
|
||||
return q, false
|
||||
}
|
||||
switch q.Category {
|
||||
case "", "all", "available", "pending_slice", "slicing", "pending_annotation", "rework", "pending_quality", "rework_review", "pending_acceptance", "accepted", "submitting", "submission_conflicted":
|
||||
case "", "all", "available", "pending_slice", "slicing", "pending_annotation", "rework", "pending_quality", "rework_review", "pending_acceptance", "accepted", "submitting", "submission_conflicted", "source_failed":
|
||||
default:
|
||||
renderValidationError(w, validationIssue{Field: "category", Message: "unknown task category"})
|
||||
return q, false
|
||||
|
||||
77
internal/api/admin_source_exceptions_test.go
Normal file
77
internal/api/admin_source_exceptions_test.go
Normal file
@ -0,0 +1,77 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAdminSourceFailureRemainsDiscoverableButUnavailable(t *testing.T) {
|
||||
f := randomClaimFixture(t, 1)
|
||||
task, _ := f.store.GetAnnotationTask(f.ctx, "random-task-0")
|
||||
task.SourceIsFullVideo, task.TemplateVersion, task.ArtifactID = true, 5, "original-left"
|
||||
if err := f.store.UpsertAnnotationTask(f.ctx, *task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item, _ := f.store.GetHumanItem(f.ctx, "random-item-0")
|
||||
item.Subject.Payload = map[string]any{"draft": "keep annotation"}
|
||||
if err := f.store.UpsertHumanItem(f.ctx, *item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
job, err := f.store.ClaimAnnotationIndex(f.ctx, "invalid-source", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if err != nil || job == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job.Status, job.Error = "failed", "IMU invalid type gyr at row 418451"
|
||||
if ok, err := f.store.FinishAnnotationIndex(f.ctx, *job); err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := "/api/plugins/annotation-qa/admin-pool?projectId=" + task.ProjectID
|
||||
read := func(suffix string) struct {
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
SourceValidationStatus string `json:"sourceValidationStatus"`
|
||||
} `json:"items"`
|
||||
Counts struct {
|
||||
Available int `json:"available"`
|
||||
All int `json:"all"`
|
||||
} `json:"counts"`
|
||||
} {
|
||||
t.Helper()
|
||||
var out struct {
|
||||
Items []struct {
|
||||
ID string `json:"id"`
|
||||
SourceValidationStatus string `json:"sourceValidationStatus"`
|
||||
} `json:"items"`
|
||||
Counts struct {
|
||||
Available int `json:"available"`
|
||||
All int `json:"all"`
|
||||
} `json:"counts"`
|
||||
}
|
||||
r := f.request(http.MethodGet, base+suffix, f.admin, nil)
|
||||
if r.Code != 200 || json.Unmarshal(r.Body.Bytes(), &out) != nil {
|
||||
t.Fatalf("%d %s", r.Code, r.Body)
|
||||
}
|
||||
return out
|
||||
}
|
||||
available := read("&pool=available")
|
||||
if len(available.Items) != 0 || available.Counts.Available != 0 || available.Counts.All != 2 {
|
||||
t.Fatalf("failed task still available or disappeared: %+v", available)
|
||||
}
|
||||
failed := read("&pool=all&category=source_failed")
|
||||
if len(failed.Items) != 1 || failed.Items[0].ID != item.ID || failed.Items[0].SourceValidationStatus != "failed" {
|
||||
t.Fatalf("exception not discoverable: %+v", failed)
|
||||
}
|
||||
for _, user := range []domain.UserRecord{f.annotator, f.reviewer} {
|
||||
if r := f.request(http.MethodGet, base+"&pool=all&category=source_failed", user, nil); r.Code != 403 {
|
||||
t.Fatalf("nonadmin exception access: %d", r.Code)
|
||||
}
|
||||
}
|
||||
after, _ := f.store.GetHumanItem(f.ctx, item.ID)
|
||||
if after.Version != item.Version || after.AssignedUserID != item.AssignedUserID || after.Subject.Payload["draft"] != "keep annotation" {
|
||||
t.Fatal("diagnostic read altered task")
|
||||
}
|
||||
}
|
||||
@ -1,18 +1,22 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func TestDelegatedReadOnlyFrameTimelineIncludesIMUSource(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
||||
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "validation")
|
||||
f, root := annotationProcessorFixture(t)
|
||||
workspace := filepath.Dir(root)
|
||||
prefix := filepath.Base(root)
|
||||
@ -49,6 +53,30 @@ func TestDelegatedReadOnlyFrameTimelineIncludesIMUSource(t *testing.T) {
|
||||
}
|
||||
t.Setenv("ANNOTATION_WORKSPACE_MAPPINGS", string(raw))
|
||||
f.handler = NewHandlerWithConfig(f.store, nil, config.AppConfig{WorkspaceMountPath: workspace})
|
||||
pending := f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", f.annotator, nil)
|
||||
if pending.Code != http.StatusUnprocessableEntity || !strings.Contains(pending.Body.String(), "source_validation_pending") {
|
||||
t.Fatalf("unvalidated delegated source was admitted: %d %s", pending.Code, pending.Body.String())
|
||||
}
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: workspace}, 1, false)
|
||||
}()
|
||||
defer func() { cancel(); <-done }()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
job, err := f.store.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job != nil && job.Status == "completed" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("delegated read-only validation did not complete: %+v", job)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
for _, camera := range []string{"left", "right"} {
|
||||
r := f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline?camera="+camera+"&encoding=compact", f.annotator, nil)
|
||||
if r.Code != 200 {
|
||||
|
||||
@ -957,8 +957,13 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"task": item})
|
||||
return
|
||||
}
|
||||
sourceValidation, sourceErr := server.annotationSourceValidationForTask(request.Context(), *item, linkedHumanItem, annotationManager(principal))
|
||||
if sourceErr != nil {
|
||||
renderError(writer, http.StatusInternalServerError, sourceErr)
|
||||
return
|
||||
}
|
||||
if request.URL.Query().Get("view") == "status" {
|
||||
status := map[string]any{"taskId": item.ID, "taskStatus": item.Status}
|
||||
status := map[string]any{"taskId": item.ID, "taskStatus": item.Status, "sourceValidation": sourceValidation}
|
||||
if progress, ok := server.materializationProgress.Load(item.OrganizationID + ":" + item.ID); ok {
|
||||
status["progress"] = progress
|
||||
}
|
||||
@ -1096,7 +1101,7 @@ func (server *server) handleAnnotationTaskGet(writer http.ResponseWriter, reques
|
||||
renderError(writer, http.StatusInternalServerError, readinessErr)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"task": item, "sliceRevisions": sliceRevisions, "revisions": revisions, "reviews": reviews, "humanItem": linkedHumanItem, "editorState": editorState})
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"task": item, "sliceRevisions": sliceRevisions, "revisions": revisions, "reviews": reviews, "humanItem": linkedHumanItem, "editorState": editorState, "sourceValidation": sourceValidation})
|
||||
}
|
||||
|
||||
func annotationFrameTimelinePayload(camera string, rows []hardwarePTSRow) (map[string]any, error) {
|
||||
@ -1286,9 +1291,25 @@ func (server *server) handleAnnotationTaskFrameTimelineGet(writer http.ResponseW
|
||||
renderValidationError(writer, validationIssue{Field: "camera", Message: "camera must be left or right"})
|
||||
return
|
||||
}
|
||||
sourceValidation, sourceErr := server.annotationSourceValidationForTask(request.Context(), *item, linkedHumanItem, annotationManager(principal))
|
||||
if sourceErr != nil {
|
||||
renderError(writer, http.StatusInternalServerError, sourceErr)
|
||||
return
|
||||
}
|
||||
if sourceValidation.blocked() {
|
||||
renderSourceValidationBlock(writer, sourceValidation)
|
||||
return
|
||||
}
|
||||
payload, err := server.annotationFrameTimelineForTask(request.Context(), *item, camera)
|
||||
if err != nil {
|
||||
renderJSON(writer, http.StatusUnprocessableEntity, map[string]string{"error": err.Error()})
|
||||
if transientSourceValidationFailure(request.Context(), err) {
|
||||
renderJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "源数据暂时无法读取,请重试"})
|
||||
return
|
||||
}
|
||||
recorded := server.recordSourceValidationFailure(request.Context(), *item, sourceValidation, err.Error())
|
||||
sourceValidation.Status, sourceValidation.Code, sourceValidation.Message = "failed", "source_validation_failed", safeSourceValidationMessage(err.Error())
|
||||
sourceValidation.RetryAllowed = annotationManager(principal) && (recorded || sourceValidation.Required || sourceValidation.jobCreatedAt != "")
|
||||
renderSourceValidationBlock(writer, sourceValidation)
|
||||
return
|
||||
}
|
||||
if request.URL.Query().Get("encoding") == "compact" {
|
||||
|
||||
@ -39,7 +39,7 @@ func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.Annota
|
||||
if e = json.Unmarshal(b, &values); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
|
||||
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, SourceIndex: s.annotationSourceIndex, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
|
||||
}
|
||||
|
||||
func durableAnnotationActive(item *domain.HumanItemRecord) bool {
|
||||
@ -188,7 +188,7 @@ func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppC
|
||||
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
|
||||
// Legacy timeline jobs may predate this release. They must
|
||||
// not silently trigger full-media validation/source copies.
|
||||
if readErr == nil && s.annotationSourceIndex && j.TaskID != "" {
|
||||
if readErr == nil && (s.annotationSourceIndex && j.TaskID != "" || j.RevalidateSource) {
|
||||
run, err := db.GetRun(indexCtx, task.RunID)
|
||||
if err != nil {
|
||||
readErr = err
|
||||
@ -200,6 +200,9 @@ func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppC
|
||||
readErr = err
|
||||
} else {
|
||||
_, readErr = s.annotationSourceCertificate(indexCtx, *task, *run, key)
|
||||
if readErr == nil {
|
||||
j.SourceFingerprint = key
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
236
internal/api/annotation_source_admission.go
Normal file
236
internal/api/annotation_source_admission.go
Normal file
@ -0,0 +1,236 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
type annotationSourceValidation struct {
|
||||
jobCreatedAt string
|
||||
Status string `json:"status"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Required bool `json:"required"`
|
||||
RetryAllowed bool `json:"retryAllowed"`
|
||||
}
|
||||
|
||||
func (v annotationSourceValidation) blocked() bool {
|
||||
return v.Status == "failed" || v.Status == "pending" || v.Status == "unverified" && v.Required
|
||||
}
|
||||
|
||||
var sourceErrorRow = regexp.MustCompile(`(?i)(?:row|line|第)\s*([0-9]{1,12})`)
|
||||
var sourceErrorType = regexp.MustCompile(`(?i)invalid type[ :"']+([a-z0-9_-]{1,16})`)
|
||||
|
||||
func safeSourceValidationMessage(raw string) string {
|
||||
lower := strings.ToLower(raw)
|
||||
if strings.Contains(lower, "imu") {
|
||||
row := sourceErrorRow.FindStringSubmatch(raw)
|
||||
if len(row) > 1 {
|
||||
if kind := sourceErrorType.FindStringSubmatch(raw); len(kind) > 1 {
|
||||
return fmt.Sprintf("IMU 第 %s 行类型不合法(%s)", row[1], kind[1])
|
||||
}
|
||||
if strings.Contains(lower, "non-finite") {
|
||||
return "IMU 第 " + row[1] + " 行数值不合法"
|
||||
}
|
||||
return "IMU 第 " + row[1] + " 行数据异常"
|
||||
}
|
||||
if strings.Contains(lower, "pair") {
|
||||
return "IMU 数据缺少配对采样"
|
||||
}
|
||||
if strings.Contains(lower, "duplicate") {
|
||||
return "IMU 数据存在重复采样"
|
||||
}
|
||||
if strings.Contains(lower, "missing") {
|
||||
return "IMU 数据缺少必需字段"
|
||||
}
|
||||
return "IMU 数据校验失败"
|
||||
}
|
||||
if strings.Contains(lower, "pts") {
|
||||
if strings.Contains(raw, "左视频") {
|
||||
return "左视频与时间戳数据不一致"
|
||||
}
|
||||
if strings.Contains(raw, "右视频") {
|
||||
return "右视频与时间戳数据不一致"
|
||||
}
|
||||
return "视频时间戳数据校验失败"
|
||||
}
|
||||
if strings.Contains(lower, "meta") {
|
||||
return "采集信息文件校验失败"
|
||||
}
|
||||
return "源数据校验失败"
|
||||
}
|
||||
|
||||
func (s *server) annotationSourceValidationForTask(ctx context.Context, task domain.AnnotationTaskRecord, item *domain.HumanItemRecord, retryAllowed bool) (annotationSourceValidation, error) {
|
||||
v := annotationSourceValidation{Status: "unverified", Required: store.AnnotationSourceValidationRequired(task)}
|
||||
db, ok := s.store.(store.AnnotationProcessingStore)
|
||||
if !ok {
|
||||
if v.Required {
|
||||
v.Code, v.Message = "source_validation_unverified", "源数据尚未完成校验"
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
base, e := db.GetAnnotationIndexJob(ctx, task.ID)
|
||||
if e != nil {
|
||||
return v, e
|
||||
}
|
||||
if base != nil {
|
||||
v.jobCreatedAt = base.CreatedAt
|
||||
}
|
||||
jobs := []*domain.AnnotationIndexJob{base}
|
||||
if item != nil && item.ProcessingJob != nil && item.ProcessingJob.SourceFingerprint != "" && !store.AnnotationSourceJobSuperseded(*item, base) {
|
||||
source, e := db.GetAnnotationIndexJob(ctx, store.AnnotationSourceJobID(task.OrganizationID, item.ProcessingJob.SourceFingerprint))
|
||||
if e != nil {
|
||||
return v, e
|
||||
}
|
||||
jobs = append(jobs, source)
|
||||
}
|
||||
if base == nil && v.Required {
|
||||
v.Code, v.Message = "source_validation_unverified", "源数据尚未完成校验"
|
||||
}
|
||||
for _, j := range jobs {
|
||||
if j == nil {
|
||||
continue
|
||||
}
|
||||
if j.OrganizationID != task.OrganizationID || j.ProjectID != task.ProjectID || j.RunID != task.RunID {
|
||||
v.Status, v.Code, v.Message = "failed", "source_validation_failed", "源数据校验记录不匹配"
|
||||
break
|
||||
}
|
||||
if j.Status == "failed" {
|
||||
v.Status, v.Code, v.Message = "failed", "source_validation_failed", safeSourceValidationMessage(j.Error)
|
||||
break
|
||||
}
|
||||
if j.Status != "completed" {
|
||||
v.Status, v.Code, v.Message = "pending", "source_validation_pending", "源数据校验中"
|
||||
}
|
||||
}
|
||||
var source *domain.AnnotationIndexJob
|
||||
if len(jobs) > 1 {
|
||||
source = jobs[1]
|
||||
}
|
||||
ready := store.AnnotationSourceJobsReady(task, base, source)
|
||||
if item != nil {
|
||||
ready = store.AnnotationItemSourceJobsReady(task, *item, base, source)
|
||||
}
|
||||
if ready && base != nil {
|
||||
v.Status, v.Code, v.Message = "valid", "", ""
|
||||
}
|
||||
if !ready && v.Status == "unverified" {
|
||||
v.Required = true
|
||||
v.Code, v.Message = "source_validation_unverified", "源数据尚未完成校验"
|
||||
}
|
||||
|
||||
v.RetryAllowed = retryAllowed && (v.Status == "failed" || v.Status == "unverified" && v.Required)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func renderSourceValidationBlock(w http.ResponseWriter, v annotationSourceValidation) {
|
||||
renderJSON(w, http.StatusUnprocessableEntity, map[string]any{"code": v.Code, "error": v.Message, "details": map[string]any{"sourceValidation": v}})
|
||||
}
|
||||
|
||||
func (s *server) handleAnnotationSourceValidationRetry(w http.ResponseWriter, r *http.Request) {
|
||||
principal, _ := currentPrincipal(r)
|
||||
if !annotationManager(principal) {
|
||||
renderJSON(w, 403, map[string]string{"error": "仅管理员可重新校验源数据"})
|
||||
return
|
||||
}
|
||||
task, e := s.getAnnotationTaskForAccess(r.Context(), chi.URLParam(r, "id"))
|
||||
if e != nil {
|
||||
renderError(w, 500, e)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
renderJSON(w, 404, map[string]string{"error": "任务不存在"})
|
||||
return
|
||||
}
|
||||
if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *task); !ok {
|
||||
return
|
||||
}
|
||||
var input struct {
|
||||
SourceRepaired bool `json:"sourceRepaired"`
|
||||
}
|
||||
if !decodeJSON(w, r, &input) {
|
||||
return
|
||||
}
|
||||
if !input.SourceRepaired {
|
||||
renderJSON(w, 422, map[string]string{"error": "请先确认源数据已修复"})
|
||||
return
|
||||
}
|
||||
item, e := s.humanItemForLegacyTask(r.Context(), *task)
|
||||
if e != nil {
|
||||
renderError(w, 500, e)
|
||||
return
|
||||
}
|
||||
state, e := s.annotationSourceValidationForTask(r.Context(), *task, item, true)
|
||||
if e != nil {
|
||||
renderError(w, 500, e)
|
||||
return
|
||||
}
|
||||
if !state.RetryAllowed && state.Status != "valid" {
|
||||
renderJSON(w, 409, map[string]any{"error": "当前状态无需重新校验", "details": map[string]any{"sourceValidation": state}})
|
||||
return
|
||||
}
|
||||
db, ok := s.store.(store.AnnotationProcessingStore)
|
||||
if !ok {
|
||||
renderJSON(w, 503, map[string]string{"error": "校验服务暂不可用"})
|
||||
return
|
||||
}
|
||||
verifySource := s.annotationSourceIndex
|
||||
supersededFingerprint := ""
|
||||
if item != nil && store.AnnotationSourceCertificateRequired(*item) {
|
||||
verifySource = true
|
||||
}
|
||||
if item != nil && item.ProcessingJob != nil && item.ProcessingJob.SourceFingerprint != "" {
|
||||
supersededFingerprint = item.ProcessingJob.SourceFingerprint
|
||||
source, e := db.GetAnnotationIndexJob(r.Context(), store.AnnotationSourceJobID(task.OrganizationID, item.ProcessingJob.SourceFingerprint))
|
||||
if e != nil {
|
||||
renderError(w, 500, e)
|
||||
return
|
||||
}
|
||||
verifySource = verifySource || source != nil
|
||||
}
|
||||
if e = db.RetryAnnotationSourceValidation(r.Context(), *task, verifySource, supersededFingerprint); e != nil {
|
||||
if errors.Is(e, store.ErrSourceValidationActive) {
|
||||
renderJSON(w, 409, map[string]string{"error": "源数据正在校验"})
|
||||
return
|
||||
}
|
||||
renderError(w, 500, e)
|
||||
return
|
||||
}
|
||||
state, e = s.annotationSourceValidationForTask(r.Context(), *task, item, true)
|
||||
if e != nil {
|
||||
renderError(w, 500, e)
|
||||
return
|
||||
}
|
||||
renderJSON(w, http.StatusAccepted, map[string]any{"sourceValidation": state})
|
||||
}
|
||||
|
||||
func (s *server) recordSourceValidationFailure(ctx context.Context, task domain.AnnotationTaskRecord, previous annotationSourceValidation, message string) bool {
|
||||
db, ok := s.store.(store.AnnotationProcessingStore)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 3*time.Second)
|
||||
defer cancel()
|
||||
changed, err := db.FailAnnotationSourceValidation(persistCtx, task, previous.jobCreatedAt, message)
|
||||
if err != nil {
|
||||
log.Printf("[api] source validation failure persistence task=%s error=%v", task.ID, err)
|
||||
}
|
||||
return changed && err == nil
|
||||
}
|
||||
|
||||
func transientSourceValidationFailure(ctx context.Context, err error) bool {
|
||||
var networkError net.Error
|
||||
return ctx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || errors.As(err, &networkError) || mongo.IsNetworkError(err) || mongo.IsTimeout(err)
|
||||
}
|
||||
503
internal/api/annotation_source_admission_test.go
Normal file
503
internal/api/annotation_source_admission_test.go
Normal file
@ -0,0 +1,503 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSourceValidationFailureBlocksRandomClaimWithoutLosingDraft(t *testing.T) {
|
||||
f := randomClaimFixture(t, 1)
|
||||
task, err := f.store.GetAnnotationTask(f.ctx, "random-task-0")
|
||||
if err != nil || task == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task.SourceIsFullVideo, task.TemplateVersion, task.ArtifactID = true, 5, "source-left"
|
||||
if err := f.store.UpsertAnnotationTask(f.ctx, *task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item, _ := f.store.GetHumanItem(f.ctx, "random-item-0")
|
||||
item.Subject.Payload = map[string]any{"draft": "keep existing annotation"}
|
||||
if err := f.store.UpsertHumanItem(f.ctx, *item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
job, err := f.store.ClaimAnnotationIndex(f.ctx, "test-validator", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if err != nil || job == nil {
|
||||
t.Fatal("missing validation job", err)
|
||||
}
|
||||
job.Status, job.Error, job.Attempt = "failed", "IMU row 418451 invalid type gyr", 5
|
||||
if ok, err := f.store.FinishAnnotationIndex(f.ctx, *job); err != nil || !ok {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := claimRandom(t, f, "invalid-source-000001")
|
||||
if got.ID != "" {
|
||||
t.Fatalf("failed source validation was still assigned: %s", got.ID)
|
||||
}
|
||||
current, _ := f.store.GetHumanItem(f.ctx, item.ID)
|
||||
if current.AssignedUserID != "" || current.Version != item.Version || current.Subject.Payload["draft"] != "keep existing annotation" {
|
||||
t.Fatalf("quarantine altered existing content/ownership: %+v", current)
|
||||
}
|
||||
count := f.request(http.MethodGet, "/api/plugins/annotation-qa/claim-count?projectId="+task.ProjectID, f.annotator, nil)
|
||||
var payload struct {
|
||||
Count int `json:"count"`
|
||||
}
|
||||
if count.Code != 200 || json.Unmarshal(count.Body.Bytes(), &payload) != nil || payload.Count != 0 {
|
||||
t.Fatalf("invalid source counted as available: %d %s", count.Code, count.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingSourceValidationIsNotAssigned(t *testing.T) {
|
||||
f := randomClaimFixture(t, 1)
|
||||
task, _ := f.store.GetAnnotationTask(f.ctx, "random-task-0")
|
||||
task.SourceIsFullVideo, task.TemplateVersion, task.ArtifactID = true, 5, "source-left"
|
||||
if err := f.store.UpsertAnnotationTask(f.ctx, *task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if item := claimRandom(t, f, "pending-source-00001"); item.ID != "" {
|
||||
t.Fatalf("pending source was assigned: %s", item.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceValidationRetryRequiresAdminAndResumesOnlyAfterPass(t *testing.T) {
|
||||
f := randomClaimFixture(t, 1)
|
||||
task, _ := f.store.GetAnnotationTask(f.ctx, "random-task-0")
|
||||
task.SourceIsFullVideo, task.TemplateVersion, task.ArtifactID = true, 5, "source-left"
|
||||
if e := f.store.UpsertAnnotationTask(f.ctx, *task); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
job, e := f.store.ClaimAnnotationIndex(f.ctx, "validation-owner", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if e != nil || job == nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
job.Status, job.Attempt, job.Error = "failed", 5, `IMU CSV imu.csv invalid type "gyr" at row 418451`
|
||||
if ok, e := f.store.FinishAnnotationIndex(f.ctx, *job); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
route := "/api/plugins/annotation-qa/tasks/" + task.ID + "/source-validation-retries"
|
||||
for _, user := range []domain.UserRecord{f.annotator, f.reviewer} {
|
||||
r := f.request(http.MethodPost, route, user, map[string]any{"sourceRepaired": true})
|
||||
if r.Code != 403 {
|
||||
t.Fatalf("non-admin revalidation %d %s", r.Code, r.Body.String())
|
||||
}
|
||||
}
|
||||
denied := f.request(http.MethodPost, route, f.admin, map[string]any{})
|
||||
if denied.Code != 422 {
|
||||
t.Fatal(denied.Code, denied.Body.String())
|
||||
}
|
||||
response := f.request(http.MethodPost, route, f.admin, map[string]any{"sourceRepaired": true})
|
||||
if response.Code != 202 {
|
||||
t.Fatal(response.Code, response.Body.String())
|
||||
}
|
||||
if got := claimRandom(t, f, "revalidating-source-1"); got.ID != "" {
|
||||
t.Fatal("revalidation admitted before success")
|
||||
}
|
||||
// An old validator completion must not supersede a reset job.
|
||||
stale := *job
|
||||
stale.Status = "completed"
|
||||
if ok, e := f.store.FinishAnnotationIndex(f.ctx, stale); ok || e != nil {
|
||||
t.Fatalf("old validator survived reset: %v %v", ok, e)
|
||||
}
|
||||
replacement, e := f.store.ClaimAnnotationIndex(f.ctx, "replacement-validator", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if e != nil || replacement == nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
replacement.Status, replacement.Error = "completed", ""
|
||||
if ok, e := f.store.FinishAnnotationIndex(f.ctx, *replacement); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if got := claimRandom(t, f, "revalidated-source-01"); got.ID != "random-item-0" {
|
||||
t.Fatalf("validated task not resumed: %s", got.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceValidationStatusIsSafeSpecificAndPreservesOwnedTask(t *testing.T) {
|
||||
f := newCombinedDraftFixture(t)
|
||||
task := f.task
|
||||
task.SourceIsFullVideo, task.ArtifactID = true, "source-left"
|
||||
f.store.UpsertAnnotationTask(f.ctx, task)
|
||||
now := time.Now().UTC()
|
||||
job, _ := f.store.ClaimAnnotationIndex(f.ctx, "validator", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
job.Status, job.Error = "failed", `IMU CSV /private/credential-root/imu.csv invalid type "gyr" at row 418451`
|
||||
f.store.FinishAnnotationIndex(f.ctx, *job)
|
||||
for _, path := range []string{"?view=status", "/frame-timeline"} {
|
||||
r := f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+task.ID+path, f.annotator, nil)
|
||||
expected := 200
|
||||
if path == "/frame-timeline" {
|
||||
expected = 422
|
||||
}
|
||||
if r.Code != expected || !strings.Contains(r.Body.String(), "418451") || !strings.Contains(r.Body.String(), "gyr") || strings.Contains(r.Body.String(), "credential-root") {
|
||||
t.Fatalf("status not safe/specific: %d %s", r.Code, r.Body.String())
|
||||
}
|
||||
if !strings.Contains(r.Body.String(), `"retryAllowed":false`) {
|
||||
t.Fatal("worker received admin retry permission")
|
||||
}
|
||||
}
|
||||
after, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
if after.Version != f.item.Version || after.AssignedUserID != f.annotator.ID || after.Subject.Payload["preserved"] != true {
|
||||
t.Fatal("diagnostic read changed owned work")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCurrentPreparedSourceFailureBlocksClaimAfterTimelinePass(t *testing.T) {
|
||||
f := randomClaimFixture(t, 1)
|
||||
task, _ := f.store.GetAnnotationTask(f.ctx, "random-task-0")
|
||||
f.store.EnqueueAnnotationIndex(f.ctx, *task)
|
||||
now := time.Now().UTC()
|
||||
job, _ := f.store.ClaimAnnotationIndex(f.ctx, "timeline", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
job.Status = "completed"
|
||||
f.store.FinishAnnotationIndex(f.ctx, *job)
|
||||
f.store.EnqueueAnnotationSourceIndex(f.ctx, *task, "current-source", false)
|
||||
job, _ = f.store.ClaimAnnotationIndex(f.ctx, "source", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
job.Status, job.Error = "failed", "IMU row 8 invalid type gyr"
|
||||
f.store.FinishAnnotationIndex(f.ctx, *job)
|
||||
item, _ := f.store.GetHumanItem(f.ctx, "random-item-0")
|
||||
item.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: "current-source", Status: "failed"}
|
||||
f.store.UpsertHumanItem(f.ctx, *item)
|
||||
if got := claimRandom(t, f, "prepared-failed-0001"); got.ID != "" {
|
||||
t.Fatalf("source certificate failure ignored: %s", got.ID)
|
||||
}
|
||||
// An unrelated old fingerprint must not quarantine the current source.
|
||||
item.ProcessingJob.SourceFingerprint = "repaired-source"
|
||||
f.store.UpsertHumanItem(f.ctx, *item)
|
||||
f.store.EnqueueAnnotationSourceIndex(f.ctx, *task, "repaired-source", false)
|
||||
job, _ = f.store.ClaimAnnotationIndex(f.ctx, "repaired", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
job.Status = "completed"
|
||||
f.store.FinishAnnotationIndex(f.ctx, *job)
|
||||
if got := claimRandom(t, f, "prepared-passed-0001"); got.ID != "random-item-0" {
|
||||
t.Fatal("stale source failure poisoned repaired task")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceValidationLegacyWithoutJobKeepsExistingEligibility(t *testing.T) {
|
||||
f := randomClaimFixture(t, 1)
|
||||
if got := claimRandom(t, f, "legacy-source-000001"); got.ID != "random-item-0" {
|
||||
t.Fatal("unrelated legacy task blocked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceRepairRevalidatesChangedFilesWithoutReusingFailedCertificateID(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_SOURCE_INDEX", "true")
|
||||
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
||||
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "validation")
|
||||
f, root := annotationProcessorFixture(t)
|
||||
path := filepath.Join(root, "imu.csv")
|
||||
original, e := os.ReadFile(path)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
broken := strings.TrimSuffix(string(original), "gyro\n") + "gyr\n"
|
||||
if e = os.WriteFile(path, []byte(broken), 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
engine := &server{store: f.store, workspaceMountPath: root, annotationSourceIndex: true}
|
||||
run, _ := f.store.GetRun(f.ctx, f.task.RunID)
|
||||
failedFingerprint, e := engine.annotationSourceFingerprint(f.ctx, f.task, *run)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = engine.annotationFrameTimelineForTask(f.ctx, f.task, "left"); e == nil || !strings.Contains(e.Error(), "gyr") {
|
||||
t.Fatalf("bad source was not actually rejected: %v", e)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
stamp := now.Format(time.RFC3339Nano)
|
||||
until := now.Add(time.Minute).Format(time.RFC3339Nano)
|
||||
base, _ := f.store.ClaimAnnotationIndex(f.ctx, "broken-timeline", stamp, until)
|
||||
base.Status, base.Error = "failed", e.Error()
|
||||
f.store.FinishAnnotationIndex(f.ctx, *base)
|
||||
f.store.EnqueueAnnotationSourceIndex(f.ctx, f.task, failedFingerprint, false)
|
||||
source, _ := f.store.ClaimAnnotationIndex(f.ctx, "broken-source", stamp, until)
|
||||
source.Status, source.Error = "failed", e.Error()
|
||||
f.store.FinishAnnotationIndex(f.ctx, *source)
|
||||
item, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
item.ProcessingJob = &domain.AnnotationProcessingJob{ID: "preserved-failed-slice", Kind: "slice", TaskID: f.task.ID, Status: "failed", SourceFingerprint: failedFingerprint, SourceIndex: true, Input: map[string]any{"draft": "saved annotation"}}
|
||||
f.store.UpsertHumanItem(f.ctx, *item)
|
||||
if e = os.WriteFile(path, original, 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
repairedFingerprint, e := engine.annotationSourceFingerprint(f.ctx, f.task, *run)
|
||||
if e != nil || repairedFingerprint == failedFingerprint {
|
||||
t.Fatal("repair did not change source identity", e)
|
||||
}
|
||||
retry := f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/source-validation-retries", f.admin, map[string]any{"sourceRepaired": true})
|
||||
if retry.Code != 202 {
|
||||
t.Fatal(retry.Code, retry.Body.String())
|
||||
}
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false)
|
||||
}()
|
||||
defer func() {
|
||||
cancel()
|
||||
if done != nil {
|
||||
<-done
|
||||
}
|
||||
}()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
job, _ := f.store.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if job.Status == "completed" {
|
||||
if !job.RevalidateSource || job.SourceFingerprint != repairedFingerprint {
|
||||
t.Fatalf("completion lacks current source proof: %+v", job)
|
||||
}
|
||||
previous, _ := f.store.GetAnnotationIndexJob(f.ctx, source.ID)
|
||||
if previous.Status != "failed" {
|
||||
t.Fatal("old failed certificate job was relabeled successful")
|
||||
}
|
||||
current, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
if current.Version != item.Version || current.ProcessingJob.Input["draft"] != "saved annotation" {
|
||||
t.Fatal("repair overwrote saved work")
|
||||
}
|
||||
state, e := engine.annotationSourceValidationForTask(f.ctx, f.task, current, true)
|
||||
if e != nil || state.Status != "valid" {
|
||||
t.Fatalf("repaired admission state=%+v err=%v", state, e)
|
||||
}
|
||||
cert, _ := f.store.GetAnnotationTimelineIndex(f.ctx, sourceCertificateID(f.task.OrganizationID, repairedFingerprint))
|
||||
if cert == nil {
|
||||
t.Fatal("current source certificate missing")
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
done = nil
|
||||
// A later source version cannot inherit the earlier repair's exemption.
|
||||
current.ProcessingJob.SourceFingerprint = "later-failed-source"
|
||||
f.store.UpsertHumanItem(f.ctx, *current)
|
||||
f.store.EnqueueAnnotationSourceIndex(f.ctx, f.task, "later-failed-source", false)
|
||||
later, _ := f.store.ClaimAnnotationIndex(f.ctx, "later-validator", stamp, until)
|
||||
later.Status, later.Error = "failed", "IMU row 9 invalid type gyr"
|
||||
f.store.FinishAnnotationIndex(f.ctx, *later)
|
||||
state, e = engine.annotationSourceValidationForTask(f.ctx, f.task, current, true)
|
||||
if e != nil || state.Status != "failed" {
|
||||
t.Fatalf("earlier repair bypassed later failure: %+v %v", state, e)
|
||||
}
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("repaired source did not complete real validation")
|
||||
}
|
||||
|
||||
func TestRequiredSourceCertificateCannotBeMissing(t *testing.T) {
|
||||
f := randomClaimFixture(t, 1)
|
||||
task, _ := f.store.GetAnnotationTask(f.ctx, "random-task-0")
|
||||
f.store.EnqueueAnnotationIndex(f.ctx, *task)
|
||||
now := time.Now().UTC()
|
||||
job, _ := f.store.ClaimAnnotationIndex(f.ctx, "timeline", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
job.Status = "completed"
|
||||
f.store.FinishAnnotationIndex(f.ctx, *job)
|
||||
item, _ := f.store.GetHumanItem(f.ctx, "random-item-0")
|
||||
item.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: "not-enqueued-yet", SourceIndex: true}
|
||||
f.store.UpsertHumanItem(f.ctx, *item)
|
||||
if got := claimRandom(t, f, "missing-source-00001"); got.ID != "" {
|
||||
t.Fatal("required missing source certificate admitted")
|
||||
}
|
||||
item.ProcessingJob.SourceIndex = false
|
||||
f.store.UpsertHumanItem(f.ctx, *item)
|
||||
if got := claimRandom(t, f, "legacy-physical-001"); got.ID != "random-item-0" {
|
||||
t.Fatal("legacy physical outbox incorrectly required source certificate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceFailureDetectedAfterPriorSuccessRevokesAdmission(t *testing.T) {
|
||||
f, root := annotationProcessorFixture(t)
|
||||
now := time.Now().UTC()
|
||||
job, _ := f.store.ClaimAnnotationIndex(f.ctx, "old-valid", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
job.Status = "completed"
|
||||
f.store.FinishAnnotationIndex(f.ctx, *job)
|
||||
path := filepath.Join(root, "imu.csv")
|
||||
b, _ := os.ReadFile(path)
|
||||
if e := os.WriteFile(path, []byte(strings.TrimSuffix(string(b), "gyro\n")+"gyr\n"), 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
response := f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", f.admin, nil)
|
||||
if response.Code != 422 || !strings.Contains(response.Body.String(), `"retryAllowed":true`) {
|
||||
t.Fatal(response.Code, response.Body.String())
|
||||
}
|
||||
persisted, _ := f.store.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if persisted.Status != "failed" {
|
||||
t.Fatal("new source failure did not revoke admission")
|
||||
}
|
||||
response = f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/source-validation-retries", f.admin, map[string]any{"sourceRepaired": true})
|
||||
if response.Code != 202 {
|
||||
t.Fatalf("advertised recovery was unavailable: %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
if ok, e := f.store.FailAnnotationSourceValidation(f.ctx, f.task, job.CreatedAt, "stale HTTP validation failure"); ok || e != nil {
|
||||
t.Fatal("old read error overrode active revalidation", ok, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceValidationCanceledTimelineDoesNotQuarantineSource(t *testing.T) {
|
||||
f, _ := annotationProcessorFixture(t)
|
||||
now := time.Now().UTC()
|
||||
job, err := f.store.ClaimAnnotationIndex(f.ctx, "valid-before-cancel", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if err != nil || job == nil {
|
||||
t.Fatal("validation job missing", err)
|
||||
}
|
||||
job.Status = "completed"
|
||||
if ok, err := f.store.FinishAnnotationIndex(f.ctx, *job); !ok || err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
ctx = context.WithValue(ctx, authContextKey{}, &authenticatedPrincipal{User: f.admin, Organization: domain.OrganizationRecord{ID: f.task.OrganizationID, Status: domain.OrganizationStatusActive}})
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", nil).WithContext(ctx)
|
||||
response := httptest.NewRecorder()
|
||||
f.handler.ServeHTTP(response, request)
|
||||
current, err := f.store.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if err != nil || current.Status != "completed" {
|
||||
t.Fatalf("canceled read quarantined source: status=%s err=%v response=%s", current.Status, err, response.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceValidationRejectsNonFiniteIMUSamples(t *testing.T) {
|
||||
for _, value := range []string{"NaN", "+Inf", "not-a-number"} {
|
||||
t.Run(value, func(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "imu.csv")
|
||||
body := "timestamp_us,x,y,z,type\n1000000,0,0,0,accel\n1000000," + value + ",0,0,gyro\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := scanHardwareIMU(context.Background(), path, nil)
|
||||
if err == nil || safeSourceValidationMessage(err.Error()) != "IMU 第 3 行数值不合法" {
|
||||
t.Fatalf("invalid numeric sample was not identified: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceValidationKnownInvalidLegacySourceCanRecoverWithoutLosingWork(t *testing.T) {
|
||||
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
||||
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "validation")
|
||||
f, root := annotationProcessorFixture(t)
|
||||
f.task.ID, f.task.TemplateVersion = "legacy-source-task", 4
|
||||
f.item.ID, f.item.LegacyTaskID = "legacy-source-item", f.task.ID
|
||||
f.item.Subject.ItemKey = "legacy-source"
|
||||
f.item.Subject.Payload["draft"] = "legacy saved work"
|
||||
if err := f.store.UpsertAnnotationTask(f.ctx, f.task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.store.UpsertHumanItem(f.ctx, f.item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job, err := f.store.GetAnnotationIndexJob(f.ctx, f.task.ID); err != nil || job != nil {
|
||||
t.Fatal("legacy task unexpectedly requires validation", err)
|
||||
}
|
||||
read := func(ctx context.Context, user domain.UserRecord, org string) *httptest.ResponseRecorder {
|
||||
ctx = context.WithValue(ctx, authContextKey{}, &authenticatedPrincipal{User: user, Organization: domain.OrganizationRecord{ID: org, Status: domain.OrganizationStatusActive}})
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", nil).WithContext(ctx)
|
||||
response := httptest.NewRecorder()
|
||||
f.handler.ServeHTTP(response, request)
|
||||
return response
|
||||
}
|
||||
canceled, cancelRead := context.WithCancel(f.ctx)
|
||||
cancelRead()
|
||||
if response := read(canceled, f.admin, f.task.OrganizationID); response.Code != 503 {
|
||||
t.Fatalf("canceled legacy read: %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
foreign := f.admin
|
||||
foreign.OrganizationID = "other-source-org"
|
||||
if response := read(f.ctx, foreign, foreign.OrganizationID); response.Code != 403 && response.Code != 404 {
|
||||
t.Fatalf("cross-organization legacy read allowed: %d", response.Code)
|
||||
}
|
||||
if job, err := f.store.GetAnnotationIndexJob(f.ctx, f.task.ID); err != nil || job != nil {
|
||||
t.Fatal("canceled/unauthorized read quarantined legacy source", err)
|
||||
}
|
||||
path := filepath.Join(root, "imu.csv")
|
||||
original, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(strings.TrimSuffix(string(original), "gyro\n")+"gyr\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response := f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", f.admin, nil)
|
||||
if response.Code != 422 || !strings.Contains(response.Body.String(), `"retryAllowed":true`) || !strings.Contains(response.Body.String(), "gyr") {
|
||||
t.Fatalf("known bad legacy source has no recovery: %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
job, err := f.store.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if err != nil || job == nil || job.Status != "failed" {
|
||||
t.Fatalf("legacy source failure not durable: %+v %v", job, err)
|
||||
}
|
||||
response = f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/source-validation-retries", f.annotator, map[string]any{"sourceRepaired": true})
|
||||
if response.Code != 403 {
|
||||
t.Fatalf("legacy retry bypassed admin role: %d", response.Code)
|
||||
}
|
||||
if err := os.WriteFile(path, original, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response = f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/source-validation-retries", f.admin, map[string]any{"sourceRepaired": true})
|
||||
if response.Code != 202 {
|
||||
t.Fatal(response.Code, response.Body.String())
|
||||
}
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false)
|
||||
}()
|
||||
defer func() { cancel(); <-done }()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
job, err = f.store.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Status == "completed" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("legacy recovery did not finish: %+v", job)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
response = f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", f.annotator, nil)
|
||||
if response.Code != 200 {
|
||||
t.Fatal(response.Code, response.Body.String())
|
||||
}
|
||||
item, err := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
if err != nil || item.Version != f.item.Version || item.AssignedUserID != f.item.AssignedUserID || item.Subject.Payload["draft"] != "legacy saved work" {
|
||||
t.Fatal("legacy recovery changed ownership/draft", err)
|
||||
}
|
||||
item.Subject.Payload["materialization"] = map[string]any{"sourceIndex": true}
|
||||
if err := f.store.UpsertHumanItem(f.ctx, *item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response = f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", f.admin, nil)
|
||||
if response.Code != 422 || !strings.Contains(response.Body.String(), "source_validation_unverified") {
|
||||
t.Fatalf("historical source marker without proof admitted: %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
response = f.request(http.MethodPost, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/source-validation-retries", f.admin, map[string]any{"sourceRepaired": true})
|
||||
if response.Code != 202 {
|
||||
t.Fatal(response.Code, response.Body.String())
|
||||
}
|
||||
deadline = time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
job, err = f.store.GetAnnotationIndexJob(f.ctx, f.task.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if job.Status == "completed" {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("payload-only validation did not finish: %+v", job)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if !job.RevalidateSource || job.SourceFingerprint == "" || job.SupersededSourceFingerprint != "" {
|
||||
t.Fatalf("missing identity-bound historical proof: %+v", job)
|
||||
}
|
||||
response = f.request(http.MethodGet, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/frame-timeline", f.annotator, nil)
|
||||
if response.Code != 200 {
|
||||
t.Fatalf("validated payload-only source remained blocked: %d %s", response.Code, response.Body.String())
|
||||
}
|
||||
}
|
||||
96
internal/api/annotation_source_validation_browser_test.go
Normal file
96
internal/api/annotation_source_validation_browser_test.go
Normal file
@ -0,0 +1,96 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSourceValidationBrowserReplica(t *testing.T) {
|
||||
if os.Getenv("SOURCE_VALIDATION_BROWSER_TEST") != "1" {
|
||||
t.Skip("local browser opt-in")
|
||||
}
|
||||
t.Setenv("ANNOTATION_PROCESSOR_MODE", "external")
|
||||
t.Setenv("ANNOTATION_PROCESSOR_ROLE", "validation")
|
||||
f, root := annotationProcessorFixture(t)
|
||||
if err := f.store.UpsertOrganizationPlugin(f.ctx, domain.OrganizationPluginRecord{OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", Enabled: true, Config: defaultAnnotationPluginConfig()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
draft := f.request(http.MethodPut, "/api/plugins/annotation-qa/tasks/"+f.task.ID+"/drafts", f.annotator, map[string]any{
|
||||
"version": f.item.Version, "sliceSegments": []map[string]any{{"id": "source-draft-slice", "name": "测试切片", "start": 0., "end": 4.}},
|
||||
"segments": []map[string]any{{"id": "source-draft-action", "start": 0., "end": 4., "description": "已保存的测试标注", "status": "described", "source": "manual", "confidence": 1.}},
|
||||
"operations": []any{}, "statuses": map[string]string{},
|
||||
})
|
||||
if draft.Code != 200 {
|
||||
t.Fatalf("seed draft: %d %s", draft.Code, draft.Body.String())
|
||||
}
|
||||
before, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
now := time.Now().UTC()
|
||||
job, err := f.store.ClaimAnnotationIndex(f.ctx, "browser-invalid-source", now.Format(time.RFC3339Nano), now.Add(time.Minute).Format(time.RFC3339Nano))
|
||||
if err != nil || job == nil {
|
||||
t.Fatal("missing index job", err)
|
||||
}
|
||||
job.Status, job.Error, job.Attempt = "failed", "IMU row 418451 invalid type gyr", 5
|
||||
if ok, err := f.store.FinishAnnotationIndex(f.ctx, *job); err != nil || !ok {
|
||||
t.Fatal("seed invalid job", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var start sync.Once
|
||||
var processes sync.WaitGroup
|
||||
defer processes.Wait()
|
||||
defer cancel()
|
||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/__fixture/validate" && r.Method == http.MethodPost {
|
||||
start.Do(func() {
|
||||
processes.Add(1)
|
||||
go func() {
|
||||
defer processes.Done()
|
||||
_ = RunAnnotationProcessor(ctx, f.store, config.AppConfig{WorkspaceMountPath: root}, 1, false)
|
||||
}()
|
||||
})
|
||||
renderJSON(w, 202, map[string]bool{"started": true})
|
||||
return
|
||||
}
|
||||
user := f.annotator
|
||||
if cookie, err := r.Cookie("source-fixture-role"); err == nil && cookie.Value == "admin" {
|
||||
user = f.admin
|
||||
}
|
||||
if r.URL.Path == "/api/auth/me" {
|
||||
renderJSON(w, 200, map[string]any{"user": sanitizeUser(&user), "organization": domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}})
|
||||
return
|
||||
}
|
||||
if r.Method != "GET" && r.Method != "HEAD" && !strings.HasSuffix(r.URL.Path, "/source-validation-retries") {
|
||||
renderJSON(w, 403, map[string]string{"error": "fixture read-only"})
|
||||
return
|
||||
}
|
||||
csrf := "source-fixture-csrf"
|
||||
r.AddCookie(&http.Cookie{Name: csrfCookieName, Value: csrf})
|
||||
r.Header.Set(csrfHeaderName, csrf)
|
||||
r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: user, Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}}))
|
||||
f.handler.ServeHTTP(w, r)
|
||||
}))
|
||||
defer api.Close()
|
||||
commandCtx, done := context.WithTimeout(t.Context(), 3*time.Minute)
|
||||
defer done()
|
||||
cmd := exec.CommandContext(commandCtx, "node", "tests/browser/annotation-source-validation-replica.mjs")
|
||||
cmd.Dir = "../.."
|
||||
cmd.Env = append(os.Environ(), "SOURCE_VALIDATION_API="+api.URL)
|
||||
out, err := cmd.CombinedOutput()
|
||||
t.Log(string(out))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, _ := f.store.GetHumanItem(f.ctx, f.item.ID)
|
||||
if after.Version != before.Version || after.AssignedUserID != before.AssignedUserID || string(mustJSON(t, after.Subject.Payload)) != string(mustJSON(t, before.Subject.Payload)) {
|
||||
t.Fatal("source validation altered annotation ownership or saved content")
|
||||
}
|
||||
}
|
||||
@ -176,6 +176,10 @@ func parseHumanItemQuery(request *http.Request) (domain.HumanItemQuery, *validat
|
||||
PluginID: values.Get("pluginId"), BatchID: values.Get("batchId"), ProjectID: values.Get("projectId"),
|
||||
WorkflowID: values.Get("workflowId"), RunID: values.Get("runId"), CurrentStageID: values.Get("stageId"),
|
||||
Status: domain.HumanItemStatus(values.Get("status")),
|
||||
Search: strings.TrimSpace(values.Get("search")), Capture: strings.TrimSpace(values.Get("capture")), Category: strings.TrimSpace(values.Get("category")),
|
||||
}
|
||||
if len(query.Search) > 512 || len(query.Capture) > 512 {
|
||||
return query, &validationIssue{Field: "search", Message: "search and capture must not exceed 512 bytes"}
|
||||
}
|
||||
// stageIds=a,b,c pushes the role workspace's stage set into the store
|
||||
// query. The single-value stageId stays supported; combining both merges
|
||||
@ -283,7 +287,7 @@ type humanItemListPage struct {
|
||||
}
|
||||
|
||||
func (page *humanItemListPage) response() map[string]any {
|
||||
result := map[string]any{"items": page.Items}
|
||||
result := map[string]any{"items": page.Items, "filtersApplied": true}
|
||||
if page.HasMore {
|
||||
result["nextCursor"] = page.NextCursor
|
||||
result["hasMore"] = true
|
||||
@ -331,6 +335,7 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
return nil, request
|
||||
}
|
||||
query.IDs = ids
|
||||
query.ReviewedPool = true
|
||||
}
|
||||
|
||||
if pool == "mine" && principal != nil {
|
||||
@ -366,6 +371,39 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
if randomClaimAnnotator(principal) && query.ActorUserID == "" && (query.PluginID == "" || query.PluginID == "annotation-qa") {
|
||||
query.BlindAnnotationActorID = principal.User.ID
|
||||
}
|
||||
// QC always needs a membership in the item's project. Restrict candidate
|
||||
// reads to this necessary scope before pagination; supplier/stage/read/claim
|
||||
// checks remain authoritative below. Shared acceptance is deliberately excluded.
|
||||
if pool == "available" && query.PluginID == "annotation-qa" && principal != nil && principal.User.Role == domain.UserRoleQualityReviewer {
|
||||
ctx, err := server.withHumanListReads(request.Context(), server.requestOrganizationID(request), query.ProjectID, nil)
|
||||
if err != nil {
|
||||
renderError(writer, 500, err)
|
||||
return nil, request
|
||||
}
|
||||
request = request.WithContext(ctx)
|
||||
access := server.acceptanceAccessFor(ctx, principal.User.ID)
|
||||
if access.err != nil {
|
||||
renderError(writer, 500, access.err)
|
||||
return nil, request
|
||||
}
|
||||
if access.user != nil && access.user.Role == domain.UserRoleQualityReviewer {
|
||||
roles, err := server.annotationMembershipsForUser(ctx, server.requestOrganizationID(request), "", principal.User.ID)
|
||||
if err != nil {
|
||||
renderError(writer, 500, err)
|
||||
return nil, request
|
||||
}
|
||||
query.AvailableSupplierID = &access.user.SupplierID
|
||||
query.AvailableProjectIDs = []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, role := range roles {
|
||||
if !seen[role.ProjectID] {
|
||||
seen[role.ProjectID] = true
|
||||
query.AvailableProjectIDs = append(query.AvailableProjectIDs, role.ProjectID)
|
||||
}
|
||||
}
|
||||
sort.Strings(query.AvailableProjectIDs)
|
||||
}
|
||||
}
|
||||
// List responses never need the workbench draft or disposition histories.
|
||||
query.ListProjection = true
|
||||
fetchSize := limit
|
||||
@ -393,6 +431,12 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
return nil, request
|
||||
}
|
||||
request = request.WithContext(listCtx)
|
||||
if reviewedAcceptanceCategory(query) {
|
||||
if err := server.projectReviewedAcceptanceStatuses(request.Context(), server.requestOrganizationID(request), items, batchByID); err != nil {
|
||||
renderError(writer, 500, err)
|
||||
return nil, request
|
||||
}
|
||||
}
|
||||
items, err = server.hydrateHumanPoolPage(request, items, pool, query, limit-len(filtered), batchByID)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
@ -466,31 +510,11 @@ func (server *server) readHumanItemsList(writer http.ResponseWriter, request *ht
|
||||
}
|
||||
}
|
||||
}
|
||||
acceptanceIDs := []string{}
|
||||
for _, item := range filtered {
|
||||
if item.Status == domain.HumanItemAccepted && isFanyuV58ThreeRoleBatch(batchByID[item.BatchID]) {
|
||||
acceptanceIDs = append(acceptanceIDs, fanyuV5PrePackageAcceptanceItemID(item.BatchID))
|
||||
}
|
||||
}
|
||||
accepted := map[string]bool{}
|
||||
if len(acceptanceIDs) > 0 {
|
||||
downstream, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), domain.HumanItemQuery{IDs: acceptanceIDs})
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
if !reviewedAcceptanceCategory(query) {
|
||||
if err := server.projectReviewedAcceptanceStatuses(request.Context(), server.requestOrganizationID(request), filtered, batchByID); err != nil {
|
||||
renderError(writer, 500, err)
|
||||
return nil, request
|
||||
}
|
||||
for _, item := range downstream {
|
||||
accepted[item.ID] = item.Status == domain.HumanItemAccepted
|
||||
}
|
||||
}
|
||||
for index := range filtered {
|
||||
item := &filtered[index]
|
||||
if item.Status == domain.HumanItemAccepted && isFanyuV58ThreeRoleBatch(batchByID[item.BatchID]) {
|
||||
item.ReviewHistoryStatus = "pending_acceptance"
|
||||
if accepted[fanyuV5PrePackageAcceptanceItemID(item.BatchID)] {
|
||||
item.ReviewHistoryStatus = "accepted"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
resolver := newHumanDisplayResolver(server)
|
||||
@ -554,6 +578,15 @@ func (server *server) filterHumanPoolItem(request *http.Request, item domain.Hum
|
||||
if pool != "reviewed" && !server.canAccessHumanItem(request.Context(), request, item, annotationRoleForHumanStage(stage)) && !isAuthor {
|
||||
return item, false, nil
|
||||
}
|
||||
if reviewedAcceptanceCategory(query) && item.Status == domain.HumanItemAccepted {
|
||||
status := item.ReviewHistoryStatus
|
||||
if status == "" {
|
||||
status = "accepted"
|
||||
}
|
||||
if query.Category != status {
|
||||
return item, false, nil
|
||||
}
|
||||
}
|
||||
if pool == "mine" {
|
||||
isAssigned := principal != nil && item.AssignedUserID == principal.User.ID
|
||||
isMySubmitted := isAuthor && (item.Status == domain.HumanItemReady || item.Status == domain.HumanItemSubmitted) && item.CurrentStageID == "quality_review"
|
||||
@ -610,15 +643,23 @@ func (server *server) hydrateHumanPoolPage(request *http.Request, items []domain
|
||||
stable := len(full) == len(ids)
|
||||
for _, item := range full {
|
||||
index, found := selected[item.ID]
|
||||
if found {
|
||||
item.ReviewHistoryStatus = items[index].ReviewHistoryStatus
|
||||
}
|
||||
if !found || !reflect.DeepEqual(items[index].PoolCandidate(), item.PoolCandidate()) {
|
||||
stable = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !stable {
|
||||
return server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), query)
|
||||
current, err := server.store.ListHumanItems(request.Context(), server.requestOrganizationID(request), query)
|
||||
if err == nil && reviewedAcceptanceCategory(query) {
|
||||
err = server.projectReviewedAcceptanceStatuses(request.Context(), server.requestOrganizationID(request), current, batches)
|
||||
}
|
||||
return current, err
|
||||
}
|
||||
for _, item := range full {
|
||||
item.ReviewHistoryStatus = items[selected[item.ID]].ReviewHistoryStatus
|
||||
items[selected[item.ID]] = item
|
||||
}
|
||||
return items, nil
|
||||
@ -977,6 +1018,16 @@ func (server *server) canClaimHumanItem(ctx context.Context, request *http.Reque
|
||||
if !ok || principal == nil {
|
||||
return false
|
||||
}
|
||||
if requiredRole == domain.AnnotationRoleAnnotator && item.PluginID == "annotation-qa" && item.LegacyTaskID != "" {
|
||||
task, err := server.store.GetAnnotationTask(ctx, item.LegacyTaskID)
|
||||
if err != nil || task == nil {
|
||||
return false
|
||||
}
|
||||
validation, err := server.annotationSourceValidationForTask(ctx, *task, &item, false)
|
||||
if err != nil || validation.blocked() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// Role changes must not let an author claim their own quality review.
|
||||
if requiredRole == domain.AnnotationRoleQualityReviewer && item.LatestAuthorUserID != "" && item.LatestAuthorUserID == principal.User.ID {
|
||||
return false
|
||||
|
||||
144
internal/api/human_pool_project_scope_test.go
Normal file
144
internal/api/human_pool_project_scope_test.go
Normal file
@ -0,0 +1,144 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type qcProjectScopeProbe struct {
|
||||
store.Store
|
||||
legacy bool
|
||||
candidates int
|
||||
}
|
||||
|
||||
func (p *qcProjectScopeProbe) ListHumanItems(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
if p.legacy {
|
||||
q.AvailableProjectIDs = nil
|
||||
q.AvailableSupplierID = nil
|
||||
}
|
||||
rows, err := p.Store.ListHumanItems(ctx, org, q)
|
||||
if q.PoolCandidateProjection {
|
||||
p.candidates += len(rows)
|
||||
}
|
||||
return rows, err
|
||||
}
|
||||
func TestHumanPoolQCProjectScopeSkipsInvisibleBacklog(t *testing.T) {
|
||||
f := newCombinedDraftFixture(t)
|
||||
item := f.item
|
||||
item.CurrentStageID = "quality_review"
|
||||
item.AssignedUserID = ""
|
||||
item.Status = domain.HumanItemReady
|
||||
item.SupplierID = f.reviewer.SupplierID
|
||||
if err := f.store.UpsertHumanItem(f.ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 90; i++ {
|
||||
noise := item
|
||||
noise.ID = fmt.Sprintf("hidden-%03d", i)
|
||||
if i%2 == 0 {
|
||||
noise.ProjectID = "foreign-project"
|
||||
} else {
|
||||
noise.SupplierID = "foreign-supplier"
|
||||
}
|
||||
if err := f.store.UpsertHumanItem(f.ctx, noise); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
path := "/api/human-items?pluginId=annotation-qa&pool=available&stageIds=quality_review,semantic_review&limit=50"
|
||||
old := &qcProjectScopeProbe{Store: f.store, legacy: true}
|
||||
f.handler = NewHandler(old, nil)
|
||||
before := f.request(http.MethodGet, path, f.reviewer, nil)
|
||||
probe := &qcProjectScopeProbe{Store: f.store}
|
||||
f.handler = NewHandler(probe, nil)
|
||||
after := f.request(http.MethodGet, path, f.reviewer, nil)
|
||||
if before.Code != 200 || after.Code != 200 || before.Body.String() != after.Body.String() {
|
||||
t.Fatalf("changed response %d/%d", before.Code, after.Code)
|
||||
}
|
||||
var page struct{ Items []domain.HumanItemRecord }
|
||||
if err := json.Unmarshal(after.Body.Bytes(), &page); err != nil || len(page.Items) != 1 {
|
||||
t.Fatalf("expected one visible task: %s", after.Body.String())
|
||||
}
|
||||
if old.candidates != 91 || probe.candidates != 1 {
|
||||
t.Fatalf("candidate reads %d -> %d", old.candidates, probe.candidates)
|
||||
}
|
||||
// Permissions are rebuilt on each request. A disabled account cannot reuse scope.
|
||||
user := f.reviewer
|
||||
user.Status = domain.UserStatusDisabled
|
||||
if err := f.store.UpsertUser(f.ctx, user); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after = f.request(http.MethodGet, path, user, nil)
|
||||
if after.Code == 200 {
|
||||
var p struct{ Items []domain.HumanItemRecord }
|
||||
json.Unmarshal(after.Body.Bytes(), &p)
|
||||
if len(p.Items) > 0 {
|
||||
t.Fatal("disabled account retained visible tasks")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanPoolQCProjectScopePreservesMultiPageVisibleTasks(t *testing.T) {
|
||||
f := newCombinedDraftFixture(t)
|
||||
item := f.item
|
||||
item.CurrentStageID = "quality_review"
|
||||
item.AssignedUserID = ""
|
||||
item.Status = domain.HumanItemReady
|
||||
item.SupplierID = f.reviewer.SupplierID
|
||||
item.CreatedAt = "2026-01-01T00:00:00Z"
|
||||
if err := f.store.UpsertHumanItem(f.ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 700; i++ {
|
||||
noise := item
|
||||
noise.ID = fmt.Sprintf("noise-%04d", i)
|
||||
noise.SupplierID = "foreign-supplier"
|
||||
noise.CreatedAt = "2026-02-01T00:00:00Z"
|
||||
if err := f.store.UpsertHumanItem(f.ctx, noise); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
collect := func(legacy bool) []string {
|
||||
f.handler = NewHandler(&qcProjectScopeProbe{Store: f.store, legacy: legacy}, nil)
|
||||
cursor := ""
|
||||
ids := []string{}
|
||||
for page := 0; page < 10; page++ {
|
||||
path := "/api/human-items?pluginId=annotation-qa&pool=available&stageIds=quality_review,semantic_review&limit=5"
|
||||
if cursor != "" {
|
||||
path += "&cursor=" + cursor
|
||||
}
|
||||
r := f.request(http.MethodGet, path, f.reviewer, nil)
|
||||
if r.Code != 200 {
|
||||
t.Fatalf("list failed %d", r.Code)
|
||||
}
|
||||
var out struct {
|
||||
Items []domain.HumanItemRecord
|
||||
HasMore bool
|
||||
NextCursor string
|
||||
}
|
||||
if err := json.Unmarshal(r.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, it := range out.Items {
|
||||
ids = append(ids, it.ID)
|
||||
}
|
||||
if !out.HasMore {
|
||||
return ids
|
||||
}
|
||||
if out.NextCursor == "" || out.NextCursor == cursor {
|
||||
t.Fatal("cursor failed to advance")
|
||||
}
|
||||
cursor = out.NextCursor
|
||||
}
|
||||
t.Fatal("pagination did not finish")
|
||||
return nil
|
||||
}
|
||||
before, after := collect(true), collect(false)
|
||||
if len(before) != 1 || len(after) != 1 || before[0] != after[0] {
|
||||
t.Fatalf("visible tasks changed %v %v", before, after)
|
||||
}
|
||||
}
|
||||
155
internal/api/human_pool_search_test.go
Normal file
155
internal/api/human_pool_search_test.go
Normal file
@ -0,0 +1,155 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHumanPoolSearchBeyondFirst500(t *testing.T) {
|
||||
f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1)
|
||||
target := f.item
|
||||
target.ID = "old-search-target"
|
||||
target.Subject.ItemKey = "FYLX_B02_AZER76400GJ_19700116_013952_camera_left_part0002.mp4"
|
||||
target.CreatedAt = "2020-01-01T00:00:00Z"
|
||||
target.LatestAuthorUserID = f.annotator.ID
|
||||
if err := f.store.UpsertHumanItem(f.ctx, target); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i < 962; i++ {
|
||||
item := f.item
|
||||
item.ID = fmt.Sprintf("recent-%04d", i)
|
||||
item.Subject.ItemKey = fmt.Sprintf("FYLX_OTHER_%04d/part0001/camera_left.mp4", i)
|
||||
item.CreatedAt = time.Date(2026, 11, 1, 0, 0, i, 0, time.UTC).Format(time.RFC3339)
|
||||
if err := f.store.UpsertHumanItem(f.ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
response := f.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=available&stageId="+target.CurrentStageID+"&limit=200&search="+url.QueryEscape("FYLX_B02_AZER76400GJ_19700116_013952/part0002"), f.quality, nil)
|
||||
if response.Code != 200 {
|
||||
t.Fatalf("status %d: %s", response.Code, response.Body)
|
||||
}
|
||||
var page humanItemListPage
|
||||
if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Items[0].ID != target.ID || page.HasMore {
|
||||
t.Fatalf("search must find the old authorized item beyond 962 newer records, got %d items, more=%v", len(page.Items), page.HasMore)
|
||||
}
|
||||
if page.Items[0].CanClaim == nil || !*page.Items[0].CanClaim {
|
||||
t.Fatal("search lost server eligibility")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanPoolSearchPaginationPermissionsAndHistory(t *testing.T) {
|
||||
for _, pool := range []string{"available", "mine", "reviewed"} {
|
||||
t.Run(pool, func(t *testing.T) {
|
||||
f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1)
|
||||
const count = 611
|
||||
for i := 0; i < count; i++ {
|
||||
item := f.item
|
||||
item.ID = fmt.Sprintf("match-%04d", i)
|
||||
item.Subject.ItemKey = fmt.Sprintf("FYLX_MATCH_%04d/part0001/camera_left.mp4", i)
|
||||
item.CreatedAt = time.Date(2026, 11, 1, 0, 0, i, 0, time.UTC).Format(time.RFC3339)
|
||||
item.LatestAuthorUserID = f.annotator.ID
|
||||
if pool == "mine" {
|
||||
item.AssignedUserID = f.quality.ID
|
||||
item.Status = domain.HumanItemClaimed
|
||||
}
|
||||
if pool == "reviewed" {
|
||||
item.CurrentStageID = domain.FanyuV5StageSemanticAnnotation
|
||||
item.Status = domain.HumanItemRework
|
||||
}
|
||||
if e := f.store.UpsertHumanItem(f.ctx, item); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if pool == "reviewed" {
|
||||
if e := f.store.AppendHumanEvent(f.ctx, domain.HumanEventRecord{ID: "event-" + item.ID, OrganizationID: item.OrganizationID, ItemID: item.ID, ActorUserID: f.quality.ID, EventType: "rejected", StageID: domain.FanyuV5StageSemanticReview}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
foreign := f.item
|
||||
foreign.ID = "foreign"
|
||||
foreign.Subject.ItemKey = "FYLX_MATCH_FOREIGN/part0001/camera_left.mp4"
|
||||
foreign.SupplierID = "other-supplier"
|
||||
foreign.VendorID = "other-supplier"
|
||||
if e := f.store.UpsertHumanItem(f.ctx, foreign); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
cursor := ""
|
||||
for round := 0; round < 10; round++ {
|
||||
path := "/api/human-items?pluginId=annotation-qa&pool=" + pool + "&limit=200&search=FYLX_MATCH"
|
||||
if pool != "reviewed" {
|
||||
path += "&stageIds=quality_review,semantic_review"
|
||||
}
|
||||
if cursor != "" {
|
||||
path += "&cursor=" + cursor
|
||||
}
|
||||
response := f.request(http.MethodGet, path, f.quality, nil)
|
||||
if response.Code != 200 {
|
||||
t.Fatalf("%d %s", response.Code, response.Body)
|
||||
}
|
||||
var page humanItemListPage
|
||||
if e := json.Unmarshal(response.Body.Bytes(), &page); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(page.Items) > 200 {
|
||||
t.Fatal("unbounded page")
|
||||
}
|
||||
for _, item := range page.Items {
|
||||
if item.ID == "foreign" {
|
||||
t.Fatal("foreign supplier leaked")
|
||||
}
|
||||
if seen[item.ID] {
|
||||
t.Fatalf("duplicate %s", item.ID)
|
||||
}
|
||||
seen[item.ID] = true
|
||||
if pool == "reviewed" && item.ReviewHistory == nil {
|
||||
t.Fatal("review history lost")
|
||||
}
|
||||
}
|
||||
if !page.HasMore {
|
||||
break
|
||||
}
|
||||
if page.NextCursor == "" || page.NextCursor == cursor {
|
||||
t.Fatal("cursor did not advance")
|
||||
}
|
||||
cursor = page.NextCursor
|
||||
}
|
||||
if len(seen) != count {
|
||||
t.Fatalf("found %d, want %d", len(seen), count)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHumanPoolSearchKeepsSelfReviewBlockedAndSearchLiteral(t *testing.T) {
|
||||
f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1)
|
||||
item := f.item
|
||||
item.Subject.ItemKey = "FYLX_SELF/part0001/camera_left.mp4"
|
||||
item.LatestAuthorUserID = f.quality.ID
|
||||
if e := f.store.UpsertHumanItem(f.ctx, item); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
r := f.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=available&search=FYLX_SELF", f.quality, nil)
|
||||
var page humanItemListPage
|
||||
if e := json.Unmarshal(r.Body.Bytes(), &page); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Items[0].CanClaim == nil || *page.Items[0].CanClaim {
|
||||
t.Fatalf("self review eligibility changed: %s", r.Body)
|
||||
}
|
||||
r = f.request(http.MethodGet, "/api/human-items?pluginId=annotation-qa&pool=available&search="+url.QueryEscape("[.*"), f.quality, nil)
|
||||
if e := json.Unmarshal(r.Body.Bytes(), &page); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(page.Items) != 0 {
|
||||
t.Fatalf("search interpreted as regex: %s", r.Body)
|
||||
}
|
||||
}
|
||||
60
internal/api/human_review_history_category.go
Normal file
60
internal/api/human_review_history_category.go
Normal file
@ -0,0 +1,60 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func reviewedAcceptanceCategory(query domain.HumanItemQuery) bool {
|
||||
return query.ReviewedPool && (query.Category == "accepted" || query.Category == "pending_acceptance")
|
||||
}
|
||||
|
||||
// Match the existing history badge before page selection, using one compact
|
||||
// downstream lookup per candidate page. Never persist this read projection.
|
||||
func (server *server) projectReviewedAcceptanceStatuses(ctx context.Context, org string, items []domain.HumanItemRecord, batches map[string]*domain.HumanBatchRecord) error {
|
||||
ids := []string{}
|
||||
seen := map[string]bool{}
|
||||
for _, item := range items {
|
||||
if item.Status != domain.HumanItemAccepted {
|
||||
continue
|
||||
}
|
||||
batch, loaded := batches[item.BatchID]
|
||||
if !loaded {
|
||||
var err error
|
||||
batch, err = server.humanBatchForListRead(ctx, item.BatchID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
batches[item.BatchID] = batch
|
||||
}
|
||||
if !isFanyuV58ThreeRoleBatch(batch) {
|
||||
continue
|
||||
}
|
||||
id := fanyuV5PrePackageAcceptanceItemID(item.BatchID)
|
||||
if !seen[id] {
|
||||
ids = append(ids, id)
|
||||
seen[id] = true
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
downstream, err := server.store.ListHumanItems(ctx, org, domain.HumanItemQuery{IDs: ids, PoolCandidateProjection: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accepted := map[string]bool{}
|
||||
for _, item := range downstream {
|
||||
accepted[item.ID] = item.Status == domain.HumanItemAccepted
|
||||
}
|
||||
for index := range items {
|
||||
item := &items[index]
|
||||
if item.Status == domain.HumanItemAccepted && isFanyuV58ThreeRoleBatch(batches[item.BatchID]) {
|
||||
item.ReviewHistoryStatus = "pending_acceptance"
|
||||
if accepted[fanyuV5PrePackageAcceptanceItemID(item.BatchID)] {
|
||||
item.ReviewHistoryStatus = "accepted"
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -195,8 +195,17 @@ func scanHardwareIMU(ctx context.Context, path string, consume func(hardwareIMUR
|
||||
}
|
||||
timestamp, parseErr := strconv.ParseInt(strings.TrimSpace(record[timestampIndex]), 10, 64)
|
||||
kind := strings.ToLower(strings.TrimSpace(record[typeIndex]))
|
||||
if parseErr != nil || timestamp <= 0 || (summary.count > 0 && timestamp < summary.last) || (kind != "accel" && kind != "gyro") {
|
||||
return summary, fmt.Errorf("IMU CSV %s has invalid ordering or type at row %d", filepath.Base(path), rowIndex)
|
||||
if kind != "accel" && kind != "gyro" {
|
||||
return summary, fmt.Errorf("IMU CSV %s invalid type %q at row %d", filepath.Base(path), kind, rowIndex)
|
||||
}
|
||||
if parseErr != nil || timestamp <= 0 || (summary.count > 0 && timestamp < summary.last) {
|
||||
return summary, fmt.Errorf("IMU CSV %s has invalid timestamp ordering at row %d", filepath.Base(path), rowIndex)
|
||||
}
|
||||
for _, index := range []int{xIndex, yIndex, zIndex} {
|
||||
value, err := strconv.ParseFloat(strings.TrimSpace(record[index]), 64)
|
||||
if err != nil || math.IsNaN(value) || math.IsInf(value, 0) {
|
||||
return summary, fmt.Errorf("IMU CSV %s has non-finite or invalid numeric value at row %d", filepath.Base(path), rowIndex)
|
||||
}
|
||||
}
|
||||
if summary.count == 0 {
|
||||
summary.first = timestamp
|
||||
|
||||
@ -1,11 +1,15 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestReviewedPoolIncludesOwnDecisionsAcrossStages(t *testing.T) {
|
||||
@ -113,6 +117,20 @@ func TestReviewedPoolProjectsAggregateAcceptanceWithoutChangingSource(t *testing
|
||||
if res.Code != 200 || len(body.Items) != 1 || body.Items[0].ReviewHistoryStatus != want {
|
||||
t.Fatalf("want %s: %d %s", want, res.Code, res.Body.String())
|
||||
}
|
||||
for _, category := range []string{"pending_acceptance", "accepted"} {
|
||||
res := f.request(http.MethodGet, "/api/human-items?pool=reviewed&projectId="+f.task.ProjectID+"&category="+category+"&limit=1", f.reviewer, nil)
|
||||
var page humanItemListPage
|
||||
if res.Code != 200 || json.Unmarshal(res.Body.Bytes(), &page) != nil {
|
||||
t.Fatalf("category %s: %d %s", category, res.Code, res.Body)
|
||||
}
|
||||
wantCount := 0
|
||||
if category == want {
|
||||
wantCount = 1
|
||||
}
|
||||
if len(page.Items) != wantCount {
|
||||
t.Fatalf("history category %s want %d effective %s: %s", category, wantCount, want, res.Body)
|
||||
}
|
||||
}
|
||||
source, _ := f.store.GetHumanItem(f.ctx, item.ID)
|
||||
if source.ReviewHistoryStatus != "" || source.Status != domain.HumanItemAccepted {
|
||||
t.Fatal("read projection modified source")
|
||||
@ -242,3 +260,104 @@ func TestReviewedPoolLegacyDecisionDoesNotInventRevisionOrRound(t *testing.T) {
|
||||
t.Fatal("invented legacy round")
|
||||
}
|
||||
}
|
||||
|
||||
type historyCategoryReadProbe struct {
|
||||
store.Store
|
||||
downstream []domain.HumanItemQuery
|
||||
}
|
||||
|
||||
func (probe *historyCategoryReadProbe) ListHumanItems(ctx context.Context, org string, query domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
if len(query.IDs) > 0 && strings.HasPrefix(query.IDs[0], "human-item-fanyu-v5-prepackage-acceptance-") {
|
||||
probe.downstream = append(probe.downstream, query)
|
||||
}
|
||||
return probe.Store.ListHumanItems(ctx, org, query)
|
||||
}
|
||||
|
||||
func TestReviewedHistoryCategoryPaginationUsesDownstreamStatus(t *testing.T) {
|
||||
f := newCombinedDraftFixture(t)
|
||||
base, err := f.store.GetHumanBatch(f.ctx, f.item.BatchID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const count = 205
|
||||
for n := 0; n < count; n++ {
|
||||
batch := *base
|
||||
batch.ID = fmt.Sprintf("history-category-batch-%03d", n)
|
||||
if err := f.store.UpsertHumanBatch(f.ctx, batch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := f.item
|
||||
item.ID = fmt.Sprintf("history-category-%03d", n)
|
||||
item.BatchID = batch.ID
|
||||
item.CurrentStageID = "quality_review"
|
||||
item.Status = domain.HumanItemAccepted
|
||||
item.AssignedUserID = ""
|
||||
item.CreatedAt = time.Date(2026, 11, 1, 0, 0, n, 0, time.UTC).Format(time.RFC3339)
|
||||
if err := f.store.UpsertHumanItem(f.ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.store.AppendHumanEvent(f.ctx, domain.HumanEventRecord{ID: "approved-" + item.ID, OrganizationID: item.OrganizationID, ItemID: item.ID, ActorUserID: f.reviewer.ID, EventType: "approved"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n%2 == 0 {
|
||||
downstream := item
|
||||
downstream.ID = fanyuV5PrePackageAcceptanceItemID(batch.ID)
|
||||
downstream.CurrentStageID = "prepackage_acceptance"
|
||||
if err := f.store.UpsertHumanItem(f.ctx, downstream); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
probe := &historyCategoryReadProbe{Store: f.store}
|
||||
f.handler = NewHandler(probe, nil)
|
||||
for _, category := range []string{"pending_acceptance", "accepted"} {
|
||||
seen := map[string]bool{}
|
||||
cursor := ""
|
||||
for pageIndex := 0; pageIndex < 20; pageIndex++ {
|
||||
path := "/api/human-items?pool=reviewed&projectId=" + f.task.ProjectID + "&category=" + category + "&limit=17"
|
||||
if cursor != "" {
|
||||
path += "&cursor=" + cursor
|
||||
}
|
||||
res := f.request(http.MethodGet, path, f.reviewer, nil)
|
||||
var page humanItemListPage
|
||||
if res.Code != 200 || json.Unmarshal(res.Body.Bytes(), &page) != nil {
|
||||
t.Fatalf("category %s: %d %s", category, res.Code, res.Body)
|
||||
}
|
||||
if len(page.Items) > 17 {
|
||||
t.Fatal("unbounded page")
|
||||
}
|
||||
for _, item := range page.Items {
|
||||
if seen[item.ID] || item.ReviewHistoryStatus != category {
|
||||
t.Fatalf("wrong history page: %s %s", item.ID, item.ReviewHistoryStatus)
|
||||
}
|
||||
if item.Subject.Payload["preserved"] != true {
|
||||
t.Fatal("unhydrated metadata returned")
|
||||
}
|
||||
seen[item.ID] = true
|
||||
}
|
||||
if !page.HasMore {
|
||||
break
|
||||
}
|
||||
if page.NextCursor == "" || page.NextCursor == cursor {
|
||||
t.Fatal("nonadvancing cursor")
|
||||
}
|
||||
cursor = page.NextCursor
|
||||
}
|
||||
want := count / 2
|
||||
if category == "accepted" {
|
||||
want = (count + 1) / 2
|
||||
}
|
||||
if len(seen) != want {
|
||||
t.Fatalf("category %s got %d want %d", category, len(seen), want)
|
||||
}
|
||||
}
|
||||
if len(probe.downstream) == 0 {
|
||||
t.Fatal("missing downstream status reads")
|
||||
}
|
||||
for _, query := range probe.downstream {
|
||||
if len(query.IDs) > 100 || !query.PoolCandidateProjection {
|
||||
t.Fatalf("unbounded downstream detail read: %+v", query)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
@ -67,8 +68,13 @@ func TestReviewListsBoundRepeatedPermissionReads(t *testing.T) {
|
||||
t.Fatalf("N+1 reads: memberships=%d batches=%d linked=%d", counter.members, counter.batches, counter.linked)
|
||||
}
|
||||
// Cache must end with the request: revoking membership immediately removes visibility.
|
||||
before := res.Body.String()
|
||||
if before == "{\"items\":[]}\n" {
|
||||
var before struct {
|
||||
Items []json.RawMessage `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(res.Body.Bytes(), &before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(before.Items) == 0 {
|
||||
t.Fatal("fixture must expose review work")
|
||||
}
|
||||
u := f.reviewer
|
||||
@ -77,7 +83,13 @@ func TestReviewListsBoundRepeatedPermissionReads(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res = f.request(http.MethodGet, path+f.task.ProjectID, f.reviewer, nil)
|
||||
if res.Code != 200 || res.Body.String() != "{\"items\":[]}\n" {
|
||||
var after struct {
|
||||
Items []json.RawMessage `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(res.Body.Bytes(), &after); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Code != 200 || len(after.Items) != 0 {
|
||||
t.Fatalf("stale permission cache: %d %s", res.Code, res.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
74
internal/api/review_pool_search_browser_test.go
Normal file
74
internal/api/review_pool_search_browser_test.go
Normal file
@ -0,0 +1,74 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestReviewPoolSearchBrowserReplica(t *testing.T) {
|
||||
if os.Getenv("REVIEW_POOL_BROWSER_TEST") != "1" {
|
||||
t.Skip("local isolated browser opt-in")
|
||||
}
|
||||
f := newFanyuV5FourStageFixture(t, domain.FanyuV5StageSemanticReview, domain.HumanItemReady, "", 1)
|
||||
_ = f.store.UpsertOrganizationPlugin(f.ctx, domain.OrganizationPluginRecord{OrganizationID: domain.DefaultOrganizationID, PluginID: "annotation-qa", Enabled: true, Config: defaultAnnotationPluginConfig()})
|
||||
for i := 0; i < 613; i++ {
|
||||
item := f.item
|
||||
task := f.task
|
||||
item.ID = fmt.Sprintf("search-item-%04d", i)
|
||||
task.ID = fmt.Sprintf("search-task-%04d", i)
|
||||
item.LegacyTaskID = task.ID
|
||||
item.Subject.ItemKey = fmt.Sprintf("FYLX_SEARCH_%04d_camera_left_part0002.mp4", i)
|
||||
task.ArtifactID = item.Subject.ItemKey
|
||||
task.SourceVideo = item.Subject.ItemKey
|
||||
stamp := time.Date(2026, 11, 1, 0, 0, i, 0, time.UTC).Format(time.RFC3339)
|
||||
item.CreatedAt = stamp
|
||||
task.CreatedAt = stamp
|
||||
item.LatestAuthorUserID = f.annotator.ID
|
||||
if i == 611 {
|
||||
item.AssignedUserID = f.quality.ID
|
||||
item.Status = domain.HumanItemClaimed
|
||||
}
|
||||
if i == 612 {
|
||||
item.CurrentStageID = domain.FanyuV5StageSemanticAnnotation
|
||||
item.Status = domain.HumanItemRework
|
||||
task.Status = domain.AnnotationTaskStatusRework
|
||||
_ = f.store.AppendHumanEvent(f.ctx, domain.HumanEventRecord{ID: "history-search", OrganizationID: item.OrganizationID, ItemID: item.ID, ActorUserID: f.quality.ID, EventType: "rejected", StageID: domain.FanyuV5StageSemanticReview})
|
||||
}
|
||||
if e := f.store.UpsertHumanItem(f.ctx, item); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := f.store.UpsertAnnotationTask(f.ctx, task); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/auth/me" {
|
||||
renderJSON(w, 200, map[string]any{"user": sanitizeUser(&f.quality), "organization": domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}})
|
||||
return
|
||||
}
|
||||
if r.Method != "GET" && r.Method != "HEAD" {
|
||||
renderJSON(w, 403, map[string]string{"error": "fixture read-only"})
|
||||
return
|
||||
}
|
||||
r = r.WithContext(context.WithValue(r.Context(), authContextKey{}, &authenticatedPrincipal{User: f.quality, Organization: domain.OrganizationRecord{ID: domain.DefaultOrganizationID, Status: domain.OrganizationStatusActive}}))
|
||||
f.handler.ServeHTTP(w, r)
|
||||
}))
|
||||
defer server.Close()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 4*time.Minute)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "node", "tests/browser/review-pool-search-replica.mjs")
|
||||
cmd.Dir = "../.."
|
||||
cmd.Env = append(os.Environ(), "REVIEW_POOL_API="+server.URL)
|
||||
out, err := cmd.CombinedOutput()
|
||||
t.Log(string(out))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -50,6 +50,7 @@ func (server *server) mountAnnotationRoutes(router chi.Router) {
|
||||
reviewers.Post("/api/plugins/annotation-qa/tasks/{id}/reviews", server.handleAnnotationReviewCreate)
|
||||
|
||||
admin := server.withRoles(router, rolesAdmin...)
|
||||
admin.Post("/api/plugins/annotation-qa/tasks/{id}/source-validation-retries", server.handleAnnotationSourceValidationRetry)
|
||||
admin.Post("/api/annotation-templates", deprecatedAnnotationEndpoint(server.handleAnnotationTemplateCreate))
|
||||
admin.Post("/api/annotation-templates/{id}/publish", deprecatedAnnotationEndpoint(server.handleAnnotationTemplatePublish))
|
||||
admin.Post("/api/plugins/annotation-qa/templates", server.handleAnnotationTemplateCreate)
|
||||
|
||||
3
internal/api/testdata/routes.golden
vendored
3
internal/api/testdata/routes.golden
vendored
@ -1,7 +1,7 @@
|
||||
# Generated by TestRouteSnapshot; see routes_snapshot_test.go.
|
||||
# access: reach = endpoint reached; otherwise the status the middleware chain returned.
|
||||
global=[diagnosticTrace,middleware.(*Compressor).Handler-fm,(*server).invalidateDashboardCachesOnWrite-fm,(*server).apiTokenAuth-fm,(*server).sessionAuth-fm,(*server).csrfProtection-fm,(*server).projectDeletionWriteGuard-fm,(*server).supplierDeletionWriteGuard-fm,(*server).acceptanceAccessCache-fm,annotationReadCache]
|
||||
routes=375
|
||||
routes=376
|
||||
DELETE /api/admin/organizations/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminOrganizationsDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
|
||||
DELETE /api/admin/users/{id} | mw=[(*server).requireSuperAdmin-fm] | (*server).handleAdminUserDelete-fm | anon=401 token=401 super=reach admin=403 supplier=403 annotator=403 quality=403 acceptance=403
|
||||
DELETE /api/database/collections/{name} | mw=[(*server).requireBusinessRole.1] | (*server).handleDatabaseCollectionDelete-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
|
||||
@ -314,6 +314,7 @@ POST /api/plugins/annotation-qa/tasks/{id}/slice-edit-applications | mw=[(*serve
|
||||
POST /api/plugins/annotation-qa/tasks/{id}/slice-edit-restorations | mw=[(*server).requireBusinessRole.1] | (*server).handleSliceEditRestoration-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=reach quality=403 acceptance=403
|
||||
POST /api/plugins/annotation-qa/tasks/{id}/slice-reset | mw=[(*server).requireBusinessRole.1] | (*server).handleSliceResetCreate-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=reach quality=403 acceptance=403
|
||||
POST /api/plugins/annotation-qa/tasks/{id}/slice-revisions | mw=[(*server).requireBusinessRole.1] | (*server).handleSliceRevisionCreate-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=reach quality=403 acceptance=403
|
||||
POST /api/plugins/annotation-qa/tasks/{id}/source-validation-retries | mw=[(*server).requireBusinessRole.1] | (*server).handleAnnotationSourceValidationRetry-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
|
||||
POST /api/plugins/annotation-qa/templates | mw=[(*server).requireBusinessRole.1] | (*server).handleAnnotationTemplateCreate-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
|
||||
POST /api/plugins/annotation-qa/templates/{id}/publish | mw=[(*server).requireBusinessRole.1] | (*server).handleAnnotationTemplatePublish-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
|
||||
POST /api/project-deletions/batch | mw=[(*server).requireBusinessRole.1] | (*server).handleProjectDeletionBatch-fm | anon=401 token=401 super=403 admin=reach supplier=403 annotator=403 quality=403 acceptance=403
|
||||
|
||||
@ -14,6 +14,7 @@ type AnnotationProcessingJob struct {
|
||||
ActorID string `bson:"actorId" json:"actorId"`
|
||||
Input map[string]any `bson:"input,omitempty" json:"-"`
|
||||
SourceFingerprint string `bson:"sourceFingerprint,omitempty" json:"-"`
|
||||
SourceIndex bool `bson:"sourceIndex,omitempty" json:"-"`
|
||||
Revision int `bson:"revision,omitempty" json:"revision,omitempty"`
|
||||
Status string `bson:"status" json:"status"`
|
||||
Owner string `bson:"owner,omitempty" json:"-"`
|
||||
@ -69,17 +70,20 @@ type AnnotationTimelineIndex struct {
|
||||
}
|
||||
|
||||
type AnnotationIndexJob struct {
|
||||
ID string `bson:"id"`
|
||||
TaskID string `bson:"taskId,omitempty"`
|
||||
OrganizationID string `bson:"organizationId"`
|
||||
ProjectID string `bson:"projectId"`
|
||||
WorkflowID string `bson:"workflowId"`
|
||||
RunID string `bson:"runId"`
|
||||
Status string `bson:"status"`
|
||||
Owner string `bson:"owner,omitempty"`
|
||||
LeaseUntil string `bson:"leaseUntil,omitempty"`
|
||||
Attempt int `bson:"attempt"`
|
||||
NextAttemptAt string `bson:"nextAttemptAt"`
|
||||
Error string `bson:"error,omitempty"`
|
||||
CreatedAt string `bson:"createdAt"`
|
||||
RevalidateSource bool `bson:"revalidateSource,omitempty"`
|
||||
SupersededSourceFingerprint string `bson:"supersededSourceFingerprint,omitempty"`
|
||||
SourceFingerprint string `bson:"sourceFingerprint,omitempty"`
|
||||
ID string `bson:"id"`
|
||||
TaskID string `bson:"taskId,omitempty"`
|
||||
OrganizationID string `bson:"organizationId"`
|
||||
ProjectID string `bson:"projectId"`
|
||||
WorkflowID string `bson:"workflowId"`
|
||||
RunID string `bson:"runId"`
|
||||
Status string `bson:"status"`
|
||||
Owner string `bson:"owner,omitempty"`
|
||||
LeaseUntil string `bson:"leaseUntil,omitempty"`
|
||||
Attempt int `bson:"attempt"`
|
||||
NextAttemptAt string `bson:"nextAttemptAt"`
|
||||
Error string `bson:"error,omitempty"`
|
||||
CreatedAt string `bson:"createdAt"`
|
||||
}
|
||||
|
||||
@ -149,6 +149,7 @@ type HumanTraceSummary struct {
|
||||
}
|
||||
|
||||
type HumanItemRecord struct {
|
||||
SourceValidationStatus string `json:"sourceValidationStatus,omitempty" bson:"-"`
|
||||
TraceSummary *HumanTraceSummary `json:"traceSummary,omitempty" bson:"-"`
|
||||
ReviewHistory *HumanReviewHistory `json:"reviewHistory,omitempty" bson:"-"`
|
||||
CanClaim *bool `json:"canClaim,omitempty" bson:"-"`
|
||||
@ -326,6 +327,13 @@ type HumanItemCursor struct {
|
||||
}
|
||||
|
||||
type HumanItemQuery struct {
|
||||
// Server-derived necessary QC scope; nil is unrestricted, empty matches nothing.
|
||||
AvailableProjectIDs []string
|
||||
AvailableSupplierID *string
|
||||
Search string
|
||||
Capture string
|
||||
Category string
|
||||
ReviewedPool bool
|
||||
// Server-populated review history scope; nil means unrestricted.
|
||||
IDs []string
|
||||
LegacyTaskID string
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"droboticflow/internal/domain"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
var ErrAccountPendingTasks = errors.New("account has unfinished claimed tasks")
|
||||
@ -66,6 +67,9 @@ func (s *MemoryStore) ClaimHumanItemVersioned(_ context.Context, item domain.Hum
|
||||
if !ok || old.Version != version {
|
||||
return false, nil
|
||||
}
|
||||
if role == domain.UserRoleAnnotator && (!sameAnnotationClaimSourceSnapshot(old, item) || !s.annotationSourceReadyLocked(old)) {
|
||||
return false, ErrHumanClaimConflict
|
||||
}
|
||||
if batch, ok := s.humanBatches[item.BatchID]; !shared && ok && batch.ProcessingProfile == domain.ProcessingProfileRawHuman && domain.IsFanyuV58ThreeRoleHumanLoop(batch.Stages) {
|
||||
grants, err := s.listSupplierProjectAuthorizationsLocked(item.OrganizationID, item.ProjectID)
|
||||
if err != nil {
|
||||
@ -113,6 +117,35 @@ func (s *MongoStore) claimHumanItemSupplierGuarded(ctx context.Context, item dom
|
||||
return false, ErrHumanClaimConflict
|
||||
}
|
||||
result, err := s.runAccountTransaction(ctx, "user:"+item.AssignedUserID, func(sc context.Context) (any, error) {
|
||||
var sourceSnapshot *domain.HumanItemRecord
|
||||
if role == domain.UserRoleAnnotator {
|
||||
current, e := s.GetHumanItem(sc, item.ID)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
if current == nil || current.Version != version {
|
||||
return false, nil
|
||||
}
|
||||
if !sameAnnotationClaimSourceSnapshot(*current, item) {
|
||||
return false, ErrHumanClaimConflict
|
||||
}
|
||||
if current.LegacyTaskID != "" {
|
||||
task, e := s.GetAnnotationTask(sc, current.LegacyTaskID)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
if task != nil {
|
||||
ready, e := s.annotationSourceReady(sc, *current, *task)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
if !ready {
|
||||
return false, ErrHumanClaimConflict
|
||||
}
|
||||
}
|
||||
}
|
||||
sourceSnapshot = current
|
||||
}
|
||||
user, e := s.GetUser(sc, item.AssignedUserID)
|
||||
if e != nil {
|
||||
return false, e
|
||||
@ -156,6 +189,9 @@ func (s *MongoStore) claimHumanItemSupplierGuarded(ctx context.Context, item dom
|
||||
if _, e = s.users.UpdateOne(sc, bson.M{"id": user.ID}, bson.M{"$inc": bson.M{"claimVersion": 1}}); e != nil {
|
||||
return false, e
|
||||
}
|
||||
if sourceSnapshot != nil {
|
||||
return s.updateAnnotationClaimVersioned(sc, item, version, *sourceSnapshot)
|
||||
}
|
||||
return s.UpdateHumanItemVersioned(sc, item, version)
|
||||
})
|
||||
if err != nil {
|
||||
@ -163,3 +199,41 @@ func (s *MongoStore) claimHumanItemSupplierGuarded(ctx context.Context, item dom
|
||||
}
|
||||
return result.(bool), nil
|
||||
}
|
||||
|
||||
func sameAnnotationClaimSourceSnapshot(current, proposed domain.HumanItemRecord) bool {
|
||||
return current.OrganizationID == proposed.OrganizationID && current.ProjectID == proposed.ProjectID && current.RunID == proposed.RunID && current.LegacyTaskID == proposed.LegacyTaskID && current.BatchID == proposed.BatchID && current.CurrentStageID == proposed.CurrentStageID && AnnotationSourceCertificateRequired(current) == AnnotationSourceCertificateRequired(proposed) && reflect.DeepEqual(current.ProcessingJob, proposed.ProcessingJob)
|
||||
}
|
||||
|
||||
func (s *MongoStore) updateAnnotationClaimVersioned(ctx context.Context, item domain.HumanItemRecord, version int, snapshot domain.HumanItemRecord) (bool, error) {
|
||||
if err := prepareAnnotationRecord(&item); err != nil {
|
||||
return false, err
|
||||
}
|
||||
job := domain.AnnotationProcessingJob{}
|
||||
if snapshot.ProcessingJob != nil {
|
||||
job = *snapshot.ProcessingJob
|
||||
}
|
||||
matchString := func(value string) any {
|
||||
if value == "" {
|
||||
return bson.M{"$in": bson.A{nil, ""}}
|
||||
}
|
||||
return value
|
||||
}
|
||||
filter := bson.M{"id": item.ID, "version": version, "processingJob.id": matchString(job.ID), "processingJob.sourceFingerprint": matchString(job.SourceFingerprint), "processingJob.owner": matchString(job.Owner), "processingJob.updatedAt": matchString(job.UpdatedAt)}
|
||||
if job.SourceIndex {
|
||||
filter["processingJob.sourceIndex"] = true
|
||||
} else {
|
||||
filter["processingJob.sourceIndex"] = bson.M{"$in": bson.A{nil, false}}
|
||||
}
|
||||
updated := false
|
||||
err := supplierGuardedMongoWrite(ctx, s.humanItems, item.ID, item, func(ctx context.Context, d bson.M) error {
|
||||
result, err := s.humanItems.ReplaceOne(ctx, filter, d)
|
||||
if err == nil {
|
||||
updated = result.MatchedCount == 1
|
||||
}
|
||||
return err
|
||||
})
|
||||
if err == nil && !updated {
|
||||
return false, ErrHumanClaimConflict
|
||||
}
|
||||
return updated, err
|
||||
}
|
||||
|
||||
@ -30,7 +30,7 @@ func adminPoolScope(i domain.HumanItemRecord) bool {
|
||||
return i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined" || i.CurrentStageID == "quality_review" || (i.CurrentStageID == "split_annotation" && i.Status == domain.HumanItemRework)
|
||||
}
|
||||
func adminPoolAvailable(i domain.HumanItemRecord) bool {
|
||||
return i.AssignedUserID == "" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemRework || i.Status == domain.HumanItemSubmitted) && (i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined")
|
||||
return i.SourceValidationStatus == "" && i.AssignedUserID == "" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemRework || i.Status == domain.HumanItemSubmitted) && (i.CurrentStageID == "semantic_annotation" || i.CurrentStageID == "annotation_combined")
|
||||
}
|
||||
func adminPoolMine(i domain.HumanItemRecord, user string) bool {
|
||||
return i.AssignedUserID == user || (i.LatestAuthorUserID == user && i.CurrentStageID == "quality_review" && (i.Status == domain.HumanItemReady || i.Status == domain.HumanItemSubmitted))
|
||||
@ -115,6 +115,7 @@ func adminPoolCategory(i domain.HumanItemRecord, t domain.AnnotationTaskRecord)
|
||||
|
||||
type adminPoolRow struct {
|
||||
domain.HumanItemRecord `bson:",inline"`
|
||||
SourceAdmissionStatus string `bson:"_sourceAdmissionStatus"`
|
||||
Tasks []domain.AnnotationTaskRecord `bson:"poolTasks"`
|
||||
Runs []domain.RunRecord `bson:"poolRuns"`
|
||||
Captures []domain.CaptureRecord `bson:"poolCaptures"`
|
||||
@ -147,7 +148,11 @@ func adminPoolRowMatches(row adminPoolRow, q AdminAnnotationPoolQuery) bool {
|
||||
if q.ArtifactID != "" && task.ArtifactID != q.ArtifactID && task.RightArtifactID != q.ArtifactID {
|
||||
return false
|
||||
}
|
||||
if q.Category != "" && q.Category != "all" && adminPoolCategory(row.HumanItemRecord, task) != q.Category {
|
||||
if q.Category == "source_failed" {
|
||||
if row.SourceValidationStatus != "failed" {
|
||||
return false
|
||||
}
|
||||
} else if q.Category != "" && q.Category != "all" && adminPoolCategory(row.HumanItemRecord, task) != q.Category {
|
||||
return false
|
||||
}
|
||||
if q.Search == "" && q.Capture == "" {
|
||||
@ -204,6 +209,14 @@ func (s *MemoryStore) adminPoolRowsLocked(org string, q AdminAnnotationPoolQuery
|
||||
row := adminPoolRow{HumanItemRecord: humanItemListProjectionClone(i, humanItemListPayloadKeysRemoved)}
|
||||
if t, ok := s.annotationTasks[i.LegacyTaskID]; ok && humanOrganizationMatches(t.OrganizationID, org) {
|
||||
row.Tasks = []domain.AnnotationTaskRecord{t}
|
||||
var timeline, source *domain.AnnotationIndexJob
|
||||
if job, ok := s.annotationIndexJobs[t.ID]; ok {
|
||||
timeline = &job
|
||||
}
|
||||
if job, ok := s.annotationIndexJobs[sourceJobForItem(i)]; ok {
|
||||
source = &job
|
||||
}
|
||||
row.SourceValidationStatus = AnnotationSourceAdmissionStatus(t, i, timeline, source)
|
||||
}
|
||||
if r, ok := s.runs[i.RunID]; ok && humanOrganizationMatches(r.OrganizationID, org) {
|
||||
row.Runs = []domain.RunRecord{r}
|
||||
@ -282,7 +295,14 @@ func adminPoolPagePipeline(org string, q AdminAnnotationPoolQuery) mongo.Pipelin
|
||||
if q.Search != "" || q.Capture != "" {
|
||||
pipeline = append(pipeline, adminPoolLookup("runs", "runId", "poolRuns", org, bson.M{"_id": 0, "id": 1, "taskId": 1, "projectName": 1, "captureDirectories": 1, "inputs.ref": 1, "nodeExecutions.artifacts.id": 1, "nodeExecutions.artifacts.name": 1, "nodeExecutions.artifacts.path": 1, "artifacts.id": 1, "artifacts.name": 1, "artifacts.path": 1}), adminPoolLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org, bson.M{"_id": 0, "captureId": 1, "prefix": 1}), adminPoolLookup("projects", "projectId", "poolProjects", org, bson.M{"_id": 0, "name": 1}))
|
||||
}
|
||||
return pipeline
|
||||
pipeline = append(pipeline, annotationSourceAdmissionStatusPipeline()...)
|
||||
if q.Pool == "available" {
|
||||
pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"_sourceReady": true}}})
|
||||
}
|
||||
if q.Category == "source_failed" {
|
||||
pipeline = append(pipeline, bson.D{{Key: "$match", Value: bson.M{"_sourceAdmissionStatus": "failed"}}})
|
||||
}
|
||||
return append(pipeline, bson.D{{Key: "$unset", Value: "_sourceReady"}})
|
||||
}
|
||||
|
||||
// Stream a narrow metadata cursor; do not materialize the backlog to find an old
|
||||
@ -300,6 +320,7 @@ func (s *MongoStore) scanAdminAnnotationPool(ctx context.Context, org string, q
|
||||
if err := cur.Decode(&row); err != nil {
|
||||
return err
|
||||
}
|
||||
row.SourceValidationStatus = row.SourceAdmissionStatus
|
||||
if adminPoolRowMatches(row, q) && !visit(row.HumanItemRecord) {
|
||||
break
|
||||
}
|
||||
@ -337,9 +358,8 @@ func (s *MongoStore) CountAdminAnnotationPools(ctx context.Context, org string,
|
||||
}
|
||||
// Group compact routing counters before decoding anything into the API. Same
|
||||
// projected batch validation used by the existing blind-claim total.
|
||||
available := bson.M{"$and": bson.A{bson.M{"$in": bson.A{"$currentStageId", bson.A{"semantic_annotation", "annotation_combined"}}}, bson.M{"$eq": bson.A{bson.M{"$ifNull": bson.A{"$assignedUserId", ""}}, ""}}, bson.M{"$in": bson.A{"$status", bson.A{"ready", "rework", "submitted"}}}}}
|
||||
mine := bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$assignedUserId", q.UserID}}, bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$latestAuthorUserId", q.UserID}}, bson.M{"$eq": bson.A{"$currentStageId", "quality_review"}}, bson.M{"$in": bson.A{"$status", bson.A{"ready", "submitted"}}}}}}}
|
||||
pipeline := mongo.Pipeline{{{Key: "$match", Value: adminPoolScopeFilter(org, q)}}, {{Key: "$group", Value: bson.M{"_id": bson.M{"batch": "$batchId", "stage": "$currentStageId"}, "all": bson.M{"$sum": 1}, "available": bson.M{"$sum": bson.M{"$cond": bson.A{available, 1, 0}}}, "mine": bson.M{"$sum": bson.M{"$cond": bson.A{mine, 1, 0}}}}}}}
|
||||
pipeline := mongo.Pipeline{{{Key: "$match", Value: adminPoolScopeFilter(org, q)}}, {{Key: "$group", Value: bson.M{"_id": bson.M{"batch": "$batchId", "stage": "$currentStageId"}, "all": bson.M{"$sum": 1}, "mine": bson.M{"$sum": bson.M{"$cond": bson.A{mine, 1, 0}}}}}}}
|
||||
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second))
|
||||
if err != nil {
|
||||
return AdminAnnotationPoolCounts{}, err
|
||||
@ -372,9 +392,25 @@ func (s *MongoStore) CountAdminAnnotationPools(ctx context.Context, org string,
|
||||
for _, g := range groups {
|
||||
if adminPoolValidBatch(domain.HumanItemRecord{CurrentStageID: g.ID.Stage}, byID[g.ID.Batch], org) {
|
||||
c.All += g.All
|
||||
c.Available += g.Available
|
||||
c.Mine += g.Mine
|
||||
}
|
||||
}
|
||||
eligible := mongo.Pipeline{{{Key: "$match", Value: bson.M{"$and": bson.A{adminPoolScopeFilter(org, q), adminPoolAvailableFilter()}}}}}
|
||||
eligible = append(eligible, annotationSourceAdmissionPipeline()...)
|
||||
eligible = append(eligible, bson.D{{Key: "$group", Value: bson.M{"_id": bson.M{"batch": "$batchId", "stage": "$currentStageId"}, "available": bson.M{"$sum": 1}}}})
|
||||
ready, err := s.humanItems.Aggregate(ctx, eligible, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second))
|
||||
if err != nil {
|
||||
return AdminAnnotationPoolCounts{}, err
|
||||
}
|
||||
defer ready.Close(ctx)
|
||||
groups = nil
|
||||
if err := ready.All(ctx, &groups); err != nil {
|
||||
return AdminAnnotationPoolCounts{}, err
|
||||
}
|
||||
for _, g := range groups {
|
||||
if adminPoolValidBatch(domain.HumanItemRecord{CurrentStageID: g.ID.Stage}, byID[g.ID.Batch], org) {
|
||||
c.Available += g.Available
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
177
internal/store/admin_source_exceptions_test.go
Normal file
177
internal/store/admin_source_exceptions_test.go
Normal file
@ -0,0 +1,177 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func testAdminSourceExceptions(t *testing.T, s Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
const org, project, run = "admin-source-org", "admin-source-project", "admin-source-run"
|
||||
batch := domain.HumanBatchRecord{ID: "admin-source-batch", OrganizationID: org, ProjectID: project, RunID: run, PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages()}
|
||||
if err := s.UpsertHumanBatch(ctx, batch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scenarios := []struct {
|
||||
name, timeline, fingerprint, source, superseded, want string
|
||||
modern, revalidate bool
|
||||
}{
|
||||
{name: "legacy"},
|
||||
{name: "valid", modern: true, timeline: "completed"},
|
||||
{name: "failed", modern: true, timeline: "failed", want: "failed"},
|
||||
{name: "pending", modern: true, timeline: "queued", want: "pending"},
|
||||
{name: "missing", modern: true, want: "unverified"},
|
||||
{name: "payload-certificate", modern: true, timeline: "completed", want: "unverified"},
|
||||
{name: "payload-proof", modern: true, timeline: "completed", revalidate: true},
|
||||
{name: "payload-new-failure", modern: true, timeline: "completed", revalidate: true, fingerprint: "C", source: "failed", want: "failed"},
|
||||
{name: "payload-new-missing", modern: true, timeline: "completed", revalidate: true, fingerprint: "C", want: "unverified"},
|
||||
{name: "pending-current-proof", modern: true, timeline: "running", fingerprint: "B", superseded: "A", revalidate: true, want: "pending"},
|
||||
{name: "source-failed", modern: true, timeline: "completed", fingerprint: "A", source: "failed", want: "failed"},
|
||||
{name: "old-source-superseded", modern: true, timeline: "completed", fingerprint: "A", source: "failed", superseded: "A", revalidate: true},
|
||||
{name: "new-source-failed", modern: true, timeline: "completed", fingerprint: "C", source: "failed", superseded: "A", revalidate: true, want: "failed"},
|
||||
{name: "new-source-missing", modern: true, timeline: "completed", fingerprint: "C", superseded: "A", revalidate: true, want: "unverified"},
|
||||
{name: "current-proof", modern: true, timeline: "completed", fingerprint: "B", superseded: "A", revalidate: true},
|
||||
{name: "current-proof-failed", modern: true, timeline: "completed", fingerprint: "B", source: "failed", superseded: "A", revalidate: true, want: "failed"},
|
||||
}
|
||||
writeJob := func(job domain.AnnotationIndexJob) {
|
||||
switch st := s.(type) {
|
||||
case *MemoryStore:
|
||||
st.annotationIndexJobs[job.ID] = job
|
||||
case *MongoStore:
|
||||
if err := mongoUpsert(ctx, st.db.Collection("annotationIndexJobs"), job.ID, job); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
wantStatuses := map[string]string{}
|
||||
wantReady, wantFailed := 0, 0
|
||||
for n, sc := range scenarios {
|
||||
id := "admin-source-" + sc.name
|
||||
task := domain.AnnotationTaskRecord{ID: id, OrganizationID: org, ProjectID: project, RunID: run, SourceIsFullVideo: sc.modern, TemplateVersion: 6, ArtifactID: "left", Status: domain.AnnotationTaskStatusPendingAnnotation}
|
||||
if err := s.UpsertAnnotationTask(ctx, task); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, RunID: run, BatchID: batch.ID, PluginID: "annotation-qa", LegacyTaskID: id, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 7, CreatedAt: fmt.Sprintf("2026-11-02T00:00:%02dZ", n), Subject: domain.HumanSubjectRef{ItemKey: id, Payload: map[string]any{"draft": "keep"}}}
|
||||
// Distinct fingerprints prevent one scenario's certificate from affecting another.
|
||||
fp := func(v string) string {
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
return sc.name + "-" + v
|
||||
}
|
||||
if strings.HasPrefix(sc.name, "payload-") {
|
||||
item.Subject.Payload["materialization"] = map[string]any{"sourceIndex": true}
|
||||
}
|
||||
if sc.fingerprint != "" {
|
||||
item.ProcessingJob = &domain.AnnotationProcessingJob{SourceIndex: true, SourceFingerprint: fp(sc.fingerprint)}
|
||||
}
|
||||
if err := s.UpsertHumanItem(ctx, item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sc.timeline != "" {
|
||||
j := annotationIndexJob(task)
|
||||
j.Status = sc.timeline
|
||||
j.RevalidateSource = sc.revalidate
|
||||
j.SupersededSourceFingerprint = fp(sc.superseded)
|
||||
if sc.revalidate {
|
||||
j.SourceFingerprint = fp("B")
|
||||
}
|
||||
writeJob(j)
|
||||
} else {
|
||||
switch st := s.(type) {
|
||||
case *MemoryStore:
|
||||
delete(st.annotationIndexJobs, id)
|
||||
case *MongoStore:
|
||||
if _, err := st.db.Collection("annotationIndexJobs").DeleteOne(ctx, bson.M{"id": id}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if sc.source != "" {
|
||||
j := annotationIndexJob(task)
|
||||
j.ID = AnnotationSourceJobID(org, fp(sc.fingerprint))
|
||||
j.Status = sc.source
|
||||
writeJob(j)
|
||||
}
|
||||
wantStatuses[id] = sc.want
|
||||
if sc.want == "" {
|
||||
wantReady++
|
||||
}
|
||||
if sc.want == "failed" {
|
||||
wantFailed++
|
||||
}
|
||||
}
|
||||
q := AdminAnnotationPoolQuery{ProjectID: project, UserID: "admin", Pool: "all", Limit: 100}
|
||||
rows, err := s.ListAdminAnnotationPool(ctx, org, q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != len(scenarios) {
|
||||
t.Fatalf("all rows=%d", len(rows))
|
||||
}
|
||||
for _, row := range rows {
|
||||
if row.SourceValidationStatus != wantStatuses[row.ID] {
|
||||
t.Errorf("%s status=%q want=%q", row.ID, row.SourceValidationStatus, wantStatuses[row.ID])
|
||||
}
|
||||
}
|
||||
counts, err := s.CountAdminAnnotationPools(ctx, org, q)
|
||||
if err != nil || counts.All != len(scenarios) || counts.Available != wantReady {
|
||||
t.Fatalf("counts=%+v ready=%d err=%v", counts, wantReady, err)
|
||||
}
|
||||
q.Pool = "available"
|
||||
rows, err = s.ListAdminAnnotationPool(ctx, org, q)
|
||||
if err != nil || len(rows) != wantReady {
|
||||
t.Fatalf("available=%d want=%d err=%v", len(rows), wantReady, err)
|
||||
}
|
||||
for _, row := range rows {
|
||||
if wantStatuses[row.ID] != "" {
|
||||
t.Errorf("blocked task available: %s", row.ID)
|
||||
}
|
||||
}
|
||||
q.Pool = "all"
|
||||
q.Category = "source_failed"
|
||||
rows, err = s.ListAdminAnnotationPool(ctx, org, q)
|
||||
if err != nil || len(rows) != wantFailed {
|
||||
t.Fatalf("failures=%d want=%d err=%v", len(rows), wantFailed, err)
|
||||
}
|
||||
for id := range wantStatuses {
|
||||
item, err := s.GetHumanItem(ctx, id)
|
||||
if id == "admin-source-payload-certificate" && err == nil && !AnnotationSourceCertificateRequired(*item) {
|
||||
t.Fatalf("payload certificate marker lost after read: %T", item.Subject.Payload["materialization"])
|
||||
}
|
||||
if err != nil || item.Version != 7 || item.AssignedUserID != "" || item.Subject.Payload["draft"] != "keep" || item.SourceValidationStatus != "" {
|
||||
t.Fatalf("read changed item %s: %+v %v", id, item, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminSourceExceptionsMemory(t *testing.T) { testAdminSourceExceptions(t, NewMemoryStore()) }
|
||||
func TestAdminSourceExceptionsMongo(t *testing.T) {
|
||||
if os.Getenv("SOURCE_ADMISSION_LOCAL_MONGO") != "1" {
|
||||
t.Skip("isolated local Mongo opt-in")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("admin_source_test_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
s := NewMongoStore(db)
|
||||
if err := s.EnsureIndexes(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
testAdminSourceExceptions(t, s)
|
||||
}
|
||||
@ -855,7 +855,9 @@ func (store *MongoStore) UpsertAnnotationTask(ctx context.Context, item domain.A
|
||||
return err
|
||||
}
|
||||
if item.SourceIsFullVideo && item.TemplateVersion >= 5 && item.ArtifactID != "" {
|
||||
_ = store.EnqueueAnnotationIndex(ctx, item)
|
||||
if err := store.EnqueueAnnotationIndex(ctx, item); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -17,6 +17,8 @@ type AnnotationProcessingStore interface {
|
||||
EnqueueAnnotationIndex(context.Context, domain.AnnotationTaskRecord) error
|
||||
EnqueueAnnotationSourceIndex(context.Context, domain.AnnotationTaskRecord, string, bool) error
|
||||
GetAnnotationIndexJob(context.Context, string) (*domain.AnnotationIndexJob, error)
|
||||
RetryAnnotationSourceValidation(context.Context, domain.AnnotationTaskRecord, bool, string) error
|
||||
FailAnnotationSourceValidation(context.Context, domain.AnnotationTaskRecord, string, string) (bool, error)
|
||||
TouchAnnotationIndex(context.Context, string, string, string) (bool, error)
|
||||
ClaimAnnotationIndex(context.Context, string, string, string) (*domain.AnnotationIndexJob, error)
|
||||
FinishAnnotationIndex(context.Context, domain.AnnotationIndexJob) (bool, error)
|
||||
@ -157,11 +159,17 @@ func (s *MemoryStore) FinishAnnotationIndex(_ context.Context, j domain.Annotati
|
||||
return true, nil
|
||||
}
|
||||
func (s *MongoStore) FinishAnnotationIndex(ctx context.Context, j domain.AnnotationIndexJob) (bool, error) {
|
||||
r, e := s.db.Collection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j)
|
||||
result, e := s.withSupplierProjectLocks(ctx, []string{j.ProjectID}, func(ctx context.Context) (any, error) {
|
||||
r, e := s.db.Collection("annotationIndexJobs").ReplaceOne(ctx, bson.M{"id": j.ID, "owner": j.Owner, "status": "running"}, j)
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return r.MatchedCount == 1, nil
|
||||
})
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return r.MatchedCount == 1, nil
|
||||
return result.(bool), nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) ClaimAnnotationProcessing(_ context.Context, owner, now, until string) (*domain.HumanItemRecord, error) {
|
||||
|
||||
258
internal/store/annotation_source_admission.go
Normal file
258
internal/store/annotation_source_admission.go
Normal file
@ -0,0 +1,258 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
var ErrSourceValidationActive = errors.New("source validation is already running")
|
||||
|
||||
func AnnotationSourceValidationRequired(task domain.AnnotationTaskRecord) bool {
|
||||
return task.SourceIsFullVideo && task.TemplateVersion >= 5 && task.ArtifactID != ""
|
||||
}
|
||||
|
||||
func AnnotationSourceJobsReady(task domain.AnnotationTaskRecord, timeline, source *domain.AnnotationIndexJob) bool {
|
||||
if timeline == nil && AnnotationSourceValidationRequired(task) {
|
||||
return false
|
||||
}
|
||||
for _, job := range []*domain.AnnotationIndexJob{timeline, source} {
|
||||
if job != nil && (job.Status != "completed" || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func AnnotationSourceCertificateRequired(item domain.HumanItemRecord) bool {
|
||||
if item.ProcessingJob != nil && item.ProcessingJob.SourceIndex {
|
||||
return true
|
||||
}
|
||||
materialization, _ := item.Subject.Payload["materialization"].(map[string]any)
|
||||
value, _ := materialization["sourceIndex"].(bool)
|
||||
return value
|
||||
}
|
||||
|
||||
func AnnotationSourceJobSuperseded(item domain.HumanItemRecord, timeline *domain.AnnotationIndexJob) bool {
|
||||
return timeline != nil && timeline.RevalidateSource && timeline.SupersededSourceFingerprint != "" && item.ProcessingJob != nil && item.ProcessingJob.SourceFingerprint == timeline.SupersededSourceFingerprint
|
||||
}
|
||||
|
||||
func annotationSourceHasCurrentProof(item domain.HumanItemRecord, timeline *domain.AnnotationIndexJob) bool {
|
||||
if timeline == nil || !timeline.RevalidateSource || timeline.SourceFingerprint == "" {
|
||||
return false
|
||||
}
|
||||
if item.ProcessingJob != nil && item.ProcessingJob.SourceFingerprint != "" {
|
||||
return timeline.SourceFingerprint == item.ProcessingJob.SourceFingerprint
|
||||
}
|
||||
materialization, _ := item.Subject.Payload["materialization"].(map[string]any)
|
||||
return materialization["sourceIndex"] == true && timeline.SupersededSourceFingerprint == ""
|
||||
}
|
||||
|
||||
func AnnotationItemSourceJobsReady(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) bool {
|
||||
if timeline != nil && timeline.RevalidateSource {
|
||||
if timeline.SourceFingerprint == "" || !AnnotationSourceJobsReady(task, timeline, nil) {
|
||||
return false
|
||||
}
|
||||
if AnnotationSourceJobSuperseded(item, timeline) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if AnnotationSourceCertificateRequired(item) && source == nil {
|
||||
if !annotationSourceHasCurrentProof(item, timeline) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return AnnotationSourceJobsReady(task, timeline, source)
|
||||
}
|
||||
|
||||
func sourceJobForItem(item domain.HumanItemRecord) string {
|
||||
if item.ProcessingJob == nil || item.ProcessingJob.SourceFingerprint == "" {
|
||||
return ""
|
||||
}
|
||||
return AnnotationSourceJobID(item.OrganizationID, item.ProcessingJob.SourceFingerprint)
|
||||
}
|
||||
|
||||
func (s *MemoryStore) annotationSourceReadyLocked(item domain.HumanItemRecord) bool {
|
||||
task, exists := s.annotationTasks[item.LegacyTaskID]
|
||||
if !exists {
|
||||
return true
|
||||
} // Existing claim validation rejects a missing task.
|
||||
var timeline, source *domain.AnnotationIndexJob
|
||||
if job, ok := s.annotationIndexJobs[task.ID]; ok {
|
||||
timeline = &job
|
||||
}
|
||||
if job, ok := s.annotationIndexJobs[sourceJobForItem(item)]; ok && !AnnotationSourceJobSuperseded(item, timeline) {
|
||||
source = &job
|
||||
}
|
||||
return AnnotationItemSourceJobsReady(task, item, timeline, source)
|
||||
}
|
||||
|
||||
func (s *MongoStore) annotationSourceReady(ctx context.Context, item domain.HumanItemRecord, task domain.AnnotationTaskRecord) (bool, error) {
|
||||
timeline, err := s.GetAnnotationIndexJob(ctx, task.ID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var source *domain.AnnotationIndexJob
|
||||
if id := sourceJobForItem(item); id != "" && !AnnotationSourceJobSuperseded(item, timeline) {
|
||||
source, err = s.GetAnnotationIndexJob(ctx, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
return AnnotationItemSourceJobsReady(task, item, timeline, source), nil
|
||||
}
|
||||
|
||||
func AnnotationSourceAdmissionStatus(task domain.AnnotationTaskRecord, item domain.HumanItemRecord, timeline, source *domain.AnnotationIndexJob) string {
|
||||
if AnnotationSourceJobSuperseded(item, timeline) {
|
||||
source = nil
|
||||
}
|
||||
if AnnotationItemSourceJobsReady(task, item, timeline, source) {
|
||||
return ""
|
||||
}
|
||||
for _, job := range []*domain.AnnotationIndexJob{timeline, source} {
|
||||
if job != nil && (job.Status == "failed" || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID) {
|
||||
return "failed"
|
||||
}
|
||||
}
|
||||
currentProof := annotationSourceHasCurrentProof(item, timeline)
|
||||
if timeline == nil && AnnotationSourceValidationRequired(task) || timeline != nil && timeline.RevalidateSource && timeline.Status == "completed" && timeline.SourceFingerprint == "" || !currentProof && !AnnotationSourceJobSuperseded(item, timeline) && AnnotationSourceCertificateRequired(item) && source == nil {
|
||||
return "unverified"
|
||||
}
|
||||
return "pending"
|
||||
}
|
||||
|
||||
// All joins use existing unique id indexes and project only admission fields.
|
||||
// Retain caller fields so administrative exception pages can display blocked rows.
|
||||
func annotationSourceAdmissionStatusPipeline() mongo.Pipeline {
|
||||
first := func(path string) bson.M { return bson.M{"$arrayElemAt": bson.A{path, 0}} }
|
||||
nonempty := func(v any) bson.M { return bson.M{"$ne": bson.A{bson.M{"$ifNull": bson.A{v, ""}}, ""}} }
|
||||
needsTimeline := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{first("$_sourceTask.sourceIsFullVideo"), true}}, bson.M{"$gte": bson.A{first("$_sourceTask.templateVersion"), 5}}, nonempty(first("$_sourceTask.artifactId"))}}
|
||||
revalidated := bson.M{"$eq": bson.A{first("$_sourceTimeline.revalidateSource"), true}}
|
||||
superseded := bson.M{"$and": bson.A{revalidated, nonempty(first("$_sourceTimeline.supersededSourceFingerprint")), bson.M{"$eq": bson.A{"$processingJob.sourceFingerprint", first("$_sourceTimeline.supersededSourceFingerprint")}}}}
|
||||
missingReferenceProof := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$subject.payload.materialization.sourceIndex", true}}, bson.M{"$not": bson.A{nonempty("$processingJob.sourceFingerprint")}}, bson.M{"$not": bson.A{nonempty(first("$_sourceTimeline.supersededSourceFingerprint"))}}}}
|
||||
currentProof := bson.M{"$and": bson.A{revalidated, nonempty(first("$_sourceTimeline.sourceFingerprint")), bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$processingJob.sourceFingerprint", first("$_sourceTimeline.sourceFingerprint")}}, missingReferenceProof}}}}
|
||||
needsCertificate := bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$processingJob.sourceIndex", true}}, bson.M{"$eq": bson.A{"$subject.payload.materialization.sourceIndex", true}}}}
|
||||
jobs := bson.M{"$concatArrays": bson.A{"$_sourceTimeline", "$_sourcePrepared"}}
|
||||
matchingScope := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.organizationId", "$organizationId"}}, bson.M{"$eq": bson.A{"$$job.projectId", "$projectId"}}, bson.M{"$eq": bson.A{"$$job.runId", "$runId"}}}}
|
||||
safeJob := bson.M{"$and": bson.A{bson.M{"$eq": bson.A{"$$job.status", "completed"}}, matchingScope}}
|
||||
timelinePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourceTimeline"}, 0}}
|
||||
certificatePresent := bson.M{"$gt": bson.A{bson.M{"$size": "$_sourcePrepared"}, 0}}
|
||||
certificateSatisfied := bson.M{"$or": bson.A{bson.M{"$not": bson.A{needsCertificate}}, certificatePresent, superseded, currentProof}}
|
||||
missing := bson.M{"$or": bson.A{
|
||||
bson.M{"$and": bson.A{needsTimeline, bson.M{"$not": bson.A{timelinePresent}}}},
|
||||
bson.M{"$and": bson.A{revalidated, bson.M{"$eq": bson.A{first("$_sourceTimeline.status"), "completed"}}, bson.M{"$not": bson.A{nonempty(first("$_sourceTimeline.sourceFingerprint"))}}}},
|
||||
bson.M{"$not": bson.A{certificateSatisfied}},
|
||||
}}
|
||||
ready := bson.M{"$and": bson.A{
|
||||
bson.M{"$or": bson.A{timelinePresent, bson.M{"$not": bson.A{needsTimeline}}}},
|
||||
bson.M{"$or": bson.A{bson.M{"$not": bson.A{revalidated}}, nonempty(first("$_sourceTimeline.sourceFingerprint"))}},
|
||||
certificateSatisfied,
|
||||
bson.M{"$allElementsTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": safeJob}}}},
|
||||
}}
|
||||
failed := bson.M{"$anyElementTrue": bson.A{bson.M{"$map": bson.M{"input": jobs, "as": "job", "in": bson.M{"$or": bson.A{bson.M{"$eq": bson.A{"$$job.status", "failed"}}, bson.M{"$not": bson.A{matchingScope}}}}}}}}
|
||||
jobProjection := bson.M{"status": 1, "organizationId": 1, "projectId": 1, "runId": 1, "revalidateSource": 1, "sourceFingerprint": 1, "supersededSourceFingerprint": 1}
|
||||
lookup := func(collection, local, as string, projection bson.M) bson.D {
|
||||
return bson.D{{Key: "$lookup", Value: bson.M{"from": collection, "localField": local, "foreignField": "id", "pipeline": bson.A{bson.M{"$project": projection}}, "as": as}}}
|
||||
}
|
||||
return mongo.Pipeline{
|
||||
lookup("annotationTasks", "legacyTaskId", "_sourceTask", bson.M{"sourceIsFullVideo": 1, "templateVersion": 1, "artifactId": 1}),
|
||||
lookup("annotationIndexJobs", "legacyTaskId", "_sourceTimeline", jobProjection),
|
||||
{{Key: "$set", Value: bson.M{"_sourceJobID": bson.M{"$cond": bson.A{bson.M{"$and": bson.A{nonempty("$processingJob.sourceFingerprint"), bson.M{"$not": bson.A{superseded}}}}, bson.M{"$concat": bson.A{"source-v1:", "$organizationId", ":", "$processingJob.sourceFingerprint"}}, ""}}}}},
|
||||
lookup("annotationIndexJobs", "_sourceJobID", "_sourcePrepared", jobProjection),
|
||||
{{Key: "$set", Value: bson.M{"_sourceReady": ready, "_sourceAdmissionStatus": bson.M{"$switch": bson.M{"branches": bson.A{bson.M{"case": failed, "then": "failed"}, bson.M{"case": missing, "then": "unverified"}, bson.M{"case": ready, "then": ""}}, "default": "pending"}}}}},
|
||||
{{Key: "$unset", Value: bson.A{"_sourceTask", "_sourceTimeline", "_sourcePrepared", "_sourceJobID"}}},
|
||||
}
|
||||
}
|
||||
|
||||
func annotationSourceAdmissionPipeline() mongo.Pipeline {
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$project", Value: bson.M{"id": 1, "organizationId": 1, "projectId": 1, "workflowId": 1, "runId": 1, "pluginId": 1, "batchId": 1, "legacyTaskId": 1, "currentStageId": 1, "supplierId": 1, "processingScope": 1, "processingProfile": 1, "version": 1, "status": 1, "reviewRound": 1, "latestAuthorUserId": 1, "claimedAt": 1, "processingJob.sourceFingerprint": 1, "processingJob.sourceIndex": 1, "subject.payload.materialization.sourceIndex": 1}}},
|
||||
}
|
||||
pipeline = append(pipeline, annotationSourceAdmissionStatusPipeline()...)
|
||||
return append(pipeline, bson.D{{Key: "$match", Value: bson.M{"_sourceReady": true}}}, bson.D{{Key: "$unset", Value: bson.A{"_sourceReady", "_sourceAdmissionStatus"}}})
|
||||
}
|
||||
|
||||
func (s *MemoryStore) RetryAnnotationSourceValidation(_ context.Context, task domain.AnnotationTaskRecord, verifySource bool, supersededFingerprint string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if job, ok := s.annotationIndexJobs[task.ID]; ok && (job.Status == "running" || job.Status == "queued") {
|
||||
return ErrSourceValidationActive
|
||||
}
|
||||
if s.annotationIndexJobs == nil {
|
||||
s.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
|
||||
}
|
||||
job := annotationIndexJob(task)
|
||||
job.RevalidateSource = verifySource
|
||||
job.SupersededSourceFingerprint = supersededFingerprint
|
||||
s.annotationIndexJobs[task.ID] = job
|
||||
return nil
|
||||
}
|
||||
func (s *MongoStore) RetryAnnotationSourceValidation(ctx context.Context, task domain.AnnotationTaskRecord, verifySource bool, supersededFingerprint string) error {
|
||||
_, err := s.withSupplierProjectLocks(ctx, []string{task.ProjectID}, func(ctx context.Context) (any, error) {
|
||||
job, e := s.GetAnnotationIndexJob(ctx, task.ID)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if job != nil && (job.Status == "queued" || job.Status == "running") {
|
||||
return nil, ErrSourceValidationActive
|
||||
}
|
||||
next := annotationIndexJob(task)
|
||||
next.RevalidateSource = verifySource
|
||||
next.SupersededSourceFingerprint = supersededFingerprint
|
||||
return nil, mongoUpsert(ctx, s.db.Collection("annotationIndexJobs"), task.ID, next)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) FailAnnotationSourceValidation(_ context.Context, task domain.AnnotationTaskRecord, createdAt, message string) (bool, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
job, ok := s.annotationIndexJobs[task.ID]
|
||||
if createdAt == "" {
|
||||
if ok {
|
||||
return false, nil
|
||||
}
|
||||
if s.annotationIndexJobs == nil {
|
||||
s.annotationIndexJobs = map[string]domain.AnnotationIndexJob{}
|
||||
}
|
||||
job = annotationIndexJob(task)
|
||||
job.Status, job.Error = "failed", message
|
||||
s.annotationIndexJobs[task.ID] = job
|
||||
return true, nil
|
||||
}
|
||||
if !ok || job.OrganizationID != task.OrganizationID || job.ProjectID != task.ProjectID || job.RunID != task.RunID || job.CreatedAt != createdAt || job.Status != "completed" {
|
||||
return false, nil
|
||||
}
|
||||
job.Status, job.Error = "failed", message
|
||||
s.annotationIndexJobs[task.ID] = job
|
||||
return true, nil
|
||||
}
|
||||
func (s *MongoStore) FailAnnotationSourceValidation(ctx context.Context, task domain.AnnotationTaskRecord, createdAt, message string) (bool, error) {
|
||||
result, err := s.withSupplierProjectLocks(ctx, []string{task.ProjectID}, func(ctx context.Context) (any, error) {
|
||||
if createdAt == "" {
|
||||
job := annotationIndexJob(task)
|
||||
job.Status, job.Error = "failed", message
|
||||
r, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": task.ID}, bson.M{"$setOnInsert": job}, options.Update().SetUpsert(true))
|
||||
if mongo.IsDuplicateKeyError(e) {
|
||||
return false, nil
|
||||
}
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return r.UpsertedCount == 1, nil
|
||||
}
|
||||
r, e := s.db.Collection("annotationIndexJobs").UpdateOne(ctx, bson.M{"id": task.ID, "organizationId": task.OrganizationID, "projectId": task.ProjectID, "runId": task.RunID, "createdAt": createdAt, "status": "completed"}, bson.M{"$set": bson.M{"status": "failed", "error": message}})
|
||||
if e != nil {
|
||||
return false, e
|
||||
}
|
||||
return r.MatchedCount == 1, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return result.(bool), nil
|
||||
}
|
||||
371
internal/store/annotation_source_admission_test.go
Normal file
371
internal/store/annotation_source_admission_test.go
Normal file
@ -0,0 +1,371 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func testSourceAdmissionAndClaimFence(t *testing.T, s Store) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
queue := s.(AnnotationProcessingStore)
|
||||
org, project, vendor := "source-admission-org", "source-admission-project", "source-admission-vendor"
|
||||
for _, e := range []error{
|
||||
s.UpsertProject(ctx, domain.ProjectRecord{ID: project, OrganizationID: org}),
|
||||
s.UpsertSupplier(ctx, domain.SupplierRecord{ID: vendor, OrganizationID: org, Status: "active"}),
|
||||
s.UpsertUser(ctx, domain.UserRecord{ID: "source-actor", OrganizationID: org, SupplierID: vendor, Role: domain.UserRoleAnnotator, Status: domain.UserStatusActive}),
|
||||
s.UpsertProjectMember(ctx, domain.ProjectMemberRecord{ID: "source-member", OrganizationID: org, ProjectID: project, UserID: "source-actor", SupplierID: vendor, Roles: []domain.AnnotationAssignmentRole{domain.AnnotationRoleAnnotator}}),
|
||||
} {
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
batch := domain.HumanBatchRecord{ID: "source-batch", OrganizationID: org, ProjectID: project, RunID: "source-run", PluginID: "annotation-qa", Stages: domain.DefaultFanyuV58HumanStages(), Status: domain.HumanBatchActive}
|
||||
if e := s.UpsertHumanBatch(ctx, batch); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
tasks := map[string]domain.AnnotationTaskRecord{}
|
||||
for _, id := range []string{"failed", "pending", "valid", "legacy"} {
|
||||
task := domain.AnnotationTaskRecord{ID: "source-" + id, OrganizationID: org, ProjectID: project, RunID: batch.RunID, SourceIsFullVideo: id != "legacy", TemplateVersion: 6, ArtifactID: "source-left", Status: domain.AnnotationTaskStatusPendingAnnotation}
|
||||
if e := s.UpsertAnnotationTask(ctx, task); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
tasks[id] = task
|
||||
item := domain.HumanItemRecord{ID: task.ID, OrganizationID: org, ProjectID: project, RunID: batch.RunID, BatchID: batch.ID, PluginID: "annotation-qa", LegacyTaskID: task.ID, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Version: 1, Subject: domain.HumanSubjectRef{ItemKey: task.ID, Payload: map[string]any{"draft": "preserved"}}}
|
||||
if e := s.UpsertHumanItem(ctx, item); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
stamp := now.Format(time.RFC3339Nano)
|
||||
until := now.Add(time.Minute).Format(time.RFC3339Nano)
|
||||
for i := 0; i < 3; i++ {
|
||||
job, e := queue.ClaimAnnotationIndex(ctx, fmt.Sprintf("validator-%d", i), stamp, until)
|
||||
if e != nil || job == nil {
|
||||
t.Fatal("job not queued", e)
|
||||
}
|
||||
switch job.ID {
|
||||
case tasks["failed"].ID:
|
||||
job.Status, job.Error = "failed", "IMU row 7 invalid type gyr"
|
||||
case tasks["valid"].ID:
|
||||
job.Status = "completed"
|
||||
default:
|
||||
job.Status = "running"
|
||||
}
|
||||
if job.Status != "running" {
|
||||
if ok, e := queue.FinishAnnotationIndex(ctx, *job); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
candidates, e := s.ListAnnotationClaimCandidates(ctx, org, project)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
ids := map[string]bool{}
|
||||
for _, item := range candidates {
|
||||
ids[item.ID] = true
|
||||
}
|
||||
if len(ids) != 2 || !ids[tasks["valid"].ID] || !ids[tasks["legacy"].ID] {
|
||||
t.Fatalf("admission candidates=%v", ids)
|
||||
}
|
||||
q := AnnotationClaimCountQuery{ProjectID: project, ActiveSupplierIDs: []string{vendor}, Memberships: []AnnotationClaimMembership{{SupplierID: vendor, Legacy: true}}}
|
||||
count, e := s.CountAnnotationClaimCandidates(ctx, org, q)
|
||||
if e != nil || count != 2 {
|
||||
t.Fatalf("count=%d err=%v", count, e)
|
||||
}
|
||||
// Selection completed before administrator revalidation: commit must recheck.
|
||||
current, e := s.GetHumanItem(ctx, tasks["valid"].ID)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
current.ProcessingJob = &domain.AnnotationProcessingJob{SourceFingerprint: "missing-certificate", SourceIndex: true}
|
||||
if e := s.UpsertHumanItem(ctx, *current); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
missing, e := s.CountAnnotationClaimCandidates(ctx, org, q)
|
||||
if e != nil || missing != 1 {
|
||||
t.Fatalf("missing required certificate count=%d err=%v", missing, e)
|
||||
}
|
||||
current.ProcessingJob.SourceIndex = false
|
||||
if e := s.UpsertHumanItem(ctx, *current); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
physical, e := s.CountAnnotationClaimCandidates(ctx, org, q)
|
||||
if e != nil || physical != 2 {
|
||||
t.Fatalf("legacy physical fingerprint count=%d err=%v", physical, e)
|
||||
}
|
||||
previous := *current
|
||||
current.Version++
|
||||
current.Status = domain.HumanItemClaimed
|
||||
current.AssignedUserID = "source-actor"
|
||||
current.SupplierID = vendor
|
||||
current.ClaimedAt = stamp
|
||||
commit := HumanClaimCommit{Receipt: HumanClaimReceipt{ID: "fenced-source-request", ItemID: current.ID, OrganizationID: org, ProjectID: project, UserID: "source-actor"}, Item: *current, ExpectedVersion: previous.Version, BindSupplier: true, TaskStatus: domain.AnnotationTaskStatusPendingAnnotation}
|
||||
if e := queue.RetryAnnotationSourceValidation(ctx, tasks["valid"], false, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if got, e := s.CommitHumanClaim(ctx, commit); got != nil || !errors.Is(e, ErrHumanClaimConflict) {
|
||||
t.Fatalf("source change bypassed atomic claim fence: got=%v err=%v", got, e)
|
||||
}
|
||||
after, e := s.GetHumanItem(ctx, current.ID)
|
||||
if e != nil || after.AssignedUserID != "" || after.Version != previous.Version || after.Subject.Payload["draft"] != "preserved" {
|
||||
t.Fatalf("failed claim changed draft/ownership: %+v %v", after, e)
|
||||
}
|
||||
if ok, e := s.ClaimHumanItemVersioned(ctx, *current, previous.Version, domain.UserRoleAnnotator); ok || !errors.Is(e, ErrHumanClaimConflict) {
|
||||
t.Fatalf("direct claim bypassed validation: %v %v", ok, e)
|
||||
}
|
||||
repaired, e := queue.ClaimAnnotationIndex(ctx, "repair-proof", stamp, until)
|
||||
if e != nil || repaired == nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
repaired.Status, repaired.RevalidateSource, repaired.SourceFingerprint, repaired.SupersededSourceFingerprint = "completed", true, "repaired-B", "failed-A"
|
||||
if ok, e := queue.FinishAnnotationIndex(ctx, *repaired); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
previous.ProcessingJob = &domain.AnnotationProcessingJob{SourceIndex: true, SourceFingerprint: "failed-A"}
|
||||
if e := s.UpsertHumanItem(ctx, previous); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 2 {
|
||||
t.Fatalf("repair did not supersede failed A: %d %v", n, e)
|
||||
}
|
||||
previous.ProcessingJob.SourceFingerprint = "later-C"
|
||||
if e := s.UpsertHumanItem(ctx, previous); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 1 {
|
||||
t.Fatalf("repair bypassed missing later C: %d %v", n, e)
|
||||
}
|
||||
if e := queue.EnqueueAnnotationSourceIndex(ctx, tasks["valid"], "later-C", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
later, e := queue.ClaimAnnotationIndex(ctx, "later-validator", stamp, until)
|
||||
if e != nil || later == nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
later.Status, later.Error = "failed", "IMU row 9 invalid type gyr"
|
||||
if ok, e := queue.FinishAnnotationIndex(ctx, *later); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 1 {
|
||||
t.Fatalf("repair bypassed failed later C: %d %v", n, e)
|
||||
}
|
||||
|
||||
legacy := tasks["legacy"]
|
||||
if job, e := queue.GetAnnotationIndexJob(ctx, legacy.ID); e != nil || job != nil {
|
||||
t.Fatal("legacy record unexpectedly present", e)
|
||||
}
|
||||
if changed, e := queue.FailAnnotationSourceValidation(ctx, legacy, "", "IMU invalid type gyr at row 9"); e != nil || !changed {
|
||||
t.Fatalf("newly discovered legacy failure was not recorded: %v %v", changed, e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 0 {
|
||||
t.Fatalf("known bad legacy source remained claimable: %d %v", n, e)
|
||||
}
|
||||
before, e := s.GetHumanItem(ctx, legacy.ID)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
claim := *before
|
||||
claim.AssignedUserID, claim.Version = "source-actor", before.Version+1
|
||||
if ok, e := s.ClaimHumanItemVersioned(ctx, claim, before.Version, domain.UserRoleAnnotator); ok || !errors.Is(e, ErrHumanClaimConflict) {
|
||||
t.Fatalf("known bad legacy source direct-claimed: %v %v", ok, e)
|
||||
}
|
||||
if e := queue.RetryAnnotationSourceValidation(ctx, legacy, false, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if changed, e := queue.FailAnnotationSourceValidation(ctx, legacy, "", "stale read before legacy validation existed"); e != nil || changed {
|
||||
t.Fatalf("legacy failure overwrote newly queued validation: %v %v", changed, e)
|
||||
}
|
||||
job, e := queue.ClaimAnnotationIndex(ctx, "legacy-repair", stamp, until)
|
||||
if e != nil || job == nil || job.ID != legacy.ID {
|
||||
t.Fatal("missing legacy recovery job", e)
|
||||
}
|
||||
job.Status = "completed"
|
||||
if ok, e := queue.FinishAnnotationIndex(ctx, *job); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if changed, e := queue.FailAnnotationSourceValidation(ctx, legacy, "", "late legacy failure"); e != nil || changed {
|
||||
t.Fatalf("legacy failure overwrote completed new validation: %v %v", changed, e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 1 {
|
||||
t.Fatalf("repaired legacy source not available: %d %v", n, e)
|
||||
}
|
||||
afterLegacy, e := s.GetHumanItem(ctx, legacy.ID)
|
||||
if e != nil || afterLegacy.Version != before.Version || afterLegacy.AssignedUserID != before.AssignedUserID || afterLegacy.Subject.Payload["draft"] != "preserved" {
|
||||
t.Fatal("legacy failure/recovery changed saved work", e)
|
||||
}
|
||||
|
||||
afterLegacy.Subject.Payload["materialization"] = map[string]any{"sourceIndex": true}
|
||||
if e := s.UpsertHumanItem(ctx, *afterLegacy); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 0 {
|
||||
t.Fatalf("payload-only source missing proof admitted: %d %v", n, e)
|
||||
}
|
||||
if e := queue.RetryAnnotationSourceValidation(ctx, legacy, true, ""); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
job, e = queue.ClaimAnnotationIndex(ctx, "legacy-source-proof", stamp, until)
|
||||
if e != nil || job == nil || job.ID != legacy.ID {
|
||||
t.Fatal("missing payload-only recovery job", e)
|
||||
}
|
||||
job.Status, job.SourceFingerprint = "completed", "repaired-payload-source"
|
||||
if ok, e := queue.FinishAnnotationIndex(ctx, *job); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 1 {
|
||||
t.Fatalf("payload-only legacy recovery remained blocked: %d %v", n, e)
|
||||
}
|
||||
staleClaim := *afterLegacy
|
||||
staleClaim.AssignedUserID, staleClaim.Version = "source-actor", afterLegacy.Version+1
|
||||
afterLegacy.ProcessingJob = &domain.AnnotationProcessingJob{SourceIndex: true, SourceFingerprint: "new-after-payload-repair"}
|
||||
if e := s.UpsertHumanItem(ctx, *afterLegacy); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if n, e := s.CountAnnotationClaimCandidates(ctx, org, q); e != nil || n != 0 {
|
||||
t.Fatalf("payload-only repair bypassed later source: %d %v", n, e)
|
||||
}
|
||||
if ok, e := s.ClaimHumanItemVersioned(ctx, staleClaim, afterLegacy.Version, domain.UserRoleAnnotator); ok || !errors.Is(e, ErrHumanClaimConflict) {
|
||||
t.Fatalf("stale direct claim replaced current missing certificate: %v %v", ok, e)
|
||||
}
|
||||
if e := queue.EnqueueAnnotationSourceIndex(ctx, legacy, "new-after-payload-repair", false); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
newSource, e := queue.ClaimAnnotationIndex(ctx, "new-valid-source", stamp, until)
|
||||
if e != nil || newSource == nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
newSource.Status = "failed"
|
||||
if ok, e := queue.FinishAnnotationIndex(ctx, *newSource); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if ok, e := s.ClaimHumanItemVersioned(ctx, staleClaim, afterLegacy.Version, domain.UserRoleAnnotator); ok || !errors.Is(e, ErrHumanClaimConflict) {
|
||||
t.Fatalf("stale direct claim bypassed newer failed source: %v %v", ok, e)
|
||||
}
|
||||
if e := queue.EnqueueAnnotationSourceIndex(ctx, legacy, "new-after-payload-repair", true); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
newSource, e = queue.ClaimAnnotationIndex(ctx, "repaired-new-source", stamp, until)
|
||||
if e != nil || newSource == nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
newSource.Status = "completed"
|
||||
if ok, e := queue.FinishAnnotationIndex(ctx, *newSource); !ok || e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if ok, e := s.ClaimHumanItemVersioned(ctx, staleClaim, afterLegacy.Version, domain.UserRoleAnnotator); ok || !errors.Is(e, ErrHumanClaimConflict) {
|
||||
t.Fatalf("stale direct claim overwrote newer valid source job: %v %v", ok, e)
|
||||
}
|
||||
durable, e := s.GetHumanItem(ctx, legacy.ID)
|
||||
if e != nil || durable.Version != afterLegacy.Version || durable.AssignedUserID != "" || durable.ProcessingJob == nil || durable.ProcessingJob.SourceFingerprint != "new-after-payload-repair" {
|
||||
t.Fatal("stale claim overwrote durable source", e)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestSourceAdmissionMemoryClaimFence(t *testing.T) {
|
||||
testSourceAdmissionAndClaimFence(t, NewMemoryStore())
|
||||
}
|
||||
func TestSourceAdmissionMongoClaimFence(t *testing.T) {
|
||||
if os.Getenv("SOURCE_ADMISSION_LOCAL_MONGO") != "1" {
|
||||
t.Skip("local Mongo opt-in")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("source_admission_test_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
s := NewMongoStore(db)
|
||||
if e := s.EnsureIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
testSourceAdmissionAndClaimFence(t, s)
|
||||
}
|
||||
|
||||
func TestSourceAdmissionMongoLargePool(t *testing.T) {
|
||||
if os.Getenv("SOURCE_ADMISSION_LOCAL_MONGO") != "1" {
|
||||
t.Skip("local Mongo opt-in")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018/?directConnection=true").SetServerSelectionTimeout(5*time.Second))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("source_admission_load_test_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
s := NewMongoStore(db)
|
||||
if e := s.EnsureIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
const count = 1200
|
||||
org, project := "source-load-org", "source-load-project"
|
||||
batch := domain.HumanBatchRecord{ID: "source-load-batch", OrganizationID: org, ProjectID: project, PluginID: "annotation-qa", Status: domain.HumanBatchActive, Stages: domain.DefaultFanyuV58HumanStages()}
|
||||
if e := s.UpsertHumanBatch(ctx, batch); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
tasks, items, jobs := []any{}, []any{}, []any{}
|
||||
for i := 0; i < count; i++ {
|
||||
id := fmt.Sprintf("source-load-%04d", i)
|
||||
tasks = append(tasks, domain.AnnotationTaskRecord{ID: id, OrganizationID: org, ProjectID: project, RunID: id, SourceIsFullVideo: true, TemplateVersion: 6, ArtifactID: "left", Status: domain.AnnotationTaskStatusPendingAnnotation})
|
||||
items = append(items, domain.HumanItemRecord{ID: id, OrganizationID: org, ProjectID: project, RunID: id, PluginID: "annotation-qa", BatchID: batch.ID, LegacyTaskID: id, CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: domain.HumanItemReady, Subject: domain.HumanSubjectRef{ItemKey: id, Payload: map[string]any{"large": strings.Repeat("x", 64*1024)}}})
|
||||
jobs = append(jobs, domain.AnnotationIndexJob{ID: id, OrganizationID: org, ProjectID: project, RunID: id, Status: "completed"})
|
||||
}
|
||||
for name, docs := range map[string][]any{"annotationTasks": tasks, "humanItems": items, "annotationIndexJobs": jobs} {
|
||||
if _, e := db.Collection(name).InsertMany(ctx, docs); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
q := AnnotationClaimCountQuery{ProjectID: project, ActiveSupplierIDs: []string{"vendor"}, Memberships: []AnnotationClaimMembership{{SupplierID: "vendor", Legacy: true}}}
|
||||
// The original grouped-count plan on this all-eligible dataset is the baseline.
|
||||
candidate := annotationClaimCandidateCountPipeline(org, q)
|
||||
baseline := mongo.Pipeline{candidate[0], candidate[len(candidate)-3], candidate[len(candidate)-2], candidate[len(candidate)-1]}
|
||||
for run := 0; run < 3; run++ {
|
||||
started := time.Now()
|
||||
cur, e := s.humanItems.Aggregate(ctx, baseline)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
var groups []annotationClaimCandidateCount
|
||||
if e = cur.All(ctx, &groups); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
baselineElapsed := time.Since(started)
|
||||
started = time.Now()
|
||||
got, e := s.CountAnnotationClaimCandidates(ctx, org, q)
|
||||
elapsed := time.Since(started)
|
||||
if e != nil || got != count {
|
||||
t.Fatalf("count=%d err=%v", got, e)
|
||||
}
|
||||
started = time.Now()
|
||||
candidates, e := s.ListAnnotationClaimCandidates(ctx, org, project)
|
||||
listElapsed := time.Since(started)
|
||||
if e != nil || len(candidates) != count {
|
||||
t.Fatal(len(candidates), e)
|
||||
}
|
||||
body, e := json.Marshal(candidates)
|
||||
if e != nil || len(body) > 2*1024*1024 {
|
||||
t.Fatalf("candidate projection leaked large payload: bytes=%d err=%v", len(body), e)
|
||||
}
|
||||
if elapsed > 2*time.Second || listElapsed > 2*time.Second {
|
||||
t.Fatalf("source admission query exceeded local latency bound: count=%s list=%s", elapsed, listElapsed)
|
||||
}
|
||||
t.Logf("round=%d tasks=%d subject_bytes=%d baseline_count=%s gated_count=%s candidate_list=%s candidate_json_bytes=%d", run+1, count, count*64*1024, baselineElapsed, elapsed, listElapsed, len(body))
|
||||
}
|
||||
}
|
||||
@ -47,7 +47,7 @@ func (s *MemoryStore) ListAnnotationClaimCandidates(_ context.Context, org, proj
|
||||
defer s.mu.Unlock()
|
||||
out := []domain.HumanItemRecord{}
|
||||
for _, item := range s.humanItems {
|
||||
if humanOrganizationMatches(item.OrganizationID, org) && item.ProjectID == project && annotationClaimCandidate(item) {
|
||||
if humanOrganizationMatches(item.OrganizationID, org) && item.ProjectID == project && annotationClaimCandidate(item) && s.annotationSourceReadyLocked(item) {
|
||||
item.Subject = domain.HumanSubjectRef{}
|
||||
out = append(out, clone(item))
|
||||
}
|
||||
@ -56,7 +56,10 @@ func (s *MemoryStore) ListAnnotationClaimCandidates(_ context.Context, org, proj
|
||||
}
|
||||
func (s *MongoStore) ListAnnotationClaimCandidates(ctx context.Context, org, project string) ([]domain.HumanItemRecord, error) {
|
||||
filter := bson.M{"organizationId": org, "projectId": project, "pluginId": "annotation-qa", "legacyTaskId": bson.M{"$exists": true, "$ne": ""}, "assignedUserId": bson.M{"$in": bson.A{"", nil}}, "status": bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemRework, domain.HumanItemSubmitted}}}
|
||||
cursor, err := s.humanItems.Find(ctx, filter, options.Find().SetProjection(bson.M{"id": 1, "organizationId": 1, "projectId": 1, "pluginId": 1, "batchId": 1, "currentStageId": 1, "supplierId": 1, "processingScope": 1, "processingProfile": 1, "legacyTaskId": 1, "version": 1, "status": 1, "workflowId": 1, "runId": 1, "reviewRound": 1, "latestAuthorUserId": 1, "claimedAt": 1}))
|
||||
pipeline := mongo.Pipeline{{{Key: "$match", Value: filter}}}
|
||||
pipeline = append(pipeline, annotationSourceAdmissionPipeline()...)
|
||||
pipeline = append(pipeline, bson.D{{Key: "$project", Value: bson.M{"id": 1, "organizationId": 1, "projectId": 1, "pluginId": 1, "batchId": 1, "currentStageId": 1, "supplierId": 1, "processingScope": 1, "processingProfile": 1, "legacyTaskId": 1, "version": 1, "status": 1, "workflowId": 1, "runId": 1, "reviewRound": 1, "latestAuthorUserId": 1, "claimedAt": 1}}})
|
||||
cursor, err := s.humanItems.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -133,7 +136,7 @@ func (s *MemoryStore) CommitHumanClaim(_ context.Context, c HumanClaimCommit) (*
|
||||
}
|
||||
}
|
||||
old, ok := s.humanItems[c.Item.ID]
|
||||
if !ok || !validClaimSnapshot(old, c) {
|
||||
if !ok || !validClaimSnapshot(old, c) || !s.annotationSourceReadyLocked(old) {
|
||||
return nil, ErrHumanClaimConflict
|
||||
}
|
||||
project, ok := s.projects[c.Item.ProjectID]
|
||||
@ -281,6 +284,13 @@ func (s *MongoStore) commitHumanClaimTransaction(ctx context.Context, c HumanCla
|
||||
if task == nil || (task.SupplierID != "" && task.SupplierID != c.Item.SupplierID) {
|
||||
return nil, ErrHumanClaimConflict
|
||||
}
|
||||
ready, e := s.annotationSourceReady(sc, *old, *task)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if !ready {
|
||||
return nil, ErrHumanClaimConflict
|
||||
}
|
||||
if _, e = s.users.UpdateOne(sc, bson.M{"id": user.ID}, bson.M{"$inc": bson.M{"claimVersion": 1}}); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
@ -385,7 +395,7 @@ func (s *MemoryStore) CountAnnotationClaimCandidates(_ context.Context, org stri
|
||||
defer s.mu.Unlock()
|
||||
count := 0
|
||||
for _, item := range s.humanItems {
|
||||
if batch, ok := s.humanBatches[item.BatchID]; ok && annotationClaimCountMatches(item, batch, org, q) {
|
||||
if batch, ok := s.humanBatches[item.BatchID]; ok && annotationClaimCountMatches(item, batch, org, q) && s.annotationSourceReadyLocked(item) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
@ -504,8 +514,9 @@ func annotationClaimMembershipMongoExpression(memberships []AnnotationClaimMembe
|
||||
func annotationClaimCandidateCountPipeline(org string, q AnnotationClaimCountQuery) mongo.Pipeline {
|
||||
// Keep the indexed candidate match; compute only compact per-batch/stage
|
||||
// counters. Scalar string routing IDs avoid null equality matching missing IDs.
|
||||
return mongo.Pipeline{
|
||||
{{Key: "$match", Value: annotationClaimCountCandidateFilter(org, q)}},
|
||||
pipeline := mongo.Pipeline{{{Key: "$match", Value: annotationClaimCountCandidateFilter(org, q)}}}
|
||||
pipeline = append(pipeline, annotationSourceAdmissionPipeline()...)
|
||||
return append(pipeline, mongo.Pipeline{
|
||||
{{Key: "$project", Value: bson.M{
|
||||
"batchId": 1, "currentStageId": 1,
|
||||
"validIDs": bson.M{"$and": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$batchId"}, "string"}}, bson.M{"$eq": bson.A{bson.M{"$type": "$currentStageId"}, "string"}}}},
|
||||
@ -518,7 +529,7 @@ func annotationClaimCandidateCountPipeline(org string, q AnnotationClaimCountQue
|
||||
"legacyCount": bson.M{"$sum": bson.M{"$cond": bson.A{"$legacy", 1, 0}}},
|
||||
"poolCount": bson.M{"$sum": bson.M{"$cond": bson.A{"$pool", 1, 0}}},
|
||||
}}},
|
||||
}
|
||||
}...)
|
||||
}
|
||||
|
||||
func annotationClaimCountCandidateFilter(org string, q AnnotationClaimCountQuery) bson.M {
|
||||
|
||||
@ -122,6 +122,13 @@ func (s *MongoStore) commitHumanClaimStandalone(ctx context.Context, proposed Hu
|
||||
if !stageAllowed {
|
||||
return conflict()
|
||||
}
|
||||
ready, e := s.annotationSourceReady(ctx, *item, *task)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
if !ready {
|
||||
return nil, ErrHumanClaimConflict
|
||||
}
|
||||
claimed := c.Item
|
||||
claimed.Subject, claimed.ClaimRequestID = item.Subject, r.ID
|
||||
updated, e := s.UpdateHumanItemVersioned(ctx, claimed, c.ExpectedVersion)
|
||||
|
||||
@ -45,7 +45,7 @@ func matchesHumanMinePool(item domain.HumanItemRecord, query domain.HumanItemQue
|
||||
}
|
||||
|
||||
func humanItemMatches(item domain.HumanItemRecord, query domain.HumanItemQuery) bool {
|
||||
return matchesHumanMinePool(item, query) && matchesAnnotationView(item, query) && (query.IDs == nil || containsHumanItemID(query.IDs, item.ID)) && (query.LegacyTaskID == "" || item.LegacyTaskID == query.LegacyTaskID) &&
|
||||
return (query.AvailableSupplierID == nil || item.SupplierID == "" || item.SupplierID == *query.AvailableSupplierID) && (query.AvailableProjectIDs == nil || containsHumanItemID(query.AvailableProjectIDs, item.ProjectID)) && matchesHumanMinePool(item, query) && matchesAnnotationView(item, query) && (query.IDs == nil || containsHumanItemID(query.IDs, item.ID)) && (query.LegacyTaskID == "" || item.LegacyTaskID == query.LegacyTaskID) &&
|
||||
(query.ActorUserID == "" || item.AssignedUserID == query.ActorUserID || item.LatestAuthorUserID == query.ActorUserID) &&
|
||||
(query.PluginID == "" || item.PluginID == query.PluginID) &&
|
||||
(query.BatchID == "" || item.BatchID == query.BatchID) &&
|
||||
@ -246,6 +246,9 @@ func (store *MemoryStore) ListHumanItems(_ context.Context, organizationID strin
|
||||
items := make([]domain.HumanItemRecord, 0)
|
||||
for _, item := range store.humanItems {
|
||||
if humanOrganizationMatches(item.OrganizationID, organizationID) && humanItemMatches(item, query) && humanItemAfterCursor(item, query.Cursor) && blindAnnotationVisible(item, store.humanBatches[item.BatchID], query.BlindAnnotationActorID) {
|
||||
if humanPoolFiltered(query) && !humanPoolSearchRowMatches(store.humanPoolSearchRowLocked(item, organizationID), query) {
|
||||
continue
|
||||
}
|
||||
items = append(items, clone(item))
|
||||
}
|
||||
}
|
||||
@ -615,6 +618,12 @@ func humanBatchMongoFilter(organizationID string, query domain.HumanBatchQuery)
|
||||
|
||||
func humanItemMongoFilter(organizationID string, query domain.HumanItemQuery) bson.M {
|
||||
filter := bson.M{"organizationId": normalizedOrganizationID(organizationID)}
|
||||
if query.AvailableSupplierID != nil {
|
||||
filter["supplierId"] = bson.M{"$in": bson.A{"", nil, *query.AvailableSupplierID}}
|
||||
}
|
||||
if query.AvailableProjectIDs != nil {
|
||||
filter["projectId"] = bson.M{"$in": query.AvailableProjectIDs}
|
||||
}
|
||||
if query.IDs != nil {
|
||||
filter["id"] = bson.M{"$in": query.IDs}
|
||||
}
|
||||
@ -634,7 +643,11 @@ func humanItemMongoFilter(organizationID string, query domain.HumanItemQuery) bs
|
||||
filter["batchId"] = query.BatchID
|
||||
}
|
||||
if query.ProjectID != "" {
|
||||
filter["projectId"] = query.ProjectID
|
||||
if query.AvailableProjectIDs == nil || containsHumanItemID(query.AvailableProjectIDs, query.ProjectID) {
|
||||
filter["projectId"] = query.ProjectID
|
||||
} else {
|
||||
filter["projectId"] = bson.M{"$in": []string{}}
|
||||
}
|
||||
}
|
||||
if query.WorkflowID != "" {
|
||||
filter["workflowId"] = query.WorkflowID
|
||||
@ -781,6 +794,9 @@ func (store *MongoStore) UpsertHumanBatch(ctx context.Context, item domain.Human
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListHumanItems(ctx context.Context, organizationID string, query domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
if humanPoolFiltered(query) {
|
||||
return store.listHumanPoolSearch(ctx, organizationID, query)
|
||||
}
|
||||
findOptions := options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}})
|
||||
if query.Limit > 0 {
|
||||
findOptions.SetLimit(int64(query.Limit))
|
||||
|
||||
70
internal/store/human_pool_read_scope_test.go
Normal file
70
internal/store/human_pool_read_scope_test.go
Normal file
@ -0,0 +1,70 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHumanPoolReadScopeMongoParity(t *testing.T) {
|
||||
if os.Getenv("TEST_WORKFORCE_MONGO_URI") == "" {
|
||||
t.Skip("local Mongo opt-in")
|
||||
}
|
||||
if os.Getenv("TEST_WORKFORCE_MONGO_URI") != "mongodb://127.0.0.1:27018" {
|
||||
t.Fatal("loopback only")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cl, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://127.0.0.1:27018"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cl.Disconnect(context.Background())
|
||||
db := cl.Database(fmt.Sprintf("qc_scope_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
ms := NewMemoryStore()
|
||||
ds := NewMongoStore(db)
|
||||
for i, supplier := range []string{"vendor", "", "other"} {
|
||||
for j, project := range []string{"p", "other", ""} {
|
||||
it := domain.HumanItemRecord{ID: fmt.Sprintf("%d-%d", i, j), OrganizationID: "o", SupplierID: supplier, ProjectID: project, CreatedAt: "2026-10-01T00:00:00Z"}
|
||||
if err := ms.UpsertHumanItem(ctx, it); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ds.UpsertHumanItem(ctx, it); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Missing legacy supplier fields must behave exactly like empty strings.
|
||||
if _, err := db.Collection("humanItems").UpdateMany(ctx, bson.M{"supplierId": ""}, bson.M{"$unset": bson.M{"supplierId": ""}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
supplier := "vendor"
|
||||
for _, q := range []domain.HumanItemQuery{{}, {AvailableProjectIDs: []string{}}, {AvailableProjectIDs: []string{"p"}, AvailableSupplierID: &supplier}, {AvailableProjectIDs: []string{"p"}, AvailableSupplierID: &supplier, ProjectID: "other"}, {AvailableProjectIDs: []string{""}, AvailableSupplierID: &supplier}} {
|
||||
a, e := ms.ListHumanItems(ctx, "o", q)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
b, e := ds.ListHumanItems(ctx, "o", q)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
ids := func(items []domain.HumanItemRecord) []string {
|
||||
out := []string{}
|
||||
for _, it := range items {
|
||||
out = append(out, it.ID)
|
||||
}
|
||||
return out
|
||||
}
|
||||
if !reflect.DeepEqual(ids(a), ids(b)) {
|
||||
t.Fatalf("scope mismatch %v %v", ids(a), ids(b))
|
||||
}
|
||||
}
|
||||
}
|
||||
147
internal/store/human_pool_search.go
Normal file
147
internal/store/human_pool_search.go
Normal file
@ -0,0 +1,147 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var humanPoolPartPattern = regexp.MustCompile(`(?i)_(?:camera_(?:left|right)_)?(part[0-9]+)(?:_pts)?(?:\.[^/?#]+)?(?:[?#].*)?$`)
|
||||
|
||||
func humanPoolFiltered(q domain.HumanItemQuery) bool {
|
||||
return q.Search != "" || q.Capture != "" || q.Category != "" && q.Category != "all"
|
||||
}
|
||||
|
||||
func (s *MemoryStore) humanPoolSearchRowLocked(i domain.HumanItemRecord, org string) adminPoolRow {
|
||||
row := adminPoolRow{HumanItemRecord: humanItemListProjectionClone(i, humanItemListPayloadKeysRemoved)}
|
||||
if t, ok := s.annotationTasks[i.LegacyTaskID]; ok && humanOrganizationMatches(t.OrganizationID, org) {
|
||||
row.Tasks = []domain.AnnotationTaskRecord{t}
|
||||
}
|
||||
if r, ok := s.runs[i.RunID]; ok && humanOrganizationMatches(r.OrganizationID, org) {
|
||||
row.Runs = []domain.RunRecord{r}
|
||||
if c, ok := s.captureRecords[r.TaskID]; ok && humanOrganizationMatches(c.OrganizationID, org) {
|
||||
row.Captures = []domain.CaptureRecord{c}
|
||||
}
|
||||
}
|
||||
if p, ok := s.projects[i.ProjectID]; ok && humanOrganizationMatches(p.OrganizationID, org) {
|
||||
row.Projects = []domain.ProjectRecord{p}
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func humanPoolSearchRowMatches(row adminPoolRow, q domain.HumanItemQuery) bool {
|
||||
base := AdminAnnotationPoolQuery{Capture: q.Capture, Category: q.Category}
|
||||
if q.ReviewedPool && row.Status == domain.HumanItemAccepted && (q.Category == "accepted" || q.Category == "pending_acceptance") {
|
||||
base.Category = ""
|
||||
}
|
||||
if !adminPoolRowMatches(row, base) {
|
||||
return false
|
||||
}
|
||||
if q.Search == "" {
|
||||
return true
|
||||
}
|
||||
if adminPoolRowMatches(row, AdminAnnotationPoolQuery{Search: q.Search}) {
|
||||
return true
|
||||
}
|
||||
values := []string{row.ID, row.LegacyTaskID, row.Subject.ItemKey, row.Subject.ArtifactID}
|
||||
for _, t := range row.Tasks {
|
||||
values = append(values, t.ArtifactID, t.SourceVideo)
|
||||
}
|
||||
for _, c := range row.Captures {
|
||||
values = append(values, c.CaptureID, c.Prefix)
|
||||
}
|
||||
captureNames := []string{}
|
||||
for _, capture := range row.Captures {
|
||||
name := strings.TrimSpace(capture.CaptureID)
|
||||
if name == "" {
|
||||
name = domain.CaptureDirectoryFromDirectoryValue(capture.Prefix)
|
||||
}
|
||||
if name != "" {
|
||||
captureNames = append(captureNames, name)
|
||||
}
|
||||
}
|
||||
for _, value := range append([]string(nil), values...) {
|
||||
match := humanPoolPartPattern.FindStringSubmatch(value)
|
||||
if len(match) < 2 {
|
||||
continue
|
||||
}
|
||||
names := captureNames
|
||||
if capture := domain.CaptureDirectoryFromArtifactValue(value); capture != "" {
|
||||
names = append(append([]string(nil), names...), capture)
|
||||
}
|
||||
for _, capture := range names {
|
||||
values = append(values, capture+"/"+match[1])
|
||||
}
|
||||
}
|
||||
return strings.Contains(strings.ToLower(strings.Join(values, " ")), strings.ToLower(q.Search))
|
||||
}
|
||||
|
||||
// Stream metadata through the existing search rules before applying the page
|
||||
// limit. The handler still applies current visibility and claim permissions.
|
||||
func (s *MongoStore) listHumanPoolSearch(ctx context.Context, org string, q domain.HumanItemQuery) ([]domain.HumanItemRecord, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 8*time.Second)
|
||||
defer cancel()
|
||||
filter := humanItemMongoFilter(org, q)
|
||||
if q.BlindAnnotationActorID != "" {
|
||||
visibility, err := s.blindAnnotationMongoVisibility(ctx, org, q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
filter = bson.M{"$and": bson.A{filter, visibility}}
|
||||
}
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$match", Value: filter}},
|
||||
{{Key: "$sort", Value: bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}}},
|
||||
{{Key: "$project", Value: humanItemListProjection(domain.HumanItemQuery{ListProjection: true})}},
|
||||
adminPoolLookup("annotationTasks", "legacyTaskId", "poolTasks", org, bson.M{"_id": 0, "id": 1, "status": 1, "artifactId": 1, "sourceVideo": 1}),
|
||||
}
|
||||
if q.Search != "" || q.Capture != "" {
|
||||
pipeline = append(pipeline,
|
||||
adminPoolLookup("runs", "runId", "poolRuns", org, bson.M{"_id": 0, "id": 1, "taskId": 1, "projectName": 1, "captureDirectories": 1, "inputs.ref": 1, "nodeExecutions.artifacts.id": 1, "nodeExecutions.artifacts.name": 1, "nodeExecutions.artifacts.path": 1, "artifacts.id": 1, "artifacts.name": 1, "artifacts.path": 1}),
|
||||
adminPoolLookup("captureRecords", "poolRuns.taskId", "poolCaptures", org, bson.M{"_id": 0, "captureId": 1, "prefix": 1}),
|
||||
adminPoolLookup("projects", "projectId", "poolProjects", org, bson.M{"_id": 0, "name": 1}),
|
||||
)
|
||||
}
|
||||
for _, text := range []string{q.Search, q.Capture} {
|
||||
if text != "" {
|
||||
if searchFilter := humanPoolSearchPrefilter(text); searchFilter != nil {
|
||||
pipeline = append(pipeline, bson.D{{Key: "$match", Value: searchFilter}})
|
||||
}
|
||||
}
|
||||
}
|
||||
if category := humanPoolCategoryPrefilter(q.Category, q.ReviewedPool); category != nil {
|
||||
pipeline = append(pipeline, bson.D{{Key: "$match", Value: category}})
|
||||
}
|
||||
cur, err := s.humanItems.Aggregate(ctx, pipeline, options.Aggregate().SetAllowDiskUse(false).SetMaxTime(8*time.Second).SetBatchSize(200))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
out := []domain.HumanItemRecord{}
|
||||
for cur.Next(ctx) {
|
||||
var row adminPoolRow
|
||||
if err := cur.Decode(&row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !humanPoolSearchRowMatches(row, q) {
|
||||
continue
|
||||
}
|
||||
item := row.HumanItemRecord
|
||||
if q.PoolCandidateProjection {
|
||||
item = item.PoolCandidate()
|
||||
}
|
||||
if q.IdentityProjection {
|
||||
item = domain.HumanItemRecord{ID: item.ID}
|
||||
}
|
||||
out = append(out, item)
|
||||
if q.Limit > 0 && len(out) >= q.Limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, cur.Err()
|
||||
}
|
||||
72
internal/store/human_pool_search_filter.go
Normal file
72
internal/store/human_pool_search_filter.go
Normal file
@ -0,0 +1,72 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Capture records are authoritative display metadata. Rows predating that link
|
||||
// retain the existing recursive payload fallback; ordinary rows are filtered in
|
||||
// Mongo so a no-match query does not stream the entire pool into the API.
|
||||
func humanPoolSearchPrefilter(search string) bson.M {
|
||||
if strings.ContainsAny(search, " \t\n") {
|
||||
return nil
|
||||
}
|
||||
pattern := regexp.QuoteMeta(strings.ToLower(search))
|
||||
pattern = strings.ReplaceAll(pattern, "/part", `(?:/|_|_camera_(?:left|right)_)part`)
|
||||
match := bson.A{}
|
||||
for _, field := range []string{"id", "legacyTaskId", "subject.itemKey", "subject.artifactId", "poolTasks.artifactId", "poolTasks.sourceVideo", "poolCaptures.captureId", "poolCaptures.prefix", "poolProjects.name", "poolRuns.projectName", "poolRuns.captureDirectories", "poolRuns.inputs.ref.captureUnit", "poolRuns.inputs.ref.prefix", "poolRuns.inputs.ref.key", "poolRuns.inputs.ref.manifestKey", "poolRuns.artifacts.id", "poolRuns.artifacts.name", "poolRuns.artifacts.path", "poolRuns.nodeExecutions.artifacts.id", "poolRuns.nodeExecutions.artifacts.name", "poolRuns.nodeExecutions.artifacts.path"} {
|
||||
match = append(match, bson.M{field: bson.M{"$regex": pattern, "$options": "i"}})
|
||||
match = append(match, bson.M{field: bson.M{"$regex": "%[0-9a-f]{2}", "$options": "i"}})
|
||||
}
|
||||
if split := strings.LastIndex(strings.ToLower(search), "/part"); split > 0 {
|
||||
captures := bson.A{}
|
||||
for _, field := range []string{"poolCaptures.captureId", "poolCaptures.prefix"} {
|
||||
captures = append(captures, bson.M{field: bson.M{"$regex": regexp.QuoteMeta(search[:split]), "$options": "i"}})
|
||||
}
|
||||
parts := bson.A{}
|
||||
for _, field := range []string{"subject.itemKey", "subject.artifactId", "poolTasks.artifactId", "poolTasks.sourceVideo"} {
|
||||
parts = append(parts, bson.M{field: bson.M{"$regex": regexp.QuoteMeta(search[split+1:]), "$options": "i"}})
|
||||
}
|
||||
match = append(match, bson.M{"$and": bson.A{bson.M{"$or": captures}, bson.M{"$or": parts}}})
|
||||
}
|
||||
match = append(match, bson.M{"$nor": bson.A{
|
||||
bson.M{"poolCaptures.captureId": bson.M{"$type": "string", "$ne": ""}},
|
||||
bson.M{"poolCaptures.prefix": bson.M{"$type": "string", "$ne": ""}},
|
||||
}})
|
||||
return bson.M{"$or": match}
|
||||
}
|
||||
|
||||
func humanPoolCategoryPrefilter(category string, reviewed bool) bson.M {
|
||||
switch category {
|
||||
case "accepted", "rework_review", "pending_quality", "pending_acceptance", "rework", "slicing", "pending_annotation", "pending_slice", "available":
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
eq := func(a, b any) bson.M { return bson.M{"$eq": bson.A{a, b}} }
|
||||
in := func(a any, b ...any) bson.M { return bson.M{"$in": bson.A{a, bson.A(b)}} }
|
||||
or := func(v ...any) bson.M { return bson.M{"$or": bson.A(v)} }
|
||||
and := func(v ...any) bson.M { return bson.M{"$and": bson.A(v)} }
|
||||
text := func(field string) bson.M { return bson.M{"$ifNull": bson.A{field, ""}} }
|
||||
task := bson.M{"$ifNull": bson.A{bson.M{"$arrayElemAt": bson.A{"$poolTasks.status", 0}}, ""}}
|
||||
round := bson.M{"$gt": bson.A{bson.M{"$ifNull": bson.A{"$reviewRound", 0}}, 0}}
|
||||
branches := bson.A{}
|
||||
add := func(condition any, result string) {
|
||||
branches = append(branches, bson.M{"case": condition, "then": result})
|
||||
}
|
||||
add(or(eq("$status", "accepted"), eq(task, "accepted")), "accepted")
|
||||
add(bson.M{"$ne": bson.A{text("$pendingCombinedSubmissionId"), ""}}, "submitting")
|
||||
add(and(round, in("$currentStageId", "quality_review", "prepackage_acceptance", "acceptance")), "rework_review")
|
||||
add(or(eq(task, "pending_quality"), eq("$currentStageId", "quality_review"), eq("$status", "submitted")), "pending_quality")
|
||||
add(or(eq(task, "pending_acceptance"), in("$currentStageId", "acceptance", "prepackage_acceptance")), "pending_acceptance")
|
||||
add(or(eq(task, "rework"), eq("$status", "rework"), and(round, eq("$currentStageId", "annotation_combined"))), "rework")
|
||||
add(eq("$subject.payload.sliceWorkflow.status", "processing"), "slicing")
|
||||
add(or(eq("$subject.payload.sliceWorkflow.status", "completed"), eq("$subject.payload.sliceWorkflow.phase", "annotate")), "pending_annotation")
|
||||
add(or(eq("$subject.payload.sliceWorkflow.status", "editing"), eq("$subject.payload.sliceWorkflow.phase", "slice"), bson.M{"$ne": bson.A{text("$assignedUserId"), ""}}), "pending_slice")
|
||||
filter := bson.M{"$expr": eq(bson.M{"$switch": bson.M{"branches": branches, "default": "available"}}, category)}
|
||||
if reviewed && (category == "accepted" || category == "pending_acceptance") {
|
||||
return bson.M{"$or": bson.A{bson.M{"status": "accepted"}, filter}}
|
||||
}
|
||||
return filter
|
||||
}
|
||||
200
internal/store/human_pool_search_test.go
Normal file
200
internal/store/human_pool_search_test.go
Normal file
@ -0,0 +1,200 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestHumanPoolSearchMongoParity(t *testing.T) { testHumanPoolSearchMongo(t, 963, false) }
|
||||
func TestHumanPoolSearchMongoLargeModern(t *testing.T) { testHumanPoolSearchMongo(t, 10000, true) }
|
||||
|
||||
func testHumanPoolSearchMongo(t *testing.T, count int, modern bool) {
|
||||
uri := os.Getenv("POOL_LOCAL_MONGO_URI")
|
||||
if uri == "" {
|
||||
t.Skip("isolated local Mongo opt-in")
|
||||
}
|
||||
u, e := url.Parse(uri)
|
||||
if e != nil || u.Scheme != "mongodb" || (u.Host != "127.0.0.1:27018" && u.Host != "127.0.0.1:27017" && u.Host != "127.0.0.1:27019") {
|
||||
t.Fatal("local Mongo 27017/27018/27019 only")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
var measuring atomic.Bool
|
||||
var replyBytes atomic.Int64
|
||||
var lastMaxTime atomic.Int64
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "aggregate" {
|
||||
value := e.Command.Lookup("maxTimeMS")
|
||||
if v, ok := value.Int64OK(); ok {
|
||||
lastMaxTime.Store(v)
|
||||
} else if v, ok := value.Int32OK(); ok {
|
||||
lastMaxTime.Store(int64(v))
|
||||
}
|
||||
}
|
||||
}, Failed: func(_ context.Context, e *event.CommandFailedEvent) {
|
||||
t.Logf("mongo failed command=%s duration=%s maxTimeMS=%d failure=%s", e.CommandName, e.Duration, lastMaxTime.Load(), e.Failure)
|
||||
}, Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
if measuring.Load() && (e.CommandName == "aggregate" || e.CommandName == "getMore") {
|
||||
replyBytes.Add(int64(len(e.Reply)))
|
||||
}
|
||||
}}
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMonitor(monitor))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(fmt.Sprintf("human_pool_search_test_%d", time.Now().UnixNano()))
|
||||
defer db.Drop(context.Background())
|
||||
s := NewMongoStore(db)
|
||||
mem := NewMemoryStore()
|
||||
const org = "search-org"
|
||||
for _, name := range []string{"humanItems", "runs", "captureRecords", "annotationTasks", "projects"} {
|
||||
if _, e := db.Collection(name).Indexes().CreateOne(ctx, mongo.IndexModel{Keys: bson.D{{Key: "id", Value: 1}}, Options: options.Index().SetUnique(true)}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
project := domain.ProjectRecord{ID: "p", OrganizationID: org, Name: "历史项目"}
|
||||
_ = s.UpsertProject(ctx, project)
|
||||
_ = mem.UpsertProject(ctx, project)
|
||||
docs := []any{}
|
||||
runs, captures := []any{}, []any{}
|
||||
for n := 0; n < count; n++ {
|
||||
i := domain.HumanItemRecord{ID: fmt.Sprintf("i-%04d", n), OrganizationID: org, PluginID: "annotation-qa", ProjectID: "p", RunID: "run", CurrentStageID: "quality_review", Status: domain.HumanItemReady, CreatedAt: time.Date(2026, 11, 1, 0, 0, n, 0, time.UTC).Format(time.RFC3339), Subject: domain.HumanSubjectRef{ItemKey: fmt.Sprintf("FYLX_ITEM_%04d/part0002/camera_left.mp4", n), Payload: map[string]any{"annotationWorkspace": map[string]any{"secret": "not for lists"}}}}
|
||||
if modern {
|
||||
i.RunID = fmt.Sprintf("modern-run-%d", n)
|
||||
capture := domain.CaptureRecord{ID: fmt.Sprintf("modern-capture-%d", n), OrganizationID: org, CaptureID: fmt.Sprintf("FYLX_ITEM_%04d", n)}
|
||||
run := domain.RunRecord{ID: i.RunID, OrganizationID: org, TaskID: capture.ID}
|
||||
runs = append(runs, run)
|
||||
captures = append(captures, capture)
|
||||
_ = mem.UpsertRun(ctx, run)
|
||||
_ = mem.UpsertCaptureRecord(ctx, capture)
|
||||
}
|
||||
|
||||
if n >= 10 && n <= 18 {
|
||||
i.CurrentStageID = "annotation_combined"
|
||||
switch n {
|
||||
case 10:
|
||||
i.Status = domain.HumanItemAccepted
|
||||
i.PendingCombinedSubmissionID = "pending"
|
||||
case 11:
|
||||
i.PendingCombinedSubmissionID = "pending"
|
||||
case 12:
|
||||
i.CurrentStageID = "quality_review"
|
||||
i.ReviewRound = 1
|
||||
case 13:
|
||||
i.CurrentStageID = "prepackage_acceptance"
|
||||
case 14:
|
||||
i.Status = domain.HumanItemRework
|
||||
case 15:
|
||||
i.Subject.Payload["sliceWorkflow"] = map[string]any{"status": "processing"}
|
||||
case 16:
|
||||
i.Subject.Payload["sliceWorkflow"] = map[string]any{"phase": "annotate"}
|
||||
case 17:
|
||||
i.AssignedUserID = "owner"
|
||||
}
|
||||
}
|
||||
if n == 2 {
|
||||
i.Subject.ItemKey = "historic-no-capture"
|
||||
i.RunID = "legacy-run"
|
||||
}
|
||||
if n == 4 {
|
||||
i.Subject.ItemKey = "FYLX_ITEM_0004_part0002.mp4"
|
||||
}
|
||||
if n == 3 {
|
||||
i.Subject.ItemKey = "special-[.*-capture"
|
||||
}
|
||||
_ = mem.UpsertHumanItem(ctx, i)
|
||||
docs = append(docs, i)
|
||||
}
|
||||
if _, e := s.humanItems.InsertMany(ctx, docs); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if modern {
|
||||
if _, e := s.runs.InsertMany(ctx, runs); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := s.captureRecords.InsertMany(ctx, captures); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
legacy := domain.RunRecord{ID: "legacy-run", OrganizationID: org, CaptureDirectories: []string{"历史目录"}}
|
||||
_ = mem.UpsertRun(ctx, legacy)
|
||||
_ = s.UpsertRun(ctx, legacy)
|
||||
ids := func(t *testing.T, rows []domain.HumanItemRecord) []string {
|
||||
out := []string{}
|
||||
for _, i := range rows {
|
||||
out = append(out, i.ID)
|
||||
if _, ok := i.Subject.Payload["annotationWorkspace"]; ok {
|
||||
t.Fatal("heavy workspace leaked")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
for _, q := range []domain.HumanItemQuery{
|
||||
{Search: "FYLX_ITEM_0001/part0002", Limit: 200},
|
||||
{Search: "NO_SUCH_CAPTURE", Limit: 200},
|
||||
{Capture: "FYLX_ITEM_0001", Limit: 200},
|
||||
{Capture: "历史目录", Limit: 200},
|
||||
{Search: "FYLX_ITEM_0004/PART0002", Limit: 200},
|
||||
{Search: "历史目录", Limit: 200},
|
||||
{Search: "[.*", Limit: 200},
|
||||
{Search: "历史项目", Limit: 200, Cursor: &domain.HumanItemCursor{CreatedAt: "2026-11-01T00:05:00Z", ID: "i-0300"}},
|
||||
{Category: "pending_quality", Limit: 200, PoolCandidateProjection: true},
|
||||
{Category: "accepted", Limit: 200},
|
||||
{Category: "pending_acceptance", ReviewedPool: true, Limit: 200},
|
||||
{Category: "accepted", ReviewedPool: true, Limit: 200}, {Category: "rework_review", Limit: 200}, {Category: "pending_acceptance", Limit: 200}, {Category: "rework", Limit: 200}, {Category: "slicing", Limit: 200}, {Category: "pending_annotation", Limit: 200}, {Category: "pending_slice", Limit: 200}, {Category: "available", Limit: 200},
|
||||
{Search: "FYLX_ITEM", IDs: []string{}, Limit: 200},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("search=%s_capture=%s_category=%s_reviewed=%t", q.Search, q.Capture, q.Category, q.ReviewedPool), func(t *testing.T) {
|
||||
q.ListProjection = true
|
||||
var clockBefore time.Time
|
||||
readClock := func() time.Time {
|
||||
var hello struct {
|
||||
LocalTime time.Time `bson:"localTime"`
|
||||
}
|
||||
if err := db.RunCommand(ctx, bson.M{"hello": 1}).Decode(&hello); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return hello.LocalTime
|
||||
}
|
||||
if os.Getenv("POOL_MONGO_CLOCK_DIAGNOSTICS") == "1" {
|
||||
clockBefore = readClock()
|
||||
}
|
||||
start := time.Now()
|
||||
replyBytes.Store(0)
|
||||
measuring.Store(true)
|
||||
got, e := s.ListHumanItems(ctx, org, q)
|
||||
elapsed := time.Since(start)
|
||||
if !clockBefore.IsZero() {
|
||||
clockAfter := readClock()
|
||||
t.Logf("clock serverBefore=%s serverAfter=%s serverDelta=%s monotonicElapsed=%s error=%v", clockBefore.UTC().Format(time.RFC3339Nano), clockAfter.UTC().Format(time.RFC3339Nano), clockAfter.Sub(clockBefore), elapsed, e)
|
||||
}
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
measuring.Store(false)
|
||||
want, e := mem.ListHumanItems(ctx, org, q)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if !reflect.DeepEqual(ids(t, got), ids(t, want)) {
|
||||
t.Fatalf("query %+v\nMongo %v\nMemory %v", q, ids(t, got), ids(t, want))
|
||||
}
|
||||
t.Logf("records=%d modern=%v search=%q capture=%q category=%q rows=%d elapsed=%s mongoReplyBytes=%d", count, modern, q.Search, q.Capture, q.Category, len(got), elapsed, replyBytes.Load())
|
||||
if modern && q.Search == "NO_SUCH_CAPTURE" && replyBytes.Load() > 65536 {
|
||||
t.Fatal("modern no-hit search transferred backlog metadata")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -444,6 +444,9 @@ func (runtime *Runtime) launchCaptureRun(ctx context.Context, workflow domain.Wo
|
||||
if err := validateV4CaptureSourceSelection(workflow, record); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateRawHumanCaptureMetadataSelection(workflow, record); err != nil {
|
||||
return err
|
||||
}
|
||||
claimed, err := runtime.store.ClaimCaptureRecordForLaunch(ctx, record.ID, record.Fingerprint)
|
||||
if err != nil || !claimed {
|
||||
return err
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
@ -188,11 +189,38 @@ func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomat
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Keep legacy discovery unchanged. Only workflows explicitly requesting v4
|
||||
// need all eight original-source roles before a part is eligible for launch.
|
||||
func workflowUsesRawHumanCapture(workflow domain.WorkflowRecord) bool {
|
||||
for _, node := range workflow.Nodes {
|
||||
if node.Data.OperatorID == "operator-builtin-orbbec-data-normalize" && nodeHyperparamsMap(node)["PROCESSING_PROFILE"] == "raw_human" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func validateRawHumanCaptureMetadataSelection(workflow domain.WorkflowRecord, record *domain.CaptureRecord) error {
|
||||
if !workflowUsesRawHumanCapture(workflow) {
|
||||
return nil
|
||||
}
|
||||
if record == nil || record.SourcePrefix == "" {
|
||||
return fmt.Errorf("原始数据人工版缺少采集根目录")
|
||||
}
|
||||
key := record.SourcePrefix + "/meta_" + path.Base(record.SourcePrefix) + ".json"
|
||||
count := 0
|
||||
for _, selected := range record.SourceKeys {
|
||||
if selected == key {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count != 1 {
|
||||
return fmt.Errorf("原始数据人工版需要唯一的 %s", path.Base(key))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func discoverCaptureGroupsForWorkflow(objects []captureObject, config domain.CaptureAutomationConfig, workflow domain.WorkflowRecord) ([]captureGroup, error) {
|
||||
groups, err := discoverCaptureGroups(objects, config)
|
||||
if err != nil || !workflowUsesV4CaptureDelivery(workflow) {
|
||||
if err != nil || (!workflowUsesV4CaptureDelivery(workflow) && !workflowUsesRawHumanCapture(workflow)) {
|
||||
return groups, err
|
||||
}
|
||||
for i := range groups {
|
||||
@ -202,10 +230,13 @@ func discoverCaptureGroupsForWorkflow(objects []captureObject, config domain.Cap
|
||||
if err := validateV4CaptureSourceSelection(workflow, &record); err != nil {
|
||||
group.Missing = append(group.Missing, err.Error())
|
||||
}
|
||||
if err := validateRawHumanCaptureMetadataSelection(workflow, &record); err != nil {
|
||||
group.Missing = append(group.Missing, err.Error())
|
||||
}
|
||||
metadataKey := group.SourcePrefix + "/meta_" + path.Base(group.SourcePrefix) + ".json"
|
||||
for _, object := range group.Objects {
|
||||
if strings.EqualFold(object.Key, metadataKey) && object.Size <= 0 {
|
||||
group.Missing = append(group.Missing, "v4 原始源文件为空 "+object.Key)
|
||||
group.Missing = append(group.Missing, "采集 meta 文件为空 "+path.Base(object.Key))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
100
internal/worker/capture_raw_human_admission_test.go
Normal file
100
internal/worker/capture_raw_human_admission_test.go
Normal file
@ -0,0 +1,100 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"droboticflow/internal/domain"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func rawHumanCaptureWorkflow() domain.WorkflowRecord {
|
||||
return domain.WorkflowRecord{Nodes: []domain.WorkflowNode{{Data: domain.WorkflowNodeData{
|
||||
OperatorID: "operator-builtin-orbbec-data-normalize",
|
||||
Hyperparams: []domain.HyperParam{{Key: "PROCESSING_PROFILE", Value: "raw_human"}},
|
||||
}}}}
|
||||
}
|
||||
|
||||
func TestRawHumanIntakeRequiresOwnNonemptyMetadata(t *testing.T) {
|
||||
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
|
||||
for _, tc := range []string{"missing", "other-capture", "inside-part", "empty", "complete"} {
|
||||
t.Run(tc, func(t *testing.T) {
|
||||
objects := append(calibrationObjects(), partObjects("part0001")...)
|
||||
meta := captureObject{Key: "incoming/" + layoutCapture + "/meta_" + layoutCapture + ".json", Size: 10}
|
||||
switch tc {
|
||||
case "other-capture":
|
||||
meta.Key = "incoming/other/meta_" + layoutCapture + ".json"
|
||||
case "inside-part":
|
||||
meta.Key = "incoming/" + layoutCapture + "/part0001/meta_" + layoutCapture + ".json"
|
||||
case "empty":
|
||||
meta.Size = 0
|
||||
}
|
||||
if tc != "missing" {
|
||||
objects = append(objects, meta)
|
||||
}
|
||||
groups, err := discoverCaptureGroupsForWorkflow(objects, config, rawHumanCaptureWorkflow())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, group := range groups {
|
||||
if group.CaptureID != layoutCapture+"/part0001" {
|
||||
continue
|
||||
}
|
||||
if (len(group.Missing) == 0) != (tc == "complete") {
|
||||
t.Fatalf("metadata %s: missing=%v", tc, group.Missing)
|
||||
}
|
||||
if tc != "complete" && !strings.Contains(strings.Join(group.Missing, " "), "meta_") {
|
||||
t.Fatal("missing metadata reason", group.Missing)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("target not discovered")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyIntakeKeepsMetadataOptional(t *testing.T) {
|
||||
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
|
||||
objects := append(calibrationObjects(), partObjects("part0001")...)
|
||||
groups, err := discoverCaptureGroupsForWorkflow(objects, config, domain.WorkflowRecord{})
|
||||
if err != nil || len(groups) != 1 || len(groups[0].Missing) != 0 {
|
||||
t.Fatal(groups, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawHumanLaunchRequiresFrozenMetadataSelection(t *testing.T) {
|
||||
root := "incoming/" + layoutCapture
|
||||
key := root + "/meta_" + layoutCapture + ".json"
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
keys []string
|
||||
valid bool
|
||||
}{
|
||||
{"missing", nil, false},
|
||||
{"inside-part", []string{root + "/part0001/meta_" + layoutCapture + ".json"}, false},
|
||||
{"duplicate", []string{key, key}, false},
|
||||
{"complete", []string{key}, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
record := &domain.CaptureRecord{SourcePrefix: root, SourceKeys: tc.keys}
|
||||
if err := validateRawHumanCaptureMetadataSelection(rawHumanCaptureWorkflow(), record); (err == nil) != tc.valid {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := validateRawHumanCaptureMetadataSelection(domain.WorkflowRecord{}, record); err != nil {
|
||||
t.Fatal("legacy selection changed", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawHumanFlatIntakeRequiresMetadataAtCaptureRoot(t *testing.T) {
|
||||
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", Bucket: "synthetic-bucket"})
|
||||
objects := completeCaptureObjects("incoming", "Orbbec_Ego_legacy")
|
||||
groups, err := discoverCaptureGroupsForWorkflow(objects, config, rawHumanCaptureWorkflow())
|
||||
if err != nil || len(groups) != 1 || len(groups[0].Missing) == 0 {
|
||||
t.Fatal(groups, err)
|
||||
}
|
||||
objects = append(objects, captureObject{Key: "incoming/Orbbec_Ego_legacy/meta_Orbbec_Ego_legacy.json", Size: 10})
|
||||
groups, err = discoverCaptureGroupsForWorkflow(objects, config, rawHumanCaptureWorkflow())
|
||||
if err != nil || len(groups) != 1 || len(groups[0].Missing) != 0 {
|
||||
t.Fatal(groups, err)
|
||||
}
|
||||
}
|
||||
@ -120,8 +120,10 @@ def _validate_transcoded_pts(video: pathlib.Path, pts: pathlib.Path, expected_fr
|
||||
return {"videoFrames": frames, "ptsRows": count, "sourceDurationSeconds": duration, "startTimestampUs": first, "endTimestampUs": previous, "validation": "count_and_timestamp_checks_passed", "timestamps": "original_unmodified"}
|
||||
|
||||
|
||||
def _validate_raw_imu(path: pathlib.Path, start: int, end: int) -> None:
|
||||
def _validate_raw_imu(path: pathlib.Path, start: int, end: int, *, require_coverage: bool = True) -> None:
|
||||
ranges = {}
|
||||
previous = None
|
||||
pair = set()
|
||||
with path.open(encoding="utf-8-sig", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
if not {"timestamp_us", "x", "y", "z", "type"}.issubset(reader.fieldnames or []):
|
||||
@ -133,9 +135,27 @@ def _validate_raw_imu(path: pathlib.Path, start: int, end: int) -> None:
|
||||
raise ValueError("invalid value")
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行无效") from exc
|
||||
kind = row["type"].strip().lower()
|
||||
kind = (row.get("type") or "").strip().lower()
|
||||
if kind not in {"accel", "gyro"}:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行类型无效,只支持 accel/gyro")
|
||||
if previous is not None and timestamp < previous:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行时间戳未递增")
|
||||
if timestamp != previous:
|
||||
if previous is not None and pair != {"accel", "gyro"}:
|
||||
raise RuntimeError(f"原始 IMU 时间戳 {previous} 缺少 accel/gyro 配对")
|
||||
pair = set()
|
||||
if kind in pair:
|
||||
raise RuntimeError(f"原始 IMU 第 {reader.line_num} 行 {kind} 重复")
|
||||
pair.add(kind)
|
||||
previous = timestamp
|
||||
low, high = ranges.get(kind, (timestamp, timestamp))
|
||||
ranges[kind] = (min(low,timestamp), max(high,timestamp))
|
||||
if previous is None:
|
||||
raise RuntimeError("原始 IMU 没有有效数据")
|
||||
if pair != {"accel", "gyro"}:
|
||||
raise RuntimeError(f"原始 IMU 时间戳 {previous} 缺少 accel/gyro 配对")
|
||||
if not require_coverage:
|
||||
return
|
||||
for kind in ("accel", "gyro"):
|
||||
if kind not in ranges or ranges[kind][0] > start or ranges[kind][1] < end:
|
||||
raise RuntimeError(f"原始 IMU {kind} 未覆盖视频 PTS 时间范围")
|
||||
@ -218,9 +238,8 @@ def _validate_fallback_pair(path, paired, entries, source_path, output_dir, para
|
||||
if raw_human:
|
||||
if not paired.get("imu"):
|
||||
raise RuntimeError("原始数据人工版缺少 IMU")
|
||||
# Temporarily bypass the raw-human IMU content/coverage gate: source
|
||||
# sensor streams can cross part boundaries. Keep the paired IMU intact.
|
||||
# _validate_raw_imu(materialize(paired["imu"]), validation["startTimestampUs"], validation["endTimestampUs"])
|
||||
# Validate content without requiring the part's IMU to cover video endpoints.
|
||||
_validate_raw_imu(materialize(paired["imu"]), validation["startTimestampUs"], validation["endTimestampUs"], require_coverage=False)
|
||||
return validation
|
||||
|
||||
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import type { AnnotationTimelineFailure } from "../lib/annotation-source-validation";
|
||||
import {
|
||||
createSliceEditSession,
|
||||
freezeSliceEditSession,
|
||||
@ -186,6 +187,9 @@ type Props = ReviewViewingProps & {
|
||||
totalProgress?: { offset: number; duration: number; onSeekTotal?: (time: number) => void };
|
||||
frameBoundaries?: number[];
|
||||
frameTimelineStatus?: "loading" | "ready" | "error";
|
||||
frameTimelineFailure?: AnnotationTimelineFailure | null;
|
||||
sourceValidationRetrying?: boolean;
|
||||
onRetrySourceValidation?: () => void;
|
||||
onRetryFrameTimeline?: () => void;
|
||||
onAlignSliceBoundaries?: () => void;
|
||||
rightMediaBoundaries?: number[];
|
||||
@ -378,6 +382,9 @@ export function CaptureAnnotationWorkbench({
|
||||
totalProgress,
|
||||
frameBoundaries = [],
|
||||
frameTimelineStatus = "ready",
|
||||
frameTimelineFailure,
|
||||
sourceValidationRetrying = false,
|
||||
onRetrySourceValidation,
|
||||
onRetryFrameTimeline,
|
||||
onAlignSliceBoundaries,
|
||||
rightMediaBoundaries = [],
|
||||
@ -3763,18 +3770,36 @@ export function CaptureAnnotationWorkbench({
|
||||
</dialog>
|
||||
{!timelineReady && (
|
||||
<div
|
||||
role={frameTimelineStatus === "error" ? "alert" : "status"}
|
||||
role={frameTimelineStatus === "error" && frameTimelineFailure?.status !== "pending" ? "alert" : "status"}
|
||||
data-testid="frame-timeline-status"
|
||||
className="flex flex-wrap items-center gap-3 border-b border-blue-200 bg-blue-50 px-5 py-3 text-sm text-blue-800 dark:border-blue-900 dark:bg-blue-950/30 dark:text-blue-200"
|
||||
>
|
||||
<span>
|
||||
{frameTimelineStatus === "error"
|
||||
? "视频时间范围加载失败,已有内容已保留。"
|
||||
: "正在加载统一帧范围…就绪前不能切分、逐帧、编辑或确认,可先播放预览。"}
|
||||
? frameTimelineFailure?.message || "视频时间范围加载失败"
|
||||
: "正在加载视频时间范围…"}
|
||||
</span>
|
||||
{frameTimelineStatus === "error" && frameTimelineFailure?.recovery && (
|
||||
<span>{frameTimelineFailure.recovery}</span>
|
||||
)}
|
||||
{frameTimelineStatus === "error" && onRetryFrameTimeline && (
|
||||
<button type="button" onClick={onRetryFrameTimeline} className="rounded border border-blue-400 px-3 py-1">
|
||||
重试加载统一帧范围
|
||||
<button
|
||||
type="button"
|
||||
onClick={onRetryFrameTimeline}
|
||||
disabled={sourceValidationRetrying}
|
||||
className="rounded border border-blue-400 px-3 py-1 disabled:opacity-40"
|
||||
>
|
||||
{frameTimelineFailure?.sourceValidation ? "刷新状态" : "重试加载"}
|
||||
</button>
|
||||
)}
|
||||
{frameTimelineStatus === "error" && frameTimelineFailure?.retryAllowed && onRetrySourceValidation && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onRetrySourceValidation}
|
||||
disabled={sourceValidationRetrying}
|
||||
className="rounded border border-blue-400 px-3 py-1 disabled:opacity-40"
|
||||
>
|
||||
{sourceValidationRetrying ? "正在提交…" : "重新校验"}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@ -21,7 +21,8 @@ export function ReviewHistorySummary({
|
||||
// Informational, not a warning: a newer submitted version is the normal
|
||||
// rework path, so it stays in the card's neutral tone.
|
||||
<span className="mt-0.5 block text-slate-500 dark:text-slate-400">
|
||||
以上为旧版本{history.revision ? ` ${history.revision} ` : ""}的审核结果;新版本 {currentRevision} 已提交,等待复查。
|
||||
以上为旧版本{history.revision ? ` ${history.revision} ` : ""}的审核结果;新版本 {currentRevision}{" "}
|
||||
已提交,等待复查。
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
@ -177,11 +177,13 @@ function DailyDetails({ query }: { query: WorkforceReportQuery }) {
|
||||
<table className="w-full min-w-[760px] text-left text-xs">
|
||||
<thead className="border-b border-[color:var(--app-border)] text-gray-500">
|
||||
<tr>
|
||||
{["日期", "产出", "质检通过率", "终审通过率", "返工", "视频时长", "审查产出", "审查视频时长"].map((label) => (
|
||||
<th key={label} className="px-3 py-2 font-normal">
|
||||
{label}
|
||||
</th>
|
||||
))}
|
||||
{["日期", "产出", "质检通过率", "终审通过率", "返工", "视频时长", "审查产出", "审查视频时长"].map(
|
||||
(label) => (
|
||||
<th key={label} className="px-3 py-2 font-normal">
|
||||
{label}
|
||||
</th>
|
||||
),
|
||||
)}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@ -320,7 +322,9 @@ export function WorkforceStatisticsPanel({
|
||||
{cards.map(([label, value]) => (
|
||||
<div
|
||||
key={label}
|
||||
aria-label={label === "视频时长" ? "首交视频时长" : label === "审查视频时长" ? "审查视频时长" : undefined}
|
||||
aria-label={
|
||||
label === "视频时长" ? "首交视频时长" : label === "审查视频时长" ? "审查视频时长" : undefined
|
||||
}
|
||||
className="rounded-xl border border-[color:var(--app-border)] bg-[color:var(--app-panel)] p-4"
|
||||
>
|
||||
<div className="text-xs text-gray-500">{label}</div>
|
||||
@ -372,7 +376,18 @@ export function WorkforceStatisticsPanel({
|
||||
<table className="w-full min-w-[1080px] text-left text-sm">
|
||||
<thead className="border-b border-[color:var(--app-border)] text-xs text-gray-500">
|
||||
<tr>
|
||||
{["成员", "供应商", "角色", "产出", "质量通过率", "返工", "视频时长", "审查产出", "审查视频时长", "查看详情"].map((label) => (
|
||||
{[
|
||||
"成员",
|
||||
"供应商",
|
||||
"角色",
|
||||
"产出",
|
||||
"质量通过率",
|
||||
"返工",
|
||||
"视频时长",
|
||||
"审查产出",
|
||||
"审查视频时长",
|
||||
"查看详情",
|
||||
].map((label) => (
|
||||
<th key={label} className="whitespace-nowrap px-3 py-3 font-normal">
|
||||
{label}
|
||||
</th>
|
||||
|
||||
@ -28,7 +28,13 @@ const defaults = (timeZone: string): OutputFilters => ({
|
||||
timeZone,
|
||||
});
|
||||
export type OutputPageName =
|
||||
"tasks" | "suppliers" | "days" | "qualitySuppliers" | "qualityDays" | "acceptanceSuppliers" | "acceptanceDays";
|
||||
| "tasks"
|
||||
| "suppliers"
|
||||
| "days"
|
||||
| "qualitySuppliers"
|
||||
| "qualityDays"
|
||||
| "acceptanceSuppliers"
|
||||
| "acceptanceDays";
|
||||
const firstPages = {
|
||||
tasksPage: 1,
|
||||
suppliersPage: 1,
|
||||
|
||||
@ -119,7 +119,14 @@ export function useReviewContinuation(options: Options) {
|
||||
try {
|
||||
const result = await reviewContinuationApi.act(
|
||||
o.projectId,
|
||||
{ action, stage: o.stage, pageId: pageId.current, generation: snapshot.current?.generation || 0, ...(action === "renew" ? { compact: true } : {}), ...extra },
|
||||
{
|
||||
action,
|
||||
stage: o.stage,
|
||||
pageId: pageId.current,
|
||||
generation: snapshot.current?.generation || 0,
|
||||
...(action === "renew" ? { compact: true } : {}),
|
||||
...extra,
|
||||
},
|
||||
headers,
|
||||
);
|
||||
receive(result, token, expected, o, open);
|
||||
|
||||
@ -1,11 +1,13 @@
|
||||
// Keep GET joins below proxy request-line limits, with bounded concurrency.
|
||||
export async function loadAnnotationTaskJoin(annotation, filters, ids) {
|
||||
export async function loadAnnotationTaskJoin(annotation, filters, ids, signal = undefined) {
|
||||
signal?.throwIfAborted();
|
||||
const unique = [...new Set(ids)];
|
||||
const tasks = [];
|
||||
for (let start = 0; start < unique.length; start += 200) {
|
||||
signal?.throwIfAborted();
|
||||
const pages = [];
|
||||
for (let offset = start; offset < Math.min(unique.length, start + 200); offset += 50)
|
||||
pages.push(annotation.listTasksPage({ ...filters, ids: unique.slice(offset, offset + 50) }));
|
||||
pages.push(annotation.listTasksPage({ ...filters, ids: unique.slice(offset, offset + 50) }, { signal }));
|
||||
for (const page of await Promise.all(pages)) tasks.push(...page.items);
|
||||
}
|
||||
return tasks;
|
||||
@ -95,6 +97,54 @@ export async function loadOwnedAnnotationProject({
|
||||
// a failed join or an obsolete request cannot advance the caller's pagination.
|
||||
// `view` selects an existing HTTP contract; it never derives claim eligibility.
|
||||
export async function loadAnnotationPoolPage({ annotation, human, scope, view, cursor = "", signal = undefined }) {
|
||||
if (view.kind === "review") {
|
||||
const names = cursor
|
||||
? [view.pool]
|
||||
: view.isManager
|
||||
? ["available", "mine", "all"]
|
||||
: ["available", "mine", "reviewed"];
|
||||
const pages = await Promise.all(
|
||||
names.map(async (pool) => {
|
||||
const page = await human.listItemsPage(
|
||||
{
|
||||
pluginId: "annotation-qa",
|
||||
projectId: scope.projectId,
|
||||
workflowId: scope.workflowId,
|
||||
runId: scope.runId,
|
||||
pool,
|
||||
search: view.search || undefined,
|
||||
capture: view.capture || undefined,
|
||||
category: view.category,
|
||||
stageIds: view.acceptance
|
||||
? ["acceptance", "prepackage_acceptance"]
|
||||
: pool === "reviewed"
|
||||
? undefined
|
||||
: view.stageIds,
|
||||
...(view.trace ? { trace: 1 } : {}),
|
||||
limit: view.limit || 200,
|
||||
...(cursor ? { cursor } : {}),
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
if ((view.search || view.capture || (view.category && view.category !== "all")) && !page.filtersApplied)
|
||||
throw new Error("服务版本暂不支持此筛选,请稍后重试。");
|
||||
return [pool, page];
|
||||
}),
|
||||
);
|
||||
const pools = { available: [], mine: [], all: [], reviewed: [] },
|
||||
poolCursors = {};
|
||||
for (const [pool, page] of pages) {
|
||||
pools[pool] = page.items;
|
||||
poolCursors[pool] = page.hasMore === false ? "" : page.nextCursor || "";
|
||||
}
|
||||
const ids = Object.values(pools)
|
||||
.flat()
|
||||
.map((item) => item.legacyTaskId)
|
||||
.filter(Boolean);
|
||||
const tasks = await loadAnnotationTaskJoin(annotation, scope, ids, signal);
|
||||
return { pools, poolCursors, tasks, taskCursor: "", mineCursor: "", counts: null };
|
||||
}
|
||||
|
||||
if (view.kind === "admin") {
|
||||
const page = await annotation.adminPool({ ...view.filters, ...(cursor ? { cursor } : {}) }, { signal });
|
||||
return {
|
||||
|
||||
8
src/lib/annotation-source-validation.d.ts
vendored
Normal file
8
src/lib/annotation-source-validation.d.ts
vendored
Normal file
@ -0,0 +1,8 @@
|
||||
export interface AnnotationTimelineFailure {
|
||||
sourceValidation: boolean;
|
||||
status: "failed" | "pending" | "unverified";
|
||||
message: string;
|
||||
recovery: string;
|
||||
retryAllowed: boolean;
|
||||
}
|
||||
export function annotationTimelineFailure(error: unknown): AnnotationTimelineFailure;
|
||||
27
src/lib/annotation-source-validation.js
Normal file
27
src/lib/annotation-source-validation.js
Normal file
@ -0,0 +1,27 @@
|
||||
import { userErrorMessage } from "./user-error-message.js";
|
||||
|
||||
export function annotationTimelineFailure(error) {
|
||||
const validation = error?.details?.sourceValidation;
|
||||
const known = /^source_validation_(failed|pending|unverified)$/.test(error?.code || "");
|
||||
if (known && validation && ["failed", "pending", "unverified"].includes(validation.status)) {
|
||||
return {
|
||||
sourceValidation: true,
|
||||
status: validation.status,
|
||||
message: userErrorMessage(validation.message, "源数据暂时无法校验"),
|
||||
recovery:
|
||||
validation.status === "failed"
|
||||
? "数据提供方修复源数据后,由管理员重新校验。"
|
||||
: validation.status === "unverified"
|
||||
? "需由管理员发起源数据校验。"
|
||||
: "",
|
||||
retryAllowed: validation.retryAllowed === true && validation.status !== "pending",
|
||||
};
|
||||
}
|
||||
return {
|
||||
sourceValidation: false,
|
||||
status: "failed",
|
||||
message: userErrorMessage(error, "视频时间范围加载失败"),
|
||||
recovery: "",
|
||||
retryAllowed: false,
|
||||
};
|
||||
}
|
||||
@ -1,3 +1,5 @@
|
||||
import { annotationTimelineFailure } from "../lib/annotation-source-validation.js";
|
||||
import type { AnnotationTimelineFailure } from "../lib/annotation-source-validation";
|
||||
import "../styles/review-workspace.css";
|
||||
import { useReviewTrace, ReviewTracePanel, ReviewListTrace } from "../components/ReviewTrace";
|
||||
import {
|
||||
@ -777,6 +779,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
const [adminSearch, setAdminSearch] = useState("");
|
||||
const adminPoolAbortRef = useRef<AbortController | null>(null);
|
||||
const taskPoolCursorRef = useRef("");
|
||||
const reviewPoolCursorsRef = useRef<Partial<Record<TaskPoolView, string>>>({});
|
||||
const taskPoolFetchOwnerRef = useRef<symbol | null>(null);
|
||||
const [taskPoolFetchBusy, setTaskPoolFetchBusy] = useState(false);
|
||||
useEffect(() => () => adminPoolAbortRef.current?.abort(), []);
|
||||
@ -883,7 +886,8 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
annotationPlayheadRef.current = time;
|
||||
}, []);
|
||||
const [frameTimeline, setFrameTimeline] = useState<AnnotationFrameTimeline | null>(null);
|
||||
const [frameTimelineNotice, setFrameTimelineNotice] = useState("");
|
||||
const [frameTimelineFailure, setFrameTimelineFailure] = useState<AnnotationTimelineFailure | null>(null);
|
||||
const [sourceValidationRetryTask, setSourceValidationRetryTask] = useState("");
|
||||
const [timelineState, setTimelineState] = useState<{ taskId: string; status: "loading" | "ready" | "error" }>({
|
||||
taskId: "",
|
||||
status: "loading",
|
||||
@ -988,18 +992,18 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
!isManager && (workspaceMode === "quality" || workspaceMode === "clip-quality" || workspaceMode === "acceptance");
|
||||
const acceptanceWorkspace = workspaceMode === "acceptance";
|
||||
const deferReviewReports = reviewOnly || acceptanceWorkspace;
|
||||
// Acceptance pools are loaded together. Switching their local view must not
|
||||
// reload the same project, evidence prerequisites and package history.
|
||||
const projectPoolView = acceptanceWorkspace ? "all" : taskPoolView;
|
||||
// Review pools retain independent pages; changing tabs reuses their rows.
|
||||
const projectPoolView = deferReviewReports ? "all" : taskPoolView;
|
||||
const managerAnnotationPool = isManager && workspaceMode === "annotate";
|
||||
const pagedReviewPool = reviewOnly || acceptanceWorkspace;
|
||||
useEffect(() => {
|
||||
if (!managerAnnotationPool) {
|
||||
if (!managerAnnotationPool && !pagedReviewPool) {
|
||||
adminPoolAbortRef.current?.abort();
|
||||
return;
|
||||
}
|
||||
const timer = window.setTimeout(() => setAdminSearch(taskSearch.trim()), 350);
|
||||
return () => window.clearTimeout(timer);
|
||||
}, [managerAnnotationPool, taskSearch]);
|
||||
}, [managerAnnotationPool, pagedReviewPool, taskSearch]);
|
||||
const managerPoolFilters = useMemo(
|
||||
() =>
|
||||
managerAnnotationPool
|
||||
@ -1043,6 +1047,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
requestedWorkflowId,
|
||||
annotatorTaskView,
|
||||
...(managerAnnotationPool ? [managerPoolFilters] : []),
|
||||
...(pagedReviewPool ? [adminSearch, taskStatusFilter, requestedCaptureDirectory] : []),
|
||||
]);
|
||||
const reviewActionScopeRef = useRef(projectDataScope);
|
||||
if (reviewActionScopeRef.current !== projectDataScope) skipClaimPoolReload.current = "";
|
||||
@ -1214,12 +1219,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
setTaskPoolFetchBusy(false);
|
||||
taskPoolCursorRef.current = "";
|
||||
minePoolCursorRef.current = "";
|
||||
reviewPoolCursorsRef.current = {};
|
||||
adminPoolAbortRef.current?.abort();
|
||||
adminPoolAbortRef.current = null;
|
||||
const foreground = !deferReviewReports || loadedProjectScope.current !== projectDataScope;
|
||||
setLoading(foreground);
|
||||
setRefreshingProject(!foreground);
|
||||
setProjectLoadError("");
|
||||
if (managerAnnotationPool) {
|
||||
if (managerAnnotationPool || pagedReviewPool) {
|
||||
adminPoolAbortRef.current = new AbortController();
|
||||
setHumanPools(emptyHumanPools);
|
||||
setTasks([]);
|
||||
@ -1284,7 +1291,10 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
view: managerAnnotationPool
|
||||
? { kind: "admin", filters: managerPoolFilters }
|
||||
: {
|
||||
kind: "workspace",
|
||||
kind: pagedReviewPool ? "review" : "workspace",
|
||||
search: adminSearch,
|
||||
capture: requestedCaptureDirectory,
|
||||
category: taskStatusFilter,
|
||||
acceptance: acceptanceWorkspace,
|
||||
userId: currentUser?.id || "",
|
||||
isManager,
|
||||
@ -1328,6 +1338,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
const { tasks: nextTasks, pools: nextHumanPools } = poolPage;
|
||||
taskPoolCursorRef.current = poolPage.taskCursor;
|
||||
minePoolCursorRef.current = poolPage.mineCursor;
|
||||
if (poolPage.poolCursors) reviewPoolCursorsRef.current = poolPage.poolCursors;
|
||||
if (poolPage.counts) setAdminPoolTotals({ scope: projectDataScope, counts: poolPage.counts });
|
||||
setProjectMembers(nextProjectMembers);
|
||||
setWorkflows(nextWorkflows);
|
||||
@ -1408,6 +1419,10 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
annotatorTaskView,
|
||||
managerAnnotationPool,
|
||||
managerPoolFilters,
|
||||
pagedReviewPool,
|
||||
adminSearch,
|
||||
requestedCaptureDirectory,
|
||||
taskStatusFilter,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
@ -1607,7 +1622,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
parentSignal?.addEventListener("abort", abort, { once: true });
|
||||
if (parentSignal?.aborted) abort();
|
||||
setTimelineState({ taskId: task.id, status: "loading" });
|
||||
setFrameTimelineNotice("");
|
||||
setFrameTimelineFailure(null);
|
||||
try {
|
||||
const timeline = await annotationOperationsApi.getFrameTimeline(task.id, "left", controller.signal);
|
||||
if (
|
||||
@ -1662,7 +1677,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
return;
|
||||
setFrameTimeline(null);
|
||||
setTimelineState({ taskId: task.id, status: "error" });
|
||||
setFrameTimelineNotice("视频时间范围加载失败,切片暂时不能编辑。");
|
||||
setFrameTimelineFailure(annotationTimelineFailure(cause));
|
||||
} finally {
|
||||
parentSignal?.removeEventListener("abort", abort);
|
||||
}
|
||||
@ -1671,6 +1686,24 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
);
|
||||
useEffect(() => () => timelineAbortRef.current?.abort(), []);
|
||||
|
||||
const retrySourceValidation = async () => {
|
||||
const task = detail?.task;
|
||||
if (!task || !frameTimelineFailure?.retryAllowed || sourceValidationRetryTask) return;
|
||||
if (!window.confirm("确认源数据已修复,并重新校验?现有标注保持不变。")) return;
|
||||
setSourceValidationRetryTask(task.id);
|
||||
try {
|
||||
await annotationOperationsApi.retrySourceValidation(task.id);
|
||||
if (selectedTaskDetailIDRef.current === task.id) await loadFrameTimeline(task);
|
||||
} catch (cause) {
|
||||
if (selectedTaskDetailIDRef.current === task.id) {
|
||||
setTimelineState({ taskId: task.id, status: "error" });
|
||||
setFrameTimelineFailure(annotationTimelineFailure(cause));
|
||||
}
|
||||
} finally {
|
||||
setSourceValidationRetryTask((current) => (current === task.id ? "" : current));
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const requestID = ++taskDetailRequestRef.current;
|
||||
if (selectedTaskDetailIDRef.current !== selectedTaskId) {
|
||||
@ -1685,7 +1718,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
setSliceStatuses({});
|
||||
setAnnotationPlayhead(0);
|
||||
setFrameTimeline(null);
|
||||
setFrameTimelineNotice("");
|
||||
setFrameTimelineFailure(null);
|
||||
setCombinedEditorPhase("slice");
|
||||
annotationBaselineRef.current = "";
|
||||
confirmedAnnotationWorkspaceRef.current = null;
|
||||
@ -1702,7 +1735,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
setSliceStatuses({});
|
||||
setAnnotationPlayhead(0);
|
||||
setFrameTimeline(null);
|
||||
setFrameTimelineNotice("");
|
||||
setFrameTimelineFailure(null);
|
||||
setCombinedEditorPhase("slice");
|
||||
}
|
||||
annotationOperationsApi
|
||||
@ -2009,7 +2042,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
// List stage — panel search plus status category; scopes only the task list.
|
||||
const matchesPoolListFilters = useCallback(
|
||||
(item: HumanItem) => {
|
||||
if (managerAnnotationPool) return true;
|
||||
if (managerAnnotationPool || pagedReviewPool) return true;
|
||||
const task = tasks.find((t) => t.id === item.legacyTaskId);
|
||||
const directories = captureDirectoriesFromHumanTask(item, task);
|
||||
if (
|
||||
@ -2026,7 +2059,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
: "pending_acceptance";
|
||||
return taskStatusFilter === "all" || status === taskStatusFilter;
|
||||
},
|
||||
[tasks, taskSearch, taskStatusFilter, managerAnnotationPool],
|
||||
[tasks, taskSearch, taskStatusFilter, managerAnnotationPool, pagedReviewPool],
|
||||
);
|
||||
const matchesPoolFilters = useCallback(
|
||||
(item: HumanItem, pool: TaskPoolView) => matchesPoolScope(item, pool) && matchesPoolListFilters(item),
|
||||
@ -2562,6 +2595,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
task.id === selectedTaskId && detail?.humanItem
|
||||
? {
|
||||
...detail.humanItem,
|
||||
sourceValidationStatus: poolItem?.sourceValidationStatus,
|
||||
reviewHistoryStatus: poolItem?.reviewHistoryStatus,
|
||||
reviewHistory: poolItem?.reviewHistory,
|
||||
}
|
||||
@ -2583,14 +2617,14 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
);
|
||||
const filteredVisibleTasks = useMemo(
|
||||
() =>
|
||||
managerAnnotationPool
|
||||
managerAnnotationPool || pagedReviewPool
|
||||
? visibleTasks
|
||||
: filterAnnotationTasks(visibleTasks, taskContextById, taskSearch, taskStatusFilter),
|
||||
[taskContextById, taskSearch, taskStatusFilter, visibleTasks, managerAnnotationPool],
|
||||
[taskContextById, taskSearch, taskStatusFilter, visibleTasks, managerAnnotationPool, pagedReviewPool],
|
||||
);
|
||||
// Progressive rendering cap: a 17k-task pool renders fine in the store but
|
||||
// freezes the tab when every card mounts at once. The pool reveals cards in
|
||||
// windows; counts and filters still see the whole pool.
|
||||
// freezes the tab when every card mounts at once. Render loaded pages in
|
||||
// windows; review searches run on the server and badges count loaded rows.
|
||||
const [taskRenderLimit, setTaskRenderLimit] = useState(200);
|
||||
useEffect(() => {
|
||||
setTaskRenderLimit(200);
|
||||
@ -2605,7 +2639,11 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
const taskPoolLoadMore = async () => {
|
||||
if (reviewActionScopeRef.current !== projectDataScope) return;
|
||||
const sidebar = blindAnnotator && taskPoolView === "mine";
|
||||
const cursor = sidebar ? minePoolCursorRef.current : taskPoolCursorRef.current;
|
||||
const cursor = pagedReviewPool
|
||||
? reviewPoolCursorsRef.current[taskPoolView]
|
||||
: sidebar
|
||||
? minePoolCursorRef.current
|
||||
: taskPoolCursorRef.current;
|
||||
if (taskPoolFetchOwnerRef.current || !cursor) return;
|
||||
const owner = Symbol("pool-page");
|
||||
taskPoolFetchOwnerRef.current = owner;
|
||||
@ -2617,7 +2655,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
projectLoadRequestRef.current === epoch;
|
||||
setTaskPoolFetchBusy(true);
|
||||
try {
|
||||
if (managerAnnotationPool) {
|
||||
if (managerAnnotationPool || pagedReviewPool) {
|
||||
adminPoolAbortRef.current?.abort();
|
||||
adminPoolAbortRef.current = new AbortController();
|
||||
}
|
||||
@ -2635,16 +2673,30 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
? { kind: "admin", filters: managerPoolFilters }
|
||||
: sidebar
|
||||
? { kind: "sidebar", annotationView: annotatorTaskView }
|
||||
: { kind: "workspace", limit: taskPageLimit },
|
||||
: pagedReviewPool
|
||||
? {
|
||||
kind: "review",
|
||||
pool: taskPoolView,
|
||||
search: adminSearch,
|
||||
capture: requestedCaptureDirectory,
|
||||
category: taskStatusFilter,
|
||||
acceptance: acceptanceWorkspace,
|
||||
isManager,
|
||||
stageIds: humanItemStageIdsForMode(workspaceMode),
|
||||
trace: reviewOnly && !reviewFocused,
|
||||
limit: taskPageLimit,
|
||||
}
|
||||
: { kind: "workspace", limit: taskPageLimit },
|
||||
cursor,
|
||||
signal: adminPoolAbortRef.current?.signal,
|
||||
});
|
||||
if (!current()) return;
|
||||
taskPoolCursorRef.current = page.taskCursor;
|
||||
minePoolCursorRef.current = page.mineCursor;
|
||||
if (page.poolCursors) reviewPoolCursorsRef.current = { ...reviewPoolCursorsRef.current, ...page.poolCursors };
|
||||
if (page.counts) setAdminPoolTotals({ scope, counts: page.counts });
|
||||
if (page.pools) {
|
||||
const pool = managerAnnotationPool ? managerPoolFilters.pool : "mine";
|
||||
const pool = managerAnnotationPool ? managerPoolFilters.pool : pagedReviewPool ? taskPoolView : "mine";
|
||||
const items: HumanItem[] = page.pools[pool];
|
||||
if (sidebar && annotatorTaskView === "submitted")
|
||||
setSubmittedHumanItems((current) => [
|
||||
@ -2675,8 +2727,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
[visiblePoolItems],
|
||||
);
|
||||
const poolCounts = useMemo(
|
||||
// Scope stage only: the search/status panel filters scope the task list
|
||||
// below the tabs, never the pool tab totals (ticket 02).
|
||||
// Admin tabs use scoped totals; review tabs count the loaded filtered rows.
|
||||
() =>
|
||||
managerAnnotationPool
|
||||
? {
|
||||
@ -6877,7 +6928,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
aria-pressed={taskPoolView === value}
|
||||
className={`rounded-lg border px-3 py-2 text-xs transition-colors ${taskPoolView === value ? "border-blue-500 bg-blue-50 text-blue-600 dark:bg-blue-500/10 dark:text-blue-200 font-medium" : "border-[color:var(--app-border)] text-slate-600 hover:text-slate-900 dark:text-gray-400 dark:hover:text-white"}`}
|
||||
>
|
||||
{label}{" "}
|
||||
{label} {pagedReviewPool ? "已加载 " : ""}
|
||||
{managerAnnotationPool && adminPoolTotals?.scope !== projectDataScope
|
||||
? "—"
|
||||
: blindAnnotator && value === "available"
|
||||
@ -6886,15 +6937,13 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
: "—"
|
||||
: count}
|
||||
{/* Counts only loaded pages; a further page means more exist. */}
|
||||
{blindAnnotator && value === "mine" && minePoolCursorRef.current ? "+" : ""}
|
||||
{(pagedReviewPool && reviewPoolCursorsRef.current[value]) ||
|
||||
(blindAnnotator && value === "mine" && minePoolCursorRef.current)
|
||||
? "+"
|
||||
: ""}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
{reviewOnly && (taskPoolView === "mine" || taskPoolView === "reviewed") && (
|
||||
<p className="mt-2 text-xs text-slate-500" data-testid="review-pool-explanation">
|
||||
“我的任务”是当前待处理任务;“我已审核”保留本人历史审核记录,同一采集的新版本可以同时成为待办。返工复查表示打回后重新送检;待验收表示质检通过后等待最终验收。
|
||||
</p>
|
||||
)}
|
||||
{blindAnnotator && (
|
||||
<div className="mt-3 space-y-2" data-testid="project-random-claim">
|
||||
<p className="text-xs text-slate-500">先选项目,再随机领取一条任务;领取后不能主动释放。</p>
|
||||
@ -6998,12 +7047,16 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
<select
|
||||
className={inputClass}
|
||||
value={taskStatusFilter}
|
||||
onChange={(event) => setTaskStatusFilter(event.target.value)}
|
||||
onChange={(event) => {
|
||||
setTaskStatusFilter(event.target.value);
|
||||
if (managerAnnotationPool && event.target.value === "source_failed") setTaskPoolView("all");
|
||||
}}
|
||||
aria-label="按任务状态筛选"
|
||||
>
|
||||
<option value="all">
|
||||
{blindAnnotator && annotatorTaskView === "active" ? "全部待处理状态" : "全部状态"}
|
||||
</option>
|
||||
{managerAnnotationPool && <option value="source_failed">源数据异常</option>}
|
||||
<option value="pending_slice">待切片</option>
|
||||
<option value="slicing">正在切片</option>
|
||||
<option value="pending_annotation">待标注</option>
|
||||
@ -7079,6 +7132,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
task.id === selectedTaskId && detail?.humanItem
|
||||
? {
|
||||
...detail.humanItem,
|
||||
sourceValidationStatus: poolItem?.sourceValidationStatus,
|
||||
reviewHistoryStatus: poolItem?.reviewHistoryStatus,
|
||||
reviewHistory: poolItem?.reviewHistory,
|
||||
}
|
||||
@ -7141,7 +7195,20 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
{formatAnnotationTime(task.clipStart)}–{formatAnnotationTime(task.clipEnd)}
|
||||
</div>
|
||||
{(() => {
|
||||
switch (detailedStatus) {
|
||||
switch (humanItem?.sourceValidationStatus || detailedStatus) {
|
||||
case "failed":
|
||||
return (
|
||||
<span className="shrink-0 rounded-full border border-rose-500/50 px-2 py-0.5 text-[11px] font-semibold text-rose-700 dark:text-rose-300">
|
||||
源数据异常
|
||||
</span>
|
||||
);
|
||||
case "pending":
|
||||
case "unverified":
|
||||
return (
|
||||
<span className="shrink-0 rounded-full border border-amber-500/50 px-2 py-0.5 text-[11px] font-semibold text-amber-700 dark:text-amber-300">
|
||||
{humanItem.sourceValidationStatus === "pending" ? "校验中" : "待校验"}
|
||||
</span>
|
||||
);
|
||||
case "submitting":
|
||||
return (
|
||||
<span className="shrink-0 rounded-full border border-amber-500/50 bg-amber-50 px-2 py-0.5 text-[11px] font-semibold text-amber-700 dark:bg-amber-500/20 dark:text-amber-300">
|
||||
@ -7274,6 +7341,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
})}
|
||||
{(filteredVisibleTasks.length > renderedVisibleTasks.length ||
|
||||
taskPoolCursorRef.current ||
|
||||
(pagedReviewPool && reviewPoolCursorsRef.current[taskPoolView]) ||
|
||||
(blindAnnotator && taskPoolView === "mine" && minePoolCursorRef.current)) && (
|
||||
<button
|
||||
type="button"
|
||||
@ -7285,6 +7353,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
setTaskRenderLimit((current) => current + 200);
|
||||
if (
|
||||
taskPoolCursorRef.current ||
|
||||
(pagedReviewPool && reviewPoolCursorsRef.current[taskPoolView]) ||
|
||||
(blindAnnotator && taskPoolView === "mine" && minePoolCursorRef.current)
|
||||
)
|
||||
void taskPoolLoadMore();
|
||||
@ -7292,7 +7361,7 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
>
|
||||
{taskPoolFetchBusy
|
||||
? "正在加载更多任务…"
|
||||
: `显示更多任务(已显示 ${renderedVisibleTasks.length}${taskPoolCursorRef.current || (blindAnnotator && minePoolCursorRef.current) ? "+" : ""} 条)`}
|
||||
: `显示更多任务(已显示 ${renderedVisibleTasks.length + prepackageAcceptanceItems.length}${taskPoolCursorRef.current || (pagedReviewPool && reviewPoolCursorsRef.current[taskPoolView]) || (blindAnnotator && minePoolCursorRef.current) ? "+" : ""} 条)`}
|
||||
</button>
|
||||
)}
|
||||
{!sidebarLoading &&
|
||||
@ -7765,6 +7834,9 @@ export default function AnnotationOperations({ workPage = false }: { workPage?:
|
||||
progressSaved={!annotationDirty}
|
||||
onSaveProgress={(nextSegments) => confirmAnnotationDescription(nextSegments, "")}
|
||||
frameTimelineStatus={frameTimelineStatus}
|
||||
frameTimelineFailure={frameTimelineFailure}
|
||||
sourceValidationRetrying={sourceValidationRetryTask === detail.task.id}
|
||||
onRetrySourceValidation={() => void retrySourceValidation()}
|
||||
onRetryFrameTimeline={() =>
|
||||
void loadFrameTimeline(
|
||||
detail.task,
|
||||
|
||||
@ -222,9 +222,11 @@ export const annotationOperationsApi = {
|
||||
/** Explicit task-ID set for pool-item joins (comma-joined). */
|
||||
ids?: string[];
|
||||
} = {},
|
||||
options: { signal?: AbortSignal } = {},
|
||||
) =>
|
||||
apiRequest<{ items: AnnotationTaskRecord[]; nextCursor?: string; hasMore?: boolean }>(
|
||||
`/api/plugins/annotation-qa/tasks${query(filters)}`,
|
||||
options,
|
||||
),
|
||||
getTask: (id: string, signal?: AbortSignal, history = false) =>
|
||||
apiRequest<AnnotationTaskDetail>(`/api/plugins/annotation-qa/tasks/${id}${history ? "?view=history" : ""}`, {
|
||||
@ -246,6 +248,12 @@ export const annotationOperationsApi = {
|
||||
needsRefresh?: boolean;
|
||||
progress?: { stage: string; completedSlices: number; totalSlices: number; startedAt: string };
|
||||
}>(`/api/plugins/annotation-qa/tasks/${id}?view=status`, { signal }),
|
||||
retrySourceValidation: (id: string) =>
|
||||
apiRequest<{ sourceValidation: { status: string; message: string; retryAllowed: boolean } }>(
|
||||
`/api/plugins/annotation-qa/tasks/${encodeURIComponent(id)}/source-validation-retries`,
|
||||
{ method: "POST" },
|
||||
{ sourceRepaired: true },
|
||||
),
|
||||
getFrameTimeline: async (
|
||||
id: string,
|
||||
camera: "left" | "right" = "left",
|
||||
|
||||
@ -3,6 +3,9 @@ import { mergePagedManifestMembers } from "../../lib/annotation-workbench.js";
|
||||
import { apiRequest } from "./client";
|
||||
|
||||
type HumanItemFilters = {
|
||||
search?: string;
|
||||
capture?: string;
|
||||
category?: string;
|
||||
trace?: number;
|
||||
pluginId?: string;
|
||||
batchId?: string;
|
||||
@ -36,13 +39,14 @@ const queryString = (filters: Record<string, string | number | string[] | undefi
|
||||
|
||||
type HumanItemPage = {
|
||||
items: HumanItem[];
|
||||
filtersApplied?: boolean;
|
||||
/** Opaque cursor for the next page; absent when there are no more pages. */
|
||||
nextCursor?: string;
|
||||
hasMore?: boolean;
|
||||
};
|
||||
|
||||
const listItemsPage = async (filters: HumanItemFilters = {}) =>
|
||||
apiRequest<HumanItemPage>(`/api/human-items${queryString(filters)}`);
|
||||
const listItemsPage = async (filters: HumanItemFilters = {}, options: { signal?: AbortSignal } = {}) =>
|
||||
apiRequest<HumanItemPage>(`/api/human-items${queryString(filters)}`, options);
|
||||
|
||||
const listItems = async (filters: HumanItemFilters = {}) => (await listItemsPage(filters)).items;
|
||||
|
||||
|
||||
@ -2216,6 +2216,7 @@ export interface HumanTaskDisplayContext {
|
||||
}
|
||||
|
||||
export interface HumanItem {
|
||||
sourceValidationStatus?: "failed" | "pending" | "unverified";
|
||||
traceSummary?: { author: string; supplier: string; rejections: number | null };
|
||||
/** Server-computed acceptance claim capability; never used as a write credential. */
|
||||
canClaim?: boolean;
|
||||
|
||||
48
tests/backend/test_raw_human_admission.py
Normal file
48
tests/backend/test_raw_human_admission.py
Normal file
@ -0,0 +1,48 @@
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.operators.builtin.orbbec_data_normalize import operator as op
|
||||
|
||||
|
||||
def run_raw_pair(tmp_path, monkeypatch, imu):
|
||||
video = "part0001/capture_camera_left_part0001.mp4"
|
||||
contents = {
|
||||
video: "video",
|
||||
"part0001/capture_camera_left_part0001_pts.csv": "timestamp_us\n100\n200\n",
|
||||
"part0001/capture_imu_part0001.csv": imu,
|
||||
"meta_capture.json": json.dumps({"streams": {"color_left": {"segments": [{"index": 1, "frame_count": 2}]}}}),
|
||||
}
|
||||
entries = []
|
||||
for name, content in contents.items():
|
||||
file = tmp_path / name
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file.write_text(content)
|
||||
entries.append({"path": name, "source": {"path": str(file)}})
|
||||
monkeypatch.setattr(op.subprocess, "run", lambda *a, **k: SimpleNamespace(
|
||||
stdout='{"streams":[{"nb_read_frames":"2","duration":"0.2"}]}', stderr=""))
|
||||
try:
|
||||
return op._validate_fallback_pair(video, op._paired_files(video, entries), entries,
|
||||
str(tmp_path), tmp_path, {"PROCESSING_PROFILE": "raw_human"}, {})
|
||||
finally:
|
||||
assert all((tmp_path / name).read_text() == content for name, content in contents.items())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rows,error", [
|
||||
("90,0,0,1,accel\n90,0,0,1,gyro\n190,0,0,1,accel\n190,0,0,1,gyr\n", "第 5 行.*类型"),
|
||||
("90,0,0,1,accel\n90,0,0,1,gyro\n190,nan,0,1,accel\n190,0,0,1,gyro\n", "第 4 行"),
|
||||
("90,0,0,1,accel\n90,0,0,1,gyro\n190,0,0,1,accel\n", "配对"),
|
||||
("90,0,0,1,accel\n90,0,0,1,accel\n", "重复"),
|
||||
("190,0,0,1,accel\n190,0,0,1,gyro\n90,0,0,1,accel\n90,0,0,1,gyro\n", "递增"),
|
||||
("", "有效数据"),
|
||||
])
|
||||
def test_raw_admission_rejects_invalid_imu_without_changing_sources(tmp_path, monkeypatch, rows, error):
|
||||
with pytest.raises(RuntimeError, match=error):
|
||||
run_raw_pair(tmp_path, monkeypatch, "timestamp_us,x,y,z,type\n" + rows)
|
||||
|
||||
|
||||
def test_raw_admission_keeps_legal_part_boundary_gap(tmp_path, monkeypatch):
|
||||
result = run_raw_pair(tmp_path, monkeypatch,
|
||||
"timestamp_us,x,y,z,type\n110,0,0,1,accel\n110,0,0,1,gyro\n190,0,0,1,accel\n190,0,0,1,gyro\n")
|
||||
assert result["ptsRows"] == 2
|
||||
@ -23,7 +23,7 @@ def test_raw_profile_validates_original_files_without_modifying_them(tmp_path, m
|
||||
|
||||
|
||||
@pytest.mark.parametrize("side", ["left", "right"])
|
||||
def test_raw_profile_bypasses_imu_gate_for_real_part_boundary(tmp_path, monkeypatch, side):
|
||||
def test_raw_profile_checks_imu_content_without_requiring_part_boundary_coverage(tmp_path, monkeypatch, side):
|
||||
name = f"capture_camera_{side}_part0006_silent_480p_wm.mp4"
|
||||
pts_name = f"capture_camera_{side}_part0006_pts.csv"
|
||||
imu_name = "capture_imu_part0006.csv"
|
||||
@ -43,16 +43,16 @@ def test_raw_profile_bypasses_imu_gate_for_real_part_boundary(tmp_path, monkeypa
|
||||
monkeypatch.setattr(op.subprocess, "run", lambda *a, **k: SimpleNamespace(
|
||||
stdout='{"streams":[{"nb_read_frames":"2","duration":"900"}]}', stderr=""))
|
||||
|
||||
def unexpected_imu_validation(*args):
|
||||
pytest.fail("raw-human IMU gate must remain bypassed")
|
||||
|
||||
monkeypatch.setattr(op, "_validate_raw_imu", unexpected_imu_validation)
|
||||
pairs = op._paired_files(name, entries)
|
||||
validate = lambda: op._validate_fallback_pair(
|
||||
name, pairs, entries, str(tmp_path), tmp_path, {"PROCESSING_PROFILE": "raw_human"}, {})
|
||||
assert validate()["endTimestampUs"] == end
|
||||
assert all((tmp_path / filename).read_text() == text for filename, text in contents.items())
|
||||
# Bypassing the IMU gate must not bypass video/PTS alignment or required inputs.
|
||||
# Part boundary tolerance does not bypass content or video/PTS validation.
|
||||
(tmp_path / imu_name).write_text(contents[imu_name].replace(",gyro\n", ",gyr\n"))
|
||||
with pytest.raises(RuntimeError, match="类型无效"):
|
||||
validate()
|
||||
(tmp_path / imu_name).write_text(contents[imu_name])
|
||||
(tmp_path / pts_name).write_text(f"timestamp_us\n4747034637\n5000000000\n{end}\n")
|
||||
with pytest.raises(RuntimeError, match="视频与 PTS 数量不一致"):
|
||||
validate()
|
||||
@ -70,7 +70,7 @@ def _meta_fixture(tmp_path, segments, side="left"):
|
||||
contents = {
|
||||
name: "video",
|
||||
f"part0001/capture_camera_{side}_part0001_pts.csv": "timestamp_us\n100\n200\n",
|
||||
"part0001/capture_imu_part0001.csv": "original imu",
|
||||
"part0001/capture_imu_part0001.csv": "timestamp_us,x,y,z,type\n90,0,0,1,accel\n90,0,0,1,gyro\n190,0,0,1,accel\n190,0,0,1,gyro\n",
|
||||
"meta_capture.json": json.dumps({"streams": {f"color_{side}": {"segments": segments}}}),
|
||||
}
|
||||
entries = []
|
||||
|
||||
106
tests/browser/annotation-source-validation-replica.mjs
Normal file
106
tests/browser/annotation-source-validation-replica.mjs
Normal file
@ -0,0 +1,106 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import { createServer } from "vite";
|
||||
import { chromium } from "playwright";
|
||||
const api = process.env.SOURCE_VALIDATION_API;
|
||||
assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/);
|
||||
const output = "local-data/test-results/source-validation-browser";
|
||||
fs.mkdirSync(output, { recursive: true });
|
||||
const vite = await createServer({
|
||||
server: {
|
||||
host: "127.0.0.1",
|
||||
port: 5208,
|
||||
strictPort: true,
|
||||
hmr: false,
|
||||
watch: { ignored: ["**/*"] },
|
||||
proxy: { "/api": { target: api }, "/__fixture": { target: api } },
|
||||
},
|
||||
});
|
||||
let browser, page;
|
||||
const report = { pageErrors: [], writes: [], cases: [] };
|
||||
try {
|
||||
await vite.listen();
|
||||
browser = await chromium.launch({
|
||||
executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
||||
headless: true,
|
||||
args: ["--no-proxy-server"],
|
||||
});
|
||||
const context = await browser.newContext({ viewport: { width: 1600, height: 1000 } });
|
||||
page = await context.newPage();
|
||||
page.setDefaultTimeout(20000);
|
||||
page.on("pageerror", (error) => report.pageErrors.push(error.message));
|
||||
await page.route("**/*", (route) => {
|
||||
const request = route.request(),
|
||||
url = new URL(request.url());
|
||||
if (url.hostname !== "127.0.0.1") return route.abort();
|
||||
if (url.pathname.includes("/worker-api/")) return route.fulfill({ json: { status: "ok" } });
|
||||
if (!["GET", "HEAD"].includes(request.method())) report.writes.push(url.pathname);
|
||||
return route.continue();
|
||||
});
|
||||
const base = "http://127.0.0.1:5208";
|
||||
const path = "/plugins/annotation-qa/workflows/all/annotate?projectId=draft-project&taskId=draft-task";
|
||||
await page.goto(base + path);
|
||||
const notice = page.getByTestId("frame-timeline-status");
|
||||
await notice.getByText(/418451.*gyr/).waitFor();
|
||||
assert.match(await notice.innerText(), /数据提供方.*管理员重新校验/);
|
||||
assert.equal(await notice.getAttribute("role"), "alert");
|
||||
assert.equal(await notice.getByRole("button", { name: "重新校验", exact: true }).count(), 0);
|
||||
await notice.getByRole("button", { name: "刷新状态", exact: true }).click();
|
||||
await notice.getByText(/418451.*gyr/).waitFor();
|
||||
assert.equal(report.writes.filter((path) => path.endsWith("source-validation-retries")).length, 0);
|
||||
report.cases.push("annotator sees exact source fault and repair responsibility; refresh does not enqueue revalidation");
|
||||
await page.screenshot({ path: output + "/annotator-source-failed.png", fullPage: true });
|
||||
await context.addCookies([{ name: "source-fixture-role", value: "admin", url: base }]);
|
||||
await page.goto(base + path + "&pool=all");
|
||||
await page.getByTestId("human-task-pool-all").click();
|
||||
const retry = notice.getByRole("button", { name: "重新校验", exact: true });
|
||||
await retry.waitFor();
|
||||
page.once("dialog", (dialog) => dialog.dismiss());
|
||||
await retry.click();
|
||||
assert.equal(report.writes.filter((path) => path.endsWith("source-validation-retries")).length, 0);
|
||||
page.once("dialog", (dialog) => dialog.accept());
|
||||
await retry.click();
|
||||
await notice.getByText("源数据校验中", { exact: true }).waitFor();
|
||||
assert.equal(await notice.getAttribute("role"), "status");
|
||||
assert.equal(await notice.getByRole("button", { name: "重新校验", exact: true }).count(), 0);
|
||||
const start = await context.request.post(base + "/__fixture/validate");
|
||||
assert.equal(start.status(), 202);
|
||||
let validation;
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
const response = await context.request.get(base + "/api/plugins/annotation-qa/tasks/draft-task?view=status");
|
||||
assert.equal(response.status(), 200);
|
||||
validation = (await response.json()).sourceValidation;
|
||||
if (validation?.status === "valid") break;
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
assert.equal(validation?.status, "valid", "actual source validation completed");
|
||||
const readyFrame = await context.request.get(base + "/api/plugins/annotation-qa/tasks/draft-task/frame-timeline?camera=left&encoding=compact");
|
||||
assert.equal(readyFrame.status(), 200, "validated source exposes its frame timeline");
|
||||
await notice.getByRole("button", { name: "刷新状态", exact: true }).click();
|
||||
await notice.waitFor({ state: "detached" });
|
||||
await page.reload();
|
||||
await page.getByTestId("human-task-pool-all").click();
|
||||
await page.getByTestId("capture-annotation-workbench").waitFor();
|
||||
await page.waitForFunction(() => !document.querySelector('[data-testid="frame-timeline-status"]'));
|
||||
assert.equal(report.writes.filter((path) => path.endsWith("source-validation-retries")).length, 1);
|
||||
assert.equal(
|
||||
report.writes.some((path) => /drafts|combined-submissions|slice-confirmations/.test(path)),
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(report.pageErrors, []);
|
||||
report.cases.push(
|
||||
"admin confirmation, queued status, actual source validation, refresh and reload restore workspace without annotation writes",
|
||||
);
|
||||
await page.screenshot({ path: output + "/source-recovered.png", fullPage: true });
|
||||
console.log(JSON.stringify(report));
|
||||
} catch (error) {
|
||||
if (page) {
|
||||
await page.screenshot({ path: output + "/failure.png", fullPage: true }).catch(() => {});
|
||||
console.error((await page.locator("body").innerText()).slice(-2800));
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
fs.writeFileSync(output + "/report.json", JSON.stringify(report, null, 2));
|
||||
await browser?.close();
|
||||
await vite.close();
|
||||
}
|
||||
114
tests/browser/review-pool-search-replica.mjs
Normal file
114
tests/browser/review-pool-search-replica.mjs
Normal file
@ -0,0 +1,114 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { chromium } from "playwright";
|
||||
import { createServer } from "vite";
|
||||
const api = process.env.REVIEW_POOL_API;
|
||||
assert.match(api, /^http:\/\/127\.0\.0\.1:\d+$/);
|
||||
const out = path.resolve("local-data/test-results/review-pool-search");
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
const report = { cases: [], requests: [], pageErrors: [], writes: [] };
|
||||
const server = await createServer({
|
||||
server: { host: "127.0.0.1", port: 5207, strictPort: true, proxy: { "/api": { target: api, changeOrigin: true } } },
|
||||
});
|
||||
let browser,
|
||||
page,
|
||||
failNext = false,
|
||||
delaySearch = "",
|
||||
releaseSlow;
|
||||
try {
|
||||
await server.listen();
|
||||
browser = await chromium.launch({
|
||||
executablePath: process.env.BROWSER_EXECUTABLE || "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
||||
headless: true,
|
||||
args: ["--no-proxy-server"],
|
||||
});
|
||||
page = await browser.newPage({ viewport: { width: 1600, height: 1100 } });
|
||||
page.setDefaultTimeout(20000);
|
||||
page.on("pageerror", (e) => report.pageErrors.push(e.message));
|
||||
await page.route("**/*", async (route) => {
|
||||
const r = route.request(),
|
||||
u = new URL(r.url());
|
||||
if (u.hostname !== "127.0.0.1") return route.abort();
|
||||
if (u.pathname.includes("/worker-api/")) return route.fulfill({ json: { status: "ok" } });
|
||||
if (!["GET", "HEAD"].includes(r.method())) report.writes.push(u.pathname);
|
||||
if (u.pathname === "/api/human-items") {
|
||||
report.requests.push(Object.fromEntries(u.searchParams));
|
||||
if (u.searchParams.get("cursor") && failNext) {
|
||||
failNext = false;
|
||||
return route.fulfill({ status: 500, json: { error: "合成分页失败" } });
|
||||
}
|
||||
if (delaySearch && u.searchParams.get("pool") === "available" && u.searchParams.get("search") === delaySearch) {
|
||||
delaySearch = "";
|
||||
const response = await route.fetch();
|
||||
await new Promise((r) => (releaseSlow = r));
|
||||
return route.fulfill({ response });
|
||||
}
|
||||
}
|
||||
return route.continue();
|
||||
});
|
||||
await page.goto(
|
||||
"http://127.0.0.1:5207/plugins/annotation-qa/workflows/all/quality?projectId=project-v5&pool=available",
|
||||
);
|
||||
const available = page.getByTestId("human-task-pool-available"),
|
||||
cards = page.getByTestId("annotation-task-card"),
|
||||
more = page.getByTestId("task-pool-show-more");
|
||||
await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 200);
|
||||
assert.match(await available.innerText(), /已加载 200\+/);
|
||||
report.cases.push("bounded initial200 and explicitly loaded count");
|
||||
await more.click();
|
||||
await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 400);
|
||||
report.cases.push("available cursor loads next200");
|
||||
failNext = true;
|
||||
await more.click();
|
||||
await page.getByText("合成分页失败", { exact: true }).waitFor();
|
||||
assert.equal(await cards.count(), 400);
|
||||
await more.click();
|
||||
await page.waitForFunction(() => document.querySelectorAll('[data-testid="annotation-task-card"]').length === 600);
|
||||
report.cases.push("failed page retry retains cursor and rows");
|
||||
const search = page.getByRole("textbox", { name: "搜索标注任务", exact: true });
|
||||
await search.fill("FYLX_SEARCH_0001/part0002");
|
||||
await cards.filter({ hasText: "FYLX_SEARCH_0001" }).waitFor();
|
||||
assert.equal(await cards.count(), 1);
|
||||
assert.equal(await more.count(), 0);
|
||||
report.cases.push("server search finds old part beyond500");
|
||||
delaySearch = "FYLX_SEARCH_0002";
|
||||
await search.fill(delaySearch);
|
||||
for (let n = 0; n < 60 && !releaseSlow; n++) await page.waitForTimeout(50);
|
||||
assert.ok(releaseSlow);
|
||||
await search.fill("FYLX_SEARCH_0003");
|
||||
await cards.filter({ hasText: "FYLX_SEARCH_0003" }).waitFor();
|
||||
releaseSlow();
|
||||
releaseSlow = null;
|
||||
await page.waitForTimeout(250);
|
||||
assert.equal(await cards.count(), 1);
|
||||
assert.match(await cards.first().innerText(), /FYLX_SEARCH_0003/);
|
||||
report.cases.push("late search cannot overwrite new query");
|
||||
await search.fill("FYLX_SEARCH_0611");
|
||||
await page.getByTestId("human-task-pool-mine").click();
|
||||
await cards.filter({ hasText: "FYLX_SEARCH_0611" }).waitFor();
|
||||
assert.equal(await cards.count(), 1);
|
||||
report.cases.push("mine uses same server search");
|
||||
await search.fill("FYLX_SEARCH_0612");
|
||||
await page.getByTestId("human-task-pool-reviewed").click();
|
||||
await cards.filter({ hasText: "FYLX_SEARCH_0612" }).waitFor();
|
||||
assert.equal(await cards.count(), 1);
|
||||
report.cases.push("reviewed search retains old returned-stage history");
|
||||
assert.ok(report.requests.every((r) => Number(r.limit) <= 200));
|
||||
assert.deepEqual(report.pageErrors, []);
|
||||
assert.deepEqual(report.writes, []);
|
||||
await page.screenshot({ path: path.join(out, "review-search.png") });
|
||||
console.log(JSON.stringify(report));
|
||||
} catch (e) {
|
||||
report.failure = e.message;
|
||||
if (page) {
|
||||
fs.writeFileSync(path.join(out, "failure.txt"), await page.locator("body").innerText());
|
||||
await page.screenshot({ path: path.join(out, "failure.png") });
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
releaseSlow?.();
|
||||
fs.writeFileSync(path.join(out, "report.json"), JSON.stringify(report, null, 2));
|
||||
await browser?.close();
|
||||
await server.close();
|
||||
}
|
||||
@ -42,13 +42,21 @@ function fixture(kind) {
|
||||
requestedRunId: "r",
|
||||
taskPageLimit: 200,
|
||||
taskPoolFetchOwnerRef: { current: null },
|
||||
reviewPoolCursorsRef: { current: kind === "review" ? { mine: "review-old" } : {} },
|
||||
pagedReviewPool: kind === "review",
|
||||
acceptanceWorkspace: false,
|
||||
isManager: false,
|
||||
reviewOnly: true,
|
||||
reviewFocused: false,
|
||||
workspaceMode: "quality",
|
||||
humanItemStageIdsForMode: () => ["quality_review"],
|
||||
taskPoolCursorRef: { current: kind === "sidebar" ? "" : "old-cursor" },
|
||||
minePoolCursorRef: { current: kind === "sidebar" ? "old-cursor" : "" },
|
||||
projectLoadRequestRef: { current: 1 },
|
||||
reviewActionScopeRef: { current: "scope-a" },
|
||||
adminPoolAbortRef: { current: null },
|
||||
annotationOperationsApi: { adminPool: read, getSidebar: read, listTasksPage: read },
|
||||
humanLoopApi: {},
|
||||
humanLoopApi: { listItemsPage: read },
|
||||
setTaskPoolFetchBusy: (value) => updates.push(["busy", value]),
|
||||
setHumanPools: () => updates.push(["pools"]),
|
||||
setTasks: () => updates.push(["tasks"]),
|
||||
@ -244,3 +252,25 @@ test("an old captured loader cannot invalidate the new scope before making reque
|
||||
assert.equal(f.requests(), 0);
|
||||
assert.deepEqual(f.updates, []);
|
||||
});
|
||||
|
||||
for (const invalidation of ["scope", "refresh"]) {
|
||||
test(`review cursor ignores late ${invalidation} response`, async () => {
|
||||
const f = fixture("review");
|
||||
const pending = f.run();
|
||||
if (invalidation === "scope") f.env.reviewActionScopeRef.current = "new-query";
|
||||
else f.env.projectLoadRequestRef.current++;
|
||||
f.resolve();
|
||||
await pending;
|
||||
assert.equal(f.env.reviewPoolCursorsRef.current.mine, "review-old");
|
||||
assert.ok(!f.updates.some(([kind]) => kind === "pools"));
|
||||
});
|
||||
}
|
||||
test("review continuation is single-flight and only advances its pool cursor", async () => {
|
||||
const f = fixture("review");
|
||||
const pending = f.run();
|
||||
await f.run();
|
||||
assert.equal(f.requests(), 1);
|
||||
f.resolve();
|
||||
await pending;
|
||||
assert.equal(f.env.reviewPoolCursorsRef.current.mine, "new-cursor");
|
||||
});
|
||||
|
||||
@ -345,5 +345,6 @@ test("my-tasks count marks itself partial while another sidebar page exists", as
|
||||
page.indexOf('className="annotation-task-pool-tabs'),
|
||||
page.indexOf('data-testid="project-random-claim"'),
|
||||
);
|
||||
assert.match(tabs, /blindAnnotator && value === "mine" && minePoolCursorRef\.current \? "\+" : ""/);
|
||||
assert.match(tabs, /blindAnnotator && value === "mine" && minePoolCursorRef\.current/);
|
||||
assert.match(tabs, /\? "\+"\s*:\s*""/);
|
||||
});
|
||||
|
||||
109
tests/frontend/annotation-source-validation-ui.test.mjs
Normal file
109
tests/frontend/annotation-source-validation-ui.test.mjs
Normal file
@ -0,0 +1,109 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { annotationTimelineFailure } from "../../src/lib/annotation-source-validation.js";
|
||||
import { reviewSeekHarness } from "./review-seek-harness.mjs";
|
||||
|
||||
function text(node) {
|
||||
if (node == null || typeof node === "boolean") return "";
|
||||
if (Array.isArray(node)) return node.map(text).join("");
|
||||
return typeof node === "object" ? text(node.props?.children) : String(node);
|
||||
}
|
||||
|
||||
test("terminal source failure renders the exact safe cause and recovery instead of loading", () => {
|
||||
let reads = 0;
|
||||
const f = reviewSeekHarness({
|
||||
frameTimelineStatus: "error",
|
||||
frameTimelineFailure: {
|
||||
sourceValidation: true,
|
||||
status: "failed",
|
||||
message: "IMU 第 418451 行类型无效:gyr",
|
||||
recovery: "数据提供方修复源数据后,由管理员重新校验。",
|
||||
retryAllowed: false,
|
||||
},
|
||||
onRetryFrameTimeline: () => reads++,
|
||||
});
|
||||
const initial = structuredClone(f.props.slices);
|
||||
try {
|
||||
const notice = f.nodes().find((node) => node.props["data-testid"] === "frame-timeline-status");
|
||||
assert.equal(notice.props.role, "alert");
|
||||
assert.match(text(notice), /418451.*gyr/);
|
||||
assert.match(text(notice), /数据提供方.*管理员重新校验/);
|
||||
assert.doesNotMatch(text(notice), /正在加载/);
|
||||
assert.equal(f.nodes().filter((node) => node.type === "button" && text(node) === "重新校验").length, 0);
|
||||
const refresh = f.nodes().find((node) => node.type === "button" && text(node) === "刷新状态");
|
||||
refresh.props.onClick();
|
||||
assert.equal(reads, 1);
|
||||
assert.deepEqual(f.props.slices, initial);
|
||||
} finally {
|
||||
f.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test("source retry requires the server permission and disappears after recovery", () => {
|
||||
let retries = 0;
|
||||
const f = reviewSeekHarness({
|
||||
frameTimelineStatus: "error",
|
||||
frameTimelineFailure: {
|
||||
sourceValidation: true,
|
||||
status: "failed",
|
||||
message: "源数据校验失败",
|
||||
recovery: "数据提供方修复源数据后,由管理员重新校验。",
|
||||
retryAllowed: true,
|
||||
},
|
||||
onRetrySourceValidation: () => retries++,
|
||||
});
|
||||
try {
|
||||
const retry = f.nodes().find((node) => node.type === "button" && text(node) === "重新校验");
|
||||
assert.ok(retry);
|
||||
retry.props.onClick();
|
||||
assert.equal(retries, 1);
|
||||
f.props.frameTimelineStatus = "ready";
|
||||
f.flush();
|
||||
assert.equal(
|
||||
f.nodes().some((node) => node.props["data-testid"] === "frame-timeline-status"),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
f.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test("source status mapping preserves actionable facts and limits retry to server permission", () => {
|
||||
const error = {
|
||||
code: "source_validation_failed",
|
||||
details: { sourceValidation: { status: "failed", message: "IMU 第 418451 行类型无效:gyr", retryAllowed: false } },
|
||||
};
|
||||
assert.match(annotationTimelineFailure(error).message, /418451.*gyr/);
|
||||
assert.equal(annotationTimelineFailure(error).retryAllowed, false);
|
||||
error.details.sourceValidation.retryAllowed = true;
|
||||
assert.equal(annotationTimelineFailure(error).retryAllowed, true);
|
||||
error.code = "source_validation_pending";
|
||||
error.details.sourceValidation.status = "pending";
|
||||
error.details.sourceValidation.message = "源数据校验中";
|
||||
assert.equal(annotationTimelineFailure(error).retryAllowed, false);
|
||||
assert.equal(annotationTimelineFailure(error).recovery, "");
|
||||
assert.equal(annotationTimelineFailure({ code: "other", details: error.details }).sourceValidation, false);
|
||||
assert.equal(annotationTimelineFailure(new Error("Failed to fetch")).retryAllowed, false);
|
||||
});
|
||||
|
||||
test("pending validation is a refreshable state, not an endless loading spinner or a failure", () => {
|
||||
const f = reviewSeekHarness({
|
||||
frameTimelineStatus: "error",
|
||||
frameTimelineFailure: {
|
||||
sourceValidation: true,
|
||||
status: "pending",
|
||||
message: "源数据校验中",
|
||||
recovery: "",
|
||||
retryAllowed: false,
|
||||
},
|
||||
onRetryFrameTimeline: () => {},
|
||||
});
|
||||
try {
|
||||
const notice = f.nodes().find((node) => node.props["data-testid"] === "frame-timeline-status");
|
||||
assert.equal(notice.props.role, "status");
|
||||
assert.match(text(notice), /源数据校验中.*刷新状态/);
|
||||
assert.doesNotMatch(text(notice), /加载失败|正在加载/);
|
||||
} finally {
|
||||
f.destroy();
|
||||
}
|
||||
});
|
||||
79
tests/frontend/review-pool-search.test.mjs
Normal file
79
tests/frontend/review-pool-search.test.mjs
Normal file
@ -0,0 +1,79 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { loadAnnotationPoolPage } from "../../src/lib/annotation-project-loading.js";
|
||||
|
||||
function fixture(pool = "available") {
|
||||
const calls = [];
|
||||
return {
|
||||
calls,
|
||||
options: {
|
||||
scope: { projectId: "p" },
|
||||
view: {
|
||||
kind: "review",
|
||||
pool,
|
||||
search: "old/part0002",
|
||||
category: "all",
|
||||
stageIds: ["quality_review", "split_quality_review"],
|
||||
limit: 200,
|
||||
},
|
||||
human: {
|
||||
listItemsPage: async (filters, options) => {
|
||||
calls.push({ filters, options });
|
||||
return {
|
||||
filtersApplied: true,
|
||||
items: [{ id: filters.pool, legacyTaskId: filters.pool + "-task", canClaim: filters.pool === "available" }],
|
||||
hasMore: true,
|
||||
nextCursor: filters.pool + "-next",
|
||||
};
|
||||
},
|
||||
},
|
||||
annotation: { listTasksPage: async ({ ids }) => ({ items: ids.map((id) => ({ id })) }) },
|
||||
},
|
||||
};
|
||||
}
|
||||
test("review initial reads retain all pool cursors and push search before pagination", async () => {
|
||||
const f = fixture();
|
||||
const signal = new AbortController().signal;
|
||||
const page = await loadAnnotationPoolPage({ ...f.options, signal });
|
||||
assert.equal(f.calls.length, 3);
|
||||
assert.deepEqual(page.poolCursors, { available: "available-next", mine: "mine-next", reviewed: "reviewed-next" });
|
||||
assert.equal(page.pools.available[0].canClaim, true);
|
||||
for (const { filters, options } of f.calls) {
|
||||
assert.equal(filters.search, "old/part0002");
|
||||
assert.equal(filters.limit, 200);
|
||||
assert.equal(options.signal, signal);
|
||||
}
|
||||
assert.equal(f.calls.find(({ filters }) => filters.pool === "reviewed").filters.stageIds, undefined);
|
||||
});
|
||||
for (const pool of ["available", "mine", "reviewed"])
|
||||
test(`${pool} consumes only its own cursor and joins just this page`, async () => {
|
||||
const f = fixture(pool);
|
||||
const page = await loadAnnotationPoolPage({ ...f.options, cursor: "previous" });
|
||||
assert.equal(f.calls.length, 1);
|
||||
assert.equal(f.calls[0].filters.pool, pool);
|
||||
assert.equal(f.calls[0].filters.cursor, "previous");
|
||||
assert.equal(page.poolCursors[pool], pool + "-next");
|
||||
assert.deepEqual(page.tasks, [{ id: pool + "-task" }]);
|
||||
});
|
||||
test("acceptance searches both stages in a single bounded page", async () => {
|
||||
const f = fixture();
|
||||
f.options.view.acceptance = true;
|
||||
await loadAnnotationPoolPage(f.options);
|
||||
for (const { filters } of f.calls) assert.deepEqual(filters.stageIds, ["acceptance", "prepackage_acceptance"]);
|
||||
});
|
||||
test("failed join cannot publish a partial page or advance its cursor", async () => {
|
||||
const f = fixture();
|
||||
f.options.annotation.listTasksPage = async () => {
|
||||
throw Error("join failed");
|
||||
};
|
||||
await assert.rejects(loadAnnotationPoolPage(f.options), /join failed/);
|
||||
});
|
||||
|
||||
test("old servers cannot silently return unfiltered rows for a search", async () => {
|
||||
const f = fixture();
|
||||
f.options.human.listItemsPage = async () => ({ items: [{ id: "unfiltered" }] });
|
||||
await assert.rejects(loadAnnotationPoolPage(f.options), /服务版本暂不支持此筛选/);
|
||||
f.options.view.search = "";
|
||||
const page = await loadAnnotationPoolPage(f.options);
|
||||
assert.equal(page.pools.available[0].id, "unfiltered");
|
||||
});
|
||||
@ -137,7 +137,8 @@ test("source guard: hook wires the target model (switch, freeze anchor, F direct
|
||||
test("source guard: resident editor binds the selected target with a tier label", () => {
|
||||
const panel = read("../../src/components/ReviewProblemPanel.tsx");
|
||||
assert.match(panel, /const resident =/);
|
||||
assert.match(panel, /onFocus=\{\(\) => review\.selectScope\(resident\.scope, true\)\}/);
|
||||
assert.match(panel, /onFocus=\{\(\) => review\.editSelected\(\)\}/);
|
||||
assert.doesNotMatch(panel, /onFocus=\{\(\) => review\.selectScope\(resident\.scope, true\)\}/);
|
||||
// 意见区常驻显示所选目标:档位标签(当前切片/当前动作/当前时间点)+目标说明。
|
||||
assert.match(panel, /当前目标:\{targetTierNames\[resident\.scope\]\}/);
|
||||
assert.match(panel, /const targetTierNames = \{ slice: "当前切片", action: "当前动作", timestamp: "当前时间点" \};/);
|
||||
|
||||
@ -38,6 +38,7 @@ test("source guard: hook wires the target model and resident editor", () => {
|
||||
assert.match(source, /reviewTargetPosition\(scope, m\.position, m\.targetAnchor\)/);
|
||||
// 常驻内联编辑器的绑定(无打开编辑器时)。
|
||||
assert.match(source, /residentEditor:/);
|
||||
assert.match(source, /editSelected: \(\) => selectScope\(defaultReviewScope\(m\.selected, m\.position\), true\)/);
|
||||
// 选择列表条目记录当前档位(时间点条目同时冻结锚点)。
|
||||
assert.match(source, /m\.selected = problem\.scope;/);
|
||||
assert.match(source, /m\.targetAnchor = \{/);
|
||||
@ -58,7 +59,8 @@ test("source guard: expander, summary UI and their state fully removed", () => {
|
||||
test("source guard: resident inline editor promotes into edit state on focus", () => {
|
||||
const panel = read("../../src/components/ReviewProblemPanel.tsx");
|
||||
assert.match(panel, /const resident =/);
|
||||
assert.match(panel, /onFocus=\{\(\) => review\.selectScope\(resident\.scope, true\)\}/);
|
||||
assert.match(panel, /onFocus=\{\(\) => review\.editSelected\(\)\}/);
|
||||
assert.doesNotMatch(panel, /onFocus=\{\(\) => review\.selectScope\(resident\.scope, true\)\}/);
|
||||
});
|
||||
|
||||
test("source guard: shortcut help describes target switching with zero popups (07 rewrite)", () => {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user