diff --git a/cmd/drosync-coordinator/main.go b/cmd/drosync-coordinator/main.go index 6062fc2c..8799d11c 100644 --- a/cmd/drosync-coordinator/main.go +++ b/cmd/drosync-coordinator/main.go @@ -104,6 +104,7 @@ func main() { "drosync_coordinator_reconciles_total %d\n"+ "drosync_coordinator_releases_total %d\n"+ "drosync_coordinator_audit_seals_total %d\n"+ + "drosync_coordinator_digest_backfills_total %d\n"+ "drosync_coordinator_last_cycle_timestamp_seconds %d\n"+ "drosync_coordinator_last_success_timestamp_seconds %d\n"+ "drosync_coordinator_cycle_duration_seconds %.3f\n", @@ -116,6 +117,7 @@ func main() { metrics.Reconciled, metrics.Released, metrics.Sealed, + metrics.Backfilled, metrics.LastCycle, metrics.LastOK, float64(metrics.DurationMS)/1000, @@ -133,7 +135,7 @@ func main() { _, _ = fmt.Fprintf(writer, "drosync_coordinator_queue_age_available{queue=%q} 0\n", name) } } - for _, name := range []string{"planning", "commit", "cleanup", "reconcile", "release", "audit"} { + for _, name := range []string{"planning", "commit", "cleanup", "reconcile", "release", "audit", "digest_backfill"} { phase := metrics.Phases[name] _, _ = fmt.Fprintf(writer, "drosync_coordinator_phase_cycles_total{phase=%q} %d\n"+ diff --git a/doc/drosync旧SHA256归档MD5核验补录设计-2026-10-30.md b/doc/drosync旧SHA256归档MD5核验补录设计-2026-10-30.md new file mode 100644 index 00000000..36471bbf --- /dev/null +++ b/doc/drosync旧SHA256归档MD5核验补录设计-2026-10-30.md @@ -0,0 +1,156 @@ +# drosync 旧 SHA256 归档的 MD5 核验补录设计 + +日期:2026-10-30(北京时间)。基线:`poc` 分支 `143a24b1`。本次只有设计与原型,未部署,未读写 POC Mongo、真实 BOS 或采集机数据,也未构建发布包。 + +## 结论 + +推荐方案:**由管理员批准、在协调器内执行的流式核验补录(digest backfill)**。 + +对缺少可信 MD5 回执的归档对象,协调器先做 HEAD,再按 ETag 发起分段条件 GET(`If-Match`),读取一遍同时算出 MD5 和 SHA-256。随后与仍然存在的旧回执交叉核验:Mongo head 的 `sha256`、对象元数据 `drosync-sha256`/`drosync-md5sum`、BOS 自身记录的摘要。全部一致才用 CAS 写入 MD5,并附上绑定该对象 ETag、版本和大小的 attestation,同时写入审计链。 + +整个过程不写 BOS,也不接受任何客户端提交的摘要。补录后,设备下一次普通同步会把这些文件规划为 `unchanged`,Session 进入 `completed`,文件即可逐个安全清理。 + +## 1. 现状与根因 + +- 内容协议 v2 只用 MD5(见[仅 MD5 校验](drosync仅MD5校验-2026-10-12.md))。`9a5f9ed5` 把回执字段从 BSON `sha256` 改为 `md5sum`;`poc-schema-cutover` 不处理 `objectSyncHeads`。按代码推断,POC 中 v1 head 仍是“有 `sha256`、无 `md5sum`”:Go 结构体不再解码前者,读出来的 `MD5Sum` 为空。这一点未在 POC 数据上直接核对。 +- 规划逻辑遇到 `len(head.MD5Sum) != 32` 就返回 `deny_archived_hash_change`,并保留云端原文件。回归测试 `TestArchivePlanningRetainsLegacyObjectsWithoutMD5` 覆盖了缺失、格式错误和只有 SHA256 三种形态。 +- 现场证据(见 [1.1.2 partial 诊断](drosync-1.1.2-partial修复与验证-2026-10-22.md)):mapping `a75fa7d2…` 共有 188 个 partial Session,每轮 denied=4198(MCAP、episode_meta.json)。后果如下: + - 这些文件永远无法清理,采集机磁盘持续增长; + - 每轮同步都是 partial,而 confirmed skip 只认 completed 回执,所以每轮都要重新规划全部文件; + - 新的同步会取代旧回执,在旧 Session 上执行 clean 会因“newer source receipt”保留文件。 +- 补充发现:即使 head 有了 MD5,清理证明仍要求实时 HEAD 返回的 `drosync-md5sum` 元数据与条目 MD5 相等,而 v1 对象只有 `drosync-sha256`。因此补救方案还必须规定:在不改写对象的前提下,清理证明如何认可服务端核验出的证据。 + +## 2. 不可打破的约束 + +1. 本地计算的 MD5 永远不能成为云端证据,只能拿来和云端证据比较。 +2. 不覆盖、不改写已归档对象,数据面只读。 +3. 每条补录都绑定唯一的对象版本,并且可审计、幂等、可中断续跑。 +4. 任何不确定都按失败关闭处理:保留本地文件,保留云端对象,条目维持 denied。 +5. 不改写已关闭的 Session 和条目,历史保持原样。 + +## 3. 方案比较 + +| 方案 | 安全性 | 大桶成本 | 幂等 | 受影响文件如何恢复 | 结论 | +|---|---|---|---|---|---| +| A 协调器流式核验补录 | MD5 由服务端从云端字节算出;用旧 SHA-256 回执交叉核验;If-Match、HEAD 和 CAS 绑定对象版本;对 BOS 只读 | 每个对象成功读取一次;可按 Session 定向;有预算上限 | CAS 加确定性审计键,可重跑 | 下一次普通同步即 completed,可以清理 | **推荐** | +| B 管理员批准的重新归档/覆盖(设备重传加 `force_overwrite`) | 用设备当前文件覆盖正式对象:本地若已被改写(同尺寸改写、截断)就会替换原归档;要求 `update_enabled` 和独占前缀,POC 为 `create_only`;已发布清单引用的 ETag 会失效 | 采集机上传全部旧文件,正好消耗最紧缺的设备带宽 | 每次都生成新版本,难以幂等 | 需逐文件人工批准,无法扩展 | 不采用;仅留作真实冲突的个案处理 | +| C 另存副本(keep-both 重传) | 原 key 仍缺 MD5;命名空间被冲突副本污染 | 存储翻倍,设备重传 | 一般 | 只有新 key 可清理,原问题仍在 | 不采用 | +| D 客户端双哈希(设备补算 SHA-256 与旧回执比较) | 推翻“SHA-256 退出内容比较与清理”的既定决策;证据仍来自设备 | 云端零读取,但每台设备每轮都要多读一遍文件 | 无法补录:写入设备 MD5 被禁止,head 永远缺 MD5 | 需要发版和回滚到全部采集机,且每轮都要双哈希 | 不采用 | +| E 直接把 BOS ETag 当 MD5 | 只有简单 PutObject 的 ETag 等于 MD5;v1 正式对象大多由分段复制发布,ETag 不是内容 MD5,无法可靠区分 | 零读取 | — | — | 不采用;至多作为后续优化的预筛选 | +| F 放宽策略:旧对象 denied 不计 partial | 掩盖真实状态;文件仍不可清理 | 零 | — | Session 变 completed,但磁盘问题仍在 | 不采用 | + +## 4. 推荐方案设计 + +### 4.1 触发与范围 + +- `POST /api/object-sync/spaces/{id}/digest-backfills`:仅管理员可调用,必须带幂等键和 `reason`(作为批准理由写入审计 `space.digest_backfill`)。三种范围恰好选一种: + - `sessionIds`:至多 100 个,**推荐**。传入每个受影响 mapping 最新的 partial Session,只核验其中 `deny_archived_hash_change` 条目指向的 key; + - `keyPrefix`:按空间内相对目录前缀核验; + - `wholeSpace`:核验整个空间。 +- 预算:`maxObjects` 默认 1 万、上限 100 万;`maxBytes` 默认 1 TiB、上限 1 PiB。`allowUnanchored` 默认 false。 +- `GET /api/object-sync/digest-backfills/{id}` 返回状态、计数、`budgetExhausted`、断点和至多 200 条逐 key 例外。 + +### 4.2 单个对象的核验步骤 + +1. head 已有合法 MD5:如果是上传回执,跳过(`notLegacy`);如果是已核验的,只补写审计(`alreadyAttested`)。 +2. head 被 reconcile 标记为 drifted,或回执没有 ETag:记为例外,不读取。 +3. 取得对象键租约。这与上传和清理证明用的是同一把锁(`ObjectLeaseResourceHash`);被占用则记为 `busy`,下次重跑再处理。 +4. HEAD 检查对象存在,且 ETag、大小、版本都与回执一致,否则记为 `drifted`。`ARCHIVE` 存储类型需要先解冻,直接跳过。 +5. 收集信任锚:head 的旧 `sha256`(或误存进 `md5sum` 的 64 位摘要)、元数据 `drosync-sha256`/`drosync-md5sum`、BOS 的 `x-bce-content-sha256`/`Content-MD5`。所有锚必须一致;drosync 自己写的锚格式错误时按失败关闭处理。没有任何 SHA-256 锚时记为 `unanchored`,只有管理员显式设置 `allowUnanchored` 才继续,此时只靠 ETag/版本/大小绑定,attestation 的锚记为 `none_admin_waived`。 +6. 检查预算。 +7. 按 64 MiB 分段 GET,每段都带 `If-Match: <回执 ETag>`,一遍同时计算 MD5 和 SHA-256。遇到 412 或段内 ETag 变化记为 `drifted`,短读或网络错误记为 `read_failed`。 +8. 计算结果与锚比对,不一致记为 `content_mismatch`,写审计 `head.md5.rejected`,head 不动。 +9. 再做一次 HEAD,确认对象身份没有变化。 +10. CAS 写 head:ETag、版本、大小、`platformVersion`、原摘要、旧 `sha256`、session/item 都没变,且未 drifted、未核验过,才写入 `md5sum` 和 `md5Attestation`。attestation 记录来源、作业、批准人、协调器、ETag/版本/大小、MD5/SHA-256、锚和原摘要。`platformVersion` 不变,因为内容没有变。 +11. 写审计 `head.md5.attested`,其 eventKey 由对象身份和 MD5 派生。 + +### 4.3 清理证明如何认可 + +对象元数据里有 `drosync-md5sum` 时,仍必须与条目 MD5 相等。没有时,只有同时满足以下条件才放行:head 的 attestation 来源为 `coordinator_stream`、带有锚,并且其 MD5/ETag/版本/大小与 head 一致;head 本身又已经与实时 HEAD 逐项一致。 + +伪造设备来源、ETag/版本/大小过期、MD5 不一致、没有锚,都会被拒绝。证明响应新增 `md5Evidence`(`upload_receipt` 或 `coordinator_attestation`),客户端会忽略未知字段,无需升级。 + +### 4.4 安全性 + +- **证据来源**:MD5 由协调器从云端字节算出。SHA-256 锚把它绑定到 v1 当时的上传回执;If-Match、前后两次 HEAD 和 CAS 把它绑定到唯一的对象版本。API 请求体没有任何摘要字段。测试中设备报告的 MD5 与归档不同时,head 记录的是归档字节的 MD5。 +- **只读**:只发 HEAD 和 GET;单元测试和端到端测试都断言对 BOS 零写入。 +- **并发**:靠对象租约加 CAS。新的 commit 会替换回执:Mongo 的 head upsert 现在会 `$unset` 新回执里没有的 `sha256`/`md5Attestation`;即使有残留,attestation 绑定的是旧 ETag,也不会被误用。 +- **已知交互**:reconcile 读出整页 head 后整文档回写,可能把刚补录的 MD5 改回旧值。这是失败关闭的:条目重新 denied,重跑补录即可。这是 reconcile 原有的问题,它同样可能覆盖并发 commit 的新回执,列为后续修复。 + +### 4.5 大桶成本 + +- 读取量等于被核验对象的总字节数,每个对象只成功读取一次。Session 范围只读设备仍持有、且被拒绝的对象。每个对象的请求数为 2 次 HEAD 加 ⌈大小 / 64 MiB⌉ 次 GET。 +- 估算:4198 个对象 × 平均 200 MiB ≈ 820 GiB。按 100 MiB/s 约 2.3 小时,按 30 MiB/s 约 7.8 小时,实际以网络为准。每个协调器副本同一时间只读一个对象,有独立的阶段循环,不占用 commit 和 planning 线程。 +- 费用:协调器若经同地域内网访问 BOS,读取不产生公网流出流量费;若走公网 endpoint,则按流出流量计费,大批量运行前必须确认。低频和冷存储有取回费;归档存储会被跳过。MD5 和 SHA-256 在同一遍读取中计算,瓶颈在网络。 +- 预算用尽时作业以 `completed` 结束并标记 `budgetExhausted=true`。用同一范围新建作业即可续跑,已核验的对象会被快速跳过。 + +### 4.6 幂等与故障恢复 + +- 已核验的 head 不会再次读取;CAS 防止重复写入;审计事件按确定性 eventKey 去重。 +- 作业在每读完一个对象和每翻一页后保存断点。协调器崩溃时其他副本按 claim 接管,从断点继续;中断的那个对象从头重读。计数是至少一次。 +- 审计写入失败时作业进入 `failed`。重跑时,已核验路径会用同一 eventKey 补写事件。 + +### 4.7 受影响的 Session 和文件如何恢复 + +- 历史不改写:旧 partial Session 和其中的 denied 条目保持原样。 +- 下一次普通同步(daemon 定时触发或 `drosync sync`):本地 MD5 和大小与核验结果一致的文件会规划为 `unchanged`,`expectedBase` 指向同一对象版本;没有其他问题时 Session 即 `completed`。 +- Session completed 之后:confirmed skip 生效,后续同步在本地直接跳过这些文件;`drosync clean` 使用新的 completed Session,逐文件证明通过。旧 Session 仍会因“newer source receipt”保留文件,这符合预期。 +- 本地内容与归档不同的文件(例如重新生成的日报)会变成真实的“同名文件内容冲突”,需要人工决定是另存、排除还是覆盖。作业的 `itemsConflicting` 会提前给出数量。 +- 云端字节与旧回执不一致的(`content_mismatch`)保持 denied,需要数据负责人介入。 + +## 5. 原型(本分支) + +| 位置 | 内容 | +|---|---| +| `internal/domain/object_sync.go` | head 新增 `LegacySHA256`(只读解码 BSON `sha256`)和 `MD5Attestation`;新增作业记录 `ObjectSyncDigestBackfillRecord` | +| `internal/store/object_sync_digest_backfill.go` | 内存与 Mongo 实现:作业的 claim/续租/受控保存,head 的 CAS `AttestObjectSyncHeadMD5`;Mongo 新集合 `objectSyncDigestBackfills` 及索引;head upsert 会清除过期锚 | +| `internal/objectsync/digest_backfill.go` | 新协调器阶段 `digest_backfill`、单对象核验、锚交叉核验、审计;`bos.go` 新增条件分段 GET,HEAD 带回存储摘要和存储类型 | +| `internal/api/object_sync_digest_backfill.go` | 管理员创建/查询路由;清理证明认可已绑定的 attestation | +| `docs/drosync/openapi.yaml`、`audit-dictionary.md` | 新路由、`md5Evidence` 字段和新审计动作 | + +客户端(Rust)无需改动:被拒绝的文件本来每轮都会重新进入清单,补录后自然规划为 `unchanged`。 + +测试只用 MemoryStore 和假对象存储: + +- `internal/objectsync/digest_backfill_test.go`: + - 四种旧回执形态; + - 分段 If-Match 读取; + - 内容与旧回执不符; + - 无锚与管理员豁免; + - 锚互相矛盾; + - ETag/大小/版本变化、对象丢失、已 drift、无 ETag、ARCHIVE、租约占用、读取中被替换、短读、写入前回执变化; + - 幂等与审计补写; + - 预算; + - Session 范围(每个 key 只读一次,记录的是云端字节而非设备摘要); + - 前缀范围; + - claim 丢失; + - 补录后的规划结果。 +- `internal/store/object_sync_digest_backfill_test.go`: + - CAS 的每个比较字段; + - 旧 BSON `sha256` 解码; + - Mongo 过滤器对缺失字段的匹配; + - upsert 时 `$unset`; + - 作业的 claim 与写入保护。 +- `internal/api/object_sync_digest_backfill_test.go`: + - 路由校验与审计; + - 清理证明的 7 种拒绝情形; + - 端到端 `TestLegacyArchiveSyncCompletesAndCleansAfterDigestBackfill`:通过真实 API 路由,补录前同步为 partial 且不可清理;管理员批准补录后,下一次普通同步为 completed,每个文件都拿到清理证明(`md5Evidence=coordinator_attestation`),旧 Session 保持不变,对 BOS 零写入。 +- 变异检查:分别移除清理证明对 attestation 的认可、SHA-256 锚比较、ETag 绑定、CAS 的 platformVersion 比较、无锚拒绝、读取前身份检查、锚冲突检查、审计补写,均有测试失败。 + +## 6. 未包含与后续 + +- 真实 BOS 上仍需验证:条件分段 GET 的 412 行为,HEAD 返回的 `Content-MD5`、`x-bce-content-sha256` 和存储类型的格式,内网 endpoint 的计费;还要在 POC 数据上只读确认 v1 head 的实际形态。Mongo 侧的 CAS 过滤器、`$unset` 和新集合目前只有纯函数测试,需要 Mongo 集成测试。 +- reconcile 改为按 ETag 条件的字段级 drift 写入,同时修复它覆盖并发 commit 的问题。 +- 作业列表/重试 API、管理界面、可选的双人审批;观察成本后,可考虑“规划时按需自动排队核验”。 +- 没有 head 的 `deny_existing_untracked` 对象需要“认领”,风险更高,不在本方案范围内。 + +## 7. 部署后的操作步骤 + +1. 只读找出受影响 mapping 最新的 partial Session(拒绝原因为“已归档对象缺少 MD5 回执”)。 +2. 用 `POST /api/object-sync/spaces/{spaceId}/digest-backfills` 提交 `{"reason": "...", "sessionIds": ["<最新 partial>"]}`,必要时调大 `maxBytes`。 +3. 用 `GET /api/object-sync/digest-backfills/{id}` 等待作业 `completed`,逐条处理 `mismatched`、`drifted`、`unanchored`、`busy` 和 `budgetExhausted`。 +4. 设备正常同步,得到新的 completed Session,先用该 Session 做 `drosync clean` 预览,再确认执行。 + +## 8. 验证 + +`go test ./internal/api ./internal/objectsync ./internal/store -count=1` 全部通过;`go vet` 通过;`scripts/generate_drosync_contract.py --check` 无差异。 diff --git a/docs/drosync/audit-dictionary.md b/docs/drosync/audit-dictionary.md index dfe656e1..5ea5df17 100644 --- a/docs/drosync/audit-dictionary.md +++ b/docs/drosync/audit-dictionary.md @@ -17,6 +17,8 @@ Each audit event contains: | `previousHash`, `eventHash` | Organization hash-chain link and SHA-256 hash of the canonical event payload. | | `occurredAt` | UTC RFC3339 timestamp. | +Legacy digest backfill records four actions. `space.digest_backfill` (user, `result=accepted`) is the administrator approval with scope, budgets and `reason`. `head.md5.attested` (system) records each archived head whose MD5 the coordinator computed by reading the object: `before` holds the previous digest and the retired v1 SHA-256, `after` the MD5, SHA-256, ETag, version, size, platform version, the receipts it was checked against and the approver. Its `eventKey` is derived from the attested object identity, so a rerun that repairs a missing event cannot duplicate it. `head.md5.rejected` (system, `result=denied`) records archived bytes or receipts that disagree; the head is left unchanged. `space.digest_backfill.completed` (system) carries the job counters. + An organization-scoped fencing lease serializes hash-chain append across API replicas. Events retain for one year. Export endpoints are read-only and available as CSV or NDJSON. The coordinator verifies every event hash and intra-day chain link, then conditionally writes the previous UTC day's count, first/last event hash, aggregate digest, Ed25519 signature, and verifier key ID to `.drosync-internal/audit/{organizationId}/{YYYY-MM-DD}.json`. Mongo stores the BOS ETag/version ID and seal index for one year. `GET /api/object-sync/audit-seals/{day}` returns the index record. diff --git a/docs/drosync/openapi.yaml b/docs/drosync/openapi.yaml index d204a432..191d5208 100644 --- a/docs/drosync/openapi.yaml +++ b/docs/drosync/openapi.yaml @@ -121,6 +121,59 @@ paths: drifted: { type: integer, format: int64 } default: $ref: "#/components/responses/Error" + /api/object-sync/spaces/{id}/digest-backfills: + post: + tags: [management] + operationId: createDigestBackfill + description: | + Administrator-approved verification of archived heads that lack a trusted + MD5 receipt (v1 SHA-256-only, missing or malformed). The coordinator reads + each archived object with conditional ranged GETs, checks the computed + SHA-256/MD5 against every surviving receipt, and records the MD5 with a + bound attestation and audit event. The request never carries a digest and + BOS objects are never written. Exactly one scope is required. + parameters: + - $ref: "#/components/parameters/ResourceID" + - $ref: "#/components/parameters/IdempotencyKey" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [reason] + properties: + reason: { type: string, maxLength: 1000 } + sessionIds: { type: array, maxItems: 100, items: { type: string }, description: "Verify keys denied with deny_archived_hash_change in these sessions." } + keyPrefix: { type: string, description: "Relative directory prefix inside the sync space." } + wholeSpace: { type: boolean } + allowUnanchored: { type: boolean, default: false, description: "Also attest heads with no surviving SHA-256 receipt, bound only to ETag/version/size." } + maxObjects: { type: integer, format: int64, default: 10000, maximum: 1000000 } + maxBytes: { type: integer, format: int64, default: 1099511627776 } + responses: + "202": + description: Verification accepted. + content: + application/json: + schema: + $ref: "#/components/schemas/DigestBackfill" + default: + $ref: "#/components/responses/Error" + /api/object-sync/digest-backfills/{id}: + get: + tags: [management] + operationId: getDigestBackfill + parameters: + - $ref: "#/components/parameters/ResourceID" + responses: + "200": + description: Durable progress, counters and per-key exceptions. + content: + application/json: + schema: + $ref: "#/components/schemas/DigestBackfill" + default: + $ref: "#/components/responses/Error" /api/object-sync/devices: get: tags: [management] @@ -808,7 +861,9 @@ paths: Device-scoped, rate-limited and never idempotency-cached. Requires an owned completed or partial non-preview Session, a committed/unchanged item with a complete MD5 receipt, matching current platform head and live BOS HEAD - (MD5 metadata, size, ETag and version). Other items remain ineligible; + (size, ETag, version, and MD5 metadata; a v1 archive without MD5 metadata + qualifies only through a coordinator digest attestation bound to that same + ETag, version and size). Other items remain ineligible; issuing a proof never changes the Session or item status. Acquires a short object lease. The local daemon must independently prove the current source identity/content before deleting anything. @@ -834,6 +889,7 @@ paths: relativePath: { type: string } targetKey: { type: string } md5sum: { type: string } + md5Evidence: { type: string, enum: [upload_receipt, coordinator_attestation] } sizeBytes: { type: integer, format: int64 } eTag: { type: string } versionId: { type: string } @@ -1943,6 +1999,53 @@ components: drifted: type: integer format: int64 + DigestBackfill: + type: object + properties: + id: { type: string } + operationId: { type: string } + syncSpaceId: { type: string } + status: { type: string, enum: [pending, running, completed, failed] } + reason: { type: string } + sessionIds: { type: array, items: { type: string } } + keyPrefix: { type: string } + wholeSpace: { type: boolean } + allowUnanchored: { type: boolean } + maxObjects: { type: integer, format: int64 } + maxBytes: { type: integer, format: int64 } + budgetExhausted: { type: boolean } + counts: + type: object + description: At-least-once counters; a resumed job may revisit its last page. + properties: + scanned: { type: integer, format: int64 } + attested: { type: integer, format: int64 } + alreadyAttested: { type: integer, format: int64 } + notLegacy: { type: integer, format: int64 } + unanchored: { type: integer, format: int64 } + mismatched: { type: integer, format: int64 } + drifted: { type: integer, format: int64 } + busy: { type: integer, format: int64 } + readFailed: { type: integer, format: int64 } + objectsRead: { type: integer, format: int64 } + bytesRead: { type: integer, format: int64 } + itemsMatched: { type: integer, format: int64 } + itemsConflicting: { type: integer, format: int64 } + exceptions: + type: array + maxItems: 200 + items: + type: object + properties: + key: { type: string } + outcome: { type: string, enum: [unanchored, anchor_conflict, content_mismatch, drifted, unbound_receipt, busy, storage_class_unreadable, read_failed, budget_exhausted, session_unavailable] } + detail: { type: string } + at: { type: string, format: date-time } + error: { type: string } + createdBy: { type: string } + createdAt: { type: string } + startedAt: { type: string } + completedAt: { type: string } DataRelease: type: object properties: diff --git a/internal/api/object_sync.go b/internal/api/object_sync.go index c8ffcd1e..8440024f 100644 --- a/internal/api/object_sync.go +++ b/internal/api/object_sync.go @@ -2073,8 +2073,7 @@ func abortObjectSyncSession(ctx context.Context, target store.ObjectSyncStore, s } func objectSyncLeaseResourceHash(space domain.ObjectSyncSpaceRecord, key string) string { - sum := sha256.Sum256([]byte(space.ConnectionID + "\x00" + space.Bucket + "\x00" + key)) - return hex.EncodeToString(sum[:]) + return objectsync.ObjectLeaseResourceHash(space, key) } func (server *server) handleExternalObjectSyncItemReserve(writer http.ResponseWriter, request *http.Request) { diff --git a/internal/api/object_sync_cleanup.go b/internal/api/object_sync_cleanup.go index c48dd908..24bcf153 100644 --- a/internal/api/object_sync_cleanup.go +++ b/internal/api/object_sync_cleanup.go @@ -119,7 +119,7 @@ func (server *server) handleExternalObjectSyncCleanupProof(writer http.ResponseW renderObjectSyncError(writer, request, 409, "cleanup_cloud_unverified", "Cloud object is missing or HEAD verification failed; local file retained", true, nil) return } - if remote.SizeBytes != item.SizeBytes || remote.ETag != head.RemoteETag || remote.VersionID != head.VersionID || objectSyncMetadataValue(remote.Metadata, "drosync-md5sum") != item.MD5Sum { + if remote.SizeBytes != item.SizeBytes || remote.ETag != head.RemoteETag || remote.VersionID != head.VersionID || !objectSyncCloudMD5Matches(remote, *head, item.MD5Sum) { renderObjectSyncError(writer, request, 409, "cleanup_cloud_changed", "Cloud object content or identity changed; local file retained", false, nil) return } @@ -129,5 +129,5 @@ func (server *server) handleExternalObjectSyncCleanupProof(writer http.ResponseW return } success = true - renderJSON(writer, 200, map[string]any{"sessionId": session.ID, "mappingId": session.MappingID, "syncSpaceId": session.SyncSpaceID, "credentialId": session.CredentialID, "deviceId": session.DeviceID, "itemId": item.ID, "relativePath": item.RelativePath, "targetKey": item.TargetKey, "md5sum": item.MD5Sum, "sizeBytes": item.SizeBytes, "eTag": remote.ETag, "versionId": remote.VersionID, "verifiedAt": nowISO(), "expiresAt": lease.ExpiresAt, "leaseId": lease.ID, "leaseOwner": lease.OwnerItemID, "fencingToken": lease.FencingToken}) + renderJSON(writer, 200, map[string]any{"sessionId": session.ID, "mappingId": session.MappingID, "syncSpaceId": session.SyncSpaceID, "credentialId": session.CredentialID, "deviceId": session.DeviceID, "itemId": item.ID, "relativePath": item.RelativePath, "targetKey": item.TargetKey, "md5sum": item.MD5Sum, "md5Evidence": objectSyncMD5Evidence(*head), "sizeBytes": item.SizeBytes, "eTag": remote.ETag, "versionId": remote.VersionID, "verifiedAt": nowISO(), "expiresAt": lease.ExpiresAt, "leaseId": lease.ID, "leaseOwner": lease.OwnerItemID, "fencingToken": lease.FencingToken}) } diff --git a/internal/api/object_sync_digest_backfill.go b/internal/api/object_sync_digest_backfill.go new file mode 100644 index 00000000..cd661f97 --- /dev/null +++ b/internal/api/object_sync_digest_backfill.go @@ -0,0 +1,179 @@ +package api + +import ( + "net/http" + "slices" + "strings" + + "droboticflow/internal/domain" + "droboticflow/internal/objectsync" + + "github.com/go-chi/chi/v5" +) + +const ( + objectSyncBackfillDefaultMaxObjects = int64(10_000) + objectSyncBackfillDefaultMaxBytes = int64(1) << 40 + objectSyncBackfillMaxObjects = int64(1_000_000) + objectSyncBackfillMaxBytes = int64(1) << 50 + objectSyncBackfillMaxSessions = 100 + objectSyncBackfillMaxReasonBytes = 1000 +) + +// A digest backfill is the only way an archived legacy head gains an MD5. The +// request names what to verify, never a digest: the coordinator streams the +// archived object and computes it, so device-reported MD5 cannot become cloud +// evidence through this route. +func (server *server) handleObjectSyncDigestBackfillCreate(writer http.ResponseWriter, request *http.Request) { + target, ok := server.objectSyncStore(writer, request) + if !ok { + return + } + principal := objectSyncManagementPrincipal(request) + space, err := target.GetObjectSyncSpace(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderObjectSyncError(writer, request, http.StatusInternalServerError, "space_lookup_failed", err.Error(), true, nil) + return + } + if space == nil || space.OrganizationID != principal.User.OrganizationID { + renderObjectSyncError(writer, request, http.StatusNotFound, "space_not_found", "Object sync space not found", false, nil) + return + } + var input struct { + Reason string `json:"reason"` + SessionIDs []string `json:"sessionIds"` + KeyPrefix string `json:"keyPrefix"` + WholeSpace bool `json:"wholeSpace"` + AllowUnanchored bool `json:"allowUnanchored"` + MaxObjects int64 `json:"maxObjects"` + MaxBytes int64 `json:"maxBytes"` + } + if !decodeJSON(writer, request, &input) { + return + } + reason := strings.TrimSpace(input.Reason) + if reason == "" || len(reason) > objectSyncBackfillMaxReasonBytes { + renderObjectSyncError(writer, request, http.StatusBadRequest, "backfill_reason_required", "An approval reason of at most 1000 bytes is required", false, nil) + return + } + sessionIDs := make([]string, 0, len(input.SessionIDs)) + for _, id := range input.SessionIDs { + if id = strings.TrimSpace(id); id != "" && !slices.Contains(sessionIDs, id) { + sessionIDs = append(sessionIDs, id) + } + } + keyPrefix := strings.TrimSpace(input.KeyPrefix) + scopes := 0 + for _, selected := range []bool{len(sessionIDs) > 0, keyPrefix != "", input.WholeSpace} { + if selected { + scopes++ + } + } + if scopes != 1 || len(sessionIDs) > objectSyncBackfillMaxSessions { + renderObjectSyncError(writer, request, http.StatusBadRequest, "backfill_scope_invalid", "Choose exactly one scope: 1-100 sessionIds, a keyPrefix, or wholeSpace", false, nil) + return + } + if keyPrefix != "" { + normalized, pathErr := normalizeObjectSyncRelativePath(strings.TrimSuffix(keyPrefix, "/")) + if pathErr != nil { + renderObjectSyncError(writer, request, http.StatusBadRequest, "backfill_prefix_invalid", pathErr.Error(), false, nil) + return + } + keyPrefix = normalized + "/" + } + for _, id := range sessionIDs { + session, err := target.GetObjectSyncSession(request.Context(), id) + if err != nil { + renderObjectSyncError(writer, request, http.StatusServiceUnavailable, "session_lookup_failed", "Could not read session state", true, nil) + return + } + if session == nil || session.OrganizationID != space.OrganizationID || session.SyncSpaceID != space.ID { + renderObjectSyncError(writer, request, http.StatusBadRequest, "backfill_session_invalid", "Every session must belong to this sync space", false, map[string]any{"sessionId": id}) + return + } + } + maxObjects, maxBytes := input.MaxObjects, input.MaxBytes + if maxObjects == 0 { + maxObjects = objectSyncBackfillDefaultMaxObjects + } + if maxBytes == 0 { + maxBytes = objectSyncBackfillDefaultMaxBytes + } + if maxObjects < 0 || maxObjects > objectSyncBackfillMaxObjects || maxBytes < 0 || maxBytes > objectSyncBackfillMaxBytes { + renderObjectSyncError(writer, request, http.StatusBadRequest, "backfill_budget_invalid", "maxObjects must be 1-1000000 and maxBytes 1 byte to 1 PiB", false, nil) + return + } + now := nowISO() + job := domain.ObjectSyncDigestBackfillRecord{ + ID: objectSyncNewID("osdb"), + OperationID: objectSyncNewID("op"), + OrganizationID: space.OrganizationID, + SyncSpaceID: space.ID, + Status: "pending", + Reason: reason, + SessionIDs: sessionIDs, + KeyPrefix: keyPrefix, + WholeSpace: input.WholeSpace, + AllowUnanchored: input.AllowUnanchored, + MaxObjects: maxObjects, + MaxBytes: maxBytes, + CreatedBy: principal.User.ID, + CreatedAt: now, + UpdatedAt: now, + } + if err := target.UpsertObjectSyncDigestBackfill(request.Context(), job); err != nil { + renderObjectSyncError(writer, request, http.StatusInternalServerError, "backfill_enqueue_failed", err.Error(), true, nil) + return + } + if !server.requireObjectSyncAudit(writer, request, target, domain.ObjectSyncAuditEvent{ + OrganizationID: space.OrganizationID, + ActorType: "user", + ActorID: principal.User.ID, + RequestID: objectSyncRequestID(request), + Action: "space.digest_backfill", + Target: map[string]any{"spaceId": space.ID, "backfillId": job.ID, "operationId": job.OperationID}, + After: map[string]any{ + "sessionIds": job.SessionIDs, "keyPrefix": job.KeyPrefix, "wholeSpace": job.WholeSpace, + "allowUnanchored": job.AllowUnanchored, "maxObjects": job.MaxObjects, "maxBytes": job.MaxBytes, + }, + Result: "accepted", + Reason: reason, + }) { + return + } + renderJSON(writer, http.StatusAccepted, job) +} + +func (server *server) handleObjectSyncDigestBackfillGet(writer http.ResponseWriter, request *http.Request) { + target, ok := server.objectSyncStore(writer, request) + if !ok { + return + } + job, err := target.GetObjectSyncDigestBackfill(request.Context(), chi.URLParam(request, "id")) + if err != nil { + renderObjectSyncError(writer, request, http.StatusServiceUnavailable, "backfill_lookup_failed", "Could not read digest backfill", true, nil) + return + } + if job == nil || job.OrganizationID != objectSyncManagementPrincipal(request).User.OrganizationID { + renderObjectSyncError(writer, request, http.StatusNotFound, "backfill_not_found", "Digest backfill not found", false, nil) + return + } + renderJSON(writer, http.StatusOK, job) +} + +// Upload receipts carry their MD5 in object metadata. A legacy archive has an +// MD5 only through a coordinator attestation bound to this exact object version; +// the caller has already matched the live object's ETag, version and size. +func objectSyncCloudMD5Matches(remote domain.ObjectStorageObjectSummary, head domain.ObjectSyncObjectHeadRecord, md5sum string) bool { + if declared := objectSyncMetadataValue(remote.Metadata, "drosync-md5sum"); declared != "" { + return declared == md5sum + } + return objectsync.AttestationBindsHead(head) && head.MD5Sum == md5sum +} + +func objectSyncMD5Evidence(head domain.ObjectSyncObjectHeadRecord) string { + if objectsync.AttestationBindsHead(head) { + return "coordinator_attestation" + } + return "upload_receipt" +} diff --git a/internal/api/object_sync_digest_backfill_test.go b/internal/api/object_sync_digest_backfill_test.go new file mode 100644 index 00000000..cee21cc7 --- /dev/null +++ b/internal/api/object_sync_digest_backfill_test.go @@ -0,0 +1,361 @@ +package api + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/objectsync" + + "github.com/baidubce/bce-sdk-go/bce" +) + +func TestDigestBackfillCreateIsAdminScopedAndAudited(t *testing.T) { + handler, data := objectSyncTestHandler(t, true) + org := domain.DefaultOrganizationID + for _, space := range []domain.ObjectSyncSpaceRecord{{ID: "space", OrganizationID: org, ConnectionID: "bos-connection", Bucket: "bucket"}, {ID: "other-space", OrganizationID: org, ConnectionID: "bos-connection", Bucket: "bucket"}} { + if err := data.UpsertObjectSyncSpace(t.Context(), space); err != nil { + t.Fatal(err) + } + } + for _, session := range []domain.ObjectSyncSessionRecord{{ID: "partial", OrganizationID: org, SyncSpaceID: "space", Status: domain.ObjectSyncSessionPartial}, {ID: "elsewhere", OrganizationID: org, SyncSpaceID: "other-space", Status: domain.ObjectSyncSessionPartial}} { + if err := data.UpsertObjectSyncSession(t.Context(), session); err != nil { + t.Fatal(err) + } + } + create := func(space string, body map[string]any, key string) *httptest.ResponseRecorder { + return objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces/"+space+"/digest-backfills", body, key) + } + for name, test := range map[string]struct { + space string + body map[string]any + code string + }{ + "missing-reason": {"space", map[string]any{"wholeSpace": true}, "backfill_reason_required"}, + "no-scope": {"space", map[string]any{"reason": "verify"}, "backfill_scope_invalid"}, + "two-scopes": {"space", map[string]any{"reason": "verify", "wholeSpace": true, "keyPrefix": "node-04"}, "backfill_scope_invalid"}, + "foreign-session": {"space", map[string]any{"reason": "verify", "sessionIds": []string{"partial", "elsewhere"}}, "backfill_session_invalid"}, + "bad-prefix": {"space", map[string]any{"reason": "verify", "keyPrefix": "../node-04"}, "backfill_prefix_invalid"}, + "bad-budget": {"space", map[string]any{"reason": "verify", "wholeSpace": true, "maxBytes": -1}, "backfill_budget_invalid"}, + "unknown-space": {"missing", map[string]any{"reason": "verify", "wholeSpace": true}, "space_not_found"}, + } { + if response := create(test.space, test.body, "invalid-"+name); response.Code/100 != 4 || !strings.Contains(response.Body.String(), test.code) { + t.Fatalf("%s: %d %s", name, response.Code, response.Body.String()) + } + } + response := create("space", map[string]any{"reason": " verify node-04 v1 archives ", "keyPrefix": "node-04", "md5sum": strings.Repeat("a", 32)}, "valid") + if response.Code != http.StatusAccepted { + t.Fatalf("valid request: %d %s", response.Code, response.Body.String()) + } + var job domain.ObjectSyncDigestBackfillRecord + if err := json.Unmarshal(response.Body.Bytes(), &job); err != nil { + t.Fatal(err) + } + if job.Status != "pending" || job.KeyPrefix != "node-04/" || job.Reason != "verify node-04 v1 archives" || job.MaxObjects != 10_000 || job.MaxBytes != 1<<40 || job.CreatedBy != "business-test-admin" || job.AllowUnanchored { + t.Fatalf("job=%#v", job) + } + events, _ := data.ListObjectSyncAudit(t.Context(), domain.ObjectSyncListQuery{OrganizationID: org, Limit: 100}) + if len(events) != 1 || events[0].Action != "space.digest_backfill" || events[0].Reason != job.Reason || events[0].Target["backfillId"] != job.ID || events[0].ActorID != "business-test-admin" { + t.Fatalf("approval audit=%#v", events) + } + got := objectSyncRequest(t, handler, http.MethodGet, "/api/object-sync/digest-backfills/"+job.ID, nil, "") + if got.Code != http.StatusOK || !strings.Contains(got.Body.String(), `"keyPrefix":"node-04/"`) { + t.Fatalf("get: %d %s", got.Code, got.Body.String()) + } + foreign := job + foreign.ID = "foreign-job" + foreign.OrganizationID = "other-org" + data.UpsertObjectSyncDigestBackfill(t.Context(), foreign) + for _, id := range []string{"foreign-job", "missing-job"} { + if response := objectSyncRequest(t, handler, http.MethodGet, "/api/object-sync/digest-backfills/"+id, nil, ""); response.Code != http.StatusNotFound { + t.Fatalf("%s visible: %d", id, response.Code) + } + } +} + +// attestedCleanupFixture turns the standard cleanup fixture into a v1 archive: +// the object carries only drosync-sha256 and the head an attested MD5. +func attestedCleanupFixture(t *testing.T, mutate func(*domain.ObjectSyncDigestAttestation, domain.ObjectStorageObjectSummary)) (*httptest.ResponseRecorder, map[string]any) { + t.Helper() + _, data, fake, call := cleanupFixture(t) + head, _ := data.GetObjectSyncHead(t.Context(), "org", "space", "prefix/capture.bin") + attestation := domain.ObjectSyncDigestAttestation{ + Source: domain.ObjectSyncDigestSourceCoordinatorStream, BackfillID: "backfill", ETag: "etag", VersionID: "v1", SizeBytes: 4, + MD5Sum: head.MD5Sum, SHA256: strings.Repeat("f", 64), Anchors: []string{domain.ObjectSyncDigestAnchorHeadSHA256}, AttestedAt: nowISO(), + } + remote := fake.objects["bucket|prefix/capture.bin"] + remote.Metadata = map[string]any{"drosync-sha256": strings.Repeat("f", 64)} + mutate(&attestation, remote) + head.MD5Attestation = &attestation + head.LegacySHA256 = strings.Repeat("f", 64) + if err := data.UpsertObjectSyncHead(t.Context(), *head); err != nil { + t.Fatal(err) + } + fake.objects["bucket|prefix/capture.bin"] = remote + response := call(http.MethodPost, "device", "") + var proof map[string]any + _ = json.Unmarshal(response.Body.Bytes(), &proof) + return response, proof +} + +func TestCleanupProofAcceptsOnlyBoundCoordinatorAttestations(t *testing.T) { + response, proof := attestedCleanupFixture(t, func(*domain.ObjectSyncDigestAttestation, domain.ObjectStorageObjectSummary) {}) + if response.Code != http.StatusOK || proof["md5Evidence"] != "coordinator_attestation" || proof["md5sum"] != strings.Repeat("a", 32) { + t.Fatalf("bound attestation rejected: %d %s", response.Code, response.Body.String()) + } + for name, mutate := range map[string]func(*domain.ObjectSyncDigestAttestation, domain.ObjectStorageObjectSummary){ + "attested-other-etag": func(a *domain.ObjectSyncDigestAttestation, _ domain.ObjectStorageObjectSummary) { + a.ETag = "older-etag" + }, + "attested-other-version": func(a *domain.ObjectSyncDigestAttestation, _ domain.ObjectStorageObjectSummary) { a.VersionID = "" }, + "attested-other-size": func(a *domain.ObjectSyncDigestAttestation, _ domain.ObjectStorageObjectSummary) { a.SizeBytes = 5 }, + "attested-other-md5": func(a *domain.ObjectSyncDigestAttestation, _ domain.ObjectStorageObjectSummary) { + a.MD5Sum = strings.Repeat("c", 32) + }, + "device-source": func(a *domain.ObjectSyncDigestAttestation, _ domain.ObjectStorageObjectSummary) { + a.Source = "device_report" + }, + "no-anchor": func(a *domain.ObjectSyncDigestAttestation, _ domain.ObjectStorageObjectSummary) { a.Anchors = nil }, + "object-declares-other-md5": func(_ *domain.ObjectSyncDigestAttestation, remote domain.ObjectStorageObjectSummary) { + remote.Metadata["drosync-md5sum"] = strings.Repeat("c", 32) + }, + } { + if response, _ := attestedCleanupFixture(t, mutate); response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), "cleanup_cloud_changed") { + t.Fatalf("%s: unsafe proof %d %s", name, response.Code, response.Body.String()) + } + } +} + +// legacyArchiveStorage stands in for BOS on the coordinator side. It serves +// conditional reads of the archived bytes and refuses every write. +type legacyArchiveStorage struct { + mu sync.Mutex + objects map[string]objectsync.ObjectMetadata + content map[string][]byte + reads int + writes int +} + +func (storage *legacyArchiveStorage) Head(_ context.Context, _ domain.StorageConnectionRecord, bucket, key string) (objectsync.ObjectMetadata, error) { + storage.mu.Lock() + defer storage.mu.Unlock() + return storage.objects[bucket+"/"+key], nil +} + +func (storage *legacyArchiveStorage) GetObjectRange(_ context.Context, _ domain.StorageConnectionRecord, bucket, key, ifMatch string, offset, length int64) (io.ReadCloser, objectsync.ObjectMetadata, error) { + storage.mu.Lock() + defer storage.mu.Unlock() + storage.reads++ + meta, ok := storage.objects[bucket+"/"+key] + if !ok || meta.ETag != ifMatch { + return nil, objectsync.ObjectMetadata{}, &bce.BceServiceError{StatusCode: http.StatusPreconditionFailed} + } + return io.NopCloser(bytes.NewReader(storage.content[bucket+"/"+key][offset : offset+length])), meta, nil +} + +func (storage *legacyArchiveStorage) refuse() error { + storage.mu.Lock() + defer storage.mu.Unlock() + storage.writes++ + return errors.New("the v1 archive is read-only") +} + +func (storage *legacyArchiveStorage) PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (objectsync.ObjectMetadata, error) { + return objectsync.ObjectMetadata{}, storage.refuse() +} + +func (storage *legacyArchiveStorage) PutInternal(context.Context, domain.StorageConnectionRecord, string, string, io.Reader, int64, string, map[string]string) (objectsync.ObjectMetadata, error) { + return objectsync.ObjectMetadata{}, storage.refuse() +} + +func (storage *legacyArchiveStorage) AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error { + return storage.refuse() +} + +func (storage *legacyArchiveStorage) Delete(context.Context, domain.StorageConnectionRecord, string, string) error { + return storage.refuse() +} + +// A device keeps v1-archived files whose receipts only have SHA-256. Its syncs +// stay partial and nothing is cleanable until an administrator approves a +// digest backfill; then the next ordinary sync completes and every file gets a +// cleanup proof. Only the BOS adapter and Mongo are in-memory fakes. +func TestLegacyArchiveSyncCompletesAndCleansAfterDigestBackfill(t *testing.T) { + handler, data := objectSyncTestHandler(t, true) + ctx := t.Context() + org := domain.DefaultOrganizationID + space := domain.ObjectSyncSpaceRecord{ + ID: "legacy-space", OrganizationID: org, Name: "v1 archive", ConnectionID: "bos-connection", Bucket: "archive", RootPrefix: "robots", + Enabled: true, Mode: "create_only", Policy: domain.DefaultObjectSyncSpacePolicy(), + Capability: domain.ObjectSyncCapabilityRecord{State: domain.ObjectSyncCapabilityCreateOnly, ConditionalCreate: true}, + } + token := "legacy-device-token" + credential := domain.ObjectSyncCredentialRecord{ + ID: "legacy-credential", OrganizationID: org, Enabled: true, + TokenBindings: []domain.ObjectSyncCredentialTokenBinding{{TokenHash: sessionTokenHash(token), DeviceID: "legacy-device"}}, + Scopes: []domain.ObjectSyncCredentialScope{{SyncSpaceID: space.ID, AllowedRelativePrefixes: []string{""}, Actions: []string{"plan", "create"}}}, + } + for _, err := range []error{ + data.UpsertObjectSyncSpace(ctx, space), + data.UpsertObjectSyncCredential(ctx, credential), + data.UpsertObjectSyncDevice(ctx, domain.ObjectSyncDeviceRecord{ID: "legacy-device", OrganizationID: org, Status: "active"}), + } { + if err != nil { + t.Fatal(err) + } + } + files := []struct { + path string + payload []byte + }{ + {"node-04/episode_0001/capture.mcap", bytes.Repeat([]byte("mcap-frame;"), 3000)}, + {"node-04/episode_0001/episode_meta.json", []byte(`{"episode":1,"robot":"node-04"}`)}, + } + archive := &legacyArchiveStorage{objects: map[string]objectsync.ObjectMetadata{}, content: map[string][]byte{}} + browser := newFakeObjectStorageBrowser() + browser.objects = map[string]domain.ObjectStorageObjectSummary{} + localMD5 := map[string]string{} + var archivedBytes int64 + for index, file := range files { + key := "robots/" + file.path + md5Sum, sha256Sum := md5.Sum(file.payload), sha256.Sum256(file.payload) + localMD5[file.path] = hex.EncodeToString(md5Sum[:]) + legacySHA := hex.EncodeToString(sha256Sum[:]) + etag := fmt.Sprintf("v1-multipart-etag-%d", index) + size := int64(len(file.payload)) + archivedBytes += size + if err := data.UpsertObjectSyncHead(ctx, domain.ObjectSyncObjectHeadRecord{OrganizationID: org, SyncSpaceID: space.ID, Key: key, PlatformVersion: 1, RemoteETag: etag, SizeBytes: size, LegacySHA256: legacySHA, SessionID: "oses_v1", ItemID: fmt.Sprintf("osi_v1_%d", index)}); err != nil { + t.Fatal(err) + } + archive.objects["archive/"+key] = objectsync.ObjectMetadata{Exists: true, ETag: etag, SizeBytes: size, UserMeta: map[string]string{"drosync-sha256": legacySHA}} + archive.content["archive/"+key] = file.payload + browser.objects["archive|"+key] = domain.ObjectStorageObjectSummary{Bucket: "archive", Key: key, ETag: etag, SizeBytes: size, Metadata: map[string]any{"drosync-sha256": legacySHA}} + } + previousResolver := storageConnectionBrowserResolver + storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) { + return &resumableObjectSyncBrowser{fakeObjectStorageBrowser: browser}, func() {}, nil + } + t.Cleanup(func() { storageConnectionBrowserResolver = previousResolver }) + coordinator, err := objectsync.NewCoordinator(data, archive, "e2e-coordinator") + if err != nil { + t.Fatal(err) + } + device := func(method, path string, body any, key string) *httptest.ResponseRecorder { + t.Helper() + var payload bytes.Buffer + if body != nil { + if err := json.NewEncoder(&payload).Encode(body); err != nil { + t.Fatal(err) + } + } + request := httptest.NewRequest(method, "/api/external/object-sync/"+path, &payload) + request.Header.Set("Content-Type", "application/json") + request.Header.Set(objectSyncCredentialHeader, token) + if key != "" { + request.Header.Set("Idempotency-Key", key) + } + writer := httptest.NewRecorder() + handler.ServeHTTP(writer, request) + return writer + } + expect := func(response *httptest.ResponseRecorder, status int, target any) { + t.Helper() + if response.Code != status { + t.Fatalf("status %d, want %d: %s", response.Code, status, response.Body.String()) + } + if target != nil { + if err := json.Unmarshal(response.Body.Bytes(), target); err != nil { + t.Fatal(err) + } + } + } + // One ordinary daemon sync: manifest of local MD5s, server planning, finish. + sync := func(label string) (domain.ObjectSyncSessionRecord, []domain.ObjectSyncItemRecord) { + t.Helper() + var session domain.ObjectSyncSessionRecord + expect(device(http.MethodPost, "sessions", map[string]any{"syncSpaceId": space.ID, "mappingId": "mapping-node-04", "clientVersion": "1.1.1", "protocolVersion": domain.ObjectSyncProtocolVersion}, label+"-create"), http.StatusCreated, &session) + manifest := make([]map[string]any, 0, len(files)) + for index, file := range files { + manifest = append(manifest, map[string]any{"clientItemId": fmt.Sprintf("file-%d", index), "relativePath": file.path, "kind": "file", "sizeBytes": len(file.payload), "md5sum": localMD5[file.path], "modifiedAt": "2026-09-20T08:00:00Z"}) + } + expect(device(http.MethodPost, "sessions/"+session.ID+"/manifest-pages", map[string]any{"pageNumber": 1, "final": true, "items": manifest}, label+"-manifest"), http.StatusAccepted, nil) + expect(device(http.MethodPost, "sessions/"+session.ID+"/plan", map[string]any{}, label+"-plan"), http.StatusAccepted, nil) + if _, err := coordinator.ProcessOnce(ctx); err != nil { + t.Fatal(err) + } + var final domain.ObjectSyncSessionRecord + expect(device(http.MethodPost, "sessions/"+session.ID+"/complete", map[string]any{}, label+"-complete"), http.StatusOK, &final) + page, err := data.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{SessionID: session.ID, Limit: 100}) + if err != nil || len(page.Items) != len(files) { + t.Fatalf("items=%#v err=%v", page.Items, err) + } + return final, page.Items + } + + before, deniedItems := sync("before") + if before.Status != domain.ObjectSyncSessionPartial || before.Counts.Denied != int64(len(files)) { + t.Fatalf("legacy archives did not keep the sync partial: %#v", before) + } + for _, item := range deniedItems { + if item.Status != domain.ObjectSyncItemDenied || item.Action != "deny_archived_hash_change" { + t.Fatalf("item=%#v", item) + } + if response := device(http.MethodPost, "items/"+item.ID+"/cleanup-proof", nil, ""); response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), "cleanup_item_not_committed") { + t.Fatalf("denied legacy file became cleanable: %d %s", response.Code, response.Body.String()) + } + } + + var job domain.ObjectSyncDigestBackfillRecord + expect(objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces/"+space.ID+"/digest-backfills", map[string]any{"reason": "node-04 v1 archives keep every sync partial", "sessionIds": []string{before.ID}}, "approve-node-04"), http.StatusAccepted, &job) + if _, err := coordinator.ProcessOnce(ctx); err != nil { + t.Fatal(err) + } + expect(objectSyncRequest(t, handler, http.MethodGet, "/api/object-sync/digest-backfills/"+job.ID, nil, ""), http.StatusOK, &job) + if job.Status != "completed" || job.Counts.Attested != int64(len(files)) || job.Counts.ItemsMatched != int64(len(files)) || job.Counts.ItemsConflicting != 0 || job.Counts.BytesRead != archivedBytes || len(job.Exceptions) != 0 { + t.Fatalf("backfill=%#v", job) + } + if archive.writes != 0 || archive.reads == 0 { + t.Fatalf("archive reads=%d writes=%d", archive.reads, archive.writes) + } + + after, items := sync("after") + if after.Status != domain.ObjectSyncSessionCompleted || after.Counts.Unchanged != int64(len(files)) || after.Counts.Denied != 0 { + t.Fatalf("sync after remediation did not complete: %#v", after) + } + for _, item := range items { + head, _ := data.GetObjectSyncHead(ctx, org, space.ID, item.TargetKey) + if item.Status != domain.ObjectSyncItemUnchanged || item.ExpectedBase.ETag != head.RemoteETag || head.MD5Sum != localMD5[item.RelativePath] || head.MD5Attestation == nil || head.MD5Attestation.BackfillID != job.ID { + t.Fatalf("item=%#v head=%#v", item, head) + } + var proof map[string]any + expect(device(http.MethodPost, "items/"+item.ID+"/cleanup-proof", nil, ""), http.StatusOK, &proof) + if proof["sessionId"] != after.ID || proof["md5sum"] != localMD5[item.RelativePath] || proof["md5Evidence"] != "coordinator_attestation" || proof["eTag"] != head.RemoteETag { + t.Fatalf("proof=%#v", proof) + } + release := map[string]any{"leaseId": proof["leaseId"], "leaseOwner": proof["leaseOwner"], "fencingToken": proof["fencingToken"]} + expect(device(http.MethodDelete, "items/"+item.ID+"/cleanup-proof", release, ""), http.StatusOK, nil) + } + + history, _ := data.GetObjectSyncSession(ctx, before.ID) + if history.Status != domain.ObjectSyncSessionPartial { + t.Fatal("remediation rewrote the closed partial session") + } + for _, item := range deniedItems { + stored, _ := data.GetObjectSyncItem(ctx, item.ID) + if stored.Status != domain.ObjectSyncItemDenied { + t.Fatal("remediation rewrote a closed denied item") + } + } +} diff --git a/internal/api/server.go b/internal/api/server.go index c3d09761..a5c104e1 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1529,6 +1529,8 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Get("/reconciles", instance.handleObjectSyncReconcilesList) router.Get("/reconciles/{id}", instance.handleObjectSyncReconcileGet) router.Post("/reconciles/{id}/retry", instance.handleObjectSyncReconcileRetry) + router.Post("/spaces/{id}/digest-backfills", instance.handleObjectSyncDigestBackfillCreate) + router.Get("/digest-backfills/{id}", instance.handleObjectSyncDigestBackfillGet) router.Get("/devices", instance.handleObjectSyncDevicesList) router.Get("/credentials", instance.handleObjectSyncCredentialsList) router.Post("/credentials", instance.handleObjectSyncCredentialCreate) diff --git a/internal/domain/object_sync.go b/internal/domain/object_sync.go index 6c53ac2e..250ace6a 100644 --- a/internal/domain/object_sync.go +++ b/internal/domain/object_sync.go @@ -252,10 +252,97 @@ type ObjectSyncObjectHeadRecord struct { VersionID string `json:"versionId,omitempty" bson:"versionId,omitempty"` SizeBytes int64 `json:"sizeBytes" bson:"sizeBytes"` MD5Sum string `json:"md5sum" bson:"md5sum"` - SessionID string `json:"sessionId" bson:"sessionId"` - ItemID string `json:"itemId" bson:"itemId"` - Drifted bool `json:"drifted" bson:"drifted"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + // LegacySHA256 is the retired v1 receipt still stored under "sha256". It is + // only an anchor for coordinator re-verification, never a content comparison. + LegacySHA256 string `json:"legacySha256,omitempty" bson:"sha256,omitempty"` + // MD5Attestation is set when the coordinator, not a device, derived MD5Sum by + // streaming this exact object version. Upload receipts leave it empty. + MD5Attestation *ObjectSyncDigestAttestation `json:"md5Attestation,omitempty" bson:"md5Attestation,omitempty"` + SessionID string `json:"sessionId" bson:"sessionId"` + ItemID string `json:"itemId" bson:"itemId"` + Drifted bool `json:"drifted" bson:"drifted"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +const ( + ObjectSyncDigestSourceCoordinatorStream = "coordinator_stream" + + ObjectSyncDigestAnchorHeadSHA256 = "head_sha256" + ObjectSyncDigestAnchorObjectSHA256 = "object_metadata_sha256" + ObjectSyncDigestAnchorStorageSHA256 = "storage_content_sha256" + ObjectSyncDigestAnchorObjectMD5 = "object_metadata_md5" + ObjectSyncDigestAnchorStorageMD5 = "storage_content_md5" + ObjectSyncDigestAnchorNoneAdminWaive = "none_admin_waived" +) + +// ObjectSyncDigestAttestation binds a coordinator-computed MD5 to one archived +// object version and records which independent receipts it was checked against. +type ObjectSyncDigestAttestation struct { + Source string `json:"source" bson:"source"` + BackfillID string `json:"backfillId" bson:"backfillId"` + ApprovedBy string `json:"approvedBy" bson:"approvedBy"` + CoordinatorID string `json:"coordinatorId" bson:"coordinatorId"` + ETag string `json:"etag" bson:"etag"` + VersionID string `json:"versionId,omitempty" bson:"versionId,omitempty"` + SizeBytes int64 `json:"sizeBytes" bson:"sizeBytes"` + MD5Sum string `json:"md5sum" bson:"md5sum"` + SHA256 string `json:"sha256" bson:"sha256"` + Anchors []string `json:"anchors" bson:"anchors"` + PreviousMD5Sum string `json:"previousMd5sum,omitempty" bson:"previousMd5sum,omitempty"` + AttestedAt string `json:"attestedAt" bson:"attestedAt"` +} + +// ObjectSyncDigestBackfillRecord is an administrator-approved coordinator job +// that re-verifies archived heads lacking a trusted MD5 receipt. +type ObjectSyncDigestBackfillRecord struct { + ID string `json:"id" bson:"id"` + OperationID string `json:"operationId" bson:"operationId"` + OrganizationID string `json:"organizationId" bson:"organizationId"` + SyncSpaceID string `json:"syncSpaceId" bson:"syncSpaceId"` + Status string `json:"status" bson:"status"` + Reason string `json:"reason" bson:"reason"` + SessionIDs []string `json:"sessionIds,omitempty" bson:"sessionIds,omitempty"` + KeyPrefix string `json:"keyPrefix,omitempty" bson:"keyPrefix,omitempty"` + WholeSpace bool `json:"wholeSpace" bson:"wholeSpace"` + AllowUnanchored bool `json:"allowUnanchored" bson:"allowUnanchored"` + MaxObjects int64 `json:"maxObjects" bson:"maxObjects"` + MaxBytes int64 `json:"maxBytes" bson:"maxBytes"` + Cursor string `json:"-" bson:"cursor,omitempty"` + Counts ObjectSyncDigestBackfillCounts `json:"counts" bson:"counts"` + BudgetExhausted bool `json:"budgetExhausted" bson:"budgetExhausted"` + Exceptions []ObjectSyncDigestBackfillException `json:"exceptions,omitempty" bson:"exceptions,omitempty"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + CreatedBy string `json:"createdBy" bson:"createdBy"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + CompletedAt string `json:"completedAt,omitempty" bson:"completedAt,omitempty"` + CoordinatorOwner string `json:"-" bson:"coordinatorOwner,omitempty"` + CoordinatorLeaseUntil time.Time `json:"-" bson:"coordinatorLeaseUntil,omitempty"` +} + +// Counts are at-least-once: a resumed job may revisit keys from its last page. +type ObjectSyncDigestBackfillCounts struct { + Scanned int64 `json:"scanned" bson:"scanned"` + Attested int64 `json:"attested" bson:"attested"` + AlreadyAttested int64 `json:"alreadyAttested" bson:"alreadyAttested"` + NotLegacy int64 `json:"notLegacy" bson:"notLegacy"` + Unanchored int64 `json:"unanchored" bson:"unanchored"` + Mismatched int64 `json:"mismatched" bson:"mismatched"` + Drifted int64 `json:"drifted" bson:"drifted"` + Busy int64 `json:"busy" bson:"busy"` + ReadFailed int64 `json:"readFailed" bson:"readFailed"` + ObjectsRead int64 `json:"objectsRead" bson:"objectsRead"` + BytesRead int64 `json:"bytesRead" bson:"bytesRead"` + ItemsMatched int64 `json:"itemsMatched" bson:"itemsMatched"` + ItemsConflicting int64 `json:"itemsConflicting" bson:"itemsConflicting"` +} + +type ObjectSyncDigestBackfillException struct { + Key string `json:"key" bson:"key"` + Outcome string `json:"outcome" bson:"outcome"` + Detail string `json:"detail,omitempty" bson:"detail,omitempty"` + At string `json:"at" bson:"at"` } type ObjectSyncVersionRecord struct { diff --git a/internal/objectsync/bos.go b/internal/objectsync/bos.go index 9cec5964..68561a29 100644 --- a/internal/objectsync/bos.go +++ b/internal/objectsync/bos.go @@ -24,6 +24,10 @@ type ObjectMetadata struct { SizeBytes int64 LastModified string UserMeta map[string]string + // Storage-side digests recorded at upload time, when the provider kept them. + ContentMD5 string + ContentSHA256 string + StorageClass string } type Publisher interface { @@ -34,6 +38,13 @@ type Publisher interface { Delete(context.Context, domain.StorageConnectionRecord, string, string) error } +// ObjectReader streams byte ranges of one archived object for coordinator-side +// digest verification. Every range is conditional on the archived ETag, so a +// replacement makes the read fail instead of mixing object versions. +type ObjectReader interface { + GetObjectRange(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, ifMatchETag string, offset, length int64) (io.ReadCloser, ObjectMetadata, error) +} + type BOSPublisher struct { GlobalEndpoint string } @@ -77,20 +88,51 @@ func (publisher BOSPublisher) Head(ctx context.Context, connection domain.Storag if err != nil { return ObjectMetadata{}, bosclient.Error(ctx, "head", err) } + return objectMetadataFromBOS(result.ObjectMeta), nil +} + +func objectMetadataFromBOS(meta bosapi.ObjectMeta) ObjectMetadata { // HTTP header names are case-insensitive; the SDK returns canonicalized // metadata suffixes while publication identities use lowercase keys. - userMeta := make(map[string]string, len(result.UserMeta)) - for key, value := range result.UserMeta { + userMeta := make(map[string]string, len(meta.UserMeta)) + for key, value := range meta.UserMeta { userMeta[strings.ToLower(strings.TrimSpace(key))] = value } return ObjectMetadata{ - Exists: true, - ETag: strings.Trim(result.ETag, `"`), - VersionID: strings.TrimSpace(result.VersionId), - SizeBytes: result.ContentLength, - LastModified: strings.TrimSpace(result.LastModified), - UserMeta: userMeta, - }, nil + Exists: true, + ETag: strings.Trim(meta.ETag, `"`), + VersionID: strings.TrimSpace(meta.VersionId), + SizeBytes: meta.ContentLength, + LastModified: strings.TrimSpace(meta.LastModified), + UserMeta: userMeta, + ContentMD5: strings.TrimSpace(meta.ContentMD5), + ContentSHA256: strings.TrimSpace(meta.ContentSha256), + StorageClass: strings.TrimSpace(meta.StorageClass), + } +} + +// GetObjectRange issues a conditional ranged GET. The SDK client bounds each +// request, so callers read large objects as a sequence of bounded ranges. +func (publisher BOSPublisher) GetObjectRange(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, ifMatchETag string, offset, length int64) (io.ReadCloser, ObjectMetadata, error) { + if err := connection.CheckStoragePath(bucket, key, false); err != nil { + return nil, ObjectMetadata{}, err + } + ifMatchETag = strings.Trim(strings.TrimSpace(ifMatchETag), `"`) + if ifMatchETag == "" || offset < 0 || length <= 0 { + return nil, ObjectMetadata{}, fmt.Errorf("digest verification requires the archived ETag and a non-empty range") + } + client, err := publisher.client(connection) + if err != nil { + return nil, ObjectMetadata{}, err + } + result, err := client.GetObjectWithArgs(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), &bosapi.GetObjectArgs{ + Ranges: []int64{offset, offset + length - 1}, + IfMatch: ifMatchETag, + }) + if err != nil { + return nil, ObjectMetadata{}, bosclient.Error(ctx, "get range", err) + } + return result.Body, objectMetadataFromBOS(result.ObjectMeta), nil } func multipartCopyPartSize(sizeBytes int64) int64 { diff --git a/internal/objectsync/coordinator.go b/internal/objectsync/coordinator.go index d783da8d..1607c9f1 100644 --- a/internal/objectsync/coordinator.go +++ b/internal/objectsync/coordinator.go @@ -46,6 +46,7 @@ type Coordinator struct { reconciled atomic.Int64 released atomic.Int64 sealed atomic.Int64 + backfilled atomic.Int64 lastCycle atomic.Int64 lastOK atomic.Int64 durationMS atomic.Int64 @@ -157,6 +158,7 @@ type Metrics struct { Reconciled int64 Released int64 Sealed int64 + Backfilled int64 LastCycle int64 LastOK int64 DurationMS int64 @@ -184,6 +186,7 @@ func (coordinator *Coordinator) Metrics() Metrics { Reconciled: coordinator.reconciled.Load(), Released: coordinator.released.Load(), Sealed: coordinator.sealed.Load(), + Backfilled: coordinator.backfilled.Load(), LastCycle: coordinator.lastCycle.Load(), LastOK: coordinator.lastOK.Load(), DurationMS: coordinator.durationMS.Load(), @@ -204,6 +207,7 @@ type ProcessResult struct { Reconciled int `json:"reconciled"` Released int `json:"released"` Sealed int `json:"sealed"` + Backfilled int `json:"backfilled"` } func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, error) { @@ -225,6 +229,7 @@ func (coordinator *Coordinator) ProcessOnce(ctx context.Context) (ProcessResult, result.Reconciled += value.result.Reconciled result.Released += value.result.Released result.Sealed += value.result.Sealed + result.Backfilled += value.result.Backfilled if value.err != nil { failures = append(failures, value.err) } diff --git a/internal/objectsync/coordinator_workers.go b/internal/objectsync/coordinator_workers.go index 342dfa88..1c849949 100644 --- a/internal/objectsync/coordinator_workers.go +++ b/internal/objectsync/coordinator_workers.go @@ -18,13 +18,14 @@ const ( phaseReconcile phaseRelease phaseAudit + phaseDigestBackfill phaseCount commitConcurrency = 4 cleanupConcurrency = 2 planConcurrency = 16 ) -var phaseNames = [phaseCount]string{"planning", "commit", "cleanup", "reconcile", "release", "audit"} +var phaseNames = [phaseCount]string{"planning", "commit", "cleanup", "reconcile", "release", "audit", "digest_backfill"} var errClaimLost = errors.New("coordinator claim or object lease lost") type phaseCounters struct{ cycles, errors, completed, itemErrors, inFlight, durationMS, totalDurationMS atomic.Int64 } @@ -57,7 +58,7 @@ func (coordinator *Coordinator) measureStage(phase int, work func() (ProcessResu metrics.cycles.Add(1) metrics.durationMS.Store(elapsed) metrics.totalDurationMS.Add(elapsed) - completed := result.Planned + result.Committed + result.Cleaned + result.Reconciled + result.Released + result.Sealed + completed := result.Planned + result.Committed + result.Cleaned + result.Reconciled + result.Released + result.Sealed + result.Backfilled metrics.completed.Add(int64(completed)) coordinator.planned.Add(int64(result.Planned)) coordinator.committed.Add(int64(result.Committed)) @@ -65,6 +66,7 @@ func (coordinator *Coordinator) measureStage(phase int, work func() (ProcessResu coordinator.reconciled.Add(int64(result.Reconciled)) coordinator.released.Add(int64(result.Released)) coordinator.sealed.Add(int64(result.Sealed)) + coordinator.backfilled.Add(int64(result.Backfilled)) if err != nil && !errors.Is(err, context.Canceled) { metrics.errors.Add(1) coordinator.errors.Add(1) @@ -174,6 +176,32 @@ func (coordinator *Coordinator) processStage(ctx context.Context, phase int) (Pr result.Sealed = count return result, err } + case phaseDigestBackfill: + job, err := coordinator.syncStore.ClaimObjectSyncDigestBackfill(ctx, coordinator.owner, claimTTL) + if err != nil || job == nil { + return result, err + } + token, id := job.CoordinatorOwner, job.ID + err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) { + return coordinator.syncStore.RenewObjectSyncDigestBackfillClaim(callCtx, id, token, claimTTL) + }, func(callCtx context.Context) error { return coordinator.runDigestBackfill(callCtx, job) }) + if err != nil { + if !errors.Is(err, errClaimLost) && !errors.Is(err, context.Canceled) { + job.Status = "failed" + job.Error = err.Error() + job.UpdatedAt = nowISO() + job.CompletedAt = job.UpdatedAt + job.CoordinatorOwner = "" + job.CoordinatorLeaseUntil = time.Time{} + saved, saveErr := coordinator.syncStore.SaveClaimedObjectSyncDigestBackfill(ctx, *job, token) + if !saved && saveErr == nil { + saveErr = errClaimLost + } + err = errors.Join(err, saveErr) + } + return result, err + } + result.Backfilled = 1 } return result, nil } diff --git a/internal/objectsync/digest_backfill.go b/internal/objectsync/digest_backfill.go new file mode 100644 index 00000000..e70dcdd8 --- /dev/null +++ b/internal/objectsync/digest_backfill.go @@ -0,0 +1,704 @@ +package objectsync + +import ( + "context" + "crypto/md5" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "slices" + "strconv" + "strings" + "time" + + "droboticflow/internal/bosclient" + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +// Legacy v1 archives carry SHA-256 receipts that protocol v2 deliberately does +// not compare. A digest backfill reads the archived object itself, so the MD5 it +// records is cloud evidence; a device's local MD5 is never written to a head. +const ( + digestBackfillPageSize = 1000 + digestBackfillMaxExceptions = 200 + digestBackfillLeaseTTL = 5 * time.Minute + legacyDeniedAction = "deny_archived_hash_change" +) + +// Each ranged GET must finish inside the SDK client's request bound even on a +// slow link, so large objects are read as many conditional ranges. +var digestBackfillRangeBytes = int64(64 << 20) + +const ( + backfillAttested = "attested" + backfillAlreadyAttested = "already_attested" + backfillNotLegacy = "not_legacy" + backfillUnanchored = "unanchored" + backfillAnchorConflict = "anchor_conflict" + backfillContentMismatch = "content_mismatch" + backfillDrifted = "drifted" + backfillUnbound = "unbound_receipt" + backfillBusy = "busy" + backfillUnreadableClass = "storage_class_unreadable" + backfillReadFailed = "read_failed" + backfillBudget = "budget_exhausted" +) + +var errBackfillBudget = errors.New("digest backfill budget exhausted") + +// ValidMD5Hex accepts only the lowercase form stored by protocol v2 receipts. +func ValidMD5Hex(value string) bool { + return len(value) == 2*md5.Size && isLowerHex(value) +} + +func validSHA256Hex(value string) bool { + return len(value) == 2*sha256.Size && isLowerHex(value) +} + +func isLowerHex(value string) bool { + for _, character := range value { + if (character < '0' || character > '9') && (character < 'a' || character > 'f') { + return false + } + } + return true +} + +// ObjectLeaseResourceHash names the per-object lease shared by uploads, +// cleanup proofs and digest verification. +func ObjectLeaseResourceHash(space domain.ObjectSyncSpaceRecord, key string) string { + sum := sha256.Sum256([]byte(space.ConnectionID + "\x00" + space.Bucket + "\x00" + key)) + return hex.EncodeToString(sum[:]) +} + +// AttestationBindsHead reports whether head.MD5Sum was streamed by the +// coordinator from exactly the object identity the head still names. +func AttestationBindsHead(head domain.ObjectSyncObjectHeadRecord) bool { + attestation := head.MD5Attestation + return attestation != nil && + attestation.Source == domain.ObjectSyncDigestSourceCoordinatorStream && + ValidMD5Hex(attestation.MD5Sum) && attestation.MD5Sum == head.MD5Sum && + attestation.ETag != "" && strings.Trim(attestation.ETag, `"`) == strings.Trim(head.RemoteETag, `"`) && + attestation.VersionID == head.VersionID && attestation.SizeBytes == head.SizeBytes && + len(attestation.Anchors) > 0 +} + +type backfillVerdict struct { + outcome string + detail string + read bool + bytes int64 + head *domain.ObjectSyncObjectHeadRecord +} + +type digestBackfillRun struct { + coordinator *Coordinator + job *domain.ObjectSyncDigestBackfillRecord + token string + space *domain.ObjectSyncSpaceRecord + connection *domain.StorageConnectionRecord + reader ObjectReader +} + +func (coordinator *Coordinator) runDigestBackfill(ctx context.Context, job *domain.ObjectSyncDigestBackfillRecord) error { + token := job.CoordinatorOwner + if token == "" { + return errClaimLost + } + reader, ok := coordinator.publisher.(ObjectReader) + if !ok { + return fmt.Errorf("object storage cannot stream archived objects for digest verification") + } + space, err := coordinator.syncStore.GetObjectSyncSpace(ctx, job.SyncSpaceID) + if err != nil || space == nil || space.OrganizationID != job.OrganizationID { + return fmt.Errorf("sync space is unavailable") + } + connection, err := coordinator.store.GetStorageConnection(ctx, space.ConnectionID) + if err != nil || connection == nil { + return fmt.Errorf("storage connection is unavailable") + } + if job.StartedAt == "" { + job.StartedAt = nowISO() + } + run := &digestBackfillRun{coordinator: coordinator, job: job, token: token, space: space, connection: connection, reader: reader} + if len(job.SessionIDs) > 0 { + err = run.sessions(ctx) + } else { + err = run.heads(ctx) + } + if err != nil && !errors.Is(err, errBackfillBudget) { + return err + } + job.Status = "completed" + job.CompletedAt = nowISO() + job.UpdatedAt = job.CompletedAt + job.CoordinatorOwner = "" + job.CoordinatorLeaseUntil = time.Time{} + saved, err := coordinator.syncStore.SaveClaimedObjectSyncDigestBackfill(ctx, *job, token) + if err != nil { + return err + } + if !saved { + return errClaimLost + } + coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{ + OrganizationID: job.OrganizationID, + ActorType: "system", + ActorID: coordinator.owner, + Action: "space.digest_backfill.completed", + Target: map[string]any{"backfillId": job.ID, "operationId": job.OperationID, "spaceId": job.SyncSpaceID}, + After: map[string]any{ + "scanned": job.Counts.Scanned, "attested": job.Counts.Attested, "alreadyAttested": job.Counts.AlreadyAttested, + "unanchored": job.Counts.Unanchored, "mismatched": job.Counts.Mismatched, "drifted": job.Counts.Drifted, + "busy": job.Counts.Busy, "readFailed": job.Counts.ReadFailed, "bytesRead": job.Counts.BytesRead, + "itemsMatched": job.Counts.ItemsMatched, "itemsConflicting": job.Counts.ItemsConflicting, + "budgetExhausted": job.BudgetExhausted, + }, + Result: "success", + }) + return nil +} + +func (run *digestBackfillRun) checkpoint(ctx context.Context) error { + run.job.UpdatedAt = nowISO() + run.job.CoordinatorLeaseUntil = time.Now().UTC().Add(claimTTL) + saved, err := run.coordinator.syncStore.SaveClaimedObjectSyncDigestBackfill(ctx, *run.job, run.token) + if err != nil { + return err + } + if !saved { + return errClaimLost + } + return nil +} + +// Session scope only visits archives that devices still hold and were denied +// for, which keeps the read volume proportional to what cleanup can recover. +func (run *digestBackfillRun) sessions(ctx context.Context) error { + start, cursor := parseSessionBackfillCursor(run.job.Cursor) + seen := map[string]*domain.ObjectSyncObjectHeadRecord{} + for index := start; index < len(run.job.SessionIDs); index++ { + sessionID := run.job.SessionIDs[index] + session, err := run.coordinator.syncStore.GetObjectSyncSession(ctx, sessionID) + if err != nil { + return err + } + if session == nil || session.OrganizationID != run.job.OrganizationID || session.SyncSpaceID != run.job.SyncSpaceID { + run.addException("", "session_unavailable", "session "+sessionID+" is missing or belongs to another space") + } else { + for { + page, err := run.coordinator.syncStore.ListObjectSyncItems(ctx, domain.ObjectSyncListQuery{ + SessionID: session.ID, Status: string(domain.ObjectSyncItemDenied), Action: legacyDeniedAction, + Cursor: cursor, Limit: digestBackfillPageSize, + }) + if err != nil { + return err + } + for _, item := range page.Items { + head, visited := seen[item.TargetKey] + if !visited { + verdict, err := run.process(ctx, item.TargetKey) + if err != nil { + return err + } + head = verdict.head + seen[item.TargetKey] = head + } + run.compareDeniedItem(item, head) + } + if page.NextCursor == "" { + break + } + cursor = page.NextCursor + run.job.Cursor = formatSessionBackfillCursor(index, cursor) + if err := run.checkpoint(ctx); err != nil { + return err + } + } + } + cursor = "" + run.job.Cursor = formatSessionBackfillCursor(index+1, "") + if err := run.checkpoint(ctx); err != nil { + return err + } + } + return nil +} + +func (run *digestBackfillRun) heads(ctx context.Context) error { + prefix := "" + if !run.job.WholeSpace { + prefix = spaceKeyPrefix(*run.space, run.job.KeyPrefix) + } + cursor := run.job.Cursor + for { + page, err := run.coordinator.syncStore.ListObjectSyncHeads(ctx, run.job.OrganizationID, run.space.ID, cursor, digestBackfillPageSize) + if err != nil { + return err + } + for _, head := range page.Items { + if prefix != "" && !strings.HasPrefix(head.Key, prefix) { + if head.Key > prefix { + // Heads are key-ordered, so nothing later can match. + return nil + } + continue + } + if ValidMD5Hex(head.MD5Sum) && head.MD5Attestation == nil { + run.job.Counts.Scanned++ + run.job.Counts.NotLegacy++ + continue + } + if _, err := run.process(ctx, head.Key); err != nil { + return err + } + } + if page.NextCursor == "" { + return nil + } + cursor = page.NextCursor + run.job.Cursor = cursor + if err := run.checkpoint(ctx); err != nil { + return err + } + } +} + +func (run *digestBackfillRun) process(ctx context.Context, key string) (backfillVerdict, error) { + verdict, err := run.coordinator.backfillKey(ctx, run, key) + if err != nil { + return verdict, err + } + counts := &run.job.Counts + counts.BytesRead += verdict.bytes + if verdict.read { + counts.ObjectsRead++ + } + if verdict.outcome == backfillBudget { + run.job.BudgetExhausted = true + run.addException(key, verdict.outcome, verdict.detail) + return verdict, errBackfillBudget + } + counts.Scanned++ + switch verdict.outcome { + case backfillAttested: + counts.Attested++ + case backfillAlreadyAttested: + counts.AlreadyAttested++ + case backfillNotLegacy: + counts.NotLegacy++ + case backfillUnanchored: + counts.Unanchored++ + case backfillAnchorConflict, backfillContentMismatch: + counts.Mismatched++ + case backfillDrifted, backfillUnbound: + counts.Drifted++ + case backfillBusy: + counts.Busy++ + default: + counts.ReadFailed++ + } + switch verdict.outcome { + case backfillAttested, backfillAlreadyAttested, backfillNotLegacy: + default: + run.addException(key, verdict.outcome, verdict.detail) + } + if verdict.read { + if err := run.checkpoint(ctx); err != nil { + return verdict, err + } + } + return verdict, nil +} + +// compareDeniedItem previews recovery without touching the closed session: +// the device's next sync plans the same comparison against the trusted head. +func (run *digestBackfillRun) compareDeniedItem(item domain.ObjectSyncItemRecord, head *domain.ObjectSyncObjectHeadRecord) { + if head == nil || !ValidMD5Hex(head.MD5Sum) { + return + } + if item.MD5Sum == head.MD5Sum && item.SizeBytes == head.SizeBytes { + run.job.Counts.ItemsMatched++ + } else { + run.job.Counts.ItemsConflicting++ + } +} + +func (run *digestBackfillRun) addException(key, outcome, detail string) { + if len(run.job.Exceptions) >= digestBackfillMaxExceptions { + return + } + run.job.Exceptions = append(run.job.Exceptions, domain.ObjectSyncDigestBackfillException{Key: key, Outcome: outcome, Detail: detail, At: nowISO()}) +} + +func (coordinator *Coordinator) backfillKey(ctx context.Context, run *digestBackfillRun, key string) (backfillVerdict, error) { + head, err := coordinator.syncStore.GetObjectSyncHead(ctx, run.job.OrganizationID, run.space.ID, key) + if err != nil { + return backfillVerdict{}, err + } + if head == nil { + return backfillVerdict{outcome: backfillNotLegacy, detail: "no archived receipt"}, nil + } + if ValidMD5Hex(head.MD5Sum) { + if !AttestationBindsHead(*head) { + return backfillVerdict{outcome: backfillNotLegacy, head: head}, nil + } + // A crash between the head write and its audit event is repaired here; + // the event key is derived from the attestation, so replays are no-ops. + if err := coordinator.appendAttestationAudit(ctx, *head); err != nil { + return backfillVerdict{}, err + } + return backfillVerdict{outcome: backfillAlreadyAttested, head: head}, nil + } + if head.MD5Attestation != nil { + return backfillVerdict{outcome: backfillDrifted, detail: "archived receipt has an attestation that no longer matches it"}, nil + } + if head.Drifted { + return backfillVerdict{outcome: backfillDrifted, detail: "reconciliation marked the archived object as drifted"}, nil + } + if strings.Trim(head.RemoteETag, `"`) == "" || head.SizeBytes < 0 { + return backfillVerdict{outcome: backfillUnbound, detail: "archived receipt has no object ETag to bind a digest to"}, nil + } + ownerSession := "digest:" + run.job.ID + ownerItem := ownerSession + ":" + domain.NewID("verify") + lease, acquired, err := coordinator.syncStore.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{ + OrganizationID: run.job.OrganizationID, + ResourceHash: ObjectLeaseResourceHash(*run.space, key), + ResourceType: "object", + ConnectionID: run.space.ConnectionID, + Bucket: run.space.Bucket, + KeyOrPrefix: key, + OwnerSessionID: ownerSession, + OwnerItemID: ownerItem, + }, digestBackfillLeaseTTL) + if errors.Is(err, store.ErrObjectSyncLeaseHeld) || (err == nil && !acquired) { + return backfillVerdict{outcome: backfillBusy, detail: "another operation holds the object key"}, nil + } + if err != nil { + return backfillVerdict{}, err + } + defer func() { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if err := coordinator.syncStore.ReleaseObjectSyncLease(releaseCtx, lease.ID, ownerSession, ownerItem, lease.FencingToken); err != nil { + coordinator.recordAuxiliaryFailure(phaseDigestBackfill, "release verification lease", err) + } + }() + var verdict backfillVerdict + err = coordinator.withRenewingClaim(ctx, func(callCtx context.Context) (bool, error) { + return coordinator.syncStore.RenewObjectSyncLease(callCtx, lease.ID, ownerSession, ownerItem, lease.FencingToken, digestBackfillLeaseTTL) + }, func(callCtx context.Context) error { + var verifyErr error + verdict, verifyErr = coordinator.verifyAndAttest(callCtx, run, *head) + return verifyErr + }) + if err != nil && errors.Is(err, errClaimLost) && ctx.Err() == nil { + verdict.outcome = backfillBusy + verdict.detail = "object lease was lost during verification" + verdict.head = nil + return verdict, nil + } + return verdict, err +} + +func (coordinator *Coordinator) verifyAndAttest(ctx context.Context, run *digestBackfillRun, head domain.ObjectSyncObjectHeadRecord) (backfillVerdict, error) { + bucket := run.space.Bucket + remote, err := coordinator.publisher.Head(ctx, *run.connection, bucket, head.Key) + if err != nil { + if ctx.Err() != nil { + return backfillVerdict{}, ctx.Err() + } + return backfillVerdict{outcome: backfillReadFailed, detail: "head: " + err.Error()}, nil + } + if detail := archivedIdentityChange(head, remote); detail != "" { + return backfillVerdict{outcome: backfillDrifted, detail: detail}, nil + } + if strings.EqualFold(remote.StorageClass, "ARCHIVE") { + return backfillVerdict{outcome: backfillUnreadableClass, detail: "archive storage class must be restored before verification"}, nil + } + anchors, conflict := collectDigestAnchors(head, remote) + if conflict != "" { + coordinator.recordDigestRejection(ctx, run, head, backfillAnchorConflict, conflict) + return backfillVerdict{outcome: backfillAnchorConflict, detail: conflict}, nil + } + if anchors.sha256 == "" && !run.job.AllowUnanchored { + return backfillVerdict{outcome: backfillUnanchored, detail: "no independent SHA-256 receipt; an administrator must explicitly allow unanchored verification"}, nil + } + if run.job.MaxObjects > 0 && run.job.Counts.ObjectsRead >= run.job.MaxObjects { + return backfillVerdict{outcome: backfillBudget, detail: "maxObjects reached before this object"}, nil + } + if run.job.MaxBytes > 0 && run.job.Counts.BytesRead+head.SizeBytes > run.job.MaxBytes { + return backfillVerdict{outcome: backfillBudget, detail: fmt.Sprintf("reading %d bytes would exceed maxBytes", head.SizeBytes)}, nil + } + md5sum, sha256sum, read, err := coordinator.streamArchivedDigests(ctx, run, head) + verdict := backfillVerdict{read: true, bytes: read} + if err != nil { + if ctx.Err() != nil { + return verdict, err + } + verdict.outcome, verdict.detail = backfillReadFailed, err.Error() + if IsPreconditionFailure(err) || errors.Is(err, errArchivedIdentityChanged) { + verdict.outcome = backfillDrifted + } + return verdict, nil + } + if anchors.sha256 != "" && sha256sum != anchors.sha256 { + verdict.outcome = backfillContentMismatch + verdict.detail = fmt.Sprintf("archived bytes have SHA-256 %s but the legacy receipt records %s", sha256sum, anchors.sha256) + coordinator.recordDigestRejection(ctx, run, head, verdict.outcome, verdict.detail) + return verdict, nil + } + if anchors.md5 != "" && md5sum != anchors.md5 { + verdict.outcome = backfillContentMismatch + verdict.detail = fmt.Sprintf("archived bytes have MD5 %s but the object records %s", md5sum, anchors.md5) + coordinator.recordDigestRejection(ctx, run, head, verdict.outcome, verdict.detail) + return verdict, nil + } + after, err := coordinator.publisher.Head(ctx, *run.connection, bucket, head.Key) + if err != nil { + if ctx.Err() != nil { + return verdict, ctx.Err() + } + verdict.outcome, verdict.detail = backfillReadFailed, "head after read: "+err.Error() + return verdict, nil + } + if detail := archivedIdentityChange(head, after); detail != "" { + verdict.outcome, verdict.detail = backfillDrifted, detail + return verdict, nil + } + names := anchors.names + if len(names) == 0 { + names = []string{domain.ObjectSyncDigestAnchorNoneAdminWaive} + } + attestation := domain.ObjectSyncDigestAttestation{ + Source: domain.ObjectSyncDigestSourceCoordinatorStream, + BackfillID: run.job.ID, + ApprovedBy: run.job.CreatedBy, + CoordinatorID: coordinator.owner, + ETag: strings.Trim(head.RemoteETag, `"`), + VersionID: head.VersionID, + SizeBytes: head.SizeBytes, + MD5Sum: md5sum, + SHA256: sha256sum, + Anchors: names, + PreviousMD5Sum: head.MD5Sum, + AttestedAt: nowISO(), + } + // The shape that stored SHA-256 in md5sum keeps it as the legacy anchor. + preservedLegacy := "" + if head.LegacySHA256 == "" && validSHA256Hex(strings.ToLower(head.MD5Sum)) { + preservedLegacy = strings.ToLower(head.MD5Sum) + } + if err := ctx.Err(); err != nil { + return verdict, err + } + written, err := coordinator.syncStore.AttestObjectSyncHeadMD5(ctx, head, attestation, preservedLegacy) + if err != nil { + return verdict, err + } + if !written { + verdict.outcome, verdict.detail = backfillBusy, "archived receipt changed during verification" + return verdict, nil + } + attested := head + attested.MD5Sum = md5sum + attested.MD5Attestation = &attestation + if preservedLegacy != "" { + attested.LegacySHA256 = preservedLegacy + } + if err := coordinator.appendAttestationAudit(ctx, attested); err != nil { + return verdict, err + } + verdict.outcome = backfillAttested + verdict.head = &attested + return verdict, nil +} + +var errArchivedIdentityChanged = errors.New("archived object identity changed during verification") + +func archivedIdentityChange(head domain.ObjectSyncObjectHeadRecord, remote ObjectMetadata) string { + switch { + case !remote.Exists: + return "archived object is missing" + case strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`): + return fmt.Sprintf("archived ETag is %q, receipt records %q", remote.ETag, head.RemoteETag) + case remote.SizeBytes != head.SizeBytes: + return fmt.Sprintf("archived size is %d, receipt records %d", remote.SizeBytes, head.SizeBytes) + case remote.VersionID != head.VersionID: + return fmt.Sprintf("archived version is %q, receipt records %q", remote.VersionID, head.VersionID) + } + return "" +} + +// The stream reads bounded ranges, each conditional on the archived ETag, and +// hashes MD5 and SHA-256 in one pass so the legacy receipt can vouch for it. +func (coordinator *Coordinator) streamArchivedDigests(ctx context.Context, run *digestBackfillRun, head domain.ObjectSyncObjectHeadRecord) (string, string, int64, error) { + md5Hash, sha256Hash := md5.New(), sha256.New() + writer := io.MultiWriter(md5Hash, sha256Hash) + var offset int64 + for offset < head.SizeBytes { + length := min(digestBackfillRangeBytes, head.SizeBytes-offset) + read, err := coordinator.readArchivedRange(ctx, run, head, offset, length, writer) + offset += read + if err != nil { + return "", "", offset, err + } + } + return hex.EncodeToString(md5Hash.Sum(nil)), hex.EncodeToString(sha256Hash.Sum(nil)), offset, nil +} + +func (coordinator *Coordinator) readArchivedRange(ctx context.Context, run *digestBackfillRun, head domain.ObjectSyncObjectHeadRecord, offset, length int64, writer io.Writer) (int64, error) { + rangeCtx, cancel := context.WithTimeout(ctx, bosclient.DataTimeout) + defer cancel() + body, meta, err := run.reader.GetObjectRange(rangeCtx, *run.connection, run.space.Bucket, head.Key, head.RemoteETag, offset, length) + if err != nil { + return 0, err + } + defer body.Close() + if strings.Trim(meta.ETag, `"`) != strings.Trim(head.RemoteETag, `"`) || (meta.VersionID != "" && meta.VersionID != head.VersionID) { + return 0, errArchivedIdentityChanged + } + read, err := io.Copy(writer, io.LimitReader(body, length+1)) + if err != nil { + return read, fmt.Errorf("read range at %d: %w", offset, err) + } + if read != length { + return read, fmt.Errorf("range at %d returned %d bytes, expected %d", offset, read, length) + } + return read, nil +} + +type digestAnchors struct { + sha256 string + md5 string + names []string +} + +// Every independent receipt that is present must agree. Receipts written by +// drosync fail closed when malformed; provider headers in unknown form are +// ignored because they are optional extras, not drosync evidence. +func collectDigestAnchors(head domain.ObjectSyncObjectHeadRecord, remote ObjectMetadata) (digestAnchors, string) { + var anchors digestAnchors + var conflicts []string + add := func(slot *string, name, value string) { + if *slot != "" && *slot != value { + conflicts = append(conflicts, fmt.Sprintf("%s=%s disagrees with %s", name, value, *slot)) + return + } + *slot = value + if !slices.Contains(anchors.names, name) { + anchors.names = append(anchors.names, name) + } + } + addSHA256 := func(name, raw string, strict bool) { + value := strings.ToLower(strings.TrimSpace(raw)) + if value == "" { + return + } + if !validSHA256Hex(value) { + if strict { + conflicts = append(conflicts, fmt.Sprintf("%s is malformed", name)) + } + return + } + add(&anchors.sha256, name, value) + } + addMD5 := func(name, raw string, strict bool) { + value := strings.TrimSpace(raw) + if value == "" { + return + } + if decoded, err := base64.StdEncoding.DecodeString(value); !strict && err == nil && len(decoded) == md5.Size { + value = hex.EncodeToString(decoded) + } + value = strings.ToLower(value) + if !ValidMD5Hex(value) { + if strict { + conflicts = append(conflicts, fmt.Sprintf("%s is malformed", name)) + } + return + } + add(&anchors.md5, name, value) + } + addSHA256(domain.ObjectSyncDigestAnchorHeadSHA256, head.LegacySHA256, true) + if len(head.MD5Sum) == 2*sha256.Size { + addSHA256(domain.ObjectSyncDigestAnchorHeadSHA256, head.MD5Sum, true) + } + addSHA256(domain.ObjectSyncDigestAnchorObjectSHA256, remote.UserMeta["drosync-sha256"], true) + addSHA256(domain.ObjectSyncDigestAnchorStorageSHA256, remote.ContentSHA256, false) + addMD5(domain.ObjectSyncDigestAnchorObjectMD5, remote.UserMeta["drosync-md5sum"], true) + addMD5(domain.ObjectSyncDigestAnchorStorageMD5, remote.ContentMD5, false) + slices.Sort(anchors.names) + return anchors, strings.Join(conflicts, "; ") +} + +func attestationEventKey(head domain.ObjectSyncObjectHeadRecord) string { + attestation := head.MD5Attestation + sum := sha256.Sum256([]byte(strings.Join([]string{head.OrganizationID, head.SyncSpaceID, head.Key, attestation.ETag, attestation.VersionID, strconv.FormatInt(attestation.SizeBytes, 10), attestation.MD5Sum}, "\x00"))) + return "head.md5.attested:" + hex.EncodeToString(sum[:16]) +} + +// The event is built only from the persisted attestation, so whichever job +// repairs a missing event writes exactly the same, deduplicated record. +func (coordinator *Coordinator) appendAttestationAudit(ctx context.Context, head domain.ObjectSyncObjectHeadRecord) error { + attestation := head.MD5Attestation + return AppendAudit(ctx, coordinator.syncStore, domain.ObjectSyncAuditEvent{ + EventKey: attestationEventKey(head), + OrganizationID: head.OrganizationID, + ActorType: "system", + ActorID: attestation.CoordinatorID, + SessionID: head.SessionID, + ItemID: head.ItemID, + Action: "head.md5.attested", + Target: map[string]any{"spaceId": head.SyncSpaceID, "key": head.Key, "backfillId": attestation.BackfillID}, + Before: map[string]any{"md5sum": attestation.PreviousMD5Sum, "legacySha256": head.LegacySHA256}, + After: map[string]any{ + "md5sum": attestation.MD5Sum, "sha256": attestation.SHA256, "etag": attestation.ETag, + "versionId": attestation.VersionID, "sizeBytes": attestation.SizeBytes, "platformVersion": head.PlatformVersion, + "anchors": attestation.Anchors, "approvedBy": attestation.ApprovedBy, "source": attestation.Source, + }, + Result: "success", + }) +} + +func (coordinator *Coordinator) recordDigestRejection(ctx context.Context, run *digestBackfillRun, head domain.ObjectSyncObjectHeadRecord, outcome, detail string) { + sum := sha256.Sum256([]byte(strings.Join([]string{run.job.ID, head.Key, head.RemoteETag, outcome}, "\x00"))) + coordinator.recordSystemAudit(ctx, domain.ObjectSyncAuditEvent{ + EventKey: "head.md5.rejected:" + hex.EncodeToString(sum[:16]), + OrganizationID: head.OrganizationID, + ActorType: "system", + ActorID: coordinator.owner, + SessionID: head.SessionID, + ItemID: head.ItemID, + Action: "head.md5.rejected", + Target: map[string]any{"spaceId": head.SyncSpaceID, "key": head.Key, "backfillId": run.job.ID}, + Before: map[string]any{"md5sum": head.MD5Sum, "legacySha256": head.LegacySHA256, "etag": head.RemoteETag}, + Result: "denied", + Reason: outcome + ": " + detail, + }) +} + +func spaceKeyPrefix(space domain.ObjectSyncSpaceRecord, relativePrefix string) string { + root := strings.Trim(strings.TrimSpace(space.RootPrefix), "/") + if root == "" { + return relativePrefix + } + return root + "/" + relativePrefix +} + +func formatSessionBackfillCursor(index int, cursor string) string { + return strconv.Itoa(index) + "\x00" + cursor +} + +func parseSessionBackfillCursor(value string) (int, string) { + indexText, cursor, found := strings.Cut(value, "\x00") + index, err := strconv.Atoi(indexText) + if !found || err != nil || index < 0 { + return 0, "" + } + return index, cursor +} diff --git a/internal/objectsync/digest_backfill_test.go b/internal/objectsync/digest_backfill_test.go new file mode 100644 index 00000000..880fc188 --- /dev/null +++ b/internal/objectsync/digest_backfill_test.go @@ -0,0 +1,614 @@ +package objectsync + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "slices" + "strings" + "sync" + "testing" + "time" + + "droboticflow/internal/domain" + "droboticflow/internal/store" + + "github.com/baidubce/bce-sdk-go/bce" +) + +type rangeRead struct { + key, ifMatch string + offset, length int64 +} + +// streamingPublisher serves archived bytes and records every read and write so +// tests can prove verification is read-only and conditional on the ETag. +type streamingPublisher struct { + *fakePublisher + mu sync.Mutex + content map[string][]byte + reads []rangeRead + writes int + beforeGet func(key string, offset int64) + shortRead bool +} + +func (publisher *streamingPublisher) GetObjectRange(_ context.Context, _ domain.StorageConnectionRecord, bucket, key, ifMatch string, offset, length int64) (io.ReadCloser, ObjectMetadata, error) { + if publisher.beforeGet != nil { + publisher.beforeGet(key, offset) + } + publisher.mu.Lock() + publisher.reads = append(publisher.reads, rangeRead{key, ifMatch, offset, length}) + publisher.mu.Unlock() + meta, err := publisher.Head(context.Background(), domain.StorageConnectionRecord{}, bucket, key) + if err != nil || !meta.Exists || meta.ETag != ifMatch { + return nil, ObjectMetadata{}, &bce.BceServiceError{StatusCode: 412} + } + payload := publisher.content[bucket+"/"+key] + end := min(offset+length, int64(len(payload))) + if publisher.shortRead { + end-- + } + return io.NopCloser(bytes.NewReader(payload[offset:end])), meta, nil +} + +func (publisher *streamingPublisher) PublishConditional(ctx context.Context, connection domain.StorageConnectionRecord, bucket, source, target string, size int64, sourceETag, ifMatch, ifNoneMatch string, metadata map[string]string) (ObjectMetadata, error) { + publisher.writes++ + return publisher.fakePublisher.PublishConditional(ctx, connection, bucket, source, target, size, sourceETag, ifMatch, ifNoneMatch, metadata) +} + +func (publisher *streamingPublisher) PutInternal(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string, reader io.Reader, size int64, contentType string, metadata map[string]string) (ObjectMetadata, error) { + publisher.writes++ + return publisher.fakePublisher.PutInternal(ctx, connection, bucket, key, reader, size, contentType, metadata) +} + +func (publisher *streamingPublisher) Delete(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) error { + publisher.writes++ + return publisher.fakePublisher.Delete(ctx, connection, bucket, key) +} + +func (publisher *streamingPublisher) readsOf(key string) int { + publisher.mu.Lock() + defer publisher.mu.Unlock() + count := 0 + for _, read := range publisher.reads { + if read.key == key { + count++ + } + } + return count +} + +func digestHex(payload []byte) (string, string) { + md5Sum, sha256Sum := md5.Sum(payload), sha256.Sum256(payload) + return hex.EncodeToString(md5Sum[:]), hex.EncodeToString(sha256Sum[:]) +} + +func backfillFixture(t *testing.T) (*Coordinator, *store.MemoryStore, *streamingPublisher) { + t.Helper() + coordinator, data, fake := coordinatorFixture(t) + publisher := &streamingPublisher{fakePublisher: fake, content: map[string][]byte{}} + coordinator.publisher = publisher + return coordinator, data, publisher +} + +type legacyShape struct { + md5sum, legacySHA256, metadataSHA256 string +} + +// seedLegacyArchive stores a v1-era head whose receipt has no MD5 together +// with the archived object bytes the coordinator will verify. +func seedLegacyArchive(t *testing.T, data *store.MemoryStore, publisher *streamingPublisher, key string, payload []byte, shape legacyShape) domain.ObjectSyncObjectHeadRecord { + t.Helper() + head := domain.ObjectSyncObjectHeadRecord{ + OrganizationID: domain.DefaultOrganizationID, SyncSpaceID: "space", Key: key, PlatformVersion: 1, + RemoteETag: "legacy-" + key, SizeBytes: int64(len(payload)), MD5Sum: shape.md5sum, LegacySHA256: shape.legacySHA256, + SessionID: "v1-session", ItemID: "v1-item-" + key, UpdatedAt: "2026-09-01T00:00:00Z", + } + if err := data.UpsertObjectSyncHead(t.Context(), head); err != nil { + t.Fatal(err) + } + meta := map[string]string{} + if shape.metadataSHA256 != "" { + meta["drosync-sha256"] = shape.metadataSHA256 + } + publisher.objects["bucket/"+key] = ObjectMetadata{Exists: true, ETag: head.RemoteETag, SizeBytes: head.SizeBytes, UserMeta: meta} + publisher.content["bucket/"+key] = payload + return head +} + +func seedBackfillJob(t *testing.T, data *store.MemoryStore, job domain.ObjectSyncDigestBackfillRecord) { + t.Helper() + job.OrganizationID = domain.DefaultOrganizationID + job.SyncSpaceID = "space" + job.Status = "pending" + job.CreatedBy = "admin-user" + job.Reason = "verify v1 archives" + if job.ID == "" { + job.ID = "backfill" + } + job.OperationID = "op-" + job.ID + job.CreatedAt = nowISO() + if err := data.UpsertObjectSyncDigestBackfill(t.Context(), job); err != nil { + t.Fatal(err) + } +} + +func runBackfill(t *testing.T, coordinator *Coordinator, data *store.MemoryStore, id string) *domain.ObjectSyncDigestBackfillRecord { + t.Helper() + if _, err := coordinator.executeStage(t.Context(), phaseDigestBackfill); err != nil { + t.Fatal(err) + } + job, err := data.GetObjectSyncDigestBackfill(t.Context(), id) + if err != nil || job == nil { + t.Fatalf("job %s: %v", id, err) + } + return job +} + +func auditActions(t *testing.T, data *store.MemoryStore, action string) []domain.ObjectSyncAuditEvent { + t.Helper() + events, err := data.ListObjectSyncAudit(t.Context(), domain.ObjectSyncListQuery{OrganizationID: domain.DefaultOrganizationID, Limit: 1000}) + if err != nil { + t.Fatal(err) + } + var matched []domain.ObjectSyncAuditEvent + for _, event := range events { + if event.Action == action { + matched = append(matched, event) + } + } + return matched +} + +func TestDigestBackfillAttestsLegacyShapesFromArchivedBytes(t *testing.T) { + payload := []byte("legacy mcap payload archived by drosync v1") + md5sum, sha := digestHex(payload) + for name, test := range map[string]struct { + shape legacyShape + anchors []string + wantLegacy string + }{ + "retired-sha256-field": {legacyShape{legacySHA256: sha, metadataSHA256: sha}, []string{domain.ObjectSyncDigestAnchorHeadSHA256, domain.ObjectSyncDigestAnchorObjectSHA256}, sha}, + "sha256-in-md5-field": {legacyShape{md5sum: sha}, []string{domain.ObjectSyncDigestAnchorHeadSHA256}, sha}, + "missing-digest": {legacyShape{metadataSHA256: sha}, []string{domain.ObjectSyncDigestAnchorObjectSHA256}, ""}, + "malformed-digest": {legacyShape{md5sum: "bad", metadataSHA256: strings.ToUpper(sha)}, []string{domain.ObjectSyncDigestAnchorObjectSHA256}, ""}, + } { + t.Run(name, func(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + key := "incoming/node-04/episode.mcap" + before := seedLegacyArchive(t, data, publisher, key, payload, test.shape) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + job := runBackfill(t, coordinator, data, "backfill") + if job.Status != "completed" || job.Counts.Attested != 1 || job.Counts.BytesRead != int64(len(payload)) || job.Counts.ObjectsRead != 1 || len(job.Exceptions) != 0 || job.CoordinatorOwner != "" { + t.Fatalf("job=%#v", job) + } + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + attestation := head.MD5Attestation + if head.MD5Sum != md5sum || attestation == nil || !AttestationBindsHead(*head) { + t.Fatalf("head not attested from archived bytes: %#v", head) + } + if attestation.SHA256 != sha || attestation.BackfillID != "backfill" || attestation.ApprovedBy != "admin-user" || attestation.CoordinatorID != "test-owner" || + attestation.PreviousMD5Sum != before.MD5Sum || !slices.Equal(attestation.Anchors, test.anchors) { + t.Fatalf("attestation=%#v", attestation) + } + if head.RemoteETag != before.RemoteETag || head.PlatformVersion != before.PlatformVersion || head.SessionID != before.SessionID || head.ItemID != before.ItemID || head.LegacySHA256 != test.wantLegacy { + t.Fatalf("receipt identity changed: %#v", head) + } + if publisher.writes != 0 || len(publisher.publishedKeys) != 0 { + t.Fatal("verification wrote to object storage") + } + for _, read := range publisher.reads { + if read.ifMatch != before.RemoteETag { + t.Fatalf("unconditional read: %#v", read) + } + } + events := auditActions(t, data, "head.md5.attested") + if len(events) != 1 || events[0].After["md5sum"] != md5sum || events[0].After["sha256"] != sha || events[0].Target["backfillId"] != "backfill" || events[0].ActorType != "system" { + t.Fatalf("attestation audit=%#v", events) + } + if len(auditActions(t, data, "space.digest_backfill.completed")) != 1 { + t.Fatal("job completion not audited") + } + space, _ := data.GetObjectSyncSpace(t.Context(), "space") + if _, acquired, err := data.AcquireObjectSyncLease(t.Context(), domain.ObjectSyncLeaseRecord{ResourceHash: ObjectLeaseResourceHash(*space, key), OwnerSessionID: "next", OwnerItemID: "next"}, time.Minute); err != nil || !acquired { + t.Fatal("verification lease was not released") + } + }) + } +} + +func TestDigestBackfillReadsLargeObjectsInConditionalRanges(t *testing.T) { + previous := digestBackfillRangeBytes + digestBackfillRangeBytes = 7 + t.Cleanup(func() { digestBackfillRangeBytes = previous }) + coordinator, data, publisher := backfillFixture(t) + payload := []byte("twenty bytes of data") + md5sum, sha := digestHex(payload) + seedLegacyArchive(t, data, publisher, "incoming/big.bin", payload, legacyShape{legacySHA256: sha}) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + if job := runBackfill(t, coordinator, data, "backfill"); job.Counts.Attested != 1 { + t.Fatalf("job=%#v", job) + } + want := []rangeRead{{"incoming/big.bin", "legacy-incoming/big.bin", 0, 7}, {"incoming/big.bin", "legacy-incoming/big.bin", 7, 7}, {"incoming/big.bin", "legacy-incoming/big.bin", 14, 6}} + if !slices.Equal(publisher.reads, want) { + t.Fatalf("reads=%#v", publisher.reads) + } + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", "incoming/big.bin") + if head.MD5Sum != md5sum { + t.Fatal("ranged digest differs from whole-object MD5") + } +} + +func TestDigestBackfillRefusesBytesThatDifferFromLegacyReceipt(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + _, otherSHA := digestHex([]byte("what v1 originally uploaded")) + key := "incoming/corrupted.mcap" + before := seedLegacyArchive(t, data, publisher, key, []byte("bytes found in the archive today"), legacyShape{legacySHA256: otherSHA}) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + job := runBackfill(t, coordinator, data, "backfill") + if job.Status != "completed" || job.Counts.Mismatched != 1 || job.Counts.Attested != 0 || len(job.Exceptions) != 1 || job.Exceptions[0].Outcome != backfillContentMismatch || job.Exceptions[0].Key != key { + t.Fatalf("job=%#v", job) + } + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if head.MD5Sum != before.MD5Sum || head.MD5Attestation != nil { + t.Fatalf("mismatched archive was attested: %#v", head) + } + rejected := auditActions(t, data, "head.md5.rejected") + if len(rejected) != 1 || rejected[0].Result != "denied" || !strings.Contains(rejected[0].Reason, backfillContentMismatch) { + t.Fatalf("rejection audit=%#v", rejected) + } +} + +func TestDigestBackfillNeedsAnIndependentAnchorUnlessWaived(t *testing.T) { + for _, waived := range []bool{false, true} { + t.Run(map[bool]string{false: "refused", true: "waived"}[waived], func(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + payload := []byte("no receipt survived for this archive") + md5sum, _ := digestHex(payload) + seedLegacyArchive(t, data, publisher, "incoming/orphan.bin", payload, legacyShape{}) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true, AllowUnanchored: waived}) + job := runBackfill(t, coordinator, data, "backfill") + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", "incoming/orphan.bin") + if !waived { + if job.Counts.Unanchored != 1 || len(publisher.reads) != 0 || head.MD5Attestation != nil || job.Exceptions[0].Outcome != backfillUnanchored { + t.Fatalf("unanchored archive was read or attested: job=%#v head=%#v", job, head) + } + return + } + if job.Counts.Attested != 1 || head.MD5Sum != md5sum || !slices.Equal(head.MD5Attestation.Anchors, []string{domain.ObjectSyncDigestAnchorNoneAdminWaive}) { + t.Fatalf("waived verification: job=%#v head=%#v", job, head) + } + }) + } +} + +func TestDigestBackfillFailsClosedOnDisagreeingReceipts(t *testing.T) { + payload := []byte("archived bytes") + md5sum, sha := digestHex(payload) + _, otherSHA := digestHex([]byte("other bytes")) + for name, mutate := range map[string]func(*domain.ObjectSyncObjectHeadRecord, *ObjectMetadata){ + "head-vs-metadata-sha256": func(head *domain.ObjectSyncObjectHeadRecord, remote *ObjectMetadata) { + head.LegacySHA256 = sha + remote.UserMeta["drosync-sha256"] = otherSHA + }, + "malformed-metadata-sha256": func(head *domain.ObjectSyncObjectHeadRecord, remote *ObjectMetadata) { + head.LegacySHA256 = sha + remote.UserMeta["drosync-sha256"] = "not-a-digest" + }, + "storage-md5-disagrees": func(head *domain.ObjectSyncObjectHeadRecord, remote *ObjectMetadata) { + head.LegacySHA256 = sha + remote.UserMeta["drosync-md5sum"] = strings.Repeat("0", 32) + remote.ContentMD5 = md5sum + }, + } { + t.Run(name, func(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + key := "incoming/conflicting.bin" + head := seedLegacyArchive(t, data, publisher, key, payload, legacyShape{}) + remote := publisher.objects["bucket/"+key] + mutate(&head, &remote) + publisher.objects["bucket/"+key] = remote + data.UpsertObjectSyncHead(t.Context(), head) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + job := runBackfill(t, coordinator, data, "backfill") + stored, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if job.Counts.Mismatched != 1 || stored.MD5Attestation != nil || len(publisher.reads) != 0 { + t.Fatalf("conflicting receipts were trusted: job=%#v reads=%d", job, len(publisher.reads)) + } + }) + } +} + +type headCASStore struct { + *store.MemoryStore + beforeAttest func() +} + +func (data *headCASStore) AttestObjectSyncHeadMD5(ctx context.Context, expected domain.ObjectSyncObjectHeadRecord, attestation domain.ObjectSyncDigestAttestation, legacy string) (bool, error) { + if data.beforeAttest != nil { + data.beforeAttest() + } + return data.MemoryStore.AttestObjectSyncHeadMD5(ctx, expected, attestation, legacy) +} + +func TestDigestBackfillLeavesChangedOrBusyArchivesUntouched(t *testing.T) { + payload := []byte("archived bytes for identity checks") + _, sha := digestHex(payload) + key := "incoming/identity.bin" + for name, test := range map[string]struct { + outcome string + reads bool + arrange func(*testing.T, *Coordinator, *store.MemoryStore, *streamingPublisher) + }{ + "etag-changed": {backfillDrifted, false, func(_ *testing.T, _ *Coordinator, _ *store.MemoryStore, p *streamingPublisher) { + remote := p.objects["bucket/"+key] + remote.ETag = "rewritten" + p.objects["bucket/"+key] = remote + }}, + "size-changed": {backfillDrifted, false, func(_ *testing.T, _ *Coordinator, _ *store.MemoryStore, p *streamingPublisher) { + remote := p.objects["bucket/"+key] + remote.SizeBytes++ + p.objects["bucket/"+key] = remote + }}, + "version-changed": {backfillDrifted, false, func(_ *testing.T, _ *Coordinator, _ *store.MemoryStore, p *streamingPublisher) { + remote := p.objects["bucket/"+key] + remote.VersionID = "v9" + p.objects["bucket/"+key] = remote + }}, + "object-missing": {backfillDrifted, false, func(_ *testing.T, _ *Coordinator, _ *store.MemoryStore, p *streamingPublisher) { + delete(p.objects, "bucket/"+key) + }}, + "reconcile-marked-drift": {backfillDrifted, false, func(t *testing.T, _ *Coordinator, data *store.MemoryStore, _ *streamingPublisher) { + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + head.Drifted = true + data.UpsertObjectSyncHead(t.Context(), *head) + }}, + "receipt-without-etag": {backfillUnbound, false, func(t *testing.T, _ *Coordinator, data *store.MemoryStore, _ *streamingPublisher) { + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + head.RemoteETag = "" + data.UpsertObjectSyncHead(t.Context(), *head) + }}, + "archive-storage-class": {backfillUnreadableClass, false, func(_ *testing.T, _ *Coordinator, _ *store.MemoryStore, p *streamingPublisher) { + remote := p.objects["bucket/"+key] + remote.StorageClass = "ARCHIVE" + p.objects["bucket/"+key] = remote + }}, + "key-leased-by-upload": {backfillBusy, false, func(t *testing.T, _ *Coordinator, data *store.MemoryStore, _ *streamingPublisher) { + space, _ := data.GetObjectSyncSpace(t.Context(), "space") + if _, ok, err := data.AcquireObjectSyncLease(t.Context(), domain.ObjectSyncLeaseRecord{ResourceHash: ObjectLeaseResourceHash(*space, key), OwnerSessionID: "upload", OwnerItemID: "upload-item"}, time.Minute); err != nil || !ok { + t.Fatal(err) + } + }}, + "replaced-during-read": {backfillDrifted, true, func(_ *testing.T, _ *Coordinator, _ *store.MemoryStore, p *streamingPublisher) { + p.beforeGet = func(string, int64) { + p.fakePublisher.mu.Lock() + remote := p.objects["bucket/"+key] + remote.ETag = "replaced" + p.objects["bucket/"+key] = remote + p.fakePublisher.mu.Unlock() + } + }}, + "short-read": {backfillReadFailed, true, func(_ *testing.T, _ *Coordinator, _ *store.MemoryStore, p *streamingPublisher) { + p.shortRead = true + }}, + "receipt-changed-before-write": {backfillBusy, true, func(t *testing.T, coordinator *Coordinator, data *store.MemoryStore, _ *streamingPublisher) { + wrapped := &headCASStore{MemoryStore: data} + wrapped.beforeAttest = func() { + head, _ := data.GetObjectSyncHead(context.Background(), domain.DefaultOrganizationID, "space", key) + head.PlatformVersion++ + data.UpsertObjectSyncHead(context.Background(), *head) + } + coordinator.syncStore = wrapped + }}, + } { + t.Run(name, func(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + seedLegacyArchive(t, data, publisher, key, payload, legacyShape{legacySHA256: sha}) + test.arrange(t, coordinator, data, publisher) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + job := runBackfill(t, coordinator, data, "backfill") + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if job.Status != "completed" || job.Counts.Attested != 0 || len(job.Exceptions) != 1 || job.Exceptions[0].Outcome != test.outcome { + t.Fatalf("job=%#v", job) + } + if head.MD5Attestation != nil || ValidMD5Hex(head.MD5Sum) { + t.Fatalf("unsafe attestation: %#v", head) + } + if (len(publisher.reads) > 0) != test.reads || publisher.writes != 0 { + t.Fatalf("reads=%d writes=%d", len(publisher.reads), publisher.writes) + } + }) + } +} + +type flakyAuditStore struct { + *store.MemoryStore + failAttestationAudit bool +} + +func (data *flakyAuditStore) AppendObjectSyncAudit(ctx context.Context, event domain.ObjectSyncAuditEvent) error { + if data.failAttestationAudit && event.Action == "head.md5.attested" { + return errors.New("audit store unavailable") + } + return data.MemoryStore.AppendObjectSyncAudit(ctx, event) +} + +func TestDigestBackfillIsIdempotentAndRepairsMissingAudit(t *testing.T) { + _, data, publisher := backfillFixture(t) + wrapped := &flakyAuditStore{MemoryStore: data, failAttestationAudit: true} + coordinator, err := NewCoordinator(wrapped, publisher, "test-owner") + if err != nil { + t.Fatal(err) + } + payload := []byte("idempotent archive") + md5sum, sha := digestHex(payload) + key := "incoming/once.bin" + seedLegacyArchive(t, data, publisher, key, payload, legacyShape{legacySHA256: sha}) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{ID: "first", WholeSpace: true}) + if _, err := coordinator.executeStage(t.Context(), phaseDigestBackfill); err == nil { + t.Fatal("missing audit was not surfaced") + } + first, _ := data.GetObjectSyncDigestBackfill(t.Context(), "first") + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if first.Status != "failed" || head.MD5Sum != md5sum || len(auditActions(t, data, "head.md5.attested")) != 0 { + t.Fatalf("first=%#v head=%#v", first, head) + } + wrapped.failAttestationAudit = false + for _, id := range []string{"second", "third"} { + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{ID: id, WholeSpace: true}) + job := runBackfill(t, coordinator, data, id) + if job.Status != "completed" || job.Counts.AlreadyAttested != 1 || job.Counts.Attested != 0 || job.Counts.BytesRead != 0 { + t.Fatalf("%s=%#v", id, job) + } + } + events := auditActions(t, data, "head.md5.attested") + if len(events) != 1 || events[0].Target["backfillId"] != "first" || publisher.readsOf(key) != 1 { + t.Fatalf("events=%#v reads=%d", events, publisher.readsOf(key)) + } + after, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if after.MD5Attestation.AttestedAt != head.MD5Attestation.AttestedAt { + t.Fatal("rerun rewrote the attestation") + } +} + +func TestDigestBackfillStopsAtItsBudget(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + payload := []byte("0123456789") + _, sha := digestHex(payload) + for _, key := range []string{"incoming/a.bin", "incoming/b.bin"} { + seedLegacyArchive(t, data, publisher, key, payload, legacyShape{legacySHA256: sha}) + } + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{ID: "objects", WholeSpace: true, MaxObjects: 1}) + job := runBackfill(t, coordinator, data, "objects") + if job.Status != "completed" || !job.BudgetExhausted || job.Counts.Attested != 1 || job.Counts.ObjectsRead != 1 || publisher.readsOf("incoming/b.bin") != 0 { + t.Fatalf("object budget: %#v", job) + } + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{ID: "bytes", WholeSpace: true, MaxBytes: 9}) + job = runBackfill(t, coordinator, data, "bytes") + if !job.BudgetExhausted || job.Counts.AlreadyAttested != 1 || job.Counts.Attested != 0 || publisher.readsOf("incoming/b.bin") != 0 { + t.Fatalf("byte budget: %#v", job) + } +} + +func TestDigestBackfillSessionScopeUsesCloudBytesNotDeviceDigests(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + same, changed := []byte("episode bytes kept on the device"), []byte("daily report archived in september") + sameMD5, sameSHA := digestHex(same) + changedMD5, changedSHA := digestHex(changed) + deviceChangedMD5, _ := digestHex([]byte("daily report regenerated in october")) + seedLegacyArchive(t, data, publisher, "incoming/episode.mcap", same, legacyShape{legacySHA256: sameSHA}) + seedLegacyArchive(t, data, publisher, "incoming/report.xlsx", changed, legacyShape{legacySHA256: changedSHA}) + for _, sessionID := range []string{"partial-1", "partial-2"} { + if err := data.UpsertObjectSyncSession(t.Context(), domain.ObjectSyncSessionRecord{ID: sessionID, OrganizationID: domain.DefaultOrganizationID, SyncSpaceID: "space", Status: domain.ObjectSyncSessionPartial}); err != nil { + t.Fatal(err) + } + if err := data.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{ + {ID: sessionID + "-episode", OrganizationID: domain.DefaultOrganizationID, SessionID: sessionID, TargetKey: "incoming/episode.mcap", SizeBytes: int64(len(same)), MD5Sum: sameMD5, Status: domain.ObjectSyncItemDenied, Action: legacyDeniedAction}, + {ID: sessionID + "-report", OrganizationID: domain.DefaultOrganizationID, SessionID: sessionID, TargetKey: "incoming/report.xlsx", SizeBytes: int64(len(changed)), MD5Sum: deviceChangedMD5, Status: domain.ObjectSyncItemDenied, Action: legacyDeniedAction}, + {ID: sessionID + "-uploaded", OrganizationID: domain.DefaultOrganizationID, SessionID: sessionID, TargetKey: "incoming/new.bin", Status: domain.ObjectSyncItemCommitted, Action: "create"}, + }); err != nil { + t.Fatal(err) + } + } + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{SessionIDs: []string{"partial-1", "partial-2"}}) + job := runBackfill(t, coordinator, data, "backfill") + if job.Counts.Attested != 2 || job.Counts.Scanned != 2 || job.Counts.ItemsMatched != 2 || job.Counts.ItemsConflicting != 2 || job.Cursor != formatSessionBackfillCursor(2, "") { + t.Fatalf("job=%#v", job) + } + if publisher.readsOf("incoming/episode.mcap") != 1 || publisher.readsOf("incoming/report.xlsx") != 1 || publisher.readsOf("incoming/new.bin") != 0 { + t.Fatalf("keys were not read exactly once: %#v", publisher.reads) + } + report, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", "incoming/report.xlsx") + if report.MD5Sum != changedMD5 || report.MD5Sum == deviceChangedMD5 { + t.Fatalf("head recorded a device digest instead of the archived bytes: %#v", report) + } + for _, id := range []string{"partial-1-episode", "partial-2-report"} { + item, _ := data.GetObjectSyncItem(t.Context(), id) + if item.Status != domain.ObjectSyncItemDenied { + t.Fatal("closed session items were rewritten") + } + } + episode, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", "incoming/episode.mcap") + if episode.MD5Sum != sameMD5 { + t.Fatal("matching archive not attested") + } +} + +func TestDigestBackfillPrefixScopeOnlyVisitsItsRange(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + payload := []byte("prefix payload") + _, sha := digestHex(payload) + keys := []string{"incoming/a/1.bin", "incoming/b/1.bin", "incoming/b/2.bin", "incoming/ba/1.bin", "incoming/c/1.bin"} + for _, key := range keys { + seedLegacyArchive(t, data, publisher, key, payload, legacyShape{legacySHA256: sha}) + } + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{KeyPrefix: "b/"}) + job := runBackfill(t, coordinator, data, "backfill") + if job.Counts.Attested != 2 || job.Counts.Scanned != 2 { + t.Fatalf("job=%#v", job) + } + for _, key := range keys { + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if inPrefix := strings.HasPrefix(key, "incoming/b/"); (head.MD5Attestation != nil) != inPrefix || (publisher.readsOf(key) > 0) != inPrefix { + t.Fatalf("%s attested=%v reads=%d", key, head.MD5Attestation != nil, publisher.readsOf(key)) + } + } +} + +func TestDigestBackfillClaimLossStopsBeforeCompletion(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + payload := []byte("claim loss payload") + _, sha := digestHex(payload) + seedLegacyArchive(t, data, publisher, "incoming/claimed.bin", payload, legacyShape{legacySHA256: sha}) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + publisher.beforeGet = func(string, int64) { + job, _ := data.GetObjectSyncDigestBackfill(context.Background(), "backfill") + job.CoordinatorOwner = "replacement-coordinator" + data.UpsertObjectSyncDigestBackfill(context.Background(), *job) + } + if _, err := coordinator.executeStage(t.Context(), phaseDigestBackfill); !errors.Is(err, errClaimLost) { + t.Fatalf("claim loss not detected: %v", err) + } + job, _ := data.GetObjectSyncDigestBackfill(t.Context(), "backfill") + if job.Status != "running" || job.CoordinatorOwner != "replacement-coordinator" || job.CompletedAt != "" { + t.Fatalf("stale coordinator completed the job: %#v", job) + } +} + +func TestPlanningComparesLaterSyncsWithAttestedMD5(t *testing.T) { + coordinator, data, publisher := backfillFixture(t) + payload := []byte("planned against an attested receipt") + md5sum, sha := digestHex(payload) + key := "planned/000" + seedLegacyArchive(t, data, publisher, key, payload, legacyShape{legacySHA256: sha}) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + runBackfill(t, coordinator, data, "backfill") + for name, local := range map[string]string{"same": md5sum, "rewritten": strings.Repeat("c", 32)} { + t.Run(name, func(t *testing.T) { + seedPlanning(t, data, 1) + item, _ := data.GetObjectSyncItem(t.Context(), "plan-000") + item.MD5Sum = local + item.SizeBytes = int64(len(payload)) + data.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{*item}) + if _, err := coordinator.executeStage(t.Context(), phasePlanning); err != nil { + t.Fatal(err) + } + planned, _ := data.GetObjectSyncItem(t.Context(), "plan-000") + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if local == md5sum { + if planned.Status != domain.ObjectSyncItemUnchanged || planned.ExpectedBase.ETag != head.RemoteETag || planned.ExpectedBase.PlatformVersion != head.PlatformVersion { + t.Fatalf("matching file not unchanged: %#v", planned) + } + return + } + if planned.Status != domain.ObjectSyncItemDenied || !strings.Contains(planned.LastError, "同名文件内容冲突") { + t.Fatalf("rewritten file not reported as a content conflict: %#v", planned) + } + }) + } +} diff --git a/internal/store/memory.go b/internal/store/memory.go index cfeb658b..4742df97 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -46,6 +46,7 @@ type MemoryStore struct { objectSyncReleases map[string]domain.ObjectSyncReleaseRecord objectSyncClientReleases map[string]domain.ObjectSyncClientReleaseRecord objectSyncReconciles map[string]domain.ObjectSyncReconcileRecord + objectSyncBackfills map[string]domain.ObjectSyncDigestBackfillRecord objectSyncIdempotency map[string]domain.ObjectSyncIdempotencyRecord objectSyncRateLimits map[string]domain.ObjectSyncRateLimitRecord databaseCollections map[string][]map[string]any @@ -86,6 +87,7 @@ func NewMemoryStore() *MemoryStore { objectSyncReleases: map[string]domain.ObjectSyncReleaseRecord{}, objectSyncClientReleases: map[string]domain.ObjectSyncClientReleaseRecord{}, objectSyncReconciles: map[string]domain.ObjectSyncReconcileRecord{}, + objectSyncBackfills: map[string]domain.ObjectSyncDigestBackfillRecord{}, objectSyncIdempotency: map[string]domain.ObjectSyncIdempotencyRecord{}, objectSyncRateLimits: map[string]domain.ObjectSyncRateLimitRecord{}, databaseCollections: map[string][]map[string]any{}, diff --git a/internal/store/mongo.go b/internal/store/mongo.go index a11e07f7..65e42d83 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -51,6 +51,7 @@ type MongoStore struct { objectSyncReleases *mongo.Collection objectSyncClientReleases *mongo.Collection objectSyncReconciles *mongo.Collection + objectSyncBackfills *mongo.Collection objectSyncIdempotency *mongo.Collection objectSyncRateLimits *mongo.Collection settings *mongo.Collection @@ -91,6 +92,7 @@ func NewMongoStore(db *mongo.Database) *MongoStore { objectSyncReleases: db.Collection("objectSyncReleases"), objectSyncClientReleases: db.Collection("objectSyncClientReleases"), objectSyncReconciles: db.Collection("objectSyncReconciles"), + objectSyncBackfills: db.Collection("objectSyncDigestBackfills"), objectSyncIdempotency: db.Collection("objectSyncIdempotency"), objectSyncRateLimits: db.Collection("objectSyncRateLimits"), settings: db.Collection("settings"), @@ -236,6 +238,7 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin store.objectSyncVersions, store.objectSyncAuditEvents, store.objectSyncReleases, + store.objectSyncBackfills, store.objectSyncClientReleases, store.settings, } { diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 2bb65396..4ad03603 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -226,6 +226,12 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { uniqueIndex("operationId_unique", bson.D{{Key: "operationId", Value: 1}}), index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), }, + store.objectSyncBackfills: { + uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), + uniqueIndex("operationId_unique", bson.D{{Key: "operationId", Value: 1}}), + index("status_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "createdAt", Value: 1}}), + index("organizationId_syncSpaceId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "syncSpaceId", Value: 1}, {Key: "createdAt", Value: -1}}), + }, store.objectSyncIdempotency: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), ttlIndex("expiresAt_ttl", bson.D{{Key: "expiresAt", Value: 1}}, 0), diff --git a/internal/store/object_sync.go b/internal/store/object_sync.go index 1216f865..3755099c 100644 --- a/internal/store/object_sync.go +++ b/internal/store/object_sync.go @@ -81,8 +81,15 @@ type ObjectSyncStore interface { GetObjectSyncHead(context.Context, string, string, string) (*domain.ObjectSyncObjectHeadRecord, error) UpsertObjectSyncHead(context.Context, domain.ObjectSyncObjectHeadRecord) error ListObjectSyncHeads(context.Context, string, string, string, int) (domain.ObjectSyncHeadPage, error) + AttestObjectSyncHeadMD5(context.Context, domain.ObjectSyncObjectHeadRecord, domain.ObjectSyncDigestAttestation, string) (bool, error) InsertObjectSyncVersion(context.Context, domain.ObjectSyncVersionRecord) error + UpsertObjectSyncDigestBackfill(context.Context, domain.ObjectSyncDigestBackfillRecord) error + GetObjectSyncDigestBackfill(context.Context, string) (*domain.ObjectSyncDigestBackfillRecord, error) + ClaimObjectSyncDigestBackfill(context.Context, string, time.Duration) (*domain.ObjectSyncDigestBackfillRecord, error) + RenewObjectSyncDigestBackfillClaim(context.Context, string, string, time.Duration) (bool, error) + SaveClaimedObjectSyncDigestBackfill(context.Context, domain.ObjectSyncDigestBackfillRecord, string) (bool, error) + AppendObjectSyncAudit(context.Context, domain.ObjectSyncAuditEvent) error ListObjectSyncAudit(context.Context, domain.ObjectSyncListQuery) ([]domain.ObjectSyncAuditEvent, error) ListObjectSyncAuditRange(context.Context, string, string, string, domain.ObjectSyncListQuery) (domain.ObjectSyncAuditPage, error) @@ -611,14 +618,14 @@ func (store *MemoryStore) GetObjectSyncHead(_ context.Context, organizationID, s if !ok { return nil, nil } - copy := item + copy := cloneObjectSyncHead(item) return ©, nil } func (store *MemoryStore) UpsertObjectSyncHead(_ context.Context, item domain.ObjectSyncObjectHeadRecord) error { store.mu.Lock() defer store.mu.Unlock() - store.objectSyncHeads[objectSyncHeadKey(item.OrganizationID, item.SyncSpaceID, item.Key)] = item + store.objectSyncHeads[objectSyncHeadKey(item.OrganizationID, item.SyncSpaceID, item.Key)] = cloneObjectSyncHead(item) return nil } @@ -631,7 +638,7 @@ func (store *MemoryStore) ListObjectSyncHeads(_ context.Context, organizationID, if item.OrganizationID != normalizedOrganizationID(organizationID) || item.SyncSpaceID != syncSpaceID || (decodedCursor != "" && item.Key <= decodedCursor) { continue } - items = append(items, item) + items = append(items, cloneObjectSyncHead(item)) } sort.Slice(items, func(i, j int) bool { return items[i].Key < items[j].Key }) page := domain.ObjectSyncHeadPage{Items: items} @@ -1186,10 +1193,27 @@ func (store *MongoStore) UpsertObjectSyncHead(ctx context.Context, item domain.O "organizationId": item.OrganizationID, "syncSpaceId": item.SyncSpaceID, "key": item.Key, - }, bson.M{"$set": item}, options.Update().SetUpsert(true)) + }, objectSyncHeadReplacement(item), options.Update().SetUpsert(true)) return err } +// A head write replaces the logical receipt. Optional legacy anchors and +// attestations that the new receipt does not carry must not survive it. +func objectSyncHeadReplacement(item domain.ObjectSyncObjectHeadRecord) bson.M { + update := bson.M{"$set": item} + unset := bson.M{} + if item.LegacySHA256 == "" { + unset["sha256"] = "" + } + if item.MD5Attestation == nil { + unset["md5Attestation"] = "" + } + if len(unset) > 0 { + update["$unset"] = unset + } + return update +} + func (store *MongoStore) ListObjectSyncHeads(ctx context.Context, organizationID, syncSpaceID, cursor string, limit int) (domain.ObjectSyncHeadPage, error) { filter := bson.M{ "organizationId": normalizedOrganizationID(organizationID), diff --git a/internal/store/object_sync_digest_backfill.go b/internal/store/object_sync_digest_backfill.go new file mode 100644 index 00000000..38736a2e --- /dev/null +++ b/internal/store/object_sync_digest_backfill.go @@ -0,0 +1,220 @@ +package store + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "slices" + "time" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func cloneObjectSyncHead(item domain.ObjectSyncObjectHeadRecord) domain.ObjectSyncObjectHeadRecord { + if item.MD5Attestation != nil { + attestation := *item.MD5Attestation + attestation.Anchors = slices.Clone(attestation.Anchors) + item.MD5Attestation = &attestation + } + return item +} + +func cloneObjectSyncDigestBackfill(item domain.ObjectSyncDigestBackfillRecord) domain.ObjectSyncDigestBackfillRecord { + item.SessionIDs = slices.Clone(item.SessionIDs) + item.Exceptions = slices.Clone(item.Exceptions) + return item +} + +func validObjectSyncAttestation(attestation domain.ObjectSyncDigestAttestation) error { + digest, err := hex.DecodeString(attestation.MD5Sum) + if err != nil || len(digest) != 16 || hex.EncodeToString(digest) != attestation.MD5Sum { + return fmt.Errorf("attested MD5 must be a lowercase 32-character hex digest") + } + if attestation.Source == "" || attestation.BackfillID == "" || attestation.ETag == "" || len(attestation.Anchors) == 0 { + return fmt.Errorf("attestation must name its source, job, object ETag and anchors") + } + return nil +} + +// AttestObjectSyncHeadMD5 is a compare-and-set: the head is updated only if it +// still has exactly the identity and previous digest the coordinator verified. +func (store *MemoryStore) AttestObjectSyncHeadMD5(_ context.Context, expected domain.ObjectSyncObjectHeadRecord, attestation domain.ObjectSyncDigestAttestation, legacySHA256 string) (bool, error) { + if err := validObjectSyncAttestation(attestation); err != nil { + return false, err + } + store.mu.Lock() + defer store.mu.Unlock() + key := objectSyncHeadKey(expected.OrganizationID, expected.SyncSpaceID, expected.Key) + current, ok := store.objectSyncHeads[key] + if !ok || current.Drifted || current.MD5Attestation != nil || + current.RemoteETag != expected.RemoteETag || current.VersionID != expected.VersionID || + current.SizeBytes != expected.SizeBytes || current.PlatformVersion != expected.PlatformVersion || + current.MD5Sum != expected.MD5Sum || current.LegacySHA256 != expected.LegacySHA256 || + current.SessionID != expected.SessionID || current.ItemID != expected.ItemID { + return false, nil + } + attestation.Anchors = slices.Clone(attestation.Anchors) + current.MD5Sum = attestation.MD5Sum + current.MD5Attestation = &attestation + if legacySHA256 != "" { + current.LegacySHA256 = legacySHA256 + } + current.UpdatedAt = attestation.AttestedAt + store.objectSyncHeads[key] = current + return true, nil +} + +// Legacy documents may lack fields that the current struct always writes. +func mongoOptionalString(value string) any { + if value == "" { + return bson.M{"$in": bson.A{nil, ""}} + } + return value +} + +func objectSyncHeadAttestationFilter(expected domain.ObjectSyncObjectHeadRecord) bson.M { + return bson.M{ + "organizationId": normalizedOrganizationID(expected.OrganizationID), + "syncSpaceId": expected.SyncSpaceID, + "key": expected.Key, + "remoteETag": expected.RemoteETag, + "sizeBytes": expected.SizeBytes, + "platformVersion": expected.PlatformVersion, + "versionId": mongoOptionalString(expected.VersionID), + "md5sum": mongoOptionalString(expected.MD5Sum), + "sha256": mongoOptionalString(expected.LegacySHA256), + "sessionId": mongoOptionalString(expected.SessionID), + "itemId": mongoOptionalString(expected.ItemID), + "drifted": bson.M{"$ne": true}, + "md5Attestation": bson.M{"$exists": false}, + } +} + +func (store *MongoStore) AttestObjectSyncHeadMD5(ctx context.Context, expected domain.ObjectSyncObjectHeadRecord, attestation domain.ObjectSyncDigestAttestation, legacySHA256 string) (bool, error) { + if err := validObjectSyncAttestation(attestation); err != nil { + return false, err + } + set := bson.M{"md5sum": attestation.MD5Sum, "md5Attestation": attestation, "updatedAt": attestation.AttestedAt} + if legacySHA256 != "" { + set["sha256"] = legacySHA256 + } + result, err := store.objectSyncHeads.UpdateOne(ctx, objectSyncHeadAttestationFilter(expected), bson.M{"$set": set}) + return err == nil && result.MatchedCount == 1, err +} + +func (store *MemoryStore) UpsertObjectSyncDigestBackfill(_ context.Context, item domain.ObjectSyncDigestBackfillRecord) error { + store.mu.Lock() + defer store.mu.Unlock() + store.objectSyncBackfills[item.ID] = cloneObjectSyncDigestBackfill(item) + return nil +} + +func (store *MemoryStore) GetObjectSyncDigestBackfill(_ context.Context, id string) (*domain.ObjectSyncDigestBackfillRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.objectSyncBackfills[id] + if !ok { + return nil, nil + } + copy := cloneObjectSyncDigestBackfill(item) + return ©, nil +} + +func (store *MemoryStore) ClaimObjectSyncDigestBackfill(_ context.Context, owner string, ttl time.Duration) (*domain.ObjectSyncDigestBackfillRecord, error) { + store.mu.Lock() + defer store.mu.Unlock() + now := time.Now().UTC() + var claimed *domain.ObjectSyncDigestBackfillRecord + for _, item := range store.objectSyncBackfills { + if (item.Status != "pending" && item.Status != "running") || (item.CoordinatorOwner != "" && item.CoordinatorLeaseUntil.After(now)) { + continue + } + if claimed == nil || item.CreatedAt < claimed.CreatedAt || (item.CreatedAt == claimed.CreatedAt && item.ID < claimed.ID) { + candidate := item + claimed = &candidate + } + } + if claimed == nil { + return nil, nil + } + claimed.Status = "running" + claimed.CoordinatorOwner = owner + ":" + rand.Text() + claimed.CoordinatorLeaseUntil = now.Add(ttl) + claimed.UpdatedAt = now.Format(time.RFC3339Nano) + store.objectSyncBackfills[claimed.ID] = cloneObjectSyncDigestBackfill(*claimed) + copy := cloneObjectSyncDigestBackfill(*claimed) + return ©, nil +} + +func (store *MemoryStore) RenewObjectSyncDigestBackfillClaim(_ context.Context, id, owner string, ttl time.Duration) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + item, ok := store.objectSyncBackfills[id] + if !ok || owner == "" || item.CoordinatorOwner != owner || !item.CoordinatorLeaseUntil.After(time.Now()) || (item.Status != "pending" && item.Status != "running") { + return false, nil + } + item.CoordinatorLeaseUntil = time.Now().UTC().Add(ttl) + store.objectSyncBackfills[id] = item + return true, nil +} + +func (store *MemoryStore) SaveClaimedObjectSyncDigestBackfill(_ context.Context, item domain.ObjectSyncDigestBackfillRecord, owner string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + current, ok := store.objectSyncBackfills[item.ID] + if !ok || owner == "" || current.CoordinatorOwner != owner || !current.CoordinatorLeaseUntil.After(time.Now()) || (current.Status != "pending" && current.Status != "running") { + return false, nil + } + store.objectSyncBackfills[item.ID] = cloneObjectSyncDigestBackfill(item) + return true, nil +} + +func (store *MongoStore) UpsertObjectSyncDigestBackfill(ctx context.Context, item domain.ObjectSyncDigestBackfillRecord) error { + return mongoObjectSyncUpsert(ctx, store.objectSyncBackfills, item.ID, item) +} + +func (store *MongoStore) GetObjectSyncDigestBackfill(ctx context.Context, id string) (*domain.ObjectSyncDigestBackfillRecord, error) { + return mongoObjectSyncGet[domain.ObjectSyncDigestBackfillRecord](ctx, store.objectSyncBackfills, bson.M{"id": id}) +} + +func (store *MongoStore) ClaimObjectSyncDigestBackfill(ctx context.Context, owner string, ttl time.Duration) (*domain.ObjectSyncDigestBackfillRecord, error) { + now := time.Now().UTC() + result := store.objectSyncBackfills.FindOneAndUpdate( + ctx, + bson.M{ + "status": bson.M{"$in": bson.A{"pending", "running"}}, + "$or": bson.A{ + bson.M{"coordinatorOwner": ""}, + bson.M{"coordinatorOwner": bson.M{"$exists": false}}, + bson.M{"coordinatorLeaseUntil": bson.M{"$lte": now}}, + }, + }, + bson.M{"$set": bson.M{ + "status": "running", + "updatedAt": now.Format(time.RFC3339Nano), + "coordinatorOwner": owner + ":" + rand.Text(), + "coordinatorLeaseUntil": now.Add(ttl), + }}, + options.FindOneAndUpdate().SetSort(bson.D{{Key: "createdAt", Value: 1}}).SetReturnDocument(options.After), + ) + var item domain.ObjectSyncDigestBackfillRecord + if err := result.Decode(&item); err == mongo.ErrNoDocuments { + return nil, nil + } else if err != nil { + return nil, err + } + return &item, nil +} + +func (store *MongoStore) RenewObjectSyncDigestBackfillClaim(ctx context.Context, id, owner string, ttl time.Duration) (bool, error) { + return renewObjectSyncBackgroundClaim(ctx, store.objectSyncBackfills, id, owner, ttl) +} + +func (store *MongoStore) SaveClaimedObjectSyncDigestBackfill(ctx context.Context, item domain.ObjectSyncDigestBackfillRecord, owner string) (bool, error) { + return saveObjectSyncBackgroundClaim(ctx, store.objectSyncBackfills, item.ID, owner, item) +} diff --git a/internal/store/object_sync_digest_backfill_test.go b/internal/store/object_sync_digest_backfill_test.go new file mode 100644 index 00000000..cdd43276 --- /dev/null +++ b/internal/store/object_sync_digest_backfill_test.go @@ -0,0 +1,169 @@ +package store + +import ( + "reflect" + "strings" + "testing" + "time" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" +) + +func legacyHeadFixture() domain.ObjectSyncObjectHeadRecord { + return domain.ObjectSyncObjectHeadRecord{ + OrganizationID: domain.DefaultOrganizationID, SyncSpaceID: "space", Key: "root/a.mcap", PlatformVersion: 1, + RemoteETag: "legacy-etag", SizeBytes: 4, LegacySHA256: strings.Repeat("a", 64), SessionID: "v1-session", ItemID: "v1-item", + } +} + +func attestationFixture() domain.ObjectSyncDigestAttestation { + return domain.ObjectSyncDigestAttestation{ + Source: domain.ObjectSyncDigestSourceCoordinatorStream, BackfillID: "backfill", ETag: "legacy-etag", SizeBytes: 4, + MD5Sum: strings.Repeat("b", 32), SHA256: strings.Repeat("a", 64), Anchors: []string{domain.ObjectSyncDigestAnchorHeadSHA256}, AttestedAt: "2026-10-30T00:00:00Z", + } +} + +func TestAttestObjectSyncHeadMD5IsCompareAndSet(t *testing.T) { + data := NewMemoryStore() + head := legacyHeadFixture() + if err := data.UpsertObjectSyncHead(t.Context(), head); err != nil { + t.Fatal(err) + } + for name, change := range map[string]func(*domain.ObjectSyncObjectHeadRecord){ + "etag": func(expected *domain.ObjectSyncObjectHeadRecord) { expected.RemoteETag = "other" }, + "version": func(expected *domain.ObjectSyncObjectHeadRecord) { expected.VersionID = "v2" }, + "size": func(expected *domain.ObjectSyncObjectHeadRecord) { expected.SizeBytes = 5 }, + "platform": func(expected *domain.ObjectSyncObjectHeadRecord) { expected.PlatformVersion = 2 }, + "digest": func(expected *domain.ObjectSyncObjectHeadRecord) { expected.MD5Sum = "bad" }, + "legacy": func(expected *domain.ObjectSyncObjectHeadRecord) { expected.LegacySHA256 = strings.Repeat("c", 64) }, + "session": func(expected *domain.ObjectSyncObjectHeadRecord) { expected.SessionID = "other" }, + } { + expected := head + change(&expected) + if written, err := data.AttestObjectSyncHeadMD5(t.Context(), expected, attestationFixture(), ""); err != nil || written { + t.Fatalf("%s: stale expectation attested: %v %v", name, written, err) + } + } + invalid := attestationFixture() + invalid.MD5Sum = strings.Repeat("B", 32) + if _, err := data.AttestObjectSyncHeadMD5(t.Context(), head, invalid, ""); err == nil { + t.Fatal("non-canonical MD5 accepted") + } + if written, err := data.AttestObjectSyncHeadMD5(t.Context(), head, attestationFixture(), ""); err != nil || !written { + t.Fatalf("matching expectation rejected: %v %v", written, err) + } + stored, _ := data.GetObjectSyncHead(t.Context(), head.OrganizationID, head.SyncSpaceID, head.Key) + if stored.MD5Sum != strings.Repeat("b", 32) || stored.MD5Attestation == nil || stored.LegacySHA256 != head.LegacySHA256 || stored.RemoteETag != head.RemoteETag || stored.UpdatedAt != "2026-10-30T00:00:00Z" { + t.Fatalf("stored=%#v", stored) + } + if written, _ := data.AttestObjectSyncHeadMD5(t.Context(), head, attestationFixture(), ""); written { + t.Fatal("head was attested twice") + } + stored.MD5Attestation.Anchors[0] = "mutated" + again, _ := data.GetObjectSyncHead(t.Context(), head.OrganizationID, head.SyncSpaceID, head.Key) + if again.MD5Attestation.Anchors[0] != domain.ObjectSyncDigestAnchorHeadSHA256 { + t.Fatal("memory store leaked a mutable attestation") + } + drifted := legacyHeadFixture() + drifted.Key = "root/drifted.mcap" + drifted.Drifted = true + data.UpsertObjectSyncHead(t.Context(), drifted) + if written, _ := data.AttestObjectSyncHeadMD5(t.Context(), drifted, attestationFixture(), ""); written { + t.Fatal("drifted head attested") + } + moved := legacyHeadFixture() + moved.Key = "root/moved.mcap" + moved.LegacySHA256 = "" + moved.MD5Sum = strings.Repeat("d", 64) + data.UpsertObjectSyncHead(t.Context(), moved) + if written, _ := data.AttestObjectSyncHeadMD5(t.Context(), moved, attestationFixture(), moved.MD5Sum); !written { + t.Fatal("legacy digest stored in md5sum was not attestable") + } + stored, _ = data.GetObjectSyncHead(t.Context(), moved.OrganizationID, moved.SyncSpaceID, moved.Key) + if stored.LegacySHA256 != moved.MD5Sum || stored.MD5Attestation.PreviousMD5Sum != "" || stored.MD5Sum != strings.Repeat("b", 32) { + t.Fatalf("legacy anchor was not preserved: %#v", stored) + } +} + +func TestLegacySHA256DecodesFromRetiredBSONField(t *testing.T) { + raw, err := bson.Marshal(bson.M{"organizationId": "org", "syncSpaceId": "space", "key": "k", "remoteETag": "e", "sizeBytes": int64(4), "sha256": strings.Repeat("a", 64)}) + if err != nil { + t.Fatal(err) + } + var head domain.ObjectSyncObjectHeadRecord + if err := bson.Unmarshal(raw, &head); err != nil { + t.Fatal(err) + } + if head.LegacySHA256 != strings.Repeat("a", 64) || head.MD5Sum != "" || head.MD5Attestation != nil { + t.Fatalf("v1 head decoded as %#v", head) + } +} + +func TestMongoHeadWritesDropAbsentLegacyAnchorsAndAttestations(t *testing.T) { + fresh := legacyHeadFixture() + fresh.LegacySHA256 = "" + if update := objectSyncHeadReplacement(fresh); !reflect.DeepEqual(update["$unset"], bson.M{"sha256": "", "md5Attestation": ""}) { + t.Fatalf("new receipt keeps stale fields: %#v", update) + } + attested := legacyHeadFixture() + attestation := attestationFixture() + attested.MD5Attestation = &attestation + if update := objectSyncHeadReplacement(attested); update["$unset"] != nil { + t.Fatalf("receipt write dropped its own anchor: %#v", update) + } +} + +func TestMongoAttestationFilterMatchesLegacyDocumentsExactly(t *testing.T) { + expected := legacyHeadFixture() + filter := objectSyncHeadAttestationFilter(expected) + missing := bson.M{"$in": bson.A{nil, ""}} + for field, want := range map[string]any{ + "remoteETag": "legacy-etag", "sizeBytes": int64(4), "platformVersion": int64(1), "sha256": strings.Repeat("a", 64), + "sessionId": "v1-session", "itemId": "v1-item", "md5sum": missing, "versionId": missing, + "drifted": bson.M{"$ne": true}, "md5Attestation": bson.M{"$exists": false}, + } { + if !reflect.DeepEqual(filter[field], want) { + t.Fatalf("%s filter = %#v, want %#v", field, filter[field], want) + } + } + expected.MD5Sum = "bad" + if objectSyncHeadAttestationFilter(expected)["md5sum"] != "bad" { + t.Fatal("malformed digest must match exactly") + } +} + +func TestDigestBackfillClaimsOldestJobAndFencesWrites(t *testing.T) { + data := NewMemoryStore() + for _, job := range []domain.ObjectSyncDigestBackfillRecord{ + {ID: "newer", Status: "pending", CreatedAt: "2026-10-30T02:00:00Z"}, + {ID: "older", Status: "pending", CreatedAt: "2026-10-30T01:00:00Z"}, + {ID: "done", Status: "completed", CreatedAt: "2026-10-30T00:00:00Z"}, + } { + if err := data.UpsertObjectSyncDigestBackfill(t.Context(), job); err != nil { + t.Fatal(err) + } + } + claimed, err := data.ClaimObjectSyncDigestBackfill(t.Context(), "coordinator", time.Minute) + if err != nil || claimed == nil || claimed.ID != "older" || claimed.Status != "running" || !strings.HasPrefix(claimed.CoordinatorOwner, "coordinator:") { + t.Fatalf("claimed=%#v err=%v", claimed, err) + } + if renewed, _ := data.RenewObjectSyncDigestBackfillClaim(t.Context(), "older", "someone-else", time.Minute); renewed { + t.Fatal("foreign owner renewed the claim") + } + claimed.Status = "completed" + if saved, _ := data.SaveClaimedObjectSyncDigestBackfill(t.Context(), *claimed, "someone-else"); saved { + t.Fatal("foreign owner saved the job") + } + if saved, _ := data.SaveClaimedObjectSyncDigestBackfill(t.Context(), *claimed, claimed.CoordinatorOwner); !saved { + t.Fatal("owner could not complete the job") + } + next, _ := data.ClaimObjectSyncDigestBackfill(t.Context(), "coordinator", time.Minute) + if next == nil || next.ID != "newer" { + t.Fatalf("next=%#v", next) + } + if again, _ := data.ClaimObjectSyncDigestBackfill(t.Context(), "coordinator", time.Minute); again != nil { + t.Fatalf("live claim or completed job was claimed: %#v", again) + } +}