diff --git a/doc/drosync旧SHA256归档MD5核验补录设计-2026-10-30.md b/doc/drosync旧SHA256归档MD5核验补录设计-2026-10-30.md index 36471bbf..5c3a2407 100644 --- a/doc/drosync旧SHA256归档MD5核验补录设计-2026-10-30.md +++ b/doc/drosync旧SHA256归档MD5核验补录设计-2026-10-30.md @@ -75,7 +75,7 @@ - **证据来源**:MD5 由协调器从云端字节算出。SHA-256 锚把它绑定到 v1 当时的上传回执;If-Match、前后两次 HEAD 和 CAS 把它绑定到唯一的对象版本。API 请求体没有任何摘要字段。测试中设备报告的 MD5 与归档不同时,head 记录的是归档字节的 MD5。 - **只读**:只发 HEAD 和 GET;单元测试和端到端测试都断言对 BOS 零写入。 - **并发**:靠对象租约加 CAS。新的 commit 会替换回执:Mongo 的 head upsert 现在会 `$unset` 新回执里没有的 `sha256`/`md5Attestation`;即使有残留,attestation 绑定的是旧 ETag,也不会被误用。 -- **已知交互**:reconcile 读出整页 head 后整文档回写,可能把刚补录的 MD5 改回旧值。这是失败关闭的:条目重新 denied,重跑补录即可。这是 reconcile 原有的问题,它同样可能覆盖并发 commit 的新回执,列为后续修复。 +- **与 reconcile 的交互**:reconcile 先读出一页 head,再逐个 HEAD,耗时较长。它原先随后整文档回写,会把期间落地的补录或 commit 改回旧回执。现在它只写 `drifted` 和 `updatedAt`,并以读到的 `remoteETag`、`versionId`、`platformVersion`、`md5sum` 为条件(`SetObjectSyncHeadDrifted`;Mongo 过滤器对缺失字段按 `$in: [null, ""]` 匹配)。回执已变的 head 直接跳过,因为结论是针对旧回执得出的。跳过的 head 仍按观察结果计入 `drifted`,结束时的空间能力判定不变。 ### 4.5 大桶成本 @@ -107,6 +107,7 @@ | `internal/objectsync/digest_backfill.go` | 新协调器阶段 `digest_backfill`、单对象核验、锚交叉核验、审计;`bos.go` 新增条件分段 GET,HEAD 带回存储摘要和存储类型 | | `internal/api/object_sync_digest_backfill.go` | 管理员创建/查询路由;清理证明认可已绑定的 attestation | | `docs/drosync/openapi.yaml`、`audit-dictionary.md` | 新路由、`md5Evidence` 字段和新审计动作 | +| `internal/objectsync/coordinator.go`、`internal/store/object_sync.go` | reconcile 不再整文档回写 head,改为条件字段写 `SetObjectSyncHeadDrifted` | 客户端(Rust)无需改动:被拒绝的文件本来每轮都会重新进入清单,补录后自然规划为 `unchanged`。 @@ -135,12 +136,17 @@ - 路由校验与审计; - 清理证明的 7 种拒绝情形; - 端到端 `TestLegacyArchiveSyncCompletesAndCleansAfterDigestBackfill`:通过真实 API 路由,补录前同步为 partial 且不可清理;管理员批准补录后,下一次普通同步为 completed,每个文件都拿到清理证明(`md5Evidence=coordinator_attestation`),旧 Session 保持不变,对 BOS 零写入。 -- 变异检查:分别移除清理证明对 attestation 的认可、SHA-256 锚比较、ETag 绑定、CAS 的 platformVersion 比较、无锚拒绝、读取前身份检查、锚冲突检查、审计补写,均有测试失败。 +- `internal/objectsync/coordinator_reconcile_test.go`: + - reconcile 的 HEAD 进行中,另一协调器副本落地的 commit 和补录都原样保留; + - 真正 drift 的 head(ETag 变化、对象丢失)仍被标记,恢复一致的 head 清除旧标记,除 `drifted`/`updatedAt` 外回执不变。 +- `internal/store/object_sync_head_drift_test.go`: + - drift 写入的每个条件字段,head 不存在时不创建; + - Mongo 过滤器对缺失字段的匹配。 +- 变异检查:分别移除清理证明对 attestation 的认可、SHA-256 锚比较、ETag 绑定、CAS 的 platformVersion 比较、无锚拒绝、读取前身份检查、锚冲突检查、审计补写,把 reconcile 改回整文档回写,或去掉 drift 写入的任一条件,均有测试失败。 ## 6. 未包含与后续 -- 真实 BOS 上仍需验证:条件分段 GET 的 412 行为,HEAD 返回的 `Content-MD5`、`x-bce-content-sha256` 和存储类型的格式,内网 endpoint 的计费;还要在 POC 数据上只读确认 v1 head 的实际形态。Mongo 侧的 CAS 过滤器、`$unset` 和新集合目前只有纯函数测试,需要 Mongo 集成测试。 -- reconcile 改为按 ETag 条件的字段级 drift 写入,同时修复它覆盖并发 commit 的问题。 +- 真实 BOS 上仍需验证:条件分段 GET 的 412 行为,HEAD 返回的 `Content-MD5`、`x-bce-content-sha256` 和存储类型的格式,内网 endpoint 的计费;还要在 POC 数据上只读确认 v1 head 的实际形态。Mongo 侧的 CAS 过滤器(补录和 reconcile 的 drift 写入)、`$unset` 和新集合目前只有纯函数测试,需要 Mongo 集成测试。 - 作业列表/重试 API、管理界面、可选的双人审批;观察成本后,可考虑“规划时按需自动排队核验”。 - 没有 head 的 `deny_existing_untracked` 对象需要“认领”,风险更高,不在本方案范围内。 diff --git a/internal/objectsync/coordinator.go b/internal/objectsync/coordinator.go index 1607c9f1..4dcd238f 100644 --- a/internal/objectsync/coordinator.go +++ b/internal/objectsync/coordinator.go @@ -478,13 +478,15 @@ func (coordinator *Coordinator) reconcileSpace(ctx context.Context, reconcile *d if err := requireClaim(); err != nil { return err } - head.Drifted = !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`) - head.UpdatedAt = nowISO() - if head.Drifted { + headDrifted := !remote.Exists || strings.Trim(remote.ETag, `"`) != strings.Trim(head.RemoteETag, `"`) + if headDrifted { drifted++ } scanned++ - if err := coordinator.syncStore.UpsertObjectSyncHead(ctx, head); err != nil { + // The page was read before the slow HEADs above. A commit or digest + // attestation that replaced this receipt since then is not what the + // verdict was computed against, so the store skips it. + if _, err := coordinator.syncStore.SetObjectSyncHeadDrifted(ctx, head, headDrifted, nowISO()); err != nil { return err } } diff --git a/internal/objectsync/coordinator_reconcile_test.go b/internal/objectsync/coordinator_reconcile_test.go new file mode 100644 index 00000000..cfd031da --- /dev/null +++ b/internal/objectsync/coordinator_reconcile_test.go @@ -0,0 +1,184 @@ +package objectsync + +import ( + "context" + "reflect" + "strings" + "testing" + + "droboticflow/internal/domain" + "droboticflow/internal/store" +) + +// racingHeadPublisher lets another coordinator replace a receipt while +// reconcile's HEAD for it is in flight, after the page of heads was read. +type racingHeadPublisher struct { + Publisher + race func() +} + +func (publisher *racingHeadPublisher) Head(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) { + remote, err := publisher.Publisher.Head(ctx, connection, bucket, key) + if race := publisher.race; race != nil { + publisher.race = nil + race() + } + return remote, err +} + +// runReconcile reconciles the fixture space on its own coordinator replica, +// which shares the store but reads object metadata through publisher. +func runReconcile(t *testing.T, data *store.MemoryStore, publisher Publisher) domain.ObjectSyncReconcileRecord { + t.Helper() + if err := data.UpsertObjectSyncReconcile(t.Context(), domain.ObjectSyncReconcileRecord{ID: "reconcile", OrganizationID: domain.DefaultOrganizationID, SyncSpaceID: "space", Status: "pending"}); err != nil { + t.Fatal(err) + } + reconciler, err := NewCoordinator(data, publisher, "reconciler") + if err != nil { + t.Fatal(err) + } + if result, err := reconciler.executeStage(t.Context(), phaseReconcile); err != nil || result.Reconciled != 1 { + t.Fatalf("reconcile result=%#v err=%v", result, err) + } + reconcile, err := data.GetObjectSyncReconcile(t.Context(), "reconcile") + if err != nil || reconcile == nil || reconcile.Status != "completed" { + t.Fatalf("reconcile=%#v err=%v", reconcile, err) + } + return *reconcile +} + +func TestReconcileKeepsACommitThatLandsDuringItsScan(t *testing.T) { + writer, data, publisher := coordinatorFixture(t) + ctx := t.Context() + space, _ := data.GetObjectSyncSpace(ctx, "space") + space.Capability = domain.ObjectSyncCapabilityRecord{ + State: domain.ObjectSyncCapabilityUpdateEnabled, ConditionalCreate: true, ConditionalUpdate: true, ConditionalMultipart: true, ExclusivePrefixConfirmed: true, + } + space.Policy.AllowConditionalUpdate = true + if err := data.UpsertObjectSyncSpace(ctx, *space); err != nil { + t.Fatal(err) + } + key := "incoming/report.xlsx" + archived := domain.ObjectSyncObjectHeadRecord{ + OrganizationID: domain.DefaultOrganizationID, SyncSpaceID: "space", Key: key, PlatformVersion: 1, + RemoteETag: "archived", SizeBytes: 4, MD5Sum: strings.Repeat("a", 32), SessionID: "earlier-session", ItemID: "earlier-item", + } + if err := data.UpsertObjectSyncHead(ctx, archived); err != nil { + t.Fatal(err) + } + publisher.objects["bucket/"+key] = ObjectMetadata{Exists: true, ETag: archived.RemoteETag, SizeBytes: 4} + session := domain.ObjectSyncSessionRecord{ID: "update-session", OrganizationID: domain.DefaultOrganizationID, SyncSpaceID: "space", DeviceID: "device", Status: domain.ObjectSyncSessionRunning, Counts: domain.ObjectSyncSessionCounts{Manifest: 1}} + if err := data.UpsertObjectSyncSession(ctx, session); err != nil { + t.Fatal(err) + } + lease, ok, err := data.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{ID: "lease-update", ResourceHash: "resource-update", OrganizationID: session.OrganizationID, OwnerSessionID: session.ID, OwnerItemID: "update-item"}, claimTTL) + if err != nil || !ok { + t.Fatalf("seed lease: %v", err) + } + update := domain.ObjectSyncItemRecord{ + ID: "update-item", OrganizationID: session.OrganizationID, SessionID: session.ID, TargetKey: key, StagingKey: "staging/update-item", + Status: domain.ObjectSyncItemCommitting, Action: "update", SizeBytes: 4, MD5Sum: strings.Repeat("b", 32), + ExpectedBase: domain.ObjectSyncRemoteBase{Exists: true, ETag: archived.RemoteETag, SizeBytes: 4, PlatformVersion: 1}, + LeaseID: lease.ID, FencingToken: lease.FencingToken, UpdatedAt: nowISO(), + } + if err := data.UpsertObjectSyncItems(ctx, []domain.ObjectSyncItemRecord{update}); err != nil { + t.Fatal(err) + } + publisher.objects["bucket/"+update.StagingKey] = ObjectMetadata{Exists: true, ETag: "staged", SizeBytes: 4} + + var committed *domain.ObjectSyncObjectHeadRecord + reconcile := runReconcile(t, data, &racingHeadPublisher{Publisher: publisher, race: func() { + if result, err := writer.executeStage(ctx, phaseCommit); err != nil || result.Committed != 1 { + t.Errorf("concurrent commit result=%#v err=%v", result, err) + } + committed, _ = data.GetObjectSyncHead(ctx, domain.DefaultOrganizationID, "space", key) + }}) + head, _ := data.GetObjectSyncHead(ctx, domain.DefaultOrganizationID, "space", key) + if head.RemoteETag != "published-"+key || head.PlatformVersion != 2 || head.MD5Sum != update.MD5Sum || head.ItemID != update.ID || head.Drifted { + t.Fatalf("reconcile reverted the concurrent commit: %#v", head) + } + if !reflect.DeepEqual(head, committed) { + t.Fatalf("reconcile wrote to the committed receipt: %#v, committed %#v", head, committed) + } + if reconcile.Scanned != 1 || reconcile.Drifted != 0 { + t.Fatalf("reconcile=%#v", reconcile) + } +} + +func TestReconcileKeepsADigestAttestationThatLandsDuringItsScan(t *testing.T) { + writer, data, publisher := backfillFixture(t) + payload := []byte("legacy archive verified while reconcile was scanning") + md5sum, sha := digestHex(payload) + key := "incoming/node-04/episode.mcap" + legacy := seedLegacyArchive(t, data, publisher, key, payload, legacyShape{legacySHA256: sha}) + seedBackfillJob(t, data, domain.ObjectSyncDigestBackfillRecord{WholeSpace: true}) + + var attested *domain.ObjectSyncObjectHeadRecord + reconcile := runReconcile(t, data, &racingHeadPublisher{Publisher: publisher, race: func() { + if _, err := writer.executeStage(t.Context(), phaseDigestBackfill); err != nil { + t.Errorf("concurrent digest backfill: %v", err) + } + if job, _ := data.GetObjectSyncDigestBackfill(t.Context(), "backfill"); job == nil || job.Counts.Attested != 1 { + t.Errorf("concurrent digest backfill did not attest: %#v", job) + } + attested, _ = data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + }}) + head, _ := data.GetObjectSyncHead(t.Context(), domain.DefaultOrganizationID, "space", key) + if head.MD5Sum != md5sum || !AttestationBindsHead(*head) || head.LegacySHA256 != sha || head.Drifted { + t.Fatalf("reconcile reverted the concurrent attestation: %#v", head) + } + if head.RemoteETag != legacy.RemoteETag || head.PlatformVersion != legacy.PlatformVersion { + t.Fatalf("receipt identity changed: %#v", head) + } + if !reflect.DeepEqual(head, attested) { + t.Fatalf("reconcile wrote to the attested receipt: %#v, attested %#v", head, attested) + } + if reconcile.Scanned != 1 || reconcile.Drifted != 0 { + t.Fatalf("reconcile=%#v", reconcile) + } +} + +func TestReconcileStillFlagsDriftedHeads(t *testing.T) { + _, data, publisher := coordinatorFixture(t) + ctx := t.Context() + intact := "incoming/intact.bin" + seeded := map[string]domain.ObjectSyncObjectHeadRecord{} + for key, remote := range map[string]*ObjectMetadata{ + "incoming/rewritten.bin": {Exists: true, ETag: "rewritten-outside-drosync", SizeBytes: 4}, + "incoming/deleted.bin": nil, + intact: {Exists: true, ETag: "archived", SizeBytes: 4}, + } { + head := domain.ObjectSyncObjectHeadRecord{ + OrganizationID: domain.DefaultOrganizationID, SyncSpaceID: "space", Key: key, PlatformVersion: 3, RemoteETag: "archived", VersionID: "v3", + SizeBytes: 4, MD5Sum: strings.Repeat("a", 32), SessionID: "session", ItemID: "item-" + key, + // A flag from an earlier reconcile is cleared once the object matches again. + Drifted: key == intact, UpdatedAt: "2026-10-01T00:00:00Z", + } + if err := data.UpsertObjectSyncHead(ctx, head); err != nil { + t.Fatal(err) + } + if remote != nil { + publisher.objects["bucket/"+key] = *remote + } + seeded[key] = head + } + reconcile := runReconcile(t, data, publisher) + for key, before := range seeded { + head, _ := data.GetObjectSyncHead(ctx, domain.DefaultOrganizationID, "space", key) + if head.Drifted != (key != intact) || head.UpdatedAt == before.UpdatedAt { + t.Fatalf("%s: drifted=%v updatedAt=%s", key, head.Drifted, head.UpdatedAt) + } + after := *head + after.Drifted, after.UpdatedAt = before.Drifted, before.UpdatedAt + if !reflect.DeepEqual(after, before) { + t.Fatalf("%s: drift write changed the receipt: %#v", key, head) + } + } + if reconcile.Scanned != 3 || reconcile.Drifted != 2 { + t.Fatalf("reconcile=%#v", reconcile) + } + space, _ := data.GetObjectSyncSpace(ctx, "space") + if space.Capability.State != domain.ObjectSyncCapabilityDrifted || space.Enabled { + t.Fatalf("space=%#v", space) + } +} diff --git a/internal/store/object_sync.go b/internal/store/object_sync.go index 3755099c..445ba0ae 100644 --- a/internal/store/object_sync.go +++ b/internal/store/object_sync.go @@ -82,6 +82,7 @@ type ObjectSyncStore interface { UpsertObjectSyncHead(context.Context, domain.ObjectSyncObjectHeadRecord) error ListObjectSyncHeads(context.Context, string, string, string, int) (domain.ObjectSyncHeadPage, error) AttestObjectSyncHeadMD5(context.Context, domain.ObjectSyncObjectHeadRecord, domain.ObjectSyncDigestAttestation, string) (bool, error) + SetObjectSyncHeadDrifted(context.Context, domain.ObjectSyncObjectHeadRecord, bool, string) (bool, error) InsertObjectSyncVersion(context.Context, domain.ObjectSyncVersionRecord) error UpsertObjectSyncDigestBackfill(context.Context, domain.ObjectSyncDigestBackfillRecord) error @@ -629,6 +630,24 @@ func (store *MemoryStore) UpsertObjectSyncHead(_ context.Context, item domain.Ob return nil } +// SetObjectSyncHeadDrifted records a reconcile verdict only on the receipt it +// was computed against. A commit or digest attestation written since then is +// a different receipt, so the head is left untouched. +func (store *MemoryStore) SetObjectSyncHeadDrifted(_ context.Context, observed domain.ObjectSyncObjectHeadRecord, drifted bool, updatedAt string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + key := objectSyncHeadKey(observed.OrganizationID, observed.SyncSpaceID, observed.Key) + current, ok := store.objectSyncHeads[key] + if !ok || current.RemoteETag != observed.RemoteETag || current.VersionID != observed.VersionID || + current.PlatformVersion != observed.PlatformVersion || current.MD5Sum != observed.MD5Sum { + return false, nil + } + current.Drifted = drifted + current.UpdatedAt = updatedAt + store.objectSyncHeads[key] = current + return true, nil +} + func (store *MemoryStore) ListObjectSyncHeads(_ context.Context, organizationID, syncSpaceID, cursor string, limit int) (domain.ObjectSyncHeadPage, error) { store.mu.Lock() defer store.mu.Unlock() @@ -1214,6 +1233,23 @@ func objectSyncHeadReplacement(item domain.ObjectSyncObjectHeadRecord) bson.M { return update } +func objectSyncHeadDriftFilter(observed domain.ObjectSyncObjectHeadRecord) bson.M { + return bson.M{ + "organizationId": normalizedOrganizationID(observed.OrganizationID), + "syncSpaceId": observed.SyncSpaceID, + "key": observed.Key, + "remoteETag": mongoOptionalString(observed.RemoteETag), + "versionId": mongoOptionalString(observed.VersionID), + "platformVersion": observed.PlatformVersion, + "md5sum": mongoOptionalString(observed.MD5Sum), + } +} + +func (store *MongoStore) SetObjectSyncHeadDrifted(ctx context.Context, observed domain.ObjectSyncObjectHeadRecord, drifted bool, updatedAt string) (bool, error) { + result, err := store.objectSyncHeads.UpdateOne(ctx, objectSyncHeadDriftFilter(observed), bson.M{"$set": bson.M{"drifted": drifted, "updatedAt": updatedAt}}) + return err == nil && result.MatchedCount == 1, err +} + func (store *MongoStore) ListObjectSyncHeads(ctx context.Context, organizationID, syncSpaceID, cursor string, limit int) (domain.ObjectSyncHeadPage, error) { filter := bson.M{ "organizationId": normalizedOrganizationID(organizationID), diff --git a/internal/store/object_sync_head_drift_test.go b/internal/store/object_sync_head_drift_test.go new file mode 100644 index 00000000..9b8769af --- /dev/null +++ b/internal/store/object_sync_head_drift_test.go @@ -0,0 +1,67 @@ +package store + +import ( + "reflect" + "strings" + "testing" + + "droboticflow/internal/domain" + + "go.mongodb.org/mongo-driver/bson" +) + +func TestSetObjectSyncHeadDriftedOnlyMarksTheObservedReceipt(t *testing.T) { + data := NewMemoryStore() + head := legacyHeadFixture() + attestation := attestationFixture() + head.MD5Sum, head.MD5Attestation = attestation.MD5Sum, &attestation + if err := data.UpsertObjectSyncHead(t.Context(), head); err != nil { + t.Fatal(err) + } + for name, change := range map[string]func(*domain.ObjectSyncObjectHeadRecord){ + "etag": func(observed *domain.ObjectSyncObjectHeadRecord) { observed.RemoteETag = "other" }, + "version": func(observed *domain.ObjectSyncObjectHeadRecord) { observed.VersionID = "v2" }, + "platform": func(observed *domain.ObjectSyncObjectHeadRecord) { observed.PlatformVersion = 2 }, + "digest": func(observed *domain.ObjectSyncObjectHeadRecord) { observed.MD5Sum = "" }, + "missing": func(observed *domain.ObjectSyncObjectHeadRecord) { observed.Key = "root/missing.mcap" }, + } { + observed := head + change(&observed) + if written, err := data.SetObjectSyncHeadDrifted(t.Context(), observed, true, "2026-10-30T01:00:00Z"); err != nil || written { + t.Fatalf("%s: stale observation marked the head: %v %v", name, written, err) + } + } + if stored, _ := data.GetObjectSyncHead(t.Context(), head.OrganizationID, head.SyncSpaceID, head.Key); !reflect.DeepEqual(*stored, head) { + t.Fatalf("stale observation changed the head: %#v", stored) + } + if created, _ := data.GetObjectSyncHead(t.Context(), head.OrganizationID, head.SyncSpaceID, "root/missing.mcap"); created != nil { + t.Fatalf("drift write created a head: %#v", created) + } + for _, drifted := range []bool{true, false} { + if written, err := data.SetObjectSyncHeadDrifted(t.Context(), head, drifted, "2026-10-30T02:00:00Z"); err != nil || !written { + t.Fatalf("matching observation rejected: %v %v", written, err) + } + want := head + want.Drifted, want.UpdatedAt = drifted, "2026-10-30T02:00:00Z" + if stored, _ := data.GetObjectSyncHead(t.Context(), head.OrganizationID, head.SyncSpaceID, head.Key); !reflect.DeepEqual(*stored, want) { + t.Fatalf("drift write changed more than the verdict: %#v", stored) + } + } +} + +func TestMongoDriftFilterMatchesTheObservedReceiptOnly(t *testing.T) { + missing := bson.M{"$in": bson.A{nil, ""}} + want := bson.M{ + "organizationId": domain.DefaultOrganizationID, "syncSpaceId": "space", "key": "root/a.mcap", + "remoteETag": "legacy-etag", "versionId": missing, "platformVersion": int64(1), "md5sum": missing, + } + if filter := objectSyncHeadDriftFilter(legacyHeadFixture()); !reflect.DeepEqual(filter, want) { + t.Fatalf("legacy filter = %#v, want %#v", filter, want) + } + observed := legacyHeadFixture() + observed.RemoteETag, observed.VersionID, observed.MD5Sum = "", "v1", strings.Repeat("b", 32) + filter := objectSyncHeadDriftFilter(observed) + if !reflect.DeepEqual(filter["remoteETag"], missing) || filter["versionId"] != "v1" || filter["md5sum"] != strings.Repeat("b", 32) { + t.Fatalf("filter = %#v", filter) + } +}