forked from longtao.wu/droboticflow
Archived heads from protocol v1 carry only a SHA-256 receipt, so every sync that still holds those files ends partial and they can never be cleaned. Add an administrator-approved digest backfill job run by the coordinator. It reads each legacy object with conditional (If-Match) ranged GETs, hashes MD5 and SHA-256 in one pass, cross-checks every surviving receipt (retired head sha256, drosync object metadata, BOS stored digests), and records the MD5 on the head with a compare-and-set write, a bound attestation and an audit event. It never writes to BOS and never accepts a client digest. The cleanup proof now accepts an object without drosync-md5sum metadata only through an attestation bound to its live ETag, version and size. The next ordinary sync plans those files unchanged, completes, and they become cleanable; closed partial sessions are left untouched. Includes the design doc comparing remediation options, store/coordinator/ API unit tests and an end-to-end recovery test on MemoryStore and fake object storage. Nothing is deployed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
134 lines
7.2 KiB
Go
134 lines
7.2 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
// Cleanup proofs are deliberately not idempotency-cached: every local deletion
|
|
// requires a fresh HEAD of the current cloud object and a short object-key lease.
|
|
func (server *server) handleExternalObjectSyncCleanupProof(writer http.ResponseWriter, request *http.Request) {
|
|
target, ok := server.objectSyncStore(writer, request)
|
|
if !ok {
|
|
return
|
|
}
|
|
// The shared item reader blocks mutations on terminal sessions. Cleanup proofs
|
|
// never mutate session/item records; eligibility is checked below.
|
|
readRequest := request.Clone(request.Context())
|
|
readRequest.Method = http.MethodGet
|
|
item, session, principal, ok := server.externalObjectSyncItem(writer, readRequest, target)
|
|
if !ok {
|
|
return
|
|
}
|
|
if request.Method == http.MethodDelete {
|
|
var proof struct {
|
|
LeaseID string `json:"leaseId"`
|
|
LeaseOwner string `json:"leaseOwner"`
|
|
FencingToken int64 `json:"fencingToken"`
|
|
}
|
|
if json.NewDecoder(http.MaxBytesReader(writer, request.Body, 16<<10)).Decode(&proof) != nil || proof.LeaseID == "" || !strings.HasPrefix(proof.LeaseOwner, "clean:"+item.ID+":") || proof.FencingToken <= 0 {
|
|
renderObjectSyncError(writer, request, 400, "cleanup_proof_invalid", "A cleanup lease receipt is required", false, nil)
|
|
return
|
|
}
|
|
if err := target.ReleaseObjectSyncLease(request.Context(), proof.LeaseID, session.ID, proof.LeaseOwner, proof.FencingToken); err != nil {
|
|
renderObjectSyncError(writer, request, 503, "cleanup_lease_release_failed", "Could not release cleanup lease", true, nil)
|
|
return
|
|
}
|
|
renderJSON(writer, 200, map[string]any{"released": true})
|
|
return
|
|
}
|
|
if request.Method != http.MethodPost {
|
|
renderObjectSyncError(writer, request, 405, "method_not_allowed", "Use POST for a fresh cleanup proof", false, nil)
|
|
return
|
|
}
|
|
if (session.Status != domain.ObjectSyncSessionCompleted && session.Status != domain.ObjectSyncSessionPartial) || session.Preview {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_session_not_completed", "Only a completed or partial upload session can authorize per-file cleanup", false, nil)
|
|
return
|
|
}
|
|
if item.Status != domain.ObjectSyncItemCommitted && item.Status != domain.ObjectSyncItemUnchanged {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_item_not_committed", "Only committed or unchanged files can be cleaned", false, nil)
|
|
return
|
|
}
|
|
scope, allowed := objectSyncCredentialScopeFor(principal.Credential, session.SyncSpaceID)
|
|
if !allowed || !objectSyncScopeAllows(*scope, "create", item.RelativePath) {
|
|
renderObjectSyncError(writer, request, 403, "scope_denied", "Current credential cannot access this file", false, nil)
|
|
return
|
|
}
|
|
space, err := target.GetObjectSyncSpace(request.Context(), session.SyncSpaceID)
|
|
if err != nil {
|
|
renderObjectSyncError(writer, request, 503, "space_lookup_failed", "Could not read sync space", true, nil)
|
|
return
|
|
}
|
|
if space == nil || space.OrganizationID != session.OrganizationID || !space.Enabled {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_space_unavailable", "Sync space must remain enabled", false, nil)
|
|
return
|
|
}
|
|
normalized, pathErr := normalizeObjectSyncRelativePath(item.RelativePath)
|
|
if pathErr != nil || normalized != item.RelativePath || item.TargetKey != objectSyncTargetKey(*space, normalized) {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_target_changed", "File no longer belongs to the current sync space target", false, nil)
|
|
return
|
|
}
|
|
expected := item.CommittedVersion
|
|
if item.Status == domain.ObjectSyncItemUnchanged {
|
|
expected = item.ExpectedBase
|
|
}
|
|
digest, digestErr := hex.DecodeString(item.MD5Sum)
|
|
if !expected.Exists || strings.TrimSpace(expected.ETag) == "" || digestErr != nil || len(digest) != md5.Size || item.SizeBytes < 0 || expected.SizeBytes != item.SizeBytes {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_receipt_unverified", "Item lacks a complete cloud receipt", false, nil)
|
|
return
|
|
}
|
|
lease, acquired, err := target.AcquireObjectSyncLease(request.Context(), domain.ObjectSyncLeaseRecord{OrganizationID: session.OrganizationID, ResourceHash: objectSyncLeaseResourceHash(*space, item.TargetKey), ResourceType: "object", ConnectionID: space.ConnectionID, Bucket: space.Bucket, KeyOrPrefix: item.TargetKey, OwnerSessionID: session.ID, OwnerItemID: "clean:" + item.ID + ":" + objectSyncNewID("proof")}, 2*time.Minute)
|
|
if err != nil || !acquired {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_cloud_busy", "Cloud object is being changed; retry cleanup later", true, nil)
|
|
return
|
|
}
|
|
success := false
|
|
defer func() {
|
|
if !success {
|
|
ctx, cancel := context.WithTimeout(context.WithoutCancel(request.Context()), 5*time.Second)
|
|
defer cancel()
|
|
_ = target.ReleaseObjectSyncLease(ctx, lease.ID, session.ID, lease.OwnerItemID, lease.FencingToken)
|
|
}
|
|
}()
|
|
head, err := target.GetObjectSyncHead(request.Context(), session.OrganizationID, session.SyncSpaceID, item.TargetKey)
|
|
if err != nil {
|
|
renderObjectSyncError(writer, request, 503, "cleanup_head_failed", "Could not verify current cloud index", true, nil)
|
|
return
|
|
}
|
|
if head == nil || head.Drifted || head.MD5Sum != item.MD5Sum || head.SizeBytes != item.SizeBytes || head.RemoteETag != expected.ETag || head.VersionID != expected.VersionID || (expected.PlatformVersion > 0 && head.PlatformVersion != expected.PlatformVersion) {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_cloud_changed", "Current cloud object no longer matches the completed upload", false, nil)
|
|
return
|
|
}
|
|
browser, closeBrowser, err := server.objectSyncBrowser(request.Context(), space)
|
|
if err != nil {
|
|
renderObjectSyncError(writer, request, 503, "cleanup_storage_unavailable", "Could not verify cloud storage", true, nil)
|
|
return
|
|
}
|
|
defer closeBrowser()
|
|
ctx, cancel := context.WithTimeout(request.Context(), 45*time.Second)
|
|
defer cancel()
|
|
remote, err := browser.GetObjectSummary(ctx, space.Bucket, item.TargetKey)
|
|
if err != nil {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_cloud_unverified", "Cloud object is missing or HEAD verification failed; local file retained", true, nil)
|
|
return
|
|
}
|
|
if remote.SizeBytes != item.SizeBytes || remote.ETag != head.RemoteETag || remote.VersionID != head.VersionID || !objectSyncCloudMD5Matches(remote, *head, item.MD5Sum) {
|
|
renderObjectSyncError(writer, request, 409, "cleanup_cloud_changed", "Cloud object content or identity changed; local file retained", false, nil)
|
|
return
|
|
}
|
|
// A long HEAD must not leave an almost-expired proof in a response.
|
|
if time.Until(lease.ExpiresAtTime) < 30*time.Second {
|
|
renderObjectSyncError(writer, request, 503, "cleanup_proof_expired", "Cloud verification exceeded its safety window", true, nil)
|
|
return
|
|
}
|
|
success = true
|
|
renderJSON(writer, 200, map[string]any{"sessionId": session.ID, "mappingId": session.MappingID, "syncSpaceId": session.SyncSpaceID, "credentialId": session.CredentialID, "deviceId": session.DeviceID, "itemId": item.ID, "relativePath": item.RelativePath, "targetKey": item.TargetKey, "md5sum": item.MD5Sum, "md5Evidence": objectSyncMD5Evidence(*head), "sizeBytes": item.SizeBytes, "eTag": remote.ETag, "versionId": remote.VersionID, "verifiedAt": nowISO(), "expiresAt": lease.ExpiresAt, "leaseId": lease.ID, "leaseOwner": lease.OwnerItemID, "fencingToken": lease.FencingToken})
|
|
}
|