✨ Implement drosync production v1
This commit is contained in:
parent
aacf075400
commit
1ca8cfa5ff
77
.gitea/workflows/drosync-release.yml
Normal file
77
.gitea/workflows/drosync-release.yml
Normal file
@ -0,0 +1,77 @@
|
||||
name: drosync-release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: SemVer release version
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- run: python3 scripts/generate_drosync_contract.py --check
|
||||
- run: cargo fmt --manifest-path tools/drosync/Cargo.toml --all -- --check
|
||||
- run: cargo clippy --manifest-path tools/drosync/Cargo.toml --workspace --all-targets -- -D warnings
|
||||
- run: cargo test --manifest-path tools/drosync/Cargo.toml --workspace
|
||||
|
||||
linux:
|
||||
needs: verify
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- run: tools/drosync/packaging/build-release.sh linux-x86_64
|
||||
env:
|
||||
DROSYNC_VERSION: ${{ inputs.version }}
|
||||
DROSYNC_GPG_SIGN_KEY: ${{ secrets.DROSYNC_GPG_SIGN_KEY }}
|
||||
DROSYNC_DEB_SIGN_KEY: ${{ secrets.DROSYNC_GPG_SIGN_KEY }}
|
||||
DROSYNC_UPDATE_PUBLIC_KEY_BASE64: ${{ secrets.DROSYNC_UPDATE_PUBLIC_KEY_BASE64 }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: drosync-linux-x86_64-${{ inputs.version }}
|
||||
path: tools/drosync/dist/*
|
||||
if-no-files-found: error
|
||||
|
||||
macos:
|
||||
needs: verify
|
||||
runs-on: macos-14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- run: tools/drosync/packaging/build-release.sh macos-arm64
|
||||
env:
|
||||
DROSYNC_VERSION: ${{ inputs.version }}
|
||||
DROSYNC_APPLE_APPLICATION_IDENTITY: ${{ secrets.DROSYNC_APPLE_APPLICATION_IDENTITY }}
|
||||
DROSYNC_APPLE_INSTALLER_IDENTITY: ${{ secrets.DROSYNC_APPLE_INSTALLER_IDENTITY }}
|
||||
DROSYNC_APPLE_NOTARY_PROFILE: ${{ secrets.DROSYNC_APPLE_NOTARY_PROFILE }}
|
||||
DROSYNC_UPDATE_PUBLIC_KEY_BASE64: ${{ secrets.DROSYNC_UPDATE_PUBLIC_KEY_BASE64 }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: drosync-macos-arm64-${{ inputs.version }}
|
||||
path: tools/drosync/dist/*
|
||||
if-no-files-found: error
|
||||
|
||||
windows:
|
||||
needs: verify
|
||||
runs-on: windows-2022
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- run: cargo build --manifest-path tools/drosync/Cargo.toml --release --workspace --target x86_64-pc-windows-msvc
|
||||
env:
|
||||
DROSYNC_UPDATE_PUBLIC_KEY_BASE64: ${{ secrets.DROSYNC_UPDATE_PUBLIC_KEY_BASE64 }}
|
||||
- shell: pwsh
|
||||
run: tools/drosync/packaging/windows/build-msi.ps1
|
||||
env:
|
||||
DROSYNC_VERSION: ${{ inputs.version }}
|
||||
DROSYNC_AUTHENTICODE_CERT_THUMBPRINT: ${{ secrets.DROSYNC_AUTHENTICODE_CERT_THUMBPRINT }}
|
||||
DROSYNC_UPDATE_PUBLIC_KEY_BASE64: ${{ secrets.DROSYNC_UPDATE_PUBLIC_KEY_BASE64 }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: drosync-windows-x86_64-${{ inputs.version }}
|
||||
path: tools/drosync/dist/*
|
||||
if-no-files-found: error
|
||||
32
Dockerfile.drosync-coordinator
Normal file
32
Dockerfile.drosync-coordinator
Normal file
@ -0,0 +1,32 @@
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
|
||||
ARG DROBOTICFLOW_BUILD_TIME=unknown
|
||||
|
||||
FROM golang:1.24-bookworm AS build
|
||||
WORKDIR /src
|
||||
ARG GOPROXY=https://proxy.golang.org,direct
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN GOPROXY="${GOPROXY}" go mod download
|
||||
|
||||
COPY cmd ./cmd
|
||||
COPY internal ./internal
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/drosync-coordinator ./cmd/drosync-coordinator
|
||||
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
ARG DROBOTICFLOW_BUILD_COMMIT
|
||||
ARG DROBOTICFLOW_BUILD_TIME
|
||||
WORKDIR /app
|
||||
ENV DROSYNC_COORDINATOR_PORT=3003
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_COMMIT=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
ENV DROBOTICFLOW_IMAGE_BUILD_TIME=${DROBOTICFLOW_BUILD_TIME}
|
||||
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
|
||||
LABEL org.opencontainers.image.created=${DROBOTICFLOW_BUILD_TIME}
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
|
||||
RUN groupadd --system --gid 10001 droboticflow && useradd --system --uid 10001 --gid droboticflow --home-dir /app --shell /usr/sbin/nologin droboticflow
|
||||
|
||||
COPY --from=build /out/drosync-coordinator /usr/local/bin/drosync-coordinator
|
||||
|
||||
EXPOSE 3003
|
||||
USER 10001:10001
|
||||
CMD ["drosync-coordinator"]
|
||||
158
cmd/drosync-coordinator/main.go
Normal file
158
cmd/drosync-coordinator/main.go
Normal file
@ -0,0 +1,158 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/objectsync"
|
||||
"droboticflow/internal/retry"
|
||||
"droboticflow/internal/store"
|
||||
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) {
|
||||
return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) {
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := client.Ping(ctx, nil); err != nil {
|
||||
_ = client.Disconnect(ctx)
|
||||
return nil, err
|
||||
}
|
||||
return client, nil
|
||||
})
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
cfg, err := config.LoadValidated()
|
||||
if err != nil {
|
||||
fmt.Printf("[drosync-coordinator] invalid configuration: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
client, err := connectMongo(ctx, cfg.MongoURI, cfg.MongoConnectRetries, time.Duration(cfg.MongoConnectRetryDelayMS)*time.Millisecond)
|
||||
if err != nil {
|
||||
fmt.Printf("[drosync-coordinator] failed to connect mongo: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
|
||||
if err := dataStore.RunMigrations(ctx); err != nil {
|
||||
fmt.Printf("[drosync-coordinator] failed to run migrations: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := dataStore.EnsureIndexes(ctx); err != nil {
|
||||
fmt.Printf("[drosync-coordinator] failed to ensure indexes: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
hostname, _ := os.Hostname()
|
||||
coordinator, err := objectsync.NewCoordinator(dataStore, objectsync.BOSPublisher{GlobalEndpoint: cfg.GlobalBOSEndpoint}, hostname)
|
||||
if err != nil {
|
||||
fmt.Printf("[drosync-coordinator] initialization failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := coordinator.ConfigureAuditSigning(cfg.ObjectSyncAuditSigningPrivateKey, cfg.ObjectSyncAuditSigningKeyID); err != nil {
|
||||
fmt.Printf("[drosync-coordinator] invalid audit signing configuration: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if cfg.ObjectSyncEnabled && !coordinator.AuditSigningConfigured() {
|
||||
fmt.Println("[drosync-coordinator] DROSYNC_AUDIT_SIGNING_PRIVATE_KEY is required when object sync is enabled")
|
||||
os.Exit(1)
|
||||
}
|
||||
if cfg.ObjectSyncEnabled {
|
||||
go coordinator.Run(ctx, time.Duration(cfg.PollIntervalMS)*time.Millisecond)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health", func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(writer, `{"service":"drosync-coordinator","status":"ok","enabled":%t}`, cfg.ObjectSyncEnabled)
|
||||
})
|
||||
mux.HandleFunc("/ready", func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
readyCtx, cancel := context.WithTimeout(request.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
if err := dataStore.Ready(readyCtx); err != nil {
|
||||
writer.WriteHeader(http.StatusServiceUnavailable)
|
||||
_, _ = fmt.Fprintf(writer, `{"ready":false,"enabled":%t}`, cfg.ObjectSyncEnabled)
|
||||
return
|
||||
}
|
||||
_, _ = fmt.Fprintf(writer, `{"ready":true,"enabled":%t}`, cfg.ObjectSyncEnabled)
|
||||
})
|
||||
mux.HandleFunc("/metrics", func(writer http.ResponseWriter, _ *http.Request) {
|
||||
writer.Header().Set("Content-Type", "text/plain; version=0.0.4")
|
||||
metrics := coordinator.Metrics()
|
||||
_, _ = fmt.Fprintf(
|
||||
writer,
|
||||
"drosync_coordinator_up 1\n"+
|
||||
"drosync_coordinator_enabled %d\n"+
|
||||
"drosync_coordinator_cycles_total %d\n"+
|
||||
"drosync_coordinator_errors_total %d\n"+
|
||||
"drosync_coordinator_sessions_planned_total %d\n"+
|
||||
"drosync_coordinator_items_committed_total %d\n"+
|
||||
"drosync_coordinator_staging_cleaned_total %d\n"+
|
||||
"drosync_coordinator_reconciles_total %d\n"+
|
||||
"drosync_coordinator_releases_total %d\n"+
|
||||
"drosync_coordinator_audit_seals_total %d\n"+
|
||||
"drosync_coordinator_last_cycle_timestamp_seconds %d\n"+
|
||||
"drosync_coordinator_last_success_timestamp_seconds %d\n"+
|
||||
"drosync_coordinator_cycle_duration_seconds %.3f\n",
|
||||
boolMetric(cfg.ObjectSyncEnabled),
|
||||
metrics.Cycles,
|
||||
metrics.Errors,
|
||||
metrics.Planned,
|
||||
metrics.Committed,
|
||||
metrics.Cleaned,
|
||||
metrics.Reconciled,
|
||||
metrics.Released,
|
||||
metrics.Sealed,
|
||||
metrics.LastCycle,
|
||||
metrics.LastOK,
|
||||
float64(metrics.DurationMS)/1000,
|
||||
)
|
||||
})
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf("0.0.0.0:%d", cfg.CoordinatorPort),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 2 * time.Minute,
|
||||
}
|
||||
serverErrors := make(chan error, 1)
|
||||
go func() {
|
||||
fmt.Printf("[drosync-coordinator] listening on http://0.0.0.0:%d\n", cfg.CoordinatorPort)
|
||||
serverErrors <- server.ListenAndServe()
|
||||
}()
|
||||
select {
|
||||
case err := <-serverErrors:
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
fmt.Printf("[drosync-coordinator] server failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(shutdownCtx); err != nil {
|
||||
fmt.Printf("[drosync-coordinator] graceful shutdown failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func boolMetric(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
32
docs/drosync/README.md
Normal file
32
docs/drosync/README.md
Normal file
@ -0,0 +1,32 @@
|
||||
# drosync production v1
|
||||
|
||||
`drosync` is the supported one-way synchronization path from customer-owned local directories to Baidu Object Storage (BOS). It is not a mounted filesystem: v1 does not download, propagate remote deletion, merge file contents, preserve POSIX ownership/mode, follow symlinks, or create empty remote directories.
|
||||
|
||||
## Frozen scope
|
||||
|
||||
- Control plane: existing Go API plus the `drosync-coordinator` asynchronous worker.
|
||||
- Client: Rust CLI, daemon, tray GUI, updater, and SQLite crash journal.
|
||||
- UI: `/object-sync` with role-specific administration, operations, release, and audit surfaces.
|
||||
- Targets: Windows 10/11 x86_64, Ubuntu 22.04/24.04 x86_64, and macOS 13+ arm64.
|
||||
- Limits per session: 1,000,000 files, 1 TiB per object, 20 TiB total; 500 devices per organization and at most 16 client data requests per device.
|
||||
- Retention: session/item 90 days, versions/audit 1 year, heads indefinitely, staging/multipart 24 hours, conflicts indefinitely.
|
||||
|
||||
## Document map
|
||||
|
||||
- [Product requirements](product-requirements.md)
|
||||
- [Architecture and state machines](architecture.md)
|
||||
- [OpenAPI 3.1 contract](openapi.yaml)
|
||||
- [Installation](installation.md)
|
||||
- [CLI reference](cli-reference.md)
|
||||
- [Permissions and security](permissions-security.md)
|
||||
- [Conflict runbook](conflicts-runbook.md)
|
||||
- [Audit dictionary](audit-dictionary.md)
|
||||
- [Operations and SLO](operations-runbook.md)
|
||||
- [Disaster recovery](disaster-recovery.md)
|
||||
- [Updates and rollback](update-rollout.md)
|
||||
- [Test and acceptance matrix](testing-acceptance.md)
|
||||
- [Decisions and external prerequisites](decisions.md)
|
||||
|
||||
## Safety invariant
|
||||
|
||||
A space starts `blocked/unverified`. A successful `If-None-Match: *` probe permits create-only operation. Original-key update is available only after conditional update and conditional multipart completion probes pass and an administrator explicitly confirms that the shared prefix has no platform-external writer. Any observed drift disables the space. The UI cannot bypass these gates.
|
||||
50
docs/drosync/architecture.md
Normal file
50
docs/drosync/architecture.md
Normal file
@ -0,0 +1,50 @@
|
||||
# Architecture and state machines
|
||||
|
||||
## Components
|
||||
|
||||
1. The Rust daemon exclusively owns scheduling and the local SQLite journal. CLI and GUI use a `0600` Unix socket or a current-user Windows named pipe.
|
||||
2. The Go API authenticates a device credential, enforces scope/quota/idempotency, accepts paged manifests, creates leases, and issues integrity-bound staging URLs.
|
||||
3. `drosync-coordinator` atomically claims Mongo work across replicas. It plans items, verifies fencing tokens and staging headers, publishes conditionally, reconciles tracked heads, cleans 24-hour staging, and writes immutable release manifests.
|
||||
4. BOS contains business objects plus separate internal staging, release, and audit prefixes. Mongo contains control metadata; a million items are individual documents, never an embedded session array.
|
||||
|
||||
Each immutable release belongs to exactly one sync space. All source sessions must share that space, so the NDJSON manifest has one unambiguous BOS bucket and prefix context; Mongo stores the space ID, manifest key, SHA-256, counts, lifecycle state, and source session IDs.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
L["Local regular files"] --> D["Rust daemon and SQLite journal"]
|
||||
C["CLI / tray GUI"] -->|"local IPC"| D
|
||||
D -->|"manifest, lease, commit"| A["Go control-plane API"]
|
||||
D -->|"MD5-bound presigned upload"| S["BOS staging prefix"]
|
||||
A --> M["MongoDB"]
|
||||
Q["drosync-coordinator"] --> M
|
||||
Q -->|"multipart copy + conditional complete"| B["BOS business prefix"]
|
||||
Q --> R["Immutable release manifest"]
|
||||
Q --> X["Reconcile and staging cleanup"]
|
||||
```
|
||||
|
||||
## Session state
|
||||
|
||||
`collecting_manifest → planning → planned → running → completed | partial | conflicted`
|
||||
|
||||
Any active state may transition to `failed`, `aborted`, or `expired`. A final session is immutable except for retention cleanup.
|
||||
|
||||
## Item state
|
||||
|
||||
`discovered → planned → reserved → uploading → uploaded → committing → committed`
|
||||
|
||||
Planning may yield `unchanged`, `skipped`, or `denied`. Conditional publication may yield `kept_both`. Error and cancellation terminals are `failed`, `aborted`, and `expired`.
|
||||
|
||||
## Lease invariant
|
||||
|
||||
Each logical target hash has one permanent Mongo lease document. Acquisition atomically increments a monotonic fencing token. Release clears ownership but never deletes the counter. Coordinator commit requires the exact session, item, and fencing token; an old owner can never become valid again.
|
||||
|
||||
## Idempotency
|
||||
|
||||
Every write carries `Idempotency-Key`. The server atomically creates a request fingerprint record:
|
||||
|
||||
- same key and same body after completion: replay the stored status, headers, and body;
|
||||
- same key and different body: `409 idempotency_key_reused`;
|
||||
- same request still running: retryable `409 idempotency_request_in_progress`;
|
||||
- server-side failure before a stable result: the record is released for retry.
|
||||
|
||||
Records expire after 24 hours.
|
||||
26
docs/drosync/audit-dictionary.md
Normal file
26
docs/drosync/audit-dictionary.md
Normal file
@ -0,0 +1,26 @@
|
||||
# Audit dictionary
|
||||
|
||||
Each audit event contains:
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `id`, `eventKey` | Immutable identity and idempotent insertion key. |
|
||||
| `organizationId` | Tenant chain boundary. |
|
||||
| `actorType`, `actorId` | `user`, `device`, or system actor. |
|
||||
| `credentialId`, `deviceId` | Device authorization lineage when applicable. |
|
||||
| `requestId` | API request correlation identifier. |
|
||||
| `sessionId`, `itemId` | Transfer lineage. |
|
||||
| `action` | Stable verb such as `space.create`, `space.probe`, `session.create`, `space.reconcile`, or `lease.force_release`. |
|
||||
| `target` | Non-secret identifiers for the affected resource. |
|
||||
| `before`, `after` | Sanitized state deltas when supplied. |
|
||||
| `result`, `reason` | Outcome and operator/system rationale. |
|
||||
| `previousHash`, `eventHash` | Organization hash-chain link and SHA-256 hash of the canonical event payload. |
|
||||
| `occurredAt` | UTC RFC3339 timestamp. |
|
||||
|
||||
An organization-scoped fencing lease serializes hash-chain append across API replicas. Events retain for one year. Export endpoints are read-only and available as CSV or NDJSON.
|
||||
|
||||
The coordinator verifies every event hash and intra-day chain link, then conditionally writes the previous UTC day's count, first/last event hash, aggregate digest, Ed25519 signature, and verifier key ID to `.drosync-internal/audit/{organizationId}/{YYYY-MM-DD}.json`. Mongo stores the BOS ETag/version ID and seal index for one year. `GET /api/object-sync/audit-seals/{day}` returns the index record.
|
||||
|
||||
`DROSYNC_AUDIT_SIGNING_PRIVATE_KEY` accepts a base64 Ed25519 32-byte seed or 64-byte private key; `DROSYNC_AUDIT_SIGNING_KEY_ID` identifies the verifier key. When object sync is enabled, the coordinator refuses to start without a valid signing key. The private key must be supplied through the `drosync-signing-secret` Kubernetes Secret and must never enter the repository.
|
||||
|
||||
Production readiness still requires a live seal HEAD/readback plus independent public-key verification; source code or an unsigned export is not equivalent evidence.
|
||||
40
docs/drosync/cli-reference.md
Normal file
40
docs/drosync/cli-reference.md
Normal file
@ -0,0 +1,40 @@
|
||||
# CLI reference
|
||||
|
||||
Global flags:
|
||||
|
||||
- `--json`: one JSON value on stdout.
|
||||
- `--non-interactive`: never open or wait for an interactive approval flow.
|
||||
|
||||
Commands:
|
||||
|
||||
| Command | Purpose |
|
||||
|---|---|
|
||||
| `configure --server URL` | Persist the control-plane URL through daemon IPC. |
|
||||
| `auth --server URL [--profile NAME]` | Start short-code pairing and store the resulting device token in the OS credential store. |
|
||||
| `mapping list` | List local mappings. |
|
||||
| `mapping add ROOT --space ID [--remote-prefix P] [--include-hidden] [--require-done]` | Add a mapping. |
|
||||
| `mapping remove ID` | Remove a mapping without deleting local or remote objects. |
|
||||
| `plan ID` | Scan, upload a paged manifest, wait for the asynchronous plan, and retain the resumable session without uploading bytes. |
|
||||
| `sync ID [--dry-run]` | Resume a planned/crashed session or execute scan through final commit. |
|
||||
| `status` | Show daemon, mappings, and local crash journals. |
|
||||
| `conflicts` | List locally known `kept_both` results. |
|
||||
| `retry ID` | Resume a mapping after a retryable failure. |
|
||||
| `abort SESSION_ID` | Abort the server session; business objects already committed remain. |
|
||||
| `doctor` | Check SQLite, IPC, device credential, control plane, and scoped capabilities. |
|
||||
| `daemon --foreground` | Run a foreground per-user daemon. |
|
||||
| `update` | Check/pin/pause/rollback signed client releases. |
|
||||
| `version` | Print client, protocol, platform, and architecture. |
|
||||
|
||||
Exit codes:
|
||||
|
||||
| Code | Meaning |
|
||||
|---:|---|
|
||||
| 0 | Success. |
|
||||
| 2 | CLI usage or invalid local configuration. |
|
||||
| 10 | Authentication or credential revoked. |
|
||||
| 20 | Permission/scope/capability gate denied. |
|
||||
| 30 | Conflict or active lease; retry according to the error. |
|
||||
| 40 | Local filesystem/SQLite integrity failure. |
|
||||
| 50 | Network, server, BOS, or otherwise unclassified operation failure. |
|
||||
|
||||
The implementation emits structured server `code/message/retryable/requestId/details` errors. Automation should prefer `--json` and the stable error code over translated text.
|
||||
28
docs/drosync/conflicts-runbook.md
Normal file
28
docs/drosync/conflicts-runbook.md
Normal file
@ -0,0 +1,28 @@
|
||||
# Conflict runbook
|
||||
|
||||
## Normal conflict
|
||||
|
||||
A target conditional commit returning HTTP 412 is not data loss. The coordinator leaves the original key unchanged and conditionally creates:
|
||||
|
||||
`{original}.__conflict__{UTC}_{shortItemId}`
|
||||
|
||||
The suffix contains no user identity. Mongo records the logical target, physical conflict key, session, item, actor, expected base, and committed remote version.
|
||||
|
||||
## Administrator force overwrite
|
||||
|
||||
Force overwrite is disabled by default. It can only be enabled after the space is `update_enabled`, the administrator has confirmed exclusive ownership of the prefix, and `allowForceOverwrite` is explicitly set. The action creates a separate immutable operation item, copies from the preserved conflict object to the logical target with a fresh conditional ETag check, and keeps the conflict object. A concurrent write returns a precondition failure and must be reviewed again; it never degrades to an unconditional overwrite.
|
||||
|
||||
## Operator procedure
|
||||
|
||||
1. Open `/object-sync` → Conflicts and identify the session/item.
|
||||
2. Compare the original and conflict object using recorded SHA-256, size, source mtime, ETag, and versionId. Do not interpret multipart ETag as SHA-256.
|
||||
3. Decide outside drosync which business object should become canonical.
|
||||
4. If a manual BOS operation is required, record the change ticket and then run space reconciliation.
|
||||
5. Keep the space disabled if reconciliation reports drift. Re-run capability probe only on its dedicated temporary key.
|
||||
6. Re-enable create or update only after the gate is again satisfied.
|
||||
|
||||
Conflicts are never deleted automatically. Session abort and rollout rollback also do not delete them.
|
||||
|
||||
## Lease incident
|
||||
|
||||
Force release only after verifying the owning daemon is stopped or its credential is revoked. The permanent fencing counter remains and increments on the next acquire; releasing a lease cannot validate an older token.
|
||||
54
docs/drosync/decisions.md
Normal file
54
docs/drosync/decisions.md
Normal file
@ -0,0 +1,54 @@
|
||||
# Decisions and external prerequisites
|
||||
|
||||
## Frozen decisions
|
||||
|
||||
- Name/commands/packages/services: `drosync`.
|
||||
- Implementation: Go control plane and Rust client.
|
||||
- Direction: local to BOS only.
|
||||
- Conflict: automatic keep-both; never alter the original after a failed conditional commit.
|
||||
- BOS capability: fail closed until probed; do not auto-enable bucket versioning and do not rely on `x-bce-forbid-overwrite`.
|
||||
- Shared-prefix update: both technical conditional-update capability and administrator exclusivity confirmation are mandatory.
|
||||
- Credentials: long-lived device token until administrator revocation.
|
||||
- Existing Mongo BOS AK/SK representation remains, with the residual risk explicitly accepted.
|
||||
- Network: public direct access only in v1.
|
||||
|
||||
## External prerequisites that source code cannot satisfy
|
||||
|
||||
- Windows Authenticode certificate/private key.
|
||||
- Apple Developer ID, installer identity, notarization profile, and Apple runner.
|
||||
- Linux GPG/package signing key.
|
||||
- CI runners for all three target operating systems.
|
||||
- `droboticflow` POC namespace, registry secret, Mongo secret, and dedicated BOS acceptance credential.
|
||||
- A cost-approved window for >5 GB copy, 1 TiB, and 20 TiB real BOS certification.
|
||||
- A dedicated Ed25519 audit/update signing key in the production key vault.
|
||||
|
||||
These are release gates, not implementation defaults. Private keys must never enter Git.
|
||||
|
||||
## D-08 decision gate
|
||||
|
||||
Until a real BOS probe supplies evidence:
|
||||
|
||||
1. unprobed/unsupported spaces cannot accept sessions;
|
||||
2. conditional-create success permits only create;
|
||||
3. conditional update plus conditional multipart completion and explicit prefix exclusivity permit original-path update;
|
||||
4. any drift disables the space and requires reconcile plus explicit reopen.
|
||||
|
||||
## Open decision: `data_operator` session ownership
|
||||
|
||||
The product contract says a `data_operator` can view and operate only "their own"
|
||||
sessions, but the current identity model does not bind a user to a device, credential,
|
||||
mapping, or sync space. The management API therefore currently has only an
|
||||
organization boundary and cannot enforce personal ownership without a product choice.
|
||||
Do not enable the `data_operator` role in production until one of these models is
|
||||
selected and implemented:
|
||||
|
||||
- **A — device owner (recommended):** an administrator assigns one or more users when
|
||||
approving a device; sessions inherit the device owner and only those users can
|
||||
operate them.
|
||||
- **B — space assignment:** administrators assign operators to sync spaces; every
|
||||
assigned operator can operate all sessions in those spaces.
|
||||
- **C — organization-wide operator:** keep the current organization-wide visibility
|
||||
and explicitly redefine "own" as "within the operator's organization".
|
||||
|
||||
This decision also determines reassignment behavior, historical-session visibility,
|
||||
and whether multiple operators may own the same device.
|
||||
24
docs/drosync/disaster-recovery.md
Normal file
24
docs/drosync/disaster-recovery.md
Normal file
@ -0,0 +1,24 @@
|
||||
# Disaster recovery
|
||||
|
||||
Target RTO is 4 hours and RPO is 15 minutes.
|
||||
|
||||
## Mongo loss
|
||||
|
||||
1. Disable the object-sync feature flag and revoke exposed credentials if compromise is suspected.
|
||||
2. Restore Mongo into an isolated instance; verify `objectSync*` indexes and TTL definitions.
|
||||
3. Verify the organization audit chains and compare last release/audit BOS records with restored Mongo.
|
||||
4. Mark all nonterminal sessions expired. Do not infer successful commit from an uploading/committing state alone.
|
||||
5. Reconcile every affected space against BOS object HEAD metadata.
|
||||
6. Enable internal create-only canary before general traffic.
|
||||
|
||||
## Coordinator crash
|
||||
|
||||
Atomic claim leases expire and another replica resumes. Item commit is protected by the permanent fencing token and conditional target completion. Staging cleanup is safe to retry. A response-loss client queries item/session status instead of assuming failure.
|
||||
|
||||
## Client crash
|
||||
|
||||
The daemon reopens WAL SQLite, runs migration/integrity checks, loads the active session journal, reuses deterministic idempotency keys, resumes known multipart parts, obtains fresh presigned URLs when needed, and queries commit state. If the local file changed, start a new manifest; never continue using the old SHA-256.
|
||||
|
||||
## BOS outage
|
||||
|
||||
Pause new client sessions. Preserve staging and journals. After recovery, run capability probe only on the dedicated temporary key, then reconcile tracked heads. No automatic business-object deletion is part of recovery.
|
||||
52
docs/drosync/installation.md
Normal file
52
docs/drosync/installation.md
Normal file
@ -0,0 +1,52 @@
|
||||
# Installation
|
||||
|
||||
## Common prerequisites
|
||||
|
||||
- The control-plane base URL must be reachable over public HTTPS.
|
||||
- An administrator enters the browser short code in `/object-sync` → Devices, verifies the resolved device, and assigns one or more space/prefix/action scopes. Pending requests are not broadcast across organizations.
|
||||
- Headless service credentials must come from `DROSYNC_TOKEN` or a `0600` file named by `DROSYNC_SECRET_FILE`.
|
||||
- Do not put device credentials, BOS AK/SK, signing keys, or notarization profiles in the repository.
|
||||
|
||||
## Ubuntu 22.04/24.04 x86_64
|
||||
|
||||
Install the signed DEB and enable either one mode:
|
||||
|
||||
```bash
|
||||
sudo dpkg -i drosync_1.0.0_amd64.deb
|
||||
systemctl --user enable --now drosync.service
|
||||
# Or, for a managed machine:
|
||||
sudo systemctl enable --now drosync.service
|
||||
```
|
||||
|
||||
The portable tarball contains all three binaries and a user systemd unit. Validate the detached GPG signature and SHA-256 before extraction.
|
||||
|
||||
## macOS 13+ arm64
|
||||
|
||||
Install the notarized `.pkg`. It installs `/Applications/drosync.app`, `/usr/local/bin/drosync`, a LaunchAgent, and a LaunchDaemon definition. Enable only the per-user agent or the administrator-managed daemon:
|
||||
|
||||
```bash
|
||||
launchctl bootstrap gui/"$(id -u)" /Library/LaunchAgents/com.drobotics.drosync.agent.plist
|
||||
sudo launchctl bootstrap system /Library/LaunchDaemons/com.drobotics.drosync.daemon.plist
|
||||
```
|
||||
|
||||
The package must pass `spctl --assess --type install` and have a stapled notarization ticket.
|
||||
|
||||
## Windows 10/11 x86_64
|
||||
|
||||
The Authenticode-signed MSI installs the CLI, GUI, and Windows Service. Run it elevated for system service mode. `install-user.ps1` provides a non-admin per-user login-start alternative. Validate with:
|
||||
|
||||
```powershell
|
||||
Get-AuthenticodeSignature .\drosync-1.0.0-windows-x86_64.msi
|
||||
Get-Service drosync
|
||||
```
|
||||
|
||||
## First configuration
|
||||
|
||||
```bash
|
||||
drosync configure --server https://data.example.com
|
||||
drosync auth --server https://data.example.com --profile default
|
||||
drosync mapping add /data/capture --space oss_example --remote-prefix incoming
|
||||
drosync doctor
|
||||
drosync plan <mapping-id>
|
||||
drosync sync <mapping-id>
|
||||
```
|
||||
1155
docs/drosync/openapi.yaml
Normal file
1155
docs/drosync/openapi.yaml
Normal file
File diff suppressed because it is too large
Load Diff
46
docs/drosync/operations-runbook.md
Normal file
46
docs/drosync/operations-runbook.md
Normal file
@ -0,0 +1,46 @@
|
||||
# Operations and SLO runbook
|
||||
|
||||
## Deployment
|
||||
|
||||
On `dev`, leave `DEPLOY_ENV` unset; scripts resolve to `poc`. The feature flag defaults off:
|
||||
|
||||
```bash
|
||||
export DROBOTICFLOW_OBJECT_SYNC_ENABLED=false
|
||||
./scripts/render-manifests.sh > /tmp/droboticflow-poc.yaml
|
||||
kubectl apply -f /tmp/droboticflow-poc.yaml
|
||||
```
|
||||
|
||||
Deploy API, Web, and coordinator before enabling client access. The coordinator uses Mongo atomic claims and may run multiple replicas. Its read-only root filesystem has a size-limited `/tmp` volume for streaming release manifests.
|
||||
|
||||
Before setting the feature flag to `true`, create `drosync-signing-secret` with `DROSYNC_AUDIT_SIGNING_PRIVATE_KEY` and `DROSYNC_AUDIT_SIGNING_KEY_ID`. A missing or malformed key intentionally prevents the enabled coordinator from starting.
|
||||
|
||||
If the cluster has Prometheus Operator and a Grafana dashboard sidecar, install the optional SLO rules and dashboard:
|
||||
|
||||
```bash
|
||||
./scripts/render-drosync-observability.sh | kubectl apply -f -
|
||||
```
|
||||
|
||||
## Health and metrics
|
||||
|
||||
- API: `/api/health`, `/api/ready`, existing API metrics.
|
||||
- Coordinator: `/health`, `/ready`, `/metrics`.
|
||||
- Key counters: cycles, errors, sessions planned, items committed, staging cleaned, reconciles, releases, and daily audit seals. Cycle timestamps and duration make stalled replicas visible.
|
||||
|
||||
Alert when coordinator errors increase for 5 minutes, no cycle occurs for 2 minutes while enabled, oldest planning/committing item exceeds 15 minutes, staging exceeds 24 hours, audit chain continuity fails, or Mongo/BOS error rate threatens 99.9%.
|
||||
|
||||
## Backup check
|
||||
|
||||
Every shift:
|
||||
|
||||
1. Confirm the Mongo backup job completed within the 15-minute RPO and includes all `objectSync*` collections.
|
||||
2. Restore the latest backup into an isolated database and run index creation plus audit continuity verification.
|
||||
3. HEAD a sample immutable release manifest and daily audit seal in BOS.
|
||||
4. Record Mongo snapshot ID, BOS version/ETag/size, test restore duration, and verifier result.
|
||||
|
||||
## Cleanup
|
||||
|
||||
Coordinator aborts expired multipart uploads and deletes staging after 24 hours. Mongo TTL indexes remove session/item after 90 days and version/audit after one year. Lease documents and latest object heads are not TTL-deleted. Business objects and conflict objects are never automatic cleanup targets.
|
||||
|
||||
## Rollback
|
||||
|
||||
Disable the feature flag, revoke device tokens, and stop new sessions. Do not delete committed objects, sessions, versions, releases, or audit records. Roll server components back independently; protocol `v1` remains the compatibility boundary.
|
||||
29
docs/drosync/permissions-security.md
Normal file
29
docs/drosync/permissions-security.md
Normal file
@ -0,0 +1,29 @@
|
||||
# Permissions and security
|
||||
|
||||
## Roles
|
||||
|
||||
| Role | Access |
|
||||
|---|---|
|
||||
| `admin` | Spaces, capability probe, exclusive-prefix confirmation, device approval, credential rotation/revocation, force lease release, reconcile, and client rollout policy. |
|
||||
| `data_operator` | Intended for owned operational sessions, failures, conflict review, retry, and abort. Production enablement is blocked until the ownership model in `decisions.md` is selected; the current API boundary is organization-wide. |
|
||||
| `pipeline_engineer` | Read final sessions and create/view immutable release records. |
|
||||
| `compliance_auditor` / `auditor` | Organization-wide read-only audit and CSV/NDJSON export. |
|
||||
|
||||
Device credentials are independently scoped to sync space, allowed relative prefixes, and actions. They remain valid until administrator revocation. The API rechecks credential and device state on every new control request.
|
||||
|
||||
## Local boundaries
|
||||
|
||||
- The daemon owns SQLite writes.
|
||||
- Unix IPC is mode `0600`.
|
||||
- Windows per-user daemons use the process token's default DACL and a domain-plus-user pipe name. The LocalSystem service uses `\\.\pipe\drosync-system`; CLI/GUI try the private user pipe first and fall back to the system pipe, whose service-token DACL is expected to restrict access to administrators. Both reject remote pipe clients and enforce a single first server instance. Windows acceptance must still include standard-user/admin/cross-user denial tests because that DACL behavior cannot be certified from a macOS build.
|
||||
- User credentials are stored in Keychain, Credential Manager, or Secret Service.
|
||||
- Headless secret files must be `0600`; environment injection is accepted for orchestrated services.
|
||||
- `unsafe` Rust is forbidden in workspace crates.
|
||||
|
||||
## Accepted residual risk
|
||||
|
||||
Existing BOS AK/SK records remain stored in MongoDB using the project's current storage representation. API responses mask secrets and organization/role checks isolate access, but no new at-rest application encryption is introduced in v1. This is an explicitly accepted, not eliminated, production security risk. Compensating controls are database encryption at rest, secret-aware backups, least-privilege BOS accounts, rotation, access logging, and restricted Mongo administration.
|
||||
|
||||
## Prefix exclusivity
|
||||
|
||||
Do not enable update mode if any non-drosync writer can mutate the prefix. `x-bce-forbid-overwrite` is not a substitute, especially when bucket versioning is enabled. A drift observation immediately disables the space; reconciliation does not reopen it automatically.
|
||||
38
docs/drosync/product-requirements.md
Normal file
38
docs/drosync/product-requirements.md
Normal file
@ -0,0 +1,38 @@
|
||||
# Product requirements
|
||||
|
||||
## User outcome
|
||||
|
||||
A user selects a local directory once. The daemon waits for files to stabilize, builds a deterministic manifest, uploads new or changed regular files through short-lived BOS URLs, and reports a traceable final session. Concurrent devices never silently overwrite one another.
|
||||
|
||||
## File policy
|
||||
|
||||
- Include regular files only.
|
||||
- Exclude hidden entries by default; allow an explicit mapping opt-in.
|
||||
- Never follow symlinks or ingest FIFO, socket, block, or character devices.
|
||||
- Ignore common temporary suffixes such as `.tmp`, `.part`, `.partial`, `.crdownload`, `.swp`, and `.download`.
|
||||
- Require 60 seconds of unchanged mtime/size before hashing; verify stat again after hashing.
|
||||
- A mapping may require a sidecar `{file}.done` marker.
|
||||
- Preserve source mtime in the manifest. Do not preserve owner or mode.
|
||||
- Treat rename as a candidate after 5 seconds, still subject to the 60-second stability rule.
|
||||
|
||||
## Consistency policy
|
||||
|
||||
- Upload bytes only to a random internal staging key.
|
||||
- Bind direct uploads to Content-MD5, Content-Length, and drosync metadata.
|
||||
- Bind each multipart part to Content-MD5 and Content-Length; persist each returned part ETag.
|
||||
- Treat multipart ETag only as a transport validator, never as file SHA-256.
|
||||
- Publish with server-side multipart copy. Complete the target multipart upload with `If-None-Match: *` for create or `If-Match` for update.
|
||||
- On precondition failure, retain the original object and publish the staged content to `{original}.__conflict__{UTC}_{shortItemId}`.
|
||||
- Record remote ETag, size, versionId when available, platform version, operation, session, item, and actor.
|
||||
|
||||
## Availability and support objectives
|
||||
|
||||
- Monthly API availability: 99.9%.
|
||||
- p95: begin uploading within 15 minutes after an incremental file stabilizes.
|
||||
- New control-plane requests rejected within 60 seconds after credential revocation.
|
||||
- RTO 4 hours and RPO 15 minutes.
|
||||
- Initial support window: 8x5.
|
||||
|
||||
## Explicit non-goals
|
||||
|
||||
Two-way sync, remote deletion, file-content merging, POSIX filesystem semantics, offline-only networks, proxy/custom-CA support, VPN relay, and platform-external direct writes into update-enabled prefixes are outside v1.
|
||||
28
docs/drosync/testing-acceptance.md
Normal file
28
docs/drosync/testing-acceptance.md
Normal file
@ -0,0 +1,28 @@
|
||||
# Test and acceptance matrix
|
||||
|
||||
Completion is reported independently for source, automated tests, platform artifacts, browser, POC API, real BOS, signatures, and capacity.
|
||||
|
||||
## Automated
|
||||
|
||||
- Go: API/store/coordinator unit tests, Mongo integration, race detector, status transitions, idempotency replay/mismatch, lease fencing, cleanup, reconcile, quotas, permission gates, and audit chain.
|
||||
- Rust: format, Clippy with warnings denied, workspace/doc tests, SQLite migrations, deterministic path properties and manifest fuzz corpus, crash/reopen resume, multipart limits, and IPC permission tests. Long-running `cargo-fuzz` campaigns remain a release-certification gate rather than a claim made by routine CI.
|
||||
- React: role visibility, capability gate, pairing approval, conflict, release, audit export, and rollout policy.
|
||||
- Contract: parse OpenAPI 3.1, regenerate Go/Rust status models, and compare protocol fixtures.
|
||||
|
||||
## Real BOS
|
||||
|
||||
Record request IDs and object evidence for:
|
||||
|
||||
- conditional create/update plus HTTP 412;
|
||||
- direct and multipart staging;
|
||||
- multipart server-side copy above 5 GB;
|
||||
- two independent devices creating/updating one key;
|
||||
- response loss, disconnection, process kill, URL/lease/session expiry, and token revocation;
|
||||
- external write drift that disables the space;
|
||||
- traceability among final HEAD/version, session, item, audit, release manifest, and daily seal.
|
||||
|
||||
Verify uploads with the drosync receipt and BOS HEAD size/ETag/versionId. A generic object-list entry or UI success badge is not acceptance evidence.
|
||||
|
||||
## Capacity
|
||||
|
||||
Routine CI uses streaming/simulated million-entry manifests and sparse/fixture payloads. Formal certification must preserve the real BOS bill, elapsed time, peak client/server resources, failure/retry log, and final checksum records for 1 million files, one 1 TiB object, and a 20 TiB session.
|
||||
24
docs/drosync/update-rollout.md
Normal file
24
docs/drosync/update-rollout.md
Normal file
@ -0,0 +1,24 @@
|
||||
# Signed updates and rollback
|
||||
|
||||
The control plane returns a platform artifact URL, SHA-256, channel, protocol version, rollout percentage, pause flag, and Ed25519 signature. The updater must verify, in order:
|
||||
|
||||
1. Ed25519 signature over the canonical manifest with an embedded/pinned public key.
|
||||
2. Downloaded artifact SHA-256.
|
||||
3. Native OS signature: Authenticode, Apple Developer ID plus notarization, or Linux package signature.
|
||||
4. Platform/architecture and protocol compatibility.
|
||||
|
||||
Do not replace binaries while an upload or commit is active. Keep the previous verified package for rollback. An administrator may pause rollout or pin a device group.
|
||||
|
||||
Default progression:
|
||||
|
||||
| Stage | Population | Observation |
|
||||
|---|---:|---:|
|
||||
| Canary | 5% | 24 hours |
|
||||
| Pilot | 20% | 24 hours |
|
||||
| Stable | 100% | continuous |
|
||||
|
||||
The management API makes release payload fields immutable after registration. Administrators may only switch the channel/fixed percentage (`canary=5`, `pilot=20`, `stable=100`) or pause/resume that signed record; the Web console exposes those guarded actions. Promotion is rejected until the current active stage has been observed for 24 hours. Paused time does not count: resuming restarts the observation clock. Rollback to an earlier stage remains immediate. Stage promotion is deliberately operator-confirmed because v1 has no trustworthy client crash telemetry feed.
|
||||
|
||||
Advance only if crash-free daemon rate, upload/commit error rate, audit continuity, and conflict rate remain within the release threshold. Rollback disables the new artifact and reinstalls the previous natively signed package; it never reverts data objects.
|
||||
|
||||
The schema is [update-manifest.schema.json](../../tools/drosync/packaging/update-manifest.schema.json). CI signing keys and notarization credentials must be injected from the CI key vault.
|
||||
@ -127,6 +127,7 @@ func isSessionPublicEndpoint(request *http.Request) bool {
|
||||
path == "/api/public/settings" ||
|
||||
isWorkerAPIHealthEndpoint(request) ||
|
||||
isExternalBusinessWriteEndpoint(path) ||
|
||||
strings.HasPrefix(path, "/api/external/object-sync/") ||
|
||||
strings.HasPrefix(path, "/api/auth/")
|
||||
}
|
||||
|
||||
|
||||
@ -46,6 +46,20 @@ type objectStorageBrowser interface {
|
||||
BuildPreviewURL(context.Context, string, string) (string, error)
|
||||
}
|
||||
|
||||
type objectSyncStorageBrowser interface {
|
||||
objectStorageBrowser
|
||||
PutObjectConditional(context.Context, string, string, []byte, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
|
||||
PublishMultipartCopyConditional(context.Context, string, string, string, int64, string, string, string, map[string]string) (domain.ObjectStorageObjectSummary, error)
|
||||
GetBucketVersioning(context.Context, string) (string, error)
|
||||
}
|
||||
|
||||
type objectSyncBoundUploadBrowser interface {
|
||||
PresignPutObjectBound(context.Context, string, string, string, string, int64, map[string]string, int) (domain.ObjectStoragePresignedUploadResult, error)
|
||||
InitiateMultipartUploadBound(context.Context, string, string, string, map[string]string, int) (domain.ObjectStorageMultipartUploadCreateResult, error)
|
||||
PresignMultipartUploadPartBound(context.Context, string, string, string, int, string, int64, int) (domain.ObjectStorageMultipartUploadPartURLResult, error)
|
||||
CompleteMultipartUploadBound(context.Context, string, string, string, []domain.ObjectStorageMultipartUploadPart, map[string]string) (domain.ObjectStorageObjectSummary, error)
|
||||
}
|
||||
|
||||
type objectStoragePage struct {
|
||||
Items []domain.ObjectStorageObjectSummary
|
||||
NextMarker string
|
||||
@ -154,6 +168,7 @@ func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObj
|
||||
lastModified := ""
|
||||
etag := ""
|
||||
storageClass := ""
|
||||
versionID := ""
|
||||
metadata := map[string]any{}
|
||||
if meta != nil {
|
||||
contentType = strings.TrimSpace(meta.ContentType)
|
||||
@ -161,6 +176,7 @@ func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObj
|
||||
lastModified = strings.TrimSpace(meta.LastModified)
|
||||
etag = strings.Trim(meta.ETag, "\"")
|
||||
storageClass = strings.TrimSpace(meta.StorageClass)
|
||||
versionID = strings.TrimSpace(meta.VersionId)
|
||||
for metaKey, metaValue := range meta.UserMeta {
|
||||
metadata[metaKey] = metaValue
|
||||
}
|
||||
@ -177,6 +193,7 @@ func objectStorageSummaryFromMeta(bucket string, key string, meta *bosapi.GetObj
|
||||
ContentType: contentType,
|
||||
LastModified: lastModified,
|
||||
ETag: etag,
|
||||
VersionID: versionID,
|
||||
StorageClass: storageClass,
|
||||
Metadata: metadata,
|
||||
}
|
||||
@ -427,6 +444,29 @@ func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket st
|
||||
return browser.GetObjectSummary(ctx, bucket, objectKey)
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) PutObjectConditional(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
key string,
|
||||
payload []byte,
|
||||
contentType string,
|
||||
ifMatch string,
|
||||
ifNoneMatch string,
|
||||
userMeta map[string]string,
|
||||
) (domain.ObjectStorageObjectSummary, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
args := &bosapi.PutObjectArgs{
|
||||
ContentType: detectObjectStorageContentType(objectKey, payload, contentType),
|
||||
IfMatch: strings.TrimSpace(ifMatch),
|
||||
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
||||
UserMeta: userMeta,
|
||||
}
|
||||
if _, err := browser.client.PutObjectFromBytesWithContext(ctx, strings.TrimSpace(bucket), objectKey, payload, args); err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
return browser.GetObjectSummary(ctx, bucket, objectKey)
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, nil, contentType)}
|
||||
@ -473,6 +513,35 @@ func (browser *bosObjectStorageBrowser) PresignPutObject(_ context.Context, buck
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) PresignPutObjectBound(
|
||||
_ context.Context,
|
||||
bucket string,
|
||||
key string,
|
||||
contentType string,
|
||||
contentMD5 string,
|
||||
sizeBytes int64,
|
||||
metadata map[string]string,
|
||||
expiresInSeconds int,
|
||||
) (domain.ObjectStoragePresignedUploadResult, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
||||
headers := objectStorageUploadHeaders(contentType)
|
||||
headers["Content-MD5"] = strings.TrimSpace(contentMD5)
|
||||
headers["Content-Length"] = strconv.FormatInt(sizeBytes, 10)
|
||||
for metaKey, metaValue := range metadata {
|
||||
headers["x-bce-meta-"+strings.ToLower(strings.TrimSpace(metaKey))] = strings.TrimSpace(metaValue)
|
||||
}
|
||||
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, nil)
|
||||
return domain.ObjectStoragePresignedUploadResult{
|
||||
Bucket: strings.TrimSpace(bucket),
|
||||
Key: objectKey,
|
||||
Method: http.MethodPut,
|
||||
URL: url,
|
||||
Headers: headers,
|
||||
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) InitiateMultipartUpload(_ context.Context, bucket string, key string, contentType string, expiresInSeconds int) (domain.ObjectStorageMultipartUploadCreateResult, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil)
|
||||
@ -488,6 +557,28 @@ func (browser *bosObjectStorageBrowser) InitiateMultipartUpload(_ context.Contex
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) InitiateMultipartUploadBound(
|
||||
_ context.Context,
|
||||
bucket string,
|
||||
key string,
|
||||
contentType string,
|
||||
metadata map[string]string,
|
||||
expiresInSeconds int,
|
||||
) (domain.ObjectStorageMultipartUploadCreateResult, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
result, err := browser.client.InitiateMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(contentType), nil)
|
||||
if err != nil {
|
||||
return domain.ObjectStorageMultipartUploadCreateResult{}, err
|
||||
}
|
||||
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
||||
return domain.ObjectStorageMultipartUploadCreateResult{
|
||||
Bucket: result.Bucket,
|
||||
Key: result.Key,
|
||||
UploadID: result.UploadId,
|
||||
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) PresignMultipartUploadPart(_ context.Context, bucket string, key string, uploadID string, partNumber int, expiresInSeconds int) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
||||
@ -508,13 +599,57 @@ func (browser *bosObjectStorageBrowser) PresignMultipartUploadPart(_ context.Con
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) PresignMultipartUploadPartBound(
|
||||
_ context.Context,
|
||||
bucket string,
|
||||
key string,
|
||||
uploadID string,
|
||||
partNumber int,
|
||||
contentMD5 string,
|
||||
sizeBytes int64,
|
||||
expiresInSeconds int,
|
||||
) (domain.ObjectStorageMultipartUploadPartURLResult, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
expiresInSeconds = normalizeObjectStoragePresignExpires(expiresInSeconds)
|
||||
params := map[string]string{
|
||||
"uploadId": strings.TrimSpace(uploadID),
|
||||
"partNumber": strconv.Itoa(partNumber),
|
||||
}
|
||||
headers := map[string]string{
|
||||
"Content-MD5": strings.TrimSpace(contentMD5),
|
||||
"Content-Length": strconv.FormatInt(sizeBytes, 10),
|
||||
}
|
||||
url := browser.client.GeneratePresignedUrl(strings.TrimSpace(bucket), objectKey, expiresInSeconds, http.MethodPut, headers, params)
|
||||
return domain.ObjectStorageMultipartUploadPartURLResult{
|
||||
Bucket: strings.TrimSpace(bucket),
|
||||
Key: objectKey,
|
||||
UploadID: strings.TrimSpace(uploadID),
|
||||
PartNumber: partNumber,
|
||||
Method: http.MethodPut,
|
||||
URL: url,
|
||||
Headers: headers,
|
||||
ExpiresAt: objectStorageExpiresAt(expiresInSeconds),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, parts []domain.ObjectStorageMultipartUploadPart) (domain.ObjectStorageObjectSummary, error) {
|
||||
return browser.CompleteMultipartUploadBound(ctx, bucket, key, uploadID, parts, nil)
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) CompleteMultipartUploadBound(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
key string,
|
||||
uploadID string,
|
||||
parts []domain.ObjectStorageMultipartUploadPart,
|
||||
metadata map[string]string,
|
||||
) (domain.ObjectStorageObjectSummary, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
uploadParts := make([]bosapi.UploadInfoType, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
uploadParts = append(uploadParts, bosapi.UploadInfoType{PartNumber: part.PartNumber, ETag: strings.Trim(part.ETag, `"`)})
|
||||
}
|
||||
if _, err := browser.client.CompleteMultipartUploadFromStruct(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID), &bosapi.CompleteMultipartUploadArgs{Parts: uploadParts}); err != nil {
|
||||
if _, err := browser.client.CompleteMultipartUploadFromStruct(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID), &bosapi.CompleteMultipartUploadArgs{Parts: uploadParts, UserMeta: metadata}); err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
return browser.GetObjectSummary(ctx, bucket, objectKey)
|
||||
@ -525,6 +660,111 @@ func (browser *bosObjectStorageBrowser) AbortMultipartUpload(_ context.Context,
|
||||
return browser.client.AbortMultipartUpload(strings.TrimSpace(bucket), objectKey, strings.TrimSpace(uploadID))
|
||||
}
|
||||
|
||||
func objectSyncMultipartCopyPartSize(sizeBytes int64) int64 {
|
||||
const (
|
||||
minimum = int64(64 * 1024 * 1024)
|
||||
maximum = int64(5 * 1024 * 1024 * 1024)
|
||||
parts = int64(10_000)
|
||||
align = int64(1024 * 1024)
|
||||
)
|
||||
partSize := minimum
|
||||
if sizeBytes > partSize*parts {
|
||||
partSize = ((sizeBytes/parts + align - 1) / align) * align
|
||||
}
|
||||
if partSize > maximum {
|
||||
return maximum
|
||||
}
|
||||
return partSize
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) PublishMultipartCopyConditional(
|
||||
ctx context.Context,
|
||||
bucket string,
|
||||
sourceKey string,
|
||||
targetKey string,
|
||||
sizeBytes int64,
|
||||
sourceETag string,
|
||||
ifMatch string,
|
||||
ifNoneMatch string,
|
||||
userMeta map[string]string,
|
||||
) (domain.ObjectStorageObjectSummary, error) {
|
||||
normalizedBucket := strings.TrimSpace(bucket)
|
||||
sourceKey = exactObjectStorageKey(sourceKey)
|
||||
targetKey = exactObjectStorageKey(targetKey)
|
||||
if normalizedBucket == "" || sourceKey == "" || targetKey == "" {
|
||||
return domain.ObjectStorageObjectSummary{}, fmt.Errorf("bucket, source key, and target key are required")
|
||||
}
|
||||
if sizeBytes <= 0 {
|
||||
return browser.PutObjectConditional(ctx, normalizedBucket, targetKey, []byte{}, "application/octet-stream", ifMatch, ifNoneMatch, userMeta)
|
||||
}
|
||||
initiated, err := browser.client.InitiateMultipartUpload(normalizedBucket, targetKey, "", nil)
|
||||
if err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
abort := true
|
||||
defer func() {
|
||||
if abort {
|
||||
_ = browser.client.AbortMultipartUpload(normalizedBucket, targetKey, initiated.UploadId)
|
||||
}
|
||||
}()
|
||||
partSize := objectSyncMultipartCopyPartSize(sizeBytes)
|
||||
parts := make([]bosapi.UploadInfoType, 0, int((sizeBytes+partSize-1)/partSize))
|
||||
for start, partNumber := int64(0), 1; start < sizeBytes; start, partNumber = start+partSize, partNumber+1 {
|
||||
end := start + partSize - 1
|
||||
if end >= sizeBytes {
|
||||
end = sizeBytes - 1
|
||||
}
|
||||
result, copyErr := browser.client.UploadPartCopyWithContext(
|
||||
ctx,
|
||||
normalizedBucket,
|
||||
targetKey,
|
||||
normalizedBucket,
|
||||
sourceKey,
|
||||
initiated.UploadId,
|
||||
partNumber,
|
||||
&bosapi.UploadPartCopyArgs{
|
||||
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
|
||||
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
|
||||
},
|
||||
)
|
||||
if copyErr != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, copyErr
|
||||
}
|
||||
parts = append(parts, bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)})
|
||||
}
|
||||
completeArgs := &bosapi.CompleteMultipartUploadArgs{
|
||||
Parts: parts,
|
||||
UserMeta: userMeta,
|
||||
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
||||
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
||||
}
|
||||
completePayload, err := json.Marshal(completeArgs)
|
||||
if err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
body, err := bce.NewBodyFromBytes(completePayload)
|
||||
if err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
bosContext := browser.client.NewBosContext(ctx)
|
||||
if _, err = bosapi.CompleteMultipartUpload(browser.client, normalizedBucket, targetKey, initiated.UploadId, body, completeArgs, bosContext); err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
abort = false
|
||||
return browser.GetObjectSummary(ctx, normalizedBucket, targetKey)
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) GetBucketVersioning(_ context.Context, bucket string) (string, error) {
|
||||
result, err := browser.client.GetBucketVersioning(strings.TrimSpace(bucket))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if result == nil {
|
||||
return "", nil
|
||||
}
|
||||
return strings.ToLower(strings.TrimSpace(result.Status)), nil
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
|
||||
objectKey := exactObjectStorageKey(key)
|
||||
destinationKey := exactObjectStorageKey(newKey)
|
||||
|
||||
2602
internal/api/object_sync.go
Normal file
2602
internal/api/object_sync.go
Normal file
File diff suppressed because it is too large
Load Diff
483
internal/api/object_sync_test.go
Normal file
483
internal/api/object_sync_test.go
Normal file
@ -0,0 +1,483 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func objectSyncTestHandler(t *testing.T, enabled bool) (http.Handler, *store.MemoryStore) {
|
||||
t.Helper()
|
||||
dataStore := store.NewMemoryStore()
|
||||
if err := dataStore.UpsertStorageConnection(t.Context(), domain.StorageConnectionRecord{
|
||||
ID: "bos-connection",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Name: "BOS",
|
||||
Provider: domain.StorageProviderBOS,
|
||||
Endpoint: "https://bj.bcebos.com",
|
||||
AccessKeyID: "masked-in-api",
|
||||
SecretAccessKey: "masked-in-api",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return NewHandlerWithConfig(dataStore, nil, config.AppConfig{
|
||||
ObjectSyncEnabled: enabled,
|
||||
WorkspaceMountPath: "/workspace",
|
||||
}), dataStore
|
||||
}
|
||||
|
||||
func objectSyncRequest(t *testing.T, handler http.Handler, method, target string, payload any, idempotencyKey string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var body bytes.Buffer
|
||||
if payload != nil {
|
||||
if err := json.NewEncoder(&body).Encode(payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
request := httptest.NewRequest(method, target, &body)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
if idempotencyKey != "" {
|
||||
request.Header.Set("Idempotency-Key", idempotencyKey)
|
||||
}
|
||||
writer := httptest.NewRecorder()
|
||||
handler.ServeHTTP(writer, request)
|
||||
return writer
|
||||
}
|
||||
|
||||
func TestObjectSyncFeatureFlagBlocksAllRoutes(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, false)
|
||||
writer := objectSyncRequest(t, handler, http.MethodGet, "/api/object-sync/spaces", nil, "")
|
||||
if writer.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status = %d, want %d; body=%s", writer.Code, http.StatusServiceUnavailable, writer.Body.String())
|
||||
}
|
||||
var response objectSyncAPIError
|
||||
if err := json.Unmarshal(writer.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != "object_sync_disabled" || response.RequestID == "" {
|
||||
t.Fatalf("unexpected error envelope: %#v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncSpaceStartsBlockedAndCannotBypassCapabilityGate(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, true)
|
||||
create := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces", map[string]any{
|
||||
"name": "Robot uploads",
|
||||
"connectionId": "bos-connection",
|
||||
"bucket": "robot-data",
|
||||
"rootPrefix": "incoming",
|
||||
}, "space-create")
|
||||
if create.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d; body=%s", create.Code, create.Body.String())
|
||||
}
|
||||
var space domain.ObjectSyncSpaceRecord
|
||||
if err := json.Unmarshal(create.Body.Bytes(), &space); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if space.Enabled || space.Mode != "blocked" || space.Capability.State != domain.ObjectSyncCapabilityUnverified {
|
||||
t.Fatalf("space bypassed initial gate: %#v", space)
|
||||
}
|
||||
enable := objectSyncRequest(t, handler, http.MethodPut, "/api/object-sync/spaces/"+space.ID, map[string]any{
|
||||
"enabled": true,
|
||||
}, "space-enable")
|
||||
if enable.Code != http.StatusConflict {
|
||||
t.Fatalf("status = %d, want conflict; body=%s", enable.Code, enable.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncDriftStateCannotBeClearedBySettingsUpdate(t *testing.T) {
|
||||
handler, dataStore := objectSyncTestHandler(t, true)
|
||||
target, _ := store.AsObjectSyncStore(dataStore)
|
||||
policy := domain.DefaultObjectSyncSpacePolicy()
|
||||
policy.AllowConditionalUpdate = true
|
||||
policy.AllowForceOverwrite = true
|
||||
space := domain.ObjectSyncSpaceRecord{
|
||||
ID: "space-drifted",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ConnectionID: "bos-connection",
|
||||
Bucket: "robot-data",
|
||||
Enabled: false,
|
||||
Mode: "blocked",
|
||||
Policy: policy,
|
||||
Capability: domain.ObjectSyncCapabilityRecord{
|
||||
State: domain.ObjectSyncCapabilityDrifted,
|
||||
ConditionalCreate: true,
|
||||
ConditionalUpdate: true,
|
||||
ConditionalMultipart: true,
|
||||
ExclusivePrefixConfirmed: true,
|
||||
},
|
||||
CreatedAt: "2026-07-28T08:00:00Z",
|
||||
UpdatedAt: "2026-07-28T08:00:00Z",
|
||||
}
|
||||
if err := target.UpsertObjectSyncSpace(t.Context(), space); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer := objectSyncRequest(t, handler, http.MethodPut, "/api/object-sync/spaces/"+space.ID, map[string]any{
|
||||
"name": "renamed but still drifted",
|
||||
"enabled": true,
|
||||
}, "drift-settings-update")
|
||||
if writer.Code != http.StatusConflict {
|
||||
t.Fatalf("status = %d; body=%s", writer.Code, writer.Body.String())
|
||||
}
|
||||
stored, err := target.GetObjectSyncSpace(t.Context(), space.ID)
|
||||
if err != nil || stored == nil {
|
||||
t.Fatalf("stored space: %#v, %v", stored, err)
|
||||
}
|
||||
if stored.Capability.State != domain.ObjectSyncCapabilityDrifted || stored.Enabled {
|
||||
t.Fatalf("drift gate was bypassed: %#v", stored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncWritesRequireIdempotencyKey(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, true)
|
||||
writer := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces", map[string]any{
|
||||
"name": "Robot uploads",
|
||||
"connectionId": "bos-connection",
|
||||
"bucket": "robot-data",
|
||||
}, "")
|
||||
if writer.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d; body=%s", writer.Code, writer.Body.String())
|
||||
}
|
||||
var response objectSyncAPIError
|
||||
if err := json.Unmarshal(writer.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != "idempotency_key_required" {
|
||||
t.Fatalf("code = %q", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncIdempotencyReplaysCompletedResponse(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, true)
|
||||
payload := map[string]any{
|
||||
"name": "Robot uploads",
|
||||
"connectionId": "bos-connection",
|
||||
"bucket": "robot-data",
|
||||
}
|
||||
first := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces", payload, "replay-space")
|
||||
second := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces", payload, "replay-space")
|
||||
if first.Code != http.StatusCreated || second.Code != first.Code {
|
||||
t.Fatalf("statuses = %d, %d; first=%s second=%s", first.Code, second.Code, first.Body.String(), second.Body.String())
|
||||
}
|
||||
if first.Body.String() != second.Body.String() {
|
||||
t.Fatalf("replayed response changed:\nfirst=%s\nsecond=%s", first.Body.String(), second.Body.String())
|
||||
}
|
||||
if second.Header().Get("Idempotency-Replayed") != "true" {
|
||||
t.Fatalf("missing replay header: %#v", second.Header())
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncIdempotencyRejectsKeyReuseWithDifferentPayload(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, true)
|
||||
first := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces", map[string]any{
|
||||
"name": "Robot uploads",
|
||||
"connectionId": "bos-connection",
|
||||
"bucket": "robot-data",
|
||||
}, "reused-space-key")
|
||||
if first.Code != http.StatusCreated {
|
||||
t.Fatalf("first status = %d; body=%s", first.Code, first.Body.String())
|
||||
}
|
||||
second := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/spaces", map[string]any{
|
||||
"name": "Different uploads",
|
||||
"connectionId": "bos-connection",
|
||||
"bucket": "robot-data",
|
||||
}, "reused-space-key")
|
||||
if second.Code != http.StatusConflict {
|
||||
t.Fatalf("second status = %d; body=%s", second.Code, second.Body.String())
|
||||
}
|
||||
var response objectSyncAPIError
|
||||
if err := json.Unmarshal(second.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != "idempotency_key_reused" {
|
||||
t.Fatalf("code = %q", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncClientReleaseArtifactsAreImmutable(t *testing.T) {
|
||||
handler, dataStore := objectSyncTestHandler(t, true)
|
||||
target, _ := store.AsObjectSyncStore(dataStore)
|
||||
release := map[string]any{
|
||||
"id": "drosync-1.0.0",
|
||||
"version": "1.0.0",
|
||||
"channel": "canary",
|
||||
"protocolVersion": "v1",
|
||||
"rolloutPercent": 5,
|
||||
"paused": false,
|
||||
"artifacts": map[string]string{"linux-x86_64": "https://downloads.example/drosync.deb"},
|
||||
"sha256": map[string]string{"linux-x86_64": strings.Repeat("a", 64)},
|
||||
"signature": "signed-manifest",
|
||||
}
|
||||
created := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/client-releases", release, "release-create")
|
||||
if created.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d; body=%s", created.Code, created.Body.String())
|
||||
}
|
||||
items, err := target.ListObjectSyncClientReleases(t.Context())
|
||||
if err != nil || len(items) != 1 {
|
||||
t.Fatalf("stored releases = %#v, %v", items, err)
|
||||
}
|
||||
items[0].StageChangedAt = time.Now().UTC().Add(-25 * time.Hour).Format(time.RFC3339Nano)
|
||||
if err := target.UpsertObjectSyncClientRelease(t.Context(), items[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release["channel"] = "pilot"
|
||||
release["rolloutPercent"] = 20
|
||||
advanced := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/client-releases", release, "release-pilot")
|
||||
if advanced.Code != http.StatusOK {
|
||||
t.Fatalf("advance status = %d; body=%s", advanced.Code, advanced.Body.String())
|
||||
}
|
||||
release["artifacts"] = map[string]string{"linux-x86_64": "https://attacker.example/replaced.deb"}
|
||||
rejected := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/client-releases", release, "release-mutate")
|
||||
if rejected.Code != http.StatusConflict {
|
||||
t.Fatalf("mutation status = %d; body=%s", rejected.Code, rejected.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncClientReleasePromotionRequiresObservationWindow(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, true)
|
||||
release := map[string]any{
|
||||
"id": "drosync-2.0.0",
|
||||
"version": "2.0.0",
|
||||
"channel": "canary",
|
||||
"protocolVersion": "v1",
|
||||
"rolloutPercent": 5,
|
||||
"paused": false,
|
||||
"artifacts": map[string]string{"linux-x86_64": "https://downloads.example/drosync.deb"},
|
||||
"sha256": map[string]string{"linux-x86_64": strings.Repeat("b", 64)},
|
||||
"signature": "signed-manifest",
|
||||
}
|
||||
created := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/client-releases", release, "release-observe-create")
|
||||
if created.Code != http.StatusOK {
|
||||
t.Fatalf("create status = %d; body=%s", created.Code, created.Body.String())
|
||||
}
|
||||
release["channel"] = "pilot"
|
||||
release["rolloutPercent"] = 20
|
||||
promoted := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/client-releases", release, "release-observe-promote")
|
||||
if promoted.Code != http.StatusConflict {
|
||||
t.Fatalf("promotion status = %d; body=%s", promoted.Code, promoted.Body.String())
|
||||
}
|
||||
var response objectSyncAPIError
|
||||
if err := json.Unmarshal(promoted.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != "rollout_observation_incomplete" {
|
||||
t.Fatalf("code = %q", response.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncDeviceAuthorizationIsPublicAndReturnsShortCode(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, true)
|
||||
writer := objectSyncRequest(t, handler, http.MethodPost, "/api/external/object-sync/device-authorizations", map[string]any{
|
||||
"deviceId": "device-01",
|
||||
"deviceName": "capture-station",
|
||||
"platform": "linux",
|
||||
"architecture": "x86_64",
|
||||
"clientVersion": "1.0.0",
|
||||
}, "pair-device")
|
||||
if writer.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d; body=%s", writer.Code, writer.Body.String())
|
||||
}
|
||||
var response map[string]any
|
||||
if err := json.Unmarshal(writer.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response["deviceCode"] == "" || response["userCode"] == "" || response["verificationUri"] == "" {
|
||||
t.Fatalf("pairing response is incomplete: %#v", response)
|
||||
}
|
||||
lookup := objectSyncRequest(t, handler, http.MethodGet, "/api/object-sync/device-authorizations/by-code?userCode="+response["userCode"].(string), nil, "")
|
||||
if lookup.Code != http.StatusOK {
|
||||
t.Fatalf("lookup status = %d; body=%s", lookup.Code, lookup.Body.String())
|
||||
}
|
||||
var authorization domain.ObjectSyncDeviceAuthorizationRecord
|
||||
if err := json.Unmarshal(lookup.Body.Bytes(), &authorization); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if authorization.DeviceID != "device-01" || authorization.Status != "pending" {
|
||||
t.Fatalf("unexpected authorization lookup: %#v", authorization)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncPublicPairingUsesDistributedRateLimit(t *testing.T) {
|
||||
handler, _ := objectSyncTestHandler(t, true)
|
||||
for index := 0; index < 30; index++ {
|
||||
writer := objectSyncRequest(t, handler, http.MethodPost, "/api/external/object-sync/device-authorizations", map[string]any{
|
||||
"deviceId": "rate-device",
|
||||
"deviceName": "rate-test",
|
||||
"platform": "linux",
|
||||
"architecture": "x86_64",
|
||||
}, "rate-pair-"+strconv.Itoa(index))
|
||||
if writer.Code != http.StatusCreated {
|
||||
t.Fatalf("request %d status = %d; body=%s", index+1, writer.Code, writer.Body.String())
|
||||
}
|
||||
}
|
||||
blocked := objectSyncRequest(t, handler, http.MethodPost, "/api/external/object-sync/device-authorizations", map[string]any{
|
||||
"deviceId": "rate-device",
|
||||
"deviceName": "rate-test",
|
||||
"platform": "linux",
|
||||
"architecture": "x86_64",
|
||||
}, "rate-pair-blocked")
|
||||
if blocked.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("status = %d, want %d; body=%s", blocked.Code, http.StatusTooManyRequests, blocked.Body.String())
|
||||
}
|
||||
if blocked.Header().Get("Retry-After") == "" || blocked.Header().Get("RateLimit-Remaining") != "0" {
|
||||
t.Fatalf("missing rate limit headers: %#v", blocked.Header())
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncSessionsUseOpaqueCursorPagination(t *testing.T) {
|
||||
handler, dataStore := objectSyncTestHandler(t, true)
|
||||
target, _ := store.AsObjectSyncStore(dataStore)
|
||||
for _, id := range []string{"session-a", "session-b", "session-c"} {
|
||||
if err := target.UpsertObjectSyncSession(t.Context(), domain.ObjectSyncSessionRecord{
|
||||
ID: id,
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Status: domain.ObjectSyncSessionRunning,
|
||||
CreatedAt: "2026-07-28T08:00:00Z",
|
||||
UpdatedAt: "2026-07-28T08:00:00Z",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first := objectSyncRequest(t, handler, http.MethodGet, "/api/object-sync/sessions?limit=2", nil, "")
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("first status = %d; body=%s", first.Code, first.Body.String())
|
||||
}
|
||||
var firstPage domain.ObjectSyncSessionPage
|
||||
if err := json.Unmarshal(first.Body.Bytes(), &firstPage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(firstPage.Items) != 2 || firstPage.NextCursor == "" || strings.Contains(firstPage.NextCursor, "session-") {
|
||||
t.Fatalf("unexpected first page: %#v", firstPage)
|
||||
}
|
||||
second := objectSyncRequest(t, handler, http.MethodGet, "/api/object-sync/sessions?limit=2&cursor="+firstPage.NextCursor, nil, "")
|
||||
if second.Code != http.StatusOK {
|
||||
t.Fatalf("second status = %d; body=%s", second.Code, second.Body.String())
|
||||
}
|
||||
var secondPage domain.ObjectSyncSessionPage
|
||||
if err := json.Unmarshal(second.Body.Bytes(), &secondPage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(secondPage.Items) != 1 || secondPage.NextCursor != "" {
|
||||
t.Fatalf("unexpected second page: %#v", secondPage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncForceOverwriteQueuesSeparateAuditedItem(t *testing.T) {
|
||||
handler, dataStore := objectSyncTestHandler(t, true)
|
||||
target, _ := store.AsObjectSyncStore(dataStore)
|
||||
policy := domain.DefaultObjectSyncSpacePolicy()
|
||||
policy.AllowConditionalUpdate = true
|
||||
policy.AllowForceOverwrite = true
|
||||
space := domain.ObjectSyncSpaceRecord{
|
||||
ID: "space-force",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ConnectionID: "bos-connection",
|
||||
Bucket: "robot-data",
|
||||
Enabled: true,
|
||||
Mode: "update_enabled",
|
||||
Policy: policy,
|
||||
Capability: domain.ObjectSyncCapabilityRecord{
|
||||
State: domain.ObjectSyncCapabilityUpdateEnabled,
|
||||
ConditionalCreate: true,
|
||||
ConditionalUpdate: true,
|
||||
ConditionalMultipart: true,
|
||||
ExclusivePrefixConfirmed: true,
|
||||
},
|
||||
CreatedAt: "2026-07-28T08:00:00Z",
|
||||
UpdatedAt: "2026-07-28T08:00:00Z",
|
||||
}
|
||||
if err := target.UpsertObjectSyncSpace(t.Context(), space); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session := domain.ObjectSyncSessionRecord{
|
||||
ID: "session-force",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SyncSpaceID: space.ID,
|
||||
Status: domain.ObjectSyncSessionConflicted,
|
||||
CreatedAt: "2026-07-28T08:00:00Z",
|
||||
UpdatedAt: "2026-07-28T08:00:00Z",
|
||||
}
|
||||
if err := target.UpsertObjectSyncSession(t.Context(), session); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conflict := domain.ObjectSyncItemRecord{
|
||||
ID: "item-conflict",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SessionID: session.ID,
|
||||
ClientItemID: "client-conflict",
|
||||
TargetKey: "incoming/file.bin.__conflict__20260728T000000Z_source",
|
||||
LogicalTargetKey: "incoming/file.bin",
|
||||
SizeBytes: 4,
|
||||
SHA256: "abcd",
|
||||
Status: domain.ObjectSyncItemKeptBoth,
|
||||
CreatedAt: "2026-07-28T08:00:00Z",
|
||||
UpdatedAt: "2026-07-28T08:00:00Z",
|
||||
}
|
||||
if err := target.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{conflict}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
writer := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/items/"+conflict.ID+"/force-overwrite", map[string]any{}, "force-overwrite")
|
||||
if writer.Code != http.StatusAccepted {
|
||||
t.Fatalf("status = %d; body=%s", writer.Code, writer.Body.String())
|
||||
}
|
||||
page, err := target.ListObjectSyncItems(t.Context(), domain.ObjectSyncListQuery{SessionID: session.ID, Limit: 10})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 2 {
|
||||
t.Fatalf("items = %#v", page.Items)
|
||||
}
|
||||
original, err := target.GetObjectSyncItem(t.Context(), conflict.ID)
|
||||
if err != nil || original == nil || original.Status != domain.ObjectSyncItemKeptBoth {
|
||||
t.Fatalf("original conflict was mutated: %#v, %v", original, err)
|
||||
}
|
||||
var queued *domain.ObjectSyncItemRecord
|
||||
for index := range page.Items {
|
||||
if page.Items[index].ID != conflict.ID {
|
||||
queued = &page.Items[index]
|
||||
}
|
||||
}
|
||||
if queued == nil || queued.Action != "force_overwrite" || queued.ForceSourceKey != conflict.TargetKey || queued.Status != domain.ObjectSyncItemCommitting {
|
||||
t.Fatalf("queued force item = %#v", queued)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectSyncReleaseRejectsSessionsAcrossSpaces(t *testing.T) {
|
||||
handler, dataStore := objectSyncTestHandler(t, true)
|
||||
target, _ := store.AsObjectSyncStore(dataStore)
|
||||
for index, spaceID := range []string{"space-a", "space-b"} {
|
||||
if err := target.UpsertObjectSyncSession(t.Context(), domain.ObjectSyncSessionRecord{
|
||||
ID: "release-session-" + strconv.Itoa(index),
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SyncSpaceID: spaceID,
|
||||
Status: domain.ObjectSyncSessionCompleted,
|
||||
CreatedAt: "2026-07-28T08:00:00Z",
|
||||
UpdatedAt: "2026-07-28T08:00:00Z",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
writer := objectSyncRequest(t, handler, http.MethodPost, "/api/object-sync/releases", map[string]any{
|
||||
"name": "ambiguous-release",
|
||||
"sessionIds": []string{"release-session-0", "release-session-1"},
|
||||
}, "release-cross-space")
|
||||
if writer.Code != http.StatusConflict {
|
||||
t.Fatalf("status = %d; body=%s", writer.Code, writer.Body.String())
|
||||
}
|
||||
var response objectSyncAPIError
|
||||
if err := json.Unmarshal(writer.Body.Bytes(), &response); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Code != "release_space_mismatch" {
|
||||
t.Fatalf("code = %q", response.Code)
|
||||
}
|
||||
}
|
||||
@ -54,6 +54,7 @@ type server struct {
|
||||
apiToken string
|
||||
requireSessionAuth bool
|
||||
sessionCookieSecure bool
|
||||
objectSyncEnabled bool
|
||||
trustedProxyNets []*net.IPNet
|
||||
authLimiter *rateLimiter
|
||||
externalWriteLimiter *rateLimiter
|
||||
@ -1233,6 +1234,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
apiToken: appConfig.APIToken,
|
||||
requireSessionAuth: appConfig.RequireSessionAuth,
|
||||
sessionCookieSecure: appConfig.SessionCookieSecure,
|
||||
objectSyncEnabled: appConfig.ObjectSyncEnabled,
|
||||
trustedProxyNets: parseTrustedProxyCIDRs(appConfig.TrustedProxyCIDRs),
|
||||
authLimiter: newRateLimiter(20, time.Minute),
|
||||
externalWriteLimiter: newRateLimiter(120, time.Minute),
|
||||
@ -1500,6 +1502,75 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
router.Delete("/api/object-storage/objects", instance.handleObjectStorageObjectDelete)
|
||||
})
|
||||
|
||||
router.Route("/api/object-sync", func(router chi.Router) {
|
||||
router.Use(instance.requireObjectSyncFeature)
|
||||
router.Group(func(router chi.Router) {
|
||||
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleDataOperator, domain.UserRolePipelineEngineer, domain.UserRoleComplianceAuditor, domain.UserRoleAuditor))
|
||||
router.Get("/spaces", instance.handleObjectSyncSpacesList)
|
||||
router.Get("/sessions", instance.handleObjectSyncSessionsList)
|
||||
router.Get("/sessions/{id}", instance.handleObjectSyncSessionGet)
|
||||
router.Get("/sessions/{id}/items", instance.handleObjectSyncSessionItems)
|
||||
})
|
||||
router.Group(func(router chi.Router) {
|
||||
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin))
|
||||
router.Use(instance.requireObjectSyncIdempotency)
|
||||
router.Post("/spaces", instance.handleObjectSyncSpaceCreate)
|
||||
router.Put("/spaces/{id}", instance.handleObjectSyncSpaceUpdate)
|
||||
router.Post("/spaces/{id}/capability-probe", instance.handleObjectSyncCapabilityProbe)
|
||||
router.Post("/spaces/{id}/reconcile", instance.handleObjectSyncReconcileCreate)
|
||||
router.Get("/devices", instance.handleObjectSyncDevicesList)
|
||||
router.Get("/device-authorizations", instance.handleObjectSyncAuthorizationsList)
|
||||
router.Get("/device-authorizations/by-code", instance.handleObjectSyncAuthorizationLookup)
|
||||
router.Post("/device-authorizations/{id}/{decision}", instance.handleObjectSyncAuthorizationDecision)
|
||||
router.Get("/credentials", instance.handleObjectSyncCredentialsList)
|
||||
router.Post("/credentials/{id}/{action}", instance.handleObjectSyncCredentialMutation)
|
||||
router.Post("/sessions/{id}/admin-abort", instance.handleObjectSyncSessionAdminAbort)
|
||||
router.Post("/leases/{id}/force-release", instance.handleObjectSyncLeaseForceRelease)
|
||||
router.Post("/items/{id}/force-overwrite", instance.handleObjectSyncItemForceOverwrite)
|
||||
router.Get("/client-releases", instance.handleObjectSyncClientReleasesList)
|
||||
router.Post("/client-releases", instance.handleObjectSyncClientReleaseUpsert)
|
||||
})
|
||||
router.Group(func(router chi.Router) {
|
||||
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer))
|
||||
router.Get("/releases", instance.handleObjectSyncReleasesList)
|
||||
router.With(instance.requireObjectSyncIdempotency).Post("/releases", instance.handleObjectSyncReleaseCreate)
|
||||
})
|
||||
router.Group(func(router chi.Router) {
|
||||
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRoleComplianceAuditor, domain.UserRoleAuditor))
|
||||
router.Get("/audit-events", instance.handleObjectSyncAuditList)
|
||||
router.Get("/audit-events/export", instance.handleObjectSyncAuditExport)
|
||||
router.Get("/audit-seals/{day}", instance.handleObjectSyncAuditSealGet)
|
||||
})
|
||||
})
|
||||
|
||||
router.Route("/api/external/object-sync", func(router chi.Router) {
|
||||
router.Use(instance.requireObjectSyncFeature)
|
||||
router.With(instance.objectSyncDistributedRateLimit(30, time.Minute), instance.requireObjectSyncIdempotency).Post("/device-authorizations", instance.handleExternalObjectSyncAuthorizationCreate)
|
||||
router.With(instance.objectSyncDistributedRateLimit(30, time.Minute), instance.requireObjectSyncIdempotency).Post("/device-authorizations/token", instance.handleExternalObjectSyncAuthorizationPoll)
|
||||
router.Group(func(router chi.Router) {
|
||||
router.Use(instance.objectSyncCredentialAuth)
|
||||
router.Use(instance.objectSyncDistributedRateLimit(600, time.Minute))
|
||||
router.Use(instance.requireObjectSyncIdempotency)
|
||||
router.Get("/capabilities", instance.handleExternalObjectSyncCapabilities)
|
||||
router.Post("/sessions", instance.handleExternalObjectSyncSessionCreate)
|
||||
router.Post("/sessions/{sessionId}/manifest-pages", instance.handleExternalObjectSyncManifestPage)
|
||||
router.Post("/sessions/{sessionId}/plan", instance.handleExternalObjectSyncPlan)
|
||||
router.Get("/sessions/{sessionId}/status", instance.handleExternalObjectSyncSessionStatus)
|
||||
router.Get("/sessions/{sessionId}/items", instance.handleExternalObjectSyncSessionItems)
|
||||
router.Post("/sessions/{sessionId}/{action}", instance.handleExternalObjectSyncSessionFinish)
|
||||
router.Post("/items/{itemId}/reserve", instance.handleExternalObjectSyncItemReserve)
|
||||
router.Post("/items/{itemId}/uploads/direct", instance.handleExternalObjectSyncDirectUpload)
|
||||
router.Post("/items/{itemId}/uploads/direct/complete", instance.handleExternalObjectSyncDirectUploadComplete)
|
||||
router.Post("/items/{itemId}/uploads/multipart", instance.handleExternalObjectSyncMultipartInitiate)
|
||||
router.Post("/items/{itemId}/uploads/multipart/part-url", instance.handleExternalObjectSyncMultipartPartURL)
|
||||
router.Post("/items/{itemId}/uploads/multipart/complete", instance.handleExternalObjectSyncMultipartComplete)
|
||||
router.Post("/items/{itemId}/heartbeat", instance.handleExternalObjectSyncHeartbeat)
|
||||
router.Post("/items/{itemId}/abort", instance.handleExternalObjectSyncItemAbort)
|
||||
router.Post("/items/{itemId}/commit", instance.handleExternalObjectSyncCommit)
|
||||
router.Get("/client-updates", instance.handleExternalObjectSyncUpdateCheck)
|
||||
})
|
||||
})
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
type AppConfig struct {
|
||||
Port int
|
||||
WorkerPort int
|
||||
CoordinatorPort int
|
||||
MongoURI string
|
||||
MongoDB string
|
||||
MongoConnectRetries int
|
||||
@ -33,6 +34,9 @@ type AppConfig struct {
|
||||
APIToken string
|
||||
RequireSessionAuth bool
|
||||
SessionCookieSecure bool
|
||||
ObjectSyncEnabled bool
|
||||
ObjectSyncAuditSigningPrivateKey string
|
||||
ObjectSyncAuditSigningKeyID string
|
||||
TrustedProxyCIDRs string
|
||||
OperatorGPUNodeSelector string
|
||||
OperatorGPUTolerations string
|
||||
@ -57,6 +61,10 @@ func LoadValidated() (AppConfig, error) {
|
||||
if err != nil {
|
||||
return AppConfig{}, err
|
||||
}
|
||||
coordinatorPort, err := parseIntEnv("DROSYNC_COORDINATOR_PORT", 3003, 1, 65535)
|
||||
if err != nil {
|
||||
return AppConfig{}, err
|
||||
}
|
||||
mongoConnectRetries, err := parseIntEnv("MONGO_CONNECT_RETRIES", 30, 1, 0)
|
||||
if err != nil {
|
||||
return AppConfig{}, err
|
||||
@ -89,6 +97,10 @@ func LoadValidated() (AppConfig, error) {
|
||||
if err != nil {
|
||||
return AppConfig{}, err
|
||||
}
|
||||
objectSyncEnabled, err := parseBoolEnv("DROBOTICFLOW_OBJECT_SYNC_ENABLED", false)
|
||||
if err != nil {
|
||||
return AppConfig{}, err
|
||||
}
|
||||
workerAPIBaseURL, err := parseHTTPURLEnv("WORKER_API_BASE_URL", "http://droboticflow-worker:3002")
|
||||
if err != nil {
|
||||
return AppConfig{}, err
|
||||
@ -119,6 +131,7 @@ func LoadValidated() (AppConfig, error) {
|
||||
return AppConfig{
|
||||
Port: port,
|
||||
WorkerPort: workerPort,
|
||||
CoordinatorPort: coordinatorPort,
|
||||
MongoURI: mongoURI,
|
||||
MongoDB: getenvTrimmed("MONGO_DB", "droboticflow"),
|
||||
MongoConnectRetries: mongoConnectRetries,
|
||||
@ -140,6 +153,9 @@ func LoadValidated() (AppConfig, error) {
|
||||
APIToken: strings.TrimSpace(os.Getenv("DROBOTICFLOW_API_TOKEN")),
|
||||
RequireSessionAuth: requireSessionAuth,
|
||||
SessionCookieSecure: sessionCookieSecure,
|
||||
ObjectSyncEnabled: objectSyncEnabled,
|
||||
ObjectSyncAuditSigningPrivateKey: strings.TrimSpace(os.Getenv("DROSYNC_AUDIT_SIGNING_PRIVATE_KEY")),
|
||||
ObjectSyncAuditSigningKeyID: getenvTrimmed("DROSYNC_AUDIT_SIGNING_KEY_ID", "drosync-audit-v1"),
|
||||
TrustedProxyCIDRs: strings.TrimSpace(os.Getenv("DROBOTICFLOW_TRUSTED_PROXY_CIDRS")),
|
||||
OperatorGPUNodeSelector: strings.TrimSpace(os.Getenv("OPERATOR_GPU_NODE_SELECTOR")),
|
||||
OperatorGPUTolerations: strings.TrimSpace(os.Getenv("OPERATOR_GPU_TOLERATIONS_JSON")),
|
||||
|
||||
@ -16,6 +16,7 @@ type ObjectStorageObjectSummary struct {
|
||||
SizeBytes int64 `json:"sizeBytes,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
ETag string `json:"etag,omitempty"`
|
||||
VersionID string `json:"versionId,omitempty"`
|
||||
LastModified string `json:"lastModified,omitempty"`
|
||||
StorageClass string `json:"storageClass,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
|
||||
409
internal/domain/object_sync.go
Normal file
409
internal/domain/object_sync.go
Normal file
@ -0,0 +1,409 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
ObjectSyncCapabilityUnverified = "unverified"
|
||||
ObjectSyncCapabilityCreateOnly = "create_only"
|
||||
ObjectSyncCapabilityUpdateEnabled = "update_enabled"
|
||||
ObjectSyncCapabilityDrifted = "drifted"
|
||||
ObjectSyncCapabilityUnsupported = "unsupported"
|
||||
ObjectSyncDefaultManifestPageMaxItems = 1000
|
||||
)
|
||||
|
||||
type ObjectSyncSpacePolicy struct {
|
||||
AllowedActions []string `json:"allowedActions" bson:"allowedActions"`
|
||||
MaxObjectBytes int64 `json:"maxObjectBytes" bson:"maxObjectBytes"`
|
||||
MaxSessionFiles int64 `json:"maxSessionFiles" bson:"maxSessionFiles"`
|
||||
MaxSessionBytes int64 `json:"maxSessionBytes" bson:"maxSessionBytes"`
|
||||
MaxConcurrentUploads int `json:"maxConcurrentUploads" bson:"maxConcurrentUploads"`
|
||||
AllowDelete bool `json:"allowDelete" bson:"allowDelete"`
|
||||
AllowConditionalUpdate bool `json:"allowConditionalUpdate" bson:"allowConditionalUpdate"`
|
||||
DefaultConflictResolution string `json:"defaultConflictResolution" bson:"defaultConflictResolution"`
|
||||
AllowForceOverwrite bool `json:"allowForceOverwrite" bson:"allowForceOverwrite"`
|
||||
}
|
||||
|
||||
func DefaultObjectSyncSpacePolicy() ObjectSyncSpacePolicy {
|
||||
return ObjectSyncSpacePolicy{
|
||||
AllowedActions: []string{"plan", "list_metadata", "create", "keep_both"},
|
||||
MaxObjectBytes: 1 << 40,
|
||||
MaxSessionFiles: 1_000_000,
|
||||
MaxSessionBytes: 20 << 40,
|
||||
MaxConcurrentUploads: 16,
|
||||
AllowDelete: false,
|
||||
AllowConditionalUpdate: false,
|
||||
DefaultConflictResolution: "keep_both",
|
||||
AllowForceOverwrite: false,
|
||||
}
|
||||
}
|
||||
|
||||
type ObjectSyncCapabilityRecord struct {
|
||||
State string `json:"state" bson:"state"`
|
||||
ConditionalCreate bool `json:"conditionalCreate" bson:"conditionalCreate"`
|
||||
ConditionalUpdate bool `json:"conditionalUpdate" bson:"conditionalUpdate"`
|
||||
ConditionalMultipart bool `json:"conditionalMultipart" bson:"conditionalMultipart"`
|
||||
MultipartCopy bool `json:"multipartCopy" bson:"multipartCopy"`
|
||||
VersionID bool `json:"versionId" bson:"versionId"`
|
||||
ExclusivePrefixConfirmed bool `json:"exclusivePrefixConfirmed" bson:"exclusivePrefixConfirmed"`
|
||||
ProbedAt string `json:"probedAt,omitempty" bson:"probedAt,omitempty"`
|
||||
ProbeError string `json:"probeError,omitempty" bson:"probeError,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncSpaceRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Description string `json:"description,omitempty" bson:"description,omitempty"`
|
||||
ConnectionID string `json:"connectionId" bson:"connectionId"`
|
||||
Bucket string `json:"bucket" bson:"bucket"`
|
||||
RootPrefix string `json:"rootPrefix" bson:"rootPrefix"`
|
||||
Mode string `json:"mode" bson:"mode"`
|
||||
Enabled bool `json:"enabled" bson:"enabled"`
|
||||
Policy ObjectSyncSpacePolicy `json:"policy" bson:"policy"`
|
||||
Capability ObjectSyncCapabilityRecord `json:"capability" bson:"capability"`
|
||||
CreatedBy string `json:"createdBy,omitempty" bson:"createdBy,omitempty"`
|
||||
UpdatedBy string `json:"updatedBy,omitempty" bson:"updatedBy,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncCredentialScope struct {
|
||||
SyncSpaceID string `json:"syncSpaceId" bson:"syncSpaceId"`
|
||||
AllowedRelativePrefixes []string `json:"allowedRelativePrefixes" bson:"allowedRelativePrefixes"`
|
||||
Actions []string `json:"actions" bson:"actions"`
|
||||
}
|
||||
|
||||
type ObjectSyncDeviceRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Platform string `json:"platform,omitempty" bson:"platform,omitempty"`
|
||||
Architecture string `json:"architecture,omitempty" bson:"architecture,omitempty"`
|
||||
ClientVersion string `json:"clientVersion,omitempty" bson:"clientVersion,omitempty"`
|
||||
Status string `json:"status" bson:"status"`
|
||||
LastSeenAt string `json:"lastSeenAt,omitempty" bson:"lastSeenAt,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncDeviceAuthorizationRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
|
||||
DeviceID string `json:"deviceId" bson:"deviceId"`
|
||||
DeviceName string `json:"deviceName" bson:"deviceName"`
|
||||
Platform string `json:"platform,omitempty" bson:"platform,omitempty"`
|
||||
Architecture string `json:"architecture,omitempty" bson:"architecture,omitempty"`
|
||||
ClientVersion string `json:"clientVersion,omitempty" bson:"clientVersion,omitempty"`
|
||||
DeviceCodeHash string `json:"-" bson:"deviceCodeHash"`
|
||||
UserCode string `json:"userCode" bson:"userCode"`
|
||||
Status string `json:"status" bson:"status"`
|
||||
Scopes []ObjectSyncCredentialScope `json:"scopes,omitempty" bson:"scopes,omitempty"`
|
||||
ApprovedBy string `json:"approvedBy,omitempty" bson:"approvedBy,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
ExpiresAt string `json:"expiresAt" bson:"expiresAt"`
|
||||
ExpiresAtTime time.Time `json:"-" bson:"expiresAtTime"`
|
||||
}
|
||||
|
||||
type ObjectSyncCredentialRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
DeviceID string `json:"deviceId" bson:"deviceId"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Enabled bool `json:"enabled" bson:"enabled"`
|
||||
TokenHash string `json:"-" bson:"tokenHash"`
|
||||
TokenPreview string `json:"tokenPreview" bson:"tokenPreview"`
|
||||
Scopes []ObjectSyncCredentialScope `json:"scopes" bson:"scopes"`
|
||||
LastUsedAt string `json:"lastUsedAt,omitempty" bson:"lastUsedAt,omitempty"`
|
||||
RevokedAt string `json:"revokedAt,omitempty" bson:"revokedAt,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncRemoteBase struct {
|
||||
Exists bool `json:"exists" bson:"exists"`
|
||||
ETag string `json:"etag,omitempty" bson:"etag,omitempty"`
|
||||
SizeBytes int64 `json:"sizeBytes,omitempty" bson:"sizeBytes,omitempty"`
|
||||
LastModified string `json:"lastModified,omitempty" bson:"lastModified,omitempty"`
|
||||
PlatformVersion int64 `json:"platformVersion,omitempty" bson:"platformVersion,omitempty"`
|
||||
VersionID string `json:"versionId,omitempty" bson:"versionId,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncSessionCounts struct {
|
||||
Manifest int64 `json:"manifest" bson:"manifest"`
|
||||
Unchanged int64 `json:"unchanged" bson:"unchanged"`
|
||||
Create int64 `json:"create" bson:"create"`
|
||||
Update int64 `json:"update" bson:"update"`
|
||||
KeptBoth int64 `json:"keptBoth" bson:"keptBoth"`
|
||||
Skipped int64 `json:"skipped" bson:"skipped"`
|
||||
Denied int64 `json:"denied" bson:"denied"`
|
||||
Failed int64 `json:"failed" bson:"failed"`
|
||||
Committed int64 `json:"committed" bson:"committed"`
|
||||
}
|
||||
|
||||
type ObjectSyncSessionRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
SyncSpaceID string `json:"syncSpaceId" bson:"syncSpaceId"`
|
||||
CredentialID string `json:"credentialId" bson:"credentialId"`
|
||||
DeviceID string `json:"deviceId" bson:"deviceId"`
|
||||
MappingID string `json:"mappingId" bson:"mappingId"`
|
||||
ClientVersion string `json:"clientVersion" bson:"clientVersion"`
|
||||
ProtocolVersion string `json:"protocolVersion" bson:"protocolVersion"`
|
||||
Status ObjectSyncSessionStatus `json:"status" bson:"status"`
|
||||
PolicySnapshot ObjectSyncSpacePolicy `json:"policySnapshot" bson:"policySnapshot"`
|
||||
ManifestDigest string `json:"manifestDigest,omitempty" bson:"manifestDigest,omitempty"`
|
||||
ManifestFinalPage int `json:"manifestFinalPage,omitempty" bson:"manifestFinalPage,omitempty"`
|
||||
Counts ObjectSyncSessionCounts `json:"counts" bson:"counts"`
|
||||
TotalBytes int64 `json:"totalBytes" bson:"totalBytes"`
|
||||
OperationID string `json:"operationId,omitempty" bson:"operationId,omitempty"`
|
||||
ErrorSummary string `json:"errorSummary,omitempty" bson:"errorSummary,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
CompletedAt string `json:"completedAt,omitempty" bson:"completedAt,omitempty"`
|
||||
LastHeartbeatAt string `json:"lastHeartbeatAt,omitempty" bson:"lastHeartbeatAt,omitempty"`
|
||||
ExpiresAt string `json:"expiresAt" bson:"expiresAt"`
|
||||
PurgeAt time.Time `json:"-" bson:"purgeAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncManifestItemInput struct {
|
||||
ClientItemID string `json:"clientItemId"`
|
||||
RelativePath string `json:"relativePath"`
|
||||
Kind string `json:"kind"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
ModifiedAt string `json:"modifiedAt"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
LastKnownRemote ObjectSyncRemoteBase `json:"lastKnownRemote,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncItemRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
SessionID string `json:"sessionId" bson:"sessionId"`
|
||||
ClientItemID string `json:"clientItemId" bson:"clientItemId"`
|
||||
ManifestPage int `json:"manifestPage" bson:"manifestPage"`
|
||||
RelativePath string `json:"relativePath" bson:"relativePath"`
|
||||
NormalizedRelativePath string `json:"normalizedRelativePath" bson:"normalizedRelativePath"`
|
||||
TargetKey string `json:"targetKey" bson:"targetKey"`
|
||||
LogicalTargetKey string `json:"logicalTargetKey,omitempty" bson:"logicalTargetKey,omitempty"`
|
||||
SizeBytes int64 `json:"sizeBytes" bson:"sizeBytes"`
|
||||
ModifiedAt string `json:"modifiedAt" bson:"modifiedAt"`
|
||||
SHA256 string `json:"sha256" bson:"sha256"`
|
||||
ContentType string `json:"contentType,omitempty" bson:"contentType,omitempty"`
|
||||
ExpectedBase ObjectSyncRemoteBase `json:"expectedBase" bson:"expectedBase"`
|
||||
Action string `json:"action" bson:"action"`
|
||||
Status ObjectSyncItemStatus `json:"status" bson:"status"`
|
||||
StagingKey string `json:"stagingKey,omitempty" bson:"stagingKey,omitempty"`
|
||||
UploadMode string `json:"uploadMode,omitempty" bson:"uploadMode,omitempty"`
|
||||
UploadID string `json:"uploadId,omitempty" bson:"uploadId,omitempty"`
|
||||
UploadedParts []ObjectStorageMultipartUploadPart `json:"uploadedParts,omitempty" bson:"uploadedParts,omitempty"`
|
||||
LeaseID string `json:"leaseId,omitempty" bson:"leaseId,omitempty"`
|
||||
FencingToken int64 `json:"fencingToken,omitempty" bson:"fencingToken,omitempty"`
|
||||
CommittedVersion ObjectSyncRemoteBase `json:"committedVersion,omitempty" bson:"committedVersion,omitempty"`
|
||||
ForceSourceKey string `json:"forceSourceKey,omitempty" bson:"forceSourceKey,omitempty"`
|
||||
ForceActorID string `json:"forceActorId,omitempty" bson:"forceActorId,omitempty"`
|
||||
OperationID string `json:"operationId,omitempty" bson:"operationId,omitempty"`
|
||||
AttemptCount int `json:"attemptCount" bson:"attemptCount"`
|
||||
LastError string `json:"lastError,omitempty" bson:"lastError,omitempty"`
|
||||
CoordinatorOwner string `json:"-" bson:"coordinatorOwner,omitempty"`
|
||||
CoordinatorLeaseUntil time.Time `json:"-" bson:"coordinatorLeaseUntil,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
PurgeAt time.Time `json:"-" bson:"purgeAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncLeaseRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
ResourceHash string `json:"resourceHash" bson:"resourceHash"`
|
||||
ResourceType string `json:"resourceType" bson:"resourceType"`
|
||||
ConnectionID string `json:"connectionId" bson:"connectionId"`
|
||||
Bucket string `json:"bucket" bson:"bucket"`
|
||||
KeyOrPrefix string `json:"keyOrPrefix" bson:"keyOrPrefix"`
|
||||
OwnerSessionID string `json:"ownerSessionId,omitempty" bson:"ownerSessionId,omitempty"`
|
||||
OwnerItemID string `json:"ownerItemId,omitempty" bson:"ownerItemId,omitempty"`
|
||||
FencingToken int64 `json:"fencingToken" bson:"fencingToken"`
|
||||
ExpiresAt string `json:"expiresAt,omitempty" bson:"expiresAt,omitempty"`
|
||||
ExpiresAtTime time.Time `json:"-" bson:"expiresAtTime,omitempty"`
|
||||
LastHeartbeatAt string `json:"lastHeartbeatAt,omitempty" bson:"lastHeartbeatAt,omitempty"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncObjectHeadRecord struct {
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
SyncSpaceID string `json:"syncSpaceId" bson:"syncSpaceId"`
|
||||
Key string `json:"key" bson:"key"`
|
||||
PlatformVersion int64 `json:"platformVersion" bson:"platformVersion"`
|
||||
RemoteETag string `json:"remoteETag" bson:"remoteETag"`
|
||||
VersionID string `json:"versionId,omitempty" bson:"versionId,omitempty"`
|
||||
SizeBytes int64 `json:"sizeBytes" bson:"sizeBytes"`
|
||||
SHA256 string `json:"sha256" bson:"sha256"`
|
||||
SessionID string `json:"sessionId" bson:"sessionId"`
|
||||
ItemID string `json:"itemId" bson:"itemId"`
|
||||
Drifted bool `json:"drifted" bson:"drifted"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncVersionRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
SyncSpaceID string `json:"syncSpaceId" bson:"syncSpaceId"`
|
||||
Key string `json:"key" bson:"key"`
|
||||
LogicalTargetKey string `json:"logicalTargetKey,omitempty" bson:"logicalTargetKey,omitempty"`
|
||||
SourceKey string `json:"sourceKey,omitempty" bson:"sourceKey,omitempty"`
|
||||
PlatformVersion int64 `json:"platformVersion" bson:"platformVersion"`
|
||||
RemoteETag string `json:"remoteETag" bson:"remoteETag"`
|
||||
VersionID string `json:"versionId,omitempty" bson:"versionId,omitempty"`
|
||||
SizeBytes int64 `json:"sizeBytes" bson:"sizeBytes"`
|
||||
SHA256 string `json:"sha256" bson:"sha256"`
|
||||
SessionID string `json:"sessionId" bson:"sessionId"`
|
||||
ItemID string `json:"itemId" bson:"itemId"`
|
||||
ActorID string `json:"actorId" bson:"actorId"`
|
||||
CommittedAt string `json:"committedAt" bson:"committedAt"`
|
||||
PurgeAt time.Time `json:"-" bson:"purgeAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncAuditEvent struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
EventKey string `json:"eventKey" bson:"eventKey"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
ActorType string `json:"actorType" bson:"actorType"`
|
||||
ActorID string `json:"actorId" bson:"actorId"`
|
||||
CredentialID string `json:"credentialId,omitempty" bson:"credentialId,omitempty"`
|
||||
DeviceID string `json:"deviceId,omitempty" bson:"deviceId,omitempty"`
|
||||
RequestID string `json:"requestId,omitempty" bson:"requestId,omitempty"`
|
||||
SessionID string `json:"sessionId,omitempty" bson:"sessionId,omitempty"`
|
||||
ItemID string `json:"itemId,omitempty" bson:"itemId,omitempty"`
|
||||
Action string `json:"action" bson:"action"`
|
||||
Target map[string]any `json:"target,omitempty" bson:"target,omitempty"`
|
||||
Before map[string]any `json:"before,omitempty" bson:"before,omitempty"`
|
||||
After map[string]any `json:"after,omitempty" bson:"after,omitempty"`
|
||||
Result string `json:"result" bson:"result"`
|
||||
Reason string `json:"reason,omitempty" bson:"reason,omitempty"`
|
||||
PreviousHash string `json:"previousHash,omitempty" bson:"previousHash,omitempty"`
|
||||
EventHash string `json:"eventHash" bson:"eventHash"`
|
||||
OccurredAt string `json:"occurredAt" bson:"occurredAt"`
|
||||
PurgeAt time.Time `json:"-" bson:"purgeAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncAuditSealRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
DayUTC string `json:"dayUtc" bson:"dayUtc"`
|
||||
EventCount int64 `json:"eventCount" bson:"eventCount"`
|
||||
FirstEventHash string `json:"firstEventHash" bson:"firstEventHash"`
|
||||
LastEventHash string `json:"lastEventHash" bson:"lastEventHash"`
|
||||
AggregateHash string `json:"aggregateHash" bson:"aggregateHash"`
|
||||
Signature string `json:"signature" bson:"signature"`
|
||||
SigningKeyID string `json:"signingKeyId" bson:"signingKeyId"`
|
||||
ObjectKey string `json:"objectKey" bson:"objectKey"`
|
||||
ObjectETag string `json:"objectETag" bson:"objectETag"`
|
||||
ObjectVersionID string `json:"objectVersionId,omitempty" bson:"objectVersionId,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
PurgeAt time.Time `json:"-" bson:"purgeAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncReleaseRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
SyncSpaceID string `json:"syncSpaceId" bson:"syncSpaceId"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Description string `json:"description,omitempty" bson:"description,omitempty"`
|
||||
SessionIDs []string `json:"sessionIds" bson:"sessionIds"`
|
||||
ManifestKey string `json:"manifestKey" bson:"manifestKey"`
|
||||
ManifestSHA256 string `json:"manifestSha256" bson:"manifestSha256"`
|
||||
ObjectCount int64 `json:"objectCount" bson:"objectCount"`
|
||||
TotalBytes int64 `json:"totalBytes" bson:"totalBytes"`
|
||||
Status string `json:"status" bson:"status"`
|
||||
CreatedBy string `json:"createdBy" bson:"createdBy"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
PublishedAt string `json:"publishedAt,omitempty" bson:"publishedAt,omitempty"`
|
||||
CoordinatorOwner string `json:"-" bson:"coordinatorOwner,omitempty"`
|
||||
CoordinatorLeaseUntil time.Time `json:"-" bson:"coordinatorLeaseUntil,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncClientReleaseRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
Version string `json:"version" bson:"version"`
|
||||
Channel string `json:"channel" bson:"channel"`
|
||||
ProtocolVersion string `json:"protocolVersion" bson:"protocolVersion"`
|
||||
MinimumServerVersion string `json:"minimumServerVersion" bson:"minimumServerVersion"`
|
||||
Artifacts map[string]string `json:"artifacts" bson:"artifacts"`
|
||||
SHA256 map[string]string `json:"sha256" bson:"sha256"`
|
||||
Signature string `json:"signature" bson:"signature"`
|
||||
RolloutPercent int `json:"rolloutPercent" bson:"rolloutPercent"`
|
||||
Paused bool `json:"paused" bson:"paused"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
StageChangedAt string `json:"stageChangedAt" bson:"stageChangedAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncReconcileRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OperationID string `json:"operationId" bson:"operationId"`
|
||||
OrganizationID string `json:"organizationId" bson:"organizationId"`
|
||||
SyncSpaceID string `json:"syncSpaceId" bson:"syncSpaceId"`
|
||||
Status string `json:"status" bson:"status"`
|
||||
Scanned int64 `json:"scanned" bson:"scanned"`
|
||||
Drifted int64 `json:"drifted" bson:"drifted"`
|
||||
Error string `json:"error,omitempty" bson:"error,omitempty"`
|
||||
CreatedBy string `json:"createdBy" bson:"createdBy"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
CompletedAt string `json:"completedAt,omitempty" bson:"completedAt,omitempty"`
|
||||
CoordinatorOwner string `json:"-" bson:"coordinatorOwner,omitempty"`
|
||||
CoordinatorLeaseUntil time.Time `json:"-" bson:"coordinatorLeaseUntil,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncIdempotencyRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
|
||||
ActorID string `json:"actorId,omitempty" bson:"actorId,omitempty"`
|
||||
Key string `json:"key" bson:"key"`
|
||||
RequestMethod string `json:"requestMethod" bson:"requestMethod"`
|
||||
RequestPath string `json:"requestPath" bson:"requestPath"`
|
||||
RequestDigest string `json:"requestDigest" bson:"requestDigest"`
|
||||
State string `json:"state" bson:"state"`
|
||||
ResponseStatus int `json:"responseStatus,omitempty" bson:"responseStatus,omitempty"`
|
||||
ResponseHeader map[string]string `json:"responseHeader,omitempty" bson:"responseHeader,omitempty"`
|
||||
ResponseBody []byte `json:"responseBody,omitempty" bson:"responseBody,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
ExpiresAt time.Time `json:"-" bson:"expiresAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncRateLimitRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
Subject string `json:"subject" bson:"subject"`
|
||||
Window string `json:"window" bson:"window"`
|
||||
Count int64 `json:"count" bson:"count"`
|
||||
ExpiresAt time.Time `json:"-" bson:"expiresAt"`
|
||||
}
|
||||
|
||||
type ObjectSyncListQuery struct {
|
||||
OrganizationID string
|
||||
SessionID string
|
||||
Status string
|
||||
Cursor string
|
||||
Limit int
|
||||
}
|
||||
|
||||
type ObjectSyncItemPage struct {
|
||||
Items []ObjectSyncItemRecord `json:"items"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncSessionPage struct {
|
||||
Items []ObjectSyncSessionRecord `json:"items"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncHeadPage struct {
|
||||
Items []ObjectSyncObjectHeadRecord `json:"items"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type ObjectSyncAuditPage struct {
|
||||
Items []ObjectSyncAuditEvent `json:"items"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
39
internal/domain/object_sync_contract_generated.go
Normal file
39
internal/domain/object_sync_contract_generated.go
Normal file
@ -0,0 +1,39 @@
|
||||
// Code generated by scripts/generate_drosync_contract.py; DO NOT EDIT.
|
||||
|
||||
package domain
|
||||
|
||||
const ObjectSyncProtocolVersion = "v1"
|
||||
|
||||
type ObjectSyncSessionStatus string
|
||||
|
||||
const (
|
||||
ObjectSyncSessionCollectingManifest ObjectSyncSessionStatus = "collecting_manifest"
|
||||
ObjectSyncSessionPlanning ObjectSyncSessionStatus = "planning"
|
||||
ObjectSyncSessionPlanned ObjectSyncSessionStatus = "planned"
|
||||
ObjectSyncSessionRunning ObjectSyncSessionStatus = "running"
|
||||
ObjectSyncSessionCompleted ObjectSyncSessionStatus = "completed"
|
||||
ObjectSyncSessionPartial ObjectSyncSessionStatus = "partial"
|
||||
ObjectSyncSessionConflicted ObjectSyncSessionStatus = "conflicted"
|
||||
ObjectSyncSessionFailed ObjectSyncSessionStatus = "failed"
|
||||
ObjectSyncSessionAborted ObjectSyncSessionStatus = "aborted"
|
||||
ObjectSyncSessionExpired ObjectSyncSessionStatus = "expired"
|
||||
)
|
||||
|
||||
type ObjectSyncItemStatus string
|
||||
|
||||
const (
|
||||
ObjectSyncItemDiscovered ObjectSyncItemStatus = "discovered"
|
||||
ObjectSyncItemPlanned ObjectSyncItemStatus = "planned"
|
||||
ObjectSyncItemReserved ObjectSyncItemStatus = "reserved"
|
||||
ObjectSyncItemUploading ObjectSyncItemStatus = "uploading"
|
||||
ObjectSyncItemUploaded ObjectSyncItemStatus = "uploaded"
|
||||
ObjectSyncItemCommitting ObjectSyncItemStatus = "committing"
|
||||
ObjectSyncItemCommitted ObjectSyncItemStatus = "committed"
|
||||
ObjectSyncItemKeptBoth ObjectSyncItemStatus = "kept_both"
|
||||
ObjectSyncItemUnchanged ObjectSyncItemStatus = "unchanged"
|
||||
ObjectSyncItemSkipped ObjectSyncItemStatus = "skipped"
|
||||
ObjectSyncItemDenied ObjectSyncItemStatus = "denied"
|
||||
ObjectSyncItemFailed ObjectSyncItemStatus = "failed"
|
||||
ObjectSyncItemAborted ObjectSyncItemStatus = "aborted"
|
||||
ObjectSyncItemExpired ObjectSyncItemStatus = "expired"
|
||||
)
|
||||
85
internal/objectsync/audit.go
Normal file
85
internal/objectsync/audit.go
Normal file
@ -0,0 +1,85 @@
|
||||
package objectsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func normalizedAuditOrganization(value string) string {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
return domain.DefaultOrganizationID
|
||||
}
|
||||
|
||||
// AppendAudit serializes each organization's append through the permanent lease
|
||||
// document so multiple API and coordinator replicas cannot fork the hash chain.
|
||||
func AppendAudit(ctx context.Context, target store.ObjectSyncStore, event domain.ObjectSyncAuditEvent) error {
|
||||
if event.ID == "" {
|
||||
event.ID = domain.NewID("syncaudit")
|
||||
}
|
||||
if event.EventKey == "" {
|
||||
event.EventKey = event.ID
|
||||
}
|
||||
event.OrganizationID = normalizedAuditOrganization(event.OrganizationID)
|
||||
resourceSum := sha256.Sum256([]byte("audit-chain\x00" + event.OrganizationID))
|
||||
resourceHash := hex.EncodeToString(resourceSum[:])
|
||||
var lease *domain.ObjectSyncLeaseRecord
|
||||
for attempt := 0; attempt < 20; attempt++ {
|
||||
acquiredLease, acquired, err := target.AcquireObjectSyncLease(ctx, domain.ObjectSyncLeaseRecord{
|
||||
ID: "osl_audit_" + resourceHash,
|
||||
OrganizationID: event.OrganizationID,
|
||||
ResourceHash: resourceHash,
|
||||
ResourceType: "audit_chain",
|
||||
KeyOrPrefix: event.OrganizationID,
|
||||
OwnerSessionID: event.ID,
|
||||
OwnerItemID: event.EventKey,
|
||||
}, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if acquired {
|
||||
lease = acquiredLease
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
if lease == nil {
|
||||
return fmt.Errorf("audit chain is busy")
|
||||
}
|
||||
defer func() {
|
||||
_ = target.ReleaseObjectSyncLease(context.Background(), resourceHash, event.ID, event.EventKey, lease.FencingToken)
|
||||
}()
|
||||
latest, err := target.GetLatestObjectSyncAudit(ctx, event.OrganizationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if latest != nil {
|
||||
event.PreviousHash = latest.EventHash
|
||||
}
|
||||
if event.OccurredAt == "" {
|
||||
event.OccurredAt = nowISO()
|
||||
}
|
||||
event.PurgeAt = time.Now().UTC().Add(versionRetention)
|
||||
unsigned := event
|
||||
unsigned.EventHash = ""
|
||||
payload, err := json.Marshal(unsigned)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
|
||||
event.EventHash = hex.EncodeToString(sum[:])
|
||||
return target.AppendObjectSyncAudit(ctx, event)
|
||||
}
|
||||
234
internal/objectsync/bos.go
Normal file
234
internal/objectsync/bos.go
Normal file
@ -0,0 +1,234 @@
|
||||
package objectsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
|
||||
"github.com/baidubce/bce-sdk-go/bce"
|
||||
"github.com/baidubce/bce-sdk-go/services/bos"
|
||||
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
||||
)
|
||||
|
||||
type ObjectMetadata struct {
|
||||
Exists bool
|
||||
ETag string
|
||||
VersionID string
|
||||
SizeBytes int64
|
||||
LastModified string
|
||||
}
|
||||
|
||||
type Publisher interface {
|
||||
Head(context.Context, domain.StorageConnectionRecord, string, string) (ObjectMetadata, error)
|
||||
PublishConditional(context.Context, domain.StorageConnectionRecord, string, string, string, int64, string, string, string, map[string]string) (ObjectMetadata, error)
|
||||
PutInternal(context.Context, domain.StorageConnectionRecord, string, string, io.Reader, int64, string, map[string]string) (ObjectMetadata, error)
|
||||
AbortMultipart(context.Context, domain.StorageConnectionRecord, string, string, string) error
|
||||
Delete(context.Context, domain.StorageConnectionRecord, string, string) error
|
||||
}
|
||||
|
||||
type BOSPublisher struct {
|
||||
GlobalEndpoint string
|
||||
}
|
||||
|
||||
func (publisher BOSPublisher) client(connection domain.StorageConnectionRecord) (*bos.Client, error) {
|
||||
endpoint := strings.TrimSpace(connection.Endpoint)
|
||||
if endpoint == "" {
|
||||
endpoint = strings.TrimSpace(publisher.GlobalEndpoint)
|
||||
}
|
||||
if endpoint == "" {
|
||||
return nil, fmt.Errorf("BOS endpoint is required")
|
||||
}
|
||||
return bos.NewClient(strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint)
|
||||
}
|
||||
|
||||
func IsPreconditionFailure(err error) bool {
|
||||
var serviceError *bce.BceServiceError
|
||||
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusPreconditionFailed
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var serviceError *bce.BceServiceError
|
||||
return errors.As(err, &serviceError) && serviceError.StatusCode == http.StatusNotFound
|
||||
}
|
||||
|
||||
func (publisher BOSPublisher) Head(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) {
|
||||
client, err := publisher.client(connection)
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
result, err := client.GetObjectMetaWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
||||
if isNotFound(err) {
|
||||
return ObjectMetadata{Exists: false}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
return ObjectMetadata{
|
||||
Exists: true,
|
||||
ETag: strings.Trim(result.ETag, `"`),
|
||||
VersionID: strings.TrimSpace(result.VersionId),
|
||||
SizeBytes: result.ContentLength,
|
||||
LastModified: strings.TrimSpace(result.LastModified),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func multipartCopyPartSize(sizeBytes int64) int64 {
|
||||
const (
|
||||
minimum = int64(64 * 1024 * 1024)
|
||||
maximum = int64(5 * 1024 * 1024 * 1024)
|
||||
parts = int64(10_000)
|
||||
align = int64(1024 * 1024)
|
||||
)
|
||||
partSize := minimum
|
||||
if sizeBytes > partSize*parts {
|
||||
partSize = ((sizeBytes/parts + align - 1) / align) * align
|
||||
}
|
||||
if partSize > maximum {
|
||||
return maximum
|
||||
}
|
||||
return partSize
|
||||
}
|
||||
|
||||
func (publisher BOSPublisher) PublishConditional(
|
||||
ctx context.Context,
|
||||
connection domain.StorageConnectionRecord,
|
||||
bucket string,
|
||||
sourceKey string,
|
||||
targetKey string,
|
||||
sizeBytes int64,
|
||||
sourceETag string,
|
||||
ifMatch string,
|
||||
ifNoneMatch string,
|
||||
userMeta map[string]string,
|
||||
) (ObjectMetadata, error) {
|
||||
client, err := publisher.client(connection)
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
bucket = strings.TrimSpace(bucket)
|
||||
sourceKey = strings.Trim(strings.TrimSpace(sourceKey), "/")
|
||||
targetKey = strings.Trim(strings.TrimSpace(targetKey), "/")
|
||||
if sizeBytes == 0 {
|
||||
body, bodyErr := bce.NewBodyFromBytes([]byte{})
|
||||
if bodyErr != nil {
|
||||
return ObjectMetadata{}, bodyErr
|
||||
}
|
||||
_, err = client.PutObjectWithContext(ctx, bucket, targetKey, body, &bosapi.PutObjectArgs{
|
||||
ContentType: "application/octet-stream",
|
||||
UserMeta: userMeta,
|
||||
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
||||
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
||||
})
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
return publisher.Head(ctx, connection, bucket, targetKey)
|
||||
}
|
||||
initiated, err := client.InitiateMultipartUpload(bucket, targetKey, "", nil)
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
abort := true
|
||||
defer func() {
|
||||
if abort {
|
||||
_ = client.AbortMultipartUpload(bucket, targetKey, initiated.UploadId)
|
||||
}
|
||||
}()
|
||||
partSize := multipartCopyPartSize(sizeBytes)
|
||||
parts := make([]bosapi.UploadInfoType, 0, int((sizeBytes+partSize-1)/partSize))
|
||||
for start, partNumber := int64(0), 1; start < sizeBytes; start, partNumber = start+partSize, partNumber+1 {
|
||||
end := start + partSize - 1
|
||||
if end >= sizeBytes {
|
||||
end = sizeBytes - 1
|
||||
}
|
||||
result, copyErr := client.UploadPartCopyWithContext(ctx, bucket, targetKey, bucket, sourceKey, initiated.UploadId, partNumber, &bosapi.UploadPartCopyArgs{
|
||||
SourceRange: fmt.Sprintf("bytes=%d-%d", start, end),
|
||||
IfMatch: strings.Trim(strings.TrimSpace(sourceETag), `"`),
|
||||
})
|
||||
if copyErr != nil {
|
||||
return ObjectMetadata{}, copyErr
|
||||
}
|
||||
parts = append(parts, bosapi.UploadInfoType{PartNumber: partNumber, ETag: strings.Trim(result.ETag, `"`)})
|
||||
}
|
||||
args := &bosapi.CompleteMultipartUploadArgs{
|
||||
Parts: parts,
|
||||
UserMeta: userMeta,
|
||||
IfMatch: strings.Trim(strings.TrimSpace(ifMatch), `"`),
|
||||
IfNoneMatch: strings.TrimSpace(ifNoneMatch),
|
||||
}
|
||||
payload, err := json.Marshal(args)
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
body, err := bce.NewBodyFromBytes(payload)
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
bosContext := client.NewBosContext(ctx)
|
||||
if _, err = bosapi.CompleteMultipartUpload(client, bucket, targetKey, initiated.UploadId, body, args, bosContext); err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
abort = false
|
||||
return publisher.Head(ctx, connection, bucket, targetKey)
|
||||
}
|
||||
|
||||
func (publisher BOSPublisher) PutInternal(
|
||||
ctx context.Context,
|
||||
connection domain.StorageConnectionRecord,
|
||||
bucket string,
|
||||
key string,
|
||||
reader io.Reader,
|
||||
size int64,
|
||||
contentType string,
|
||||
userMeta map[string]string,
|
||||
) (ObjectMetadata, error) {
|
||||
client, err := publisher.client(connection)
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
body, err := bce.NewBodyFromSizedReader(reader, size)
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
_, err = client.PutObjectWithContext(ctx, strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), body, &bosapi.PutObjectArgs{
|
||||
ContentType: strings.TrimSpace(contentType),
|
||||
UserMeta: userMeta,
|
||||
IfNoneMatch: "*",
|
||||
})
|
||||
if err != nil {
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
return publisher.Head(ctx, connection, bucket, key)
|
||||
}
|
||||
|
||||
func (publisher BOSPublisher) Delete(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key string) error {
|
||||
client, err := publisher.client(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
return client.DeleteObject(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"))
|
||||
}
|
||||
}
|
||||
|
||||
func (publisher BOSPublisher) AbortMultipart(ctx context.Context, connection domain.StorageConnectionRecord, bucket, key, uploadID string) error {
|
||||
client, err := publisher.client(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
return client.AbortMultipartUpload(strings.TrimSpace(bucket), strings.Trim(strings.TrimSpace(key), "/"), strings.TrimSpace(uploadID))
|
||||
}
|
||||
}
|
||||
1069
internal/objectsync/coordinator.go
Normal file
1069
internal/objectsync/coordinator.go
Normal file
File diff suppressed because it is too large
Load Diff
386
internal/objectsync/coordinator_test.go
Normal file
386
internal/objectsync/coordinator_test.go
Normal file
@ -0,0 +1,386 @@
|
||||
package objectsync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
|
||||
"github.com/baidubce/bce-sdk-go/bce"
|
||||
)
|
||||
|
||||
type fakePublisher struct {
|
||||
objects map[string]ObjectMetadata
|
||||
precondition map[string]bool
|
||||
publishedKeys []string
|
||||
}
|
||||
|
||||
func (publisher *fakePublisher) AbortMultipart(_ context.Context, _ domain.StorageConnectionRecord, _, _, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (publisher *fakePublisher) Head(_ context.Context, _ domain.StorageConnectionRecord, bucket, key string) (ObjectMetadata, error) {
|
||||
item, ok := publisher.objects[bucket+"/"+key]
|
||||
if !ok {
|
||||
return ObjectMetadata{Exists: false}, nil
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (publisher *fakePublisher) PublishConditional(
|
||||
_ context.Context,
|
||||
_ domain.StorageConnectionRecord,
|
||||
bucket, sourceKey, targetKey string,
|
||||
sizeBytes int64,
|
||||
_, _, _ string,
|
||||
_ map[string]string,
|
||||
) (ObjectMetadata, error) {
|
||||
if publisher.precondition[targetKey] {
|
||||
delete(publisher.precondition, targetKey)
|
||||
return ObjectMetadata{}, &bce.BceServiceError{StatusCode: 412}
|
||||
}
|
||||
source := publisher.objects[bucket+"/"+sourceKey]
|
||||
result := ObjectMetadata{Exists: true, ETag: "published-" + targetKey, SizeBytes: sizeBytes}
|
||||
if source.VersionID != "" {
|
||||
result.VersionID = "version-" + targetKey
|
||||
}
|
||||
publisher.objects[bucket+"/"+targetKey] = result
|
||||
publisher.publishedKeys = append(publisher.publishedKeys, targetKey)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (publisher *fakePublisher) PutInternal(
|
||||
_ context.Context,
|
||||
_ domain.StorageConnectionRecord,
|
||||
bucket, key string,
|
||||
reader io.Reader,
|
||||
size int64,
|
||||
_ string,
|
||||
_ map[string]string,
|
||||
) (ObjectMetadata, error) {
|
||||
payload, _ := io.ReadAll(reader)
|
||||
if int64(len(payload)) != size {
|
||||
return ObjectMetadata{}, io.ErrUnexpectedEOF
|
||||
}
|
||||
result := ObjectMetadata{Exists: true, ETag: "manifest", SizeBytes: size}
|
||||
publisher.objects[bucket+"/"+key] = result
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (publisher *fakePublisher) Delete(_ context.Context, _ domain.StorageConnectionRecord, bucket, key string) error {
|
||||
delete(publisher.objects, bucket+"/"+key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func coordinatorFixture(t *testing.T) (*Coordinator, *store.MemoryStore, *fakePublisher) {
|
||||
t.Helper()
|
||||
dataStore := store.NewMemoryStore()
|
||||
connection := domain.StorageConnectionRecord{
|
||||
ID: "bos",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
Name: "BOS",
|
||||
Provider: domain.StorageProviderBOS,
|
||||
}
|
||||
if err := dataStore.UpsertStorageConnection(t.Context(), connection); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
||||
if err := syncStore.UpsertObjectSyncSpace(t.Context(), domain.ObjectSyncSpaceRecord{
|
||||
ID: "space",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ConnectionID: connection.ID,
|
||||
Bucket: "bucket",
|
||||
RootPrefix: "incoming",
|
||||
Mode: "create_only",
|
||||
Enabled: true,
|
||||
Policy: domain.DefaultObjectSyncSpacePolicy(),
|
||||
Capability: domain.ObjectSyncCapabilityRecord{
|
||||
State: domain.ObjectSyncCapabilityCreateOnly,
|
||||
ConditionalCreate: true,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publisher := &fakePublisher{objects: map[string]ObjectMetadata{}, precondition: map[string]bool{}}
|
||||
coordinator, err := NewCoordinator(dataStore, publisher, "test-owner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return coordinator, dataStore, publisher
|
||||
}
|
||||
|
||||
func testAuditEvent(t *testing.T, id, previousHash, occurredAt string) domain.ObjectSyncAuditEvent {
|
||||
t.Helper()
|
||||
event := domain.ObjectSyncAuditEvent{
|
||||
ID: id,
|
||||
EventKey: id,
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ActorType: "user",
|
||||
ActorID: "tester",
|
||||
Action: "test.audit",
|
||||
Result: "success",
|
||||
PreviousHash: previousHash,
|
||||
OccurredAt: occurredAt,
|
||||
PurgeAt: time.Now().UTC().Add(365 * 24 * time.Hour),
|
||||
}
|
||||
unsigned := event
|
||||
unsigned.EventHash = ""
|
||||
payload, err := json.Marshal(unsigned)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum := sha256.Sum256(append([]byte(event.PreviousHash), payload...))
|
||||
event.EventHash = hex.EncodeToString(sum[:])
|
||||
return event
|
||||
}
|
||||
|
||||
func TestCoordinatorSealsPreviousAuditDay(t *testing.T) {
|
||||
coordinator, dataStore, publisher := coordinatorFixture(t)
|
||||
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
||||
now := time.Date(2026, time.July, 28, 12, 0, 0, 0, time.UTC)
|
||||
first := testAuditEvent(t, "audit-1", "", "2026-07-27T01:00:00Z")
|
||||
second := testAuditEvent(t, "audit-2", first.EventHash, "2026-07-27T02:00:00Z")
|
||||
if err := syncStore.AppendObjectSyncAudit(t.Context(), first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := syncStore.AppendObjectSyncAudit(t.Context(), second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seed := make([]byte, ed25519.SeedSize)
|
||||
for index := range seed {
|
||||
seed[index] = byte(index + 1)
|
||||
}
|
||||
if err := coordinator.ConfigureAuditSigning(base64.StdEncoding.EncodeToString(seed), "test-key"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
count, err := coordinator.sealPreviousAuditDay(t.Context(), now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("expected one audit seal, got %d", count)
|
||||
}
|
||||
seal, err := syncStore.GetObjectSyncAuditSeal(t.Context(), domain.DefaultOrganizationID, "2026-07-27")
|
||||
if err != nil || seal == nil {
|
||||
t.Fatalf("expected persisted audit seal, got %#v, %v", seal, err)
|
||||
}
|
||||
if seal.EventCount != 2 || seal.FirstEventHash != first.EventHash || seal.LastEventHash != second.EventHash {
|
||||
t.Fatalf("unexpected seal contents: %#v", seal)
|
||||
}
|
||||
payload, err := json.Marshal(auditSealSigningPayload{
|
||||
OrganizationID: seal.OrganizationID,
|
||||
DayUTC: seal.DayUTC,
|
||||
EventCount: seal.EventCount,
|
||||
FirstEventHash: seal.FirstEventHash,
|
||||
LastEventHash: seal.LastEventHash,
|
||||
AggregateHash: seal.AggregateHash,
|
||||
SigningKeyID: seal.SigningKeyID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
signature, err := base64.StdEncoding.DecodeString(seal.Signature)
|
||||
if err != nil || !ed25519.Verify(ed25519.NewKeyFromSeed(seed).Public().(ed25519.PublicKey), payload, signature) {
|
||||
t.Fatal("audit seal signature did not verify")
|
||||
}
|
||||
if _, ok := publisher.objects["bucket/"+seal.ObjectKey]; !ok {
|
||||
t.Fatalf("expected immutable seal object %s", seal.ObjectKey)
|
||||
}
|
||||
secondCount, err := coordinator.sealPreviousAuditDay(t.Context(), now)
|
||||
if err != nil || secondCount != 0 {
|
||||
t.Fatalf("expected idempotent second sealing pass, got %d, %v", secondCount, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorForceOverwritePreservesConflictSource(t *testing.T) {
|
||||
coordinator, dataStore, publisher := coordinatorFixture(t)
|
||||
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
||||
space, err := syncStore.GetObjectSyncSpace(t.Context(), "space")
|
||||
if err != nil || space == nil {
|
||||
t.Fatalf("space = %#v, %v", space, err)
|
||||
}
|
||||
space.Capability.State = domain.ObjectSyncCapabilityUpdateEnabled
|
||||
space.Capability.ConditionalCreate = true
|
||||
space.Capability.ConditionalUpdate = true
|
||||
space.Capability.ConditionalMultipart = true
|
||||
space.Capability.ExclusivePrefixConfirmed = true
|
||||
space.Policy.AllowConditionalUpdate = true
|
||||
space.Policy.AllowForceOverwrite = true
|
||||
if err := syncStore.UpsertObjectSyncSpace(t.Context(), *space); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
session := domain.ObjectSyncSessionRecord{
|
||||
ID: "session-force",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SyncSpaceID: space.ID,
|
||||
DeviceID: "device-1",
|
||||
Status: domain.ObjectSyncSessionConflicted,
|
||||
CreatedAt: nowISO(),
|
||||
UpdatedAt: nowISO(),
|
||||
}
|
||||
if err := syncStore.UpsertObjectSyncSession(t.Context(), session); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := domain.ObjectSyncItemRecord{
|
||||
ID: "item-force",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SessionID: session.ID,
|
||||
ClientItemID: "client-force",
|
||||
TargetKey: "incoming/file.bin",
|
||||
LogicalTargetKey: "incoming/file.bin",
|
||||
SizeBytes: 4,
|
||||
SHA256: "abcd",
|
||||
Action: "force_overwrite",
|
||||
Status: domain.ObjectSyncItemCommitting,
|
||||
ForceSourceKey: "incoming/file.bin.__conflict__20260728T000000Z_source",
|
||||
ForceActorID: "admin-1",
|
||||
CreatedAt: nowISO(),
|
||||
UpdatedAt: nowISO(),
|
||||
}
|
||||
lease, acquired, err := syncStore.AcquireObjectSyncLease(t.Context(), domain.ObjectSyncLeaseRecord{
|
||||
OrganizationID: session.OrganizationID,
|
||||
ResourceHash: "force-resource",
|
||||
ResourceType: "object",
|
||||
OwnerSessionID: session.ID,
|
||||
OwnerItemID: item.ID,
|
||||
}, claimTTL)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("lease = %#v, acquired=%t, err=%v", lease, acquired, err)
|
||||
}
|
||||
item.LeaseID = lease.ID
|
||||
item.FencingToken = lease.FencingToken
|
||||
if err := syncStore.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{item}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceObject := "bucket/" + item.ForceSourceKey
|
||||
targetObject := "bucket/" + item.TargetKey
|
||||
publisher.objects[sourceObject] = ObjectMetadata{Exists: true, ETag: "conflict-etag", SizeBytes: item.SizeBytes}
|
||||
publisher.objects[targetObject] = ObjectMetadata{Exists: true, ETag: "current-etag", SizeBytes: item.SizeBytes}
|
||||
if err := coordinator.commitItem(t.Context(), &item); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, exists := publisher.objects[sourceObject]; !exists {
|
||||
t.Fatal("force overwrite deleted the preserved conflict source")
|
||||
}
|
||||
if publisher.objects[targetObject].ETag != "published-"+item.TargetKey {
|
||||
t.Fatalf("target was not published: %#v", publisher.objects[targetObject])
|
||||
}
|
||||
saved, err := syncStore.GetObjectSyncItem(t.Context(), item.ID)
|
||||
if err != nil || saved == nil || saved.Status != domain.ObjectSyncItemCommitted {
|
||||
t.Fatalf("saved force item = %#v, %v", saved, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorPlansCreateWithoutEmbeddingManifest(t *testing.T) {
|
||||
coordinator, dataStore, _ := coordinatorFixture(t)
|
||||
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
||||
now := time.Now().UTC()
|
||||
if err := syncStore.UpsertObjectSyncSession(t.Context(), domain.ObjectSyncSessionRecord{
|
||||
ID: "session",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SyncSpaceID: "space",
|
||||
Status: domain.ObjectSyncSessionPlanning,
|
||||
Counts: domain.ObjectSyncSessionCounts{Manifest: 1},
|
||||
UpdatedAt: now.Format(time.RFC3339Nano),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := syncStore.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{{
|
||||
ID: "item",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SessionID: "session",
|
||||
TargetKey: "incoming/file.bin",
|
||||
SizeBytes: 4,
|
||||
SHA256: strings.Repeat("a", 64),
|
||||
Status: domain.ObjectSyncItemDiscovered,
|
||||
}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := coordinator.ProcessOnce(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Planned != 1 {
|
||||
t.Fatalf("planned = %d", result.Planned)
|
||||
}
|
||||
item, _ := syncStore.GetObjectSyncItem(t.Context(), "item")
|
||||
if item.Status != domain.ObjectSyncItemPlanned || item.Action != "create" {
|
||||
t.Fatalf("unexpected planned item: %#v", item)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorKeepsBothOnConditionalConflict(t *testing.T) {
|
||||
coordinator, dataStore, publisher := coordinatorFixture(t)
|
||||
syncStore, _ := store.AsObjectSyncStore(dataStore)
|
||||
now := time.Now().UTC()
|
||||
session := domain.ObjectSyncSessionRecord{
|
||||
ID: "session",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SyncSpaceID: "space",
|
||||
DeviceID: "device",
|
||||
Status: domain.ObjectSyncSessionRunning,
|
||||
Counts: domain.ObjectSyncSessionCounts{Manifest: 1, Create: 1},
|
||||
UpdatedAt: now.Format(time.RFC3339Nano),
|
||||
}
|
||||
if err := syncStore.UpsertObjectSyncSession(t.Context(), session); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, acquired, err := syncStore.AcquireObjectSyncLease(t.Context(), domain.ObjectSyncLeaseRecord{
|
||||
ID: "lease",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ResourceHash: "resource",
|
||||
ResourceType: "object",
|
||||
ConnectionID: "bos",
|
||||
Bucket: "bucket",
|
||||
KeyOrPrefix: "incoming/file.bin",
|
||||
OwnerSessionID: session.ID,
|
||||
OwnerItemID: "item",
|
||||
}, time.Minute)
|
||||
if err != nil || !acquired {
|
||||
t.Fatalf("lease: acquired=%v err=%v", acquired, err)
|
||||
}
|
||||
item := domain.ObjectSyncItemRecord{
|
||||
ID: "item",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
SessionID: session.ID,
|
||||
TargetKey: "incoming/file.bin",
|
||||
SizeBytes: 4,
|
||||
SHA256: strings.Repeat("b", 64),
|
||||
Action: "create",
|
||||
Status: domain.ObjectSyncItemCommitting,
|
||||
StagingKey: ".drosync-internal/staging/session/item",
|
||||
LeaseID: lease.ID,
|
||||
FencingToken: lease.FencingToken,
|
||||
}
|
||||
if err := syncStore.UpsertObjectSyncItems(t.Context(), []domain.ObjectSyncItemRecord{item}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
publisher.objects["bucket/"+item.StagingKey] = ObjectMetadata{Exists: true, ETag: "staging", SizeBytes: 4}
|
||||
publisher.precondition[item.TargetKey] = true
|
||||
result, err := coordinator.ProcessOnce(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if result.Committed != 1 {
|
||||
t.Fatalf("committed = %d", result.Committed)
|
||||
}
|
||||
saved, _ := syncStore.GetObjectSyncItem(t.Context(), item.ID)
|
||||
if saved.Status != domain.ObjectSyncItemKeptBoth {
|
||||
t.Fatalf("status = %s, error=%s", saved.Status, saved.LastError)
|
||||
}
|
||||
if saved.LogicalTargetKey != item.TargetKey || !strings.Contains(saved.TargetKey, ".__conflict__") {
|
||||
t.Fatalf("unexpected keep-both keys: logical=%q target=%q", saved.LogicalTargetKey, saved.TargetKey)
|
||||
}
|
||||
if saved.CommittedVersion.ETag == "" {
|
||||
t.Fatal("committed version metadata missing")
|
||||
}
|
||||
}
|
||||
@ -30,6 +30,22 @@ type MemoryStore struct {
|
||||
sessions map[string]domain.SessionRecord
|
||||
storageConnections map[string]domain.StorageConnectionRecord
|
||||
mongoConnections map[string]domain.MongoConnectionRecord
|
||||
objectSyncSpaces map[string]domain.ObjectSyncSpaceRecord
|
||||
objectSyncDevices map[string]domain.ObjectSyncDeviceRecord
|
||||
objectSyncAuthorizations map[string]domain.ObjectSyncDeviceAuthorizationRecord
|
||||
objectSyncCredentials map[string]domain.ObjectSyncCredentialRecord
|
||||
objectSyncSessions map[string]domain.ObjectSyncSessionRecord
|
||||
objectSyncItems map[string]domain.ObjectSyncItemRecord
|
||||
objectSyncLeases map[string]domain.ObjectSyncLeaseRecord
|
||||
objectSyncHeads map[string]domain.ObjectSyncObjectHeadRecord
|
||||
objectSyncVersions map[string]domain.ObjectSyncVersionRecord
|
||||
objectSyncAuditEvents map[string]domain.ObjectSyncAuditEvent
|
||||
objectSyncAuditSeals map[string]domain.ObjectSyncAuditSealRecord
|
||||
objectSyncReleases map[string]domain.ObjectSyncReleaseRecord
|
||||
objectSyncClientReleases map[string]domain.ObjectSyncClientReleaseRecord
|
||||
objectSyncReconciles map[string]domain.ObjectSyncReconcileRecord
|
||||
objectSyncIdempotency map[string]domain.ObjectSyncIdempotencyRecord
|
||||
objectSyncRateLimits map[string]domain.ObjectSyncRateLimitRecord
|
||||
databaseCollections map[string][]map[string]any
|
||||
databaseIndexes map[string][]domain.DatabaseIndexSummary
|
||||
settings *domain.SettingsRecord
|
||||
@ -54,6 +70,22 @@ func NewMemoryStore() *MemoryStore {
|
||||
sessions: map[string]domain.SessionRecord{},
|
||||
storageConnections: map[string]domain.StorageConnectionRecord{},
|
||||
mongoConnections: map[string]domain.MongoConnectionRecord{},
|
||||
objectSyncSpaces: map[string]domain.ObjectSyncSpaceRecord{},
|
||||
objectSyncDevices: map[string]domain.ObjectSyncDeviceRecord{},
|
||||
objectSyncAuthorizations: map[string]domain.ObjectSyncDeviceAuthorizationRecord{},
|
||||
objectSyncCredentials: map[string]domain.ObjectSyncCredentialRecord{},
|
||||
objectSyncSessions: map[string]domain.ObjectSyncSessionRecord{},
|
||||
objectSyncItems: map[string]domain.ObjectSyncItemRecord{},
|
||||
objectSyncLeases: map[string]domain.ObjectSyncLeaseRecord{},
|
||||
objectSyncHeads: map[string]domain.ObjectSyncObjectHeadRecord{},
|
||||
objectSyncVersions: map[string]domain.ObjectSyncVersionRecord{},
|
||||
objectSyncAuditEvents: map[string]domain.ObjectSyncAuditEvent{},
|
||||
objectSyncAuditSeals: map[string]domain.ObjectSyncAuditSealRecord{},
|
||||
objectSyncReleases: map[string]domain.ObjectSyncReleaseRecord{},
|
||||
objectSyncClientReleases: map[string]domain.ObjectSyncClientReleaseRecord{},
|
||||
objectSyncReconciles: map[string]domain.ObjectSyncReconcileRecord{},
|
||||
objectSyncIdempotency: map[string]domain.ObjectSyncIdempotencyRecord{},
|
||||
objectSyncRateLimits: map[string]domain.ObjectSyncRateLimitRecord{},
|
||||
databaseCollections: map[string][]map[string]any{},
|
||||
databaseIndexes: map[string][]domain.DatabaseIndexSummary{},
|
||||
}
|
||||
|
||||
@ -37,6 +37,22 @@ type MongoStore struct {
|
||||
sessions *mongo.Collection
|
||||
storageConnections *mongo.Collection
|
||||
mongoConnections *mongo.Collection
|
||||
objectSyncSpaces *mongo.Collection
|
||||
objectSyncDevices *mongo.Collection
|
||||
objectSyncAuthorizations *mongo.Collection
|
||||
objectSyncCredentials *mongo.Collection
|
||||
objectSyncSessions *mongo.Collection
|
||||
objectSyncItems *mongo.Collection
|
||||
objectSyncLeases *mongo.Collection
|
||||
objectSyncHeads *mongo.Collection
|
||||
objectSyncVersions *mongo.Collection
|
||||
objectSyncAuditEvents *mongo.Collection
|
||||
objectSyncAuditSeals *mongo.Collection
|
||||
objectSyncReleases *mongo.Collection
|
||||
objectSyncClientReleases *mongo.Collection
|
||||
objectSyncReconciles *mongo.Collection
|
||||
objectSyncIdempotency *mongo.Collection
|
||||
objectSyncRateLimits *mongo.Collection
|
||||
settings *mongo.Collection
|
||||
runSummaryAggregates *runSummaryAggregateCache
|
||||
}
|
||||
@ -61,6 +77,22 @@ func NewMongoStore(db *mongo.Database) *MongoStore {
|
||||
sessions: db.Collection("sessions"),
|
||||
storageConnections: db.Collection("storageConnections"),
|
||||
mongoConnections: db.Collection("mongoConnections"),
|
||||
objectSyncSpaces: db.Collection("objectSyncSpaces"),
|
||||
objectSyncDevices: db.Collection("objectSyncDevices"),
|
||||
objectSyncAuthorizations: db.Collection("objectSyncAuthorizations"),
|
||||
objectSyncCredentials: db.Collection("objectSyncCredentials"),
|
||||
objectSyncSessions: db.Collection("objectSyncSessions"),
|
||||
objectSyncItems: db.Collection("objectSyncItems"),
|
||||
objectSyncLeases: db.Collection("objectSyncLeases"),
|
||||
objectSyncHeads: db.Collection("objectSyncHeads"),
|
||||
objectSyncVersions: db.Collection("objectSyncVersions"),
|
||||
objectSyncAuditEvents: db.Collection("objectSyncAuditEvents"),
|
||||
objectSyncAuditSeals: db.Collection("objectSyncAuditSeals"),
|
||||
objectSyncReleases: db.Collection("objectSyncReleases"),
|
||||
objectSyncClientReleases: db.Collection("objectSyncClientReleases"),
|
||||
objectSyncReconciles: db.Collection("objectSyncReconciles"),
|
||||
objectSyncIdempotency: db.Collection("objectSyncIdempotency"),
|
||||
objectSyncRateLimits: db.Collection("objectSyncRateLimits"),
|
||||
settings: db.Collection("settings"),
|
||||
runSummaryAggregates: newRunSummaryAggregateCache(runSummaryAggregateCacheTTL, runSummaryAggregateCacheMaxEntries),
|
||||
}
|
||||
@ -201,6 +233,18 @@ func (store *MongoStore) DeleteOrganizationCascade(ctx context.Context, id strin
|
||||
store.samplingTasks,
|
||||
store.storageConnections,
|
||||
store.mongoConnections,
|
||||
store.objectSyncSpaces,
|
||||
store.objectSyncDevices,
|
||||
store.objectSyncAuthorizations,
|
||||
store.objectSyncCredentials,
|
||||
store.objectSyncSessions,
|
||||
store.objectSyncItems,
|
||||
store.objectSyncLeases,
|
||||
store.objectSyncHeads,
|
||||
store.objectSyncVersions,
|
||||
store.objectSyncAuditEvents,
|
||||
store.objectSyncReleases,
|
||||
store.objectSyncClientReleases,
|
||||
store.settings,
|
||||
} {
|
||||
if _, err := collection.DeleteMany(ctx, filter); err != nil {
|
||||
|
||||
@ -139,6 +139,84 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
index("organizationId_name", bson.D{{Key: "organizationId", Value: 1}, {Key: "name", Value: 1}}),
|
||||
index("name", bson.D{{Key: "name", Value: 1}}),
|
||||
},
|
||||
store.objectSyncSpaces: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "updatedAt", Value: -1}}),
|
||||
index("organizationId_connection_bucket_prefix", bson.D{{Key: "organizationId", Value: 1}, {Key: "connectionId", Value: 1}, {Key: "bucket", Value: 1}, {Key: "rootPrefix", Value: 1}}),
|
||||
},
|
||||
store.objectSyncDevices: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "updatedAt", Value: -1}}),
|
||||
},
|
||||
store.objectSyncAuthorizations: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
uniqueIndex("deviceCodeHash_unique", bson.D{{Key: "deviceCodeHash", Value: 1}}),
|
||||
uniqueIndex("userCode_unique", bson.D{{Key: "userCode", Value: 1}}),
|
||||
ttlIndex("expiresAtTime_ttl", bson.D{{Key: "expiresAtTime", Value: 1}}, 0),
|
||||
},
|
||||
store.objectSyncCredentials: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
uniqueIndex("tokenHash_unique", bson.D{{Key: "tokenHash", Value: 1}}),
|
||||
index("organizationId_deviceId", bson.D{{Key: "organizationId", Value: 1}, {Key: "deviceId", Value: 1}}),
|
||||
},
|
||||
store.objectSyncSessions: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
index("status_updatedAt", bson.D{{Key: "status", Value: 1}, {Key: "updatedAt", Value: 1}}),
|
||||
ttlIndex("purgeAt_ttl", bson.D{{Key: "purgeAt", Value: 1}}, 0),
|
||||
},
|
||||
store.objectSyncItems: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
uniqueIndex("sessionId_clientItemId_unique", bson.D{{Key: "sessionId", Value: 1}, {Key: "clientItemId", Value: 1}}),
|
||||
index("sessionId_status_id", bson.D{{Key: "sessionId", Value: 1}, {Key: "status", Value: 1}, {Key: "id", Value: 1}}),
|
||||
index("organizationId_targetKey_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "targetKey", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
ttlIndex("purgeAt_ttl", bson.D{{Key: "purgeAt", Value: 1}}, 0),
|
||||
},
|
||||
store.objectSyncLeases: {
|
||||
uniqueIndex("organizationId_resourceHash_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "resourceHash", Value: 1}}),
|
||||
index("ownerSessionId_ownerItemId", bson.D{{Key: "ownerSessionId", Value: 1}, {Key: "ownerItemId", Value: 1}}),
|
||||
index("expiresAtTime", bson.D{{Key: "expiresAtTime", Value: 1}}),
|
||||
},
|
||||
store.objectSyncHeads: {
|
||||
uniqueIndex("organizationId_syncSpaceId_key_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "syncSpaceId", Value: 1}, {Key: "key", Value: 1}}),
|
||||
index("organizationId_drifted_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "drifted", Value: 1}, {Key: "updatedAt", Value: -1}}),
|
||||
},
|
||||
store.objectSyncVersions: {
|
||||
uniqueIndex("organizationId_syncSpaceId_key_version_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "syncSpaceId", Value: 1}, {Key: "key", Value: 1}, {Key: "platformVersion", Value: 1}}),
|
||||
index("sessionId_itemId", bson.D{{Key: "sessionId", Value: 1}, {Key: "itemId", Value: 1}}),
|
||||
ttlIndex("purgeAt_ttl", bson.D{{Key: "purgeAt", Value: 1}}, 0),
|
||||
},
|
||||
store.objectSyncAuditEvents: {
|
||||
uniqueIndex("eventKey_unique", bson.D{{Key: "eventKey", Value: 1}}),
|
||||
index("organizationId_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "occurredAt", Value: -1}}),
|
||||
index("sessionId_occurredAt", bson.D{{Key: "sessionId", Value: 1}, {Key: "occurredAt", Value: -1}}),
|
||||
ttlIndex("purgeAt_ttl", bson.D{{Key: "purgeAt", Value: 1}}, 0),
|
||||
},
|
||||
store.objectSyncAuditSeals: {
|
||||
uniqueIndex("organizationId_dayUtc_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "dayUtc", Value: 1}}),
|
||||
ttlIndex("purgeAt_ttl", bson.D{{Key: "purgeAt", Value: 1}}, 0),
|
||||
},
|
||||
store.objectSyncReleases: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
},
|
||||
store.objectSyncClientReleases: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
uniqueIndex("version_channel_unique", bson.D{{Key: "version", Value: 1}, {Key: "channel", Value: 1}}),
|
||||
},
|
||||
store.objectSyncReconciles: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
uniqueIndex("operationId_unique", bson.D{{Key: "operationId", Value: 1}}),
|
||||
index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
},
|
||||
store.objectSyncIdempotency: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
ttlIndex("expiresAt_ttl", bson.D{{Key: "expiresAt", Value: 1}}, 0),
|
||||
},
|
||||
store.objectSyncRateLimits: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
ttlIndex("expiresAt_ttl", bson.D{{Key: "expiresAt", Value: 1}}, 0),
|
||||
},
|
||||
store.settings: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
},
|
||||
|
||||
1454
internal/store/object_sync.go
Normal file
1454
internal/store/object_sync.go
Normal file
File diff suppressed because it is too large
Load Diff
@ -15,6 +15,12 @@ metadata:
|
||||
name: droboticflow-worker
|
||||
namespace: __NAMESPACE__
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: drosync-coordinator
|
||||
namespace: __NAMESPACE__
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
@ -116,6 +122,8 @@ metadata:
|
||||
data:
|
||||
PORT: "3001"
|
||||
WORKER_PORT: "3002"
|
||||
DROSYNC_COORDINATOR_PORT: "3003"
|
||||
DROBOTICFLOW_OBJECT_SYNC_ENABLED: "__DROBOTICFLOW_OBJECT_SYNC_ENABLED__"
|
||||
MONGO_DB: "__MONGO_DB__"
|
||||
MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__"
|
||||
MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__"
|
||||
@ -239,6 +247,11 @@ spec:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: MONGO_DB
|
||||
- name: DROBOTICFLOW_OBJECT_SYNC_ENABLED
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: DROBOTICFLOW_OBJECT_SYNC_ENABLED
|
||||
- name: MONGO_CONNECT_RETRIES
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
@ -644,3 +657,154 @@ spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: droboticflow-worker
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: drosync-coordinator
|
||||
namespace: __NAMESPACE__
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "3003"
|
||||
spec:
|
||||
selector:
|
||||
app: drosync-coordinator
|
||||
ports:
|
||||
- name: http
|
||||
port: 3003
|
||||
targetPort: 3003
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: drosync-coordinator
|
||||
namespace: __NAMESPACE__
|
||||
spec:
|
||||
replicas: __DROSYNC_COORDINATOR_REPLICAS__
|
||||
minReadySeconds: 5
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 0
|
||||
maxSurge: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: drosync-coordinator
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: drosync-coordinator
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 30
|
||||
serviceAccountName: drosync-coordinator
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 10001
|
||||
runAsGroup: 10001
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
imagePullSecrets:
|
||||
- name: __K8S_IMAGE_PULL_SECRET_NAME__
|
||||
containers:
|
||||
- name: coordinator
|
||||
image: __DROSYNC_COORDINATOR_IMAGE__
|
||||
imagePullPolicy: Always
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3003
|
||||
env:
|
||||
- name: DROSYNC_COORDINATOR_PORT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: DROSYNC_COORDINATOR_PORT
|
||||
- name: DROBOTICFLOW_OBJECT_SYNC_ENABLED
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: DROBOTICFLOW_OBJECT_SYNC_ENABLED
|
||||
- name: WORKER_POLL_INTERVAL_MS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: WORKER_POLL_INTERVAL_MS
|
||||
- name: BOS_ENDPOINT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: BOS_ENDPOINT
|
||||
- name: MONGO_DB
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: MONGO_DB
|
||||
- name: MONGO_CONNECT_RETRIES
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: MONGO_CONNECT_RETRIES
|
||||
- name: MONGO_CONNECT_RETRY_DELAY_MS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: MONGO_CONNECT_RETRY_DELAY_MS
|
||||
- name: MONGO_URI
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: droboticflow-mongo-secret
|
||||
key: MONGO_URI
|
||||
- name: DROSYNC_AUDIT_SIGNING_PRIVATE_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: drosync-signing-secret
|
||||
key: DROSYNC_AUDIT_SIGNING_PRIVATE_KEY
|
||||
optional: true
|
||||
- name: DROSYNC_AUDIT_SIGNING_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: drosync-signing-secret
|
||||
key: DROSYNC_AUDIT_SIGNING_KEY_ID
|
||||
optional: true
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
volumeMounts:
|
||||
- name: temporary
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: temporary
|
||||
emptyDir:
|
||||
sizeLimit: 4Gi
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: drosync-coordinator
|
||||
namespace: __NAMESPACE__
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: drosync-coordinator
|
||||
|
||||
105
k8s/drosync-observability.template.yaml
Normal file
105
k8s/drosync-observability.template.yaml
Normal file
@ -0,0 +1,105 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: drosync-slo
|
||||
namespace: __NAMESPACE__
|
||||
labels:
|
||||
app.kubernetes.io/name: drosync
|
||||
spec:
|
||||
groups:
|
||||
- name: drosync.availability
|
||||
rules:
|
||||
- alert: DrosyncCoordinatorDown
|
||||
expr: max(up{service="drosync-coordinator"}) < 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: drosync coordinator is unavailable
|
||||
- alert: DrosyncCoordinatorStalled
|
||||
expr: time() - max(drosync_coordinator_last_cycle_timestamp_seconds) > 120
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: drosync coordinator has not completed a cycle for two minutes
|
||||
- alert: DrosyncCoordinatorErrorBudgetBurn
|
||||
expr: |
|
||||
(
|
||||
sum(rate(drosync_coordinator_errors_total[1h]))
|
||||
/
|
||||
clamp_min(sum(rate(drosync_coordinator_cycles_total[1h])), 0.000001)
|
||||
) > 0.001
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: drosync coordinator exceeds the 99.9 percent monthly error budget
|
||||
- alert: DrosyncAuditSealMissing
|
||||
expr: |
|
||||
drosync_coordinator_enabled == 1
|
||||
and
|
||||
increase(drosync_coordinator_audit_seals_total[30h]) < 1
|
||||
for: 1h
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: drosync did not produce a daily audit seal
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: drosync-grafana-dashboard
|
||||
namespace: __NAMESPACE__
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
drosync-slo.json: |-
|
||||
{
|
||||
"title": "drosync production v1",
|
||||
"uid": "drosync-v1",
|
||||
"schemaVersion": 39,
|
||||
"refresh": "30s",
|
||||
"time": {"from": "now-24h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "30 day coordinator availability",
|
||||
"targets": [{"expr": "100 * avg_over_time(up{service=\"drosync-coordinator\"}[30d])"}],
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 99.9}]}}},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Seconds since successful cycle",
|
||||
"targets": [{"expr": "time() - max(drosync_coordinator_last_success_timestamp_seconds)"}],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}},
|
||||
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "Committed items",
|
||||
"targets": [{"expr": "sum(rate(drosync_coordinator_items_committed_total[5m]))", "legendFormat": "items per second"}],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Coordinator error ratio",
|
||||
"targets": [{"expr": "sum(rate(drosync_coordinator_errors_total[5m])) / clamp_min(sum(rate(drosync_coordinator_cycles_total[5m])), 0.000001)", "legendFormat": "error ratio"}],
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit"}},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Coordinator cycle duration",
|
||||
"targets": [{"expr": "max(drosync_coordinator_cycle_duration_seconds)", "legendFormat": "seconds"}],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}
|
||||
}
|
||||
]
|
||||
}
|
||||
14
package.json
14
package.json
@ -8,13 +8,16 @@
|
||||
"dev:full": "concurrently -k -n api,worker,web \"npm:dev:api\" \"npm:dev:worker\" \"npm:dev:web\"",
|
||||
"dev:api": "go run ./cmd/api",
|
||||
"dev:worker": "go run ./cmd/worker",
|
||||
"dev:drosync-coordinator": "go run ./cmd/drosync-coordinator",
|
||||
"dev:web": "vite",
|
||||
"build": "npm run build:web && npm run build:go",
|
||||
"build:web": "vite build",
|
||||
"build:go": "mkdir -p build && go build -o build/droboticflow-api ./cmd/api && go build -o build/droboticflow-worker ./cmd/worker",
|
||||
"build:go": "mkdir -p build && go build -o build/droboticflow-api ./cmd/api && go build -o build/droboticflow-worker ./cmd/worker && go build -o build/drosync-coordinator ./cmd/drosync-coordinator",
|
||||
"build:drosync": "cargo build --release --workspace --manifest-path tools/drosync/Cargo.toml",
|
||||
"start": "./build/droboticflow-api",
|
||||
"start:api": "./build/droboticflow-api",
|
||||
"start:worker": "./build/droboticflow-worker",
|
||||
"start:drosync-coordinator": "./build/drosync-coordinator",
|
||||
"start:operator-runner": "python3 backend/operator-runner/main.py",
|
||||
"preview": "vite preview",
|
||||
"clean": "rm -rf dist build",
|
||||
@ -27,6 +30,10 @@
|
||||
"test": "go test ./...",
|
||||
"test:python": "python3 -m pytest backend/operator-runner tests/backend",
|
||||
"test:frontend": "node --test tests/frontend/*.test.mjs",
|
||||
"test:drosync": "cargo test --workspace --manifest-path tools/drosync/Cargo.toml",
|
||||
"lint:drosync": "cargo fmt --all --manifest-path tools/drosync/Cargo.toml -- --check && cargo clippy --workspace --all-targets --manifest-path tools/drosync/Cargo.toml -- -D warnings",
|
||||
"generate:drosync-contract": "python3 scripts/generate_drosync_contract.py",
|
||||
"generate:drosync-contract:check": "python3 scripts/generate_drosync_contract.py --check",
|
||||
"operators:repos": "python3 scripts/generate_operator_repositories.py --write",
|
||||
"operators:repos:check": "python3 scripts/generate_operator_repositories.py --check",
|
||||
"operators:manifests": "npm run operators:repos && python3 scripts/generate_operator_repository_manifests.py --write",
|
||||
@ -39,10 +46,13 @@
|
||||
"verify": "npm run verify:local && npm run verify:security",
|
||||
"docker:build:api": "docker build -f Dockerfile.api -t droboticflow-api:local .",
|
||||
"docker:build:worker": "docker build -f Dockerfile.worker -t droboticflow-worker:local .",
|
||||
"docker:build:drosync-coordinator": "docker build -f Dockerfile.drosync-coordinator -t drosync-coordinator:local .",
|
||||
"docker:build:operator-runner": "docker build -f Dockerfile.operator-runner -t droboticflow-operator-runner:local .",
|
||||
"images:poc-kaniko": "./scripts/build-poc-images-kaniko.sh",
|
||||
"k8s:render": "./scripts/render-manifests.sh",
|
||||
"k8s:validate": "API_IMAGE=droboticflow-api:validate WORKER_IMAGE=droboticflow-worker:validate OPERATOR_RUNNER_IMAGE=droboticflow-operator-runner:validate ./scripts/render-manifests.sh | kubectl apply --dry-run=client --validate=false -f -",
|
||||
"k8s:render:drosync-observability": "./scripts/render-drosync-observability.sh",
|
||||
"k8s:validate": "API_IMAGE=droboticflow-api:validate WORKER_IMAGE=droboticflow-worker:validate DROSYNC_COORDINATOR_IMAGE=drosync-coordinator:validate OPERATOR_RUNNER_IMAGE=droboticflow-operator-runner:validate ./scripts/render-manifests.sh | kubectl create --dry-run=client --validate=false -f -",
|
||||
"k8s:validate:drosync-observability": "./scripts/render-drosync-observability.sh | python3 -c 'import sys,yaml; docs=list(yaml.safe_load_all(sys.stdin)); assert len(docs) == 2'",
|
||||
"k8s:deploy": "./scripts/deploy-k8s.sh",
|
||||
"k8s:cleanup-jobs": "./scripts/cleanup-operator-jobs.sh",
|
||||
"acceptance:gpu": "./scripts/gpu-acceptance-test.sh",
|
||||
|
||||
@ -26,6 +26,7 @@ droboticflow_resolve_image_configuration
|
||||
source "$ROOT_DIR/scripts/lib/deploy-validation.sh"
|
||||
validate_image_reference API_IMAGE "$API_IMAGE"
|
||||
validate_image_reference WORKER_IMAGE "$WORKER_IMAGE"
|
||||
validate_image_reference DROSYNC_COORDINATOR_IMAGE "$DROSYNC_COORDINATOR_IMAGE"
|
||||
validate_image_reference OPERATOR_RUNNER_IMAGE "$OPERATOR_RUNNER_IMAGE"
|
||||
|
||||
docker buildx inspect >/dev/null 2>&1 || docker buildx create --use >/dev/null
|
||||
@ -49,8 +50,10 @@ build_image() {
|
||||
|
||||
build_image "Dockerfile.api" "$API_IMAGE"
|
||||
build_image "Dockerfile.worker" "$WORKER_IMAGE"
|
||||
build_image "Dockerfile.drosync-coordinator" "$DROSYNC_COORDINATOR_IMAGE"
|
||||
build_image "Dockerfile.operator-runner" "$OPERATOR_RUNNER_IMAGE"
|
||||
|
||||
printf 'API_IMAGE=%s\n' "$API_IMAGE"
|
||||
printf 'WORKER_IMAGE=%s\n' "$WORKER_IMAGE"
|
||||
printf 'DROSYNC_COORDINATOR_IMAGE=%s\n' "$DROSYNC_COORDINATOR_IMAGE"
|
||||
printf 'OPERATOR_RUNNER_IMAGE=%s\n' "$OPERATOR_RUNNER_IMAGE"
|
||||
|
||||
@ -33,7 +33,7 @@ POC_KANIKO_NODE_NAME="${POC_KANIKO_NODE_NAME:-}"
|
||||
POC_KANIKO_BUILD_OPERATORS="${POC_KANIKO_BUILD_OPERATORS:-false}"
|
||||
POC_KANIKO_BUILD_OPERATOR_BASES="${POC_KANIKO_BUILD_OPERATOR_BASES:-true}"
|
||||
POC_KANIKO_OPERATOR_BASE_FILTER="${POC_KANIKO_OPERATOR_BASE_FILTER:-}"
|
||||
POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS:-api,worker,runner}"
|
||||
POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS:-api,worker,drosync-coordinator,runner}"
|
||||
POC_KANIKO_OPERATOR_MANIFEST="${POC_KANIKO_OPERATOR_MANIFEST:-operator-images/builtin/operator-images.json}"
|
||||
POC_KANIKO_OPERATOR_FILTER="${POC_KANIKO_OPERATOR_FILTER:-}"
|
||||
POC_KANIKO_OPERATOR_CONCURRENCY="${POC_KANIKO_OPERATOR_CONCURRENCY:-4}"
|
||||
@ -71,6 +71,7 @@ fi
|
||||
validate_image_reference POC_KANIKO_IMAGE "$POC_KANIKO_IMAGE"
|
||||
validate_image_reference API_IMAGE "$API_IMAGE"
|
||||
validate_image_reference WORKER_IMAGE "$WORKER_IMAGE"
|
||||
validate_image_reference DROSYNC_COORDINATOR_IMAGE "$DROSYNC_COORDINATOR_IMAGE"
|
||||
validate_image_reference OPERATOR_RUNNER_IMAGE "$OPERATOR_RUNNER_IMAGE"
|
||||
validate_image_reference POC_KANIKO_OPERATOR_BASE_IMAGE "$POC_KANIKO_OPERATOR_BASE_IMAGE"
|
||||
validate_image_reference POC_KANIKO_OPERATOR_GPU_BASE_IMAGE "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
|
||||
@ -336,6 +337,7 @@ enabled_platform_specs() {
|
||||
local -a specs=(
|
||||
"api"$'\t'"Dockerfile.api"$'\t'"$API_IMAGE"
|
||||
"worker"$'\t'"Dockerfile.worker"$'\t'"$WORKER_IMAGE"
|
||||
"drosync-coordinator"$'\t'"Dockerfile.drosync-coordinator"$'\t'"$DROSYNC_COORDINATOR_IMAGE"
|
||||
"runner"$'\t'"Dockerfile.operator-runner"$'\t'"$OPERATOR_RUNNER_IMAGE"
|
||||
)
|
||||
local spec
|
||||
@ -490,6 +492,7 @@ fi
|
||||
printf 'IMAGE_TAG=%s\n' "$IMAGE_TAG"
|
||||
printf 'API_IMAGE=%s\n' "$API_IMAGE"
|
||||
printf 'WORKER_IMAGE=%s\n' "$WORKER_IMAGE"
|
||||
printf 'DROSYNC_COORDINATOR_IMAGE=%s\n' "$DROSYNC_COORDINATOR_IMAGE"
|
||||
printf 'OPERATOR_RUNNER_IMAGE=%s\n' "$OPERATOR_RUNNER_IMAGE"
|
||||
if (( operator_component_count > 0 )); then
|
||||
printf 'OPERATOR_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE"
|
||||
|
||||
@ -39,6 +39,8 @@ export API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-publ
|
||||
export DROBOTICFLOW_SESSION_COOKIE_SECURE="${DROBOTICFLOW_SESSION_COOKIE_SECURE:-false}"
|
||||
export API_REPLICAS="${API_REPLICAS:-2}"
|
||||
export WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
|
||||
export DROSYNC_COORDINATOR_REPLICAS="${DROSYNC_COORDINATOR_REPLICAS:-2}"
|
||||
export DROBOTICFLOW_OBJECT_SYNC_ENABLED="${DROBOTICFLOW_OBJECT_SYNC_ENABLED:-false}"
|
||||
export OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
|
||||
export CUSTOM_OPERATOR_IMAGE_REGISTRY="${CUSTOM_OPERATOR_IMAGE_REGISTRY:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/custom-operators}"
|
||||
export CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE="${CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE:-quay.io/skopeo/stable:v1.16.1}"
|
||||
|
||||
110
scripts/generate_drosync_contract.py
Normal file
110
scripts/generate_drosync_contract.py
Normal file
@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the shared drosync status contract from OpenAPI 3.1."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
OPENAPI = ROOT / "docs" / "drosync" / "openapi.yaml"
|
||||
GO_OUTPUT = ROOT / "internal" / "domain" / "object_sync_contract_generated.go"
|
||||
RUST_OUTPUT = ROOT / "tools" / "drosync" / "crates" / "protocol" / "src" / "generated.rs"
|
||||
|
||||
|
||||
def pascal_case(value: str) -> str:
|
||||
return "".join(part.capitalize() for part in re.split(r"[_-]+", value))
|
||||
|
||||
|
||||
def load_contract() -> tuple[list[str], list[str], str]:
|
||||
contract = yaml.safe_load(OPENAPI.read_text(encoding="utf-8"))
|
||||
schemas = contract["components"]["schemas"]
|
||||
session = list(schemas["SessionStatus"]["enum"])
|
||||
item = list(schemas["ItemStatus"]["enum"])
|
||||
protocol = schemas["SessionCreate"]["properties"]["protocolVersion"]["const"]
|
||||
return session, item, protocol
|
||||
|
||||
|
||||
def generate_go(session: list[str], item: list[str], protocol: str) -> str:
|
||||
session_constants = "\n".join(
|
||||
f'\tObjectSyncSession{pascal_case(value)} ObjectSyncSessionStatus = "{value}"' for value in session
|
||||
)
|
||||
item_constants = "\n".join(
|
||||
f'\tObjectSyncItem{pascal_case(value)} ObjectSyncItemStatus = "{value}"' for value in item
|
||||
)
|
||||
return f"""// Code generated by scripts/generate_drosync_contract.py; DO NOT EDIT.
|
||||
|
||||
package domain
|
||||
|
||||
const ObjectSyncProtocolVersion = "{protocol}"
|
||||
|
||||
type ObjectSyncSessionStatus string
|
||||
|
||||
const (
|
||||
{session_constants}
|
||||
)
|
||||
|
||||
type ObjectSyncItemStatus string
|
||||
|
||||
const (
|
||||
{item_constants}
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def generate_rust(session: list[str], item: list[str], protocol: str) -> str:
|
||||
session_variants = "\n".join(f" {pascal_case(value)}," for value in session)
|
||||
item_variants = "\n".join(f" {pascal_case(value)}," for value in item)
|
||||
return f"""// Code generated by scripts/generate_drosync_contract.py; DO NOT EDIT.
|
||||
|
||||
use serde::{{Deserialize, Serialize}};
|
||||
|
||||
pub const PROTOCOL_VERSION: &str = "{protocol}";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SessionStatus {{
|
||||
{session_variants}
|
||||
}}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ItemStatus {{
|
||||
{item_variants}
|
||||
}}
|
||||
"""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--check", action="store_true", help="fail when generated files differ")
|
||||
args = parser.parse_args()
|
||||
session, item, protocol = load_contract()
|
||||
go_output = subprocess.run(
|
||||
["gofmt"],
|
||||
input=generate_go(session, item, protocol),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=True,
|
||||
).stdout
|
||||
rust_output = generate_rust(session, item, protocol)
|
||||
if args.check:
|
||||
stale = [
|
||||
str(path.relative_to(ROOT))
|
||||
for path, expected in ((GO_OUTPUT, go_output), (RUST_OUTPUT, rust_output))
|
||||
if not path.exists() or path.read_text(encoding="utf-8") != expected
|
||||
]
|
||||
if stale:
|
||||
raise SystemExit("generated drosync contract is stale: " + ", ".join(stale))
|
||||
return
|
||||
GO_OUTPUT.write_text(go_output, encoding="utf-8")
|
||||
RUST_OUTPUT.write_text(rust_output, encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -16,7 +16,7 @@ droboticflow_resolve_image_reference() {
|
||||
|
||||
if [[ -z "${REGISTRY_HOST:-}" || -z "${REGISTRY_NAMESPACE:-}" ]]; then
|
||||
echo "ERROR: $variable_name is required unless REGISTRY_HOST and REGISTRY_NAMESPACE are both set." >&2
|
||||
echo "Set explicit API_IMAGE, WORKER_IMAGE, and OPERATOR_RUNNER_IMAGE values, or configure REGISTRY_HOST and REGISTRY_NAMESPACE from local deploy secrets or CI." >&2
|
||||
echo "Set explicit API_IMAGE, WORKER_IMAGE, DROSYNC_COORDINATOR_IMAGE, and OPERATOR_RUNNER_IMAGE values, or configure REGISTRY_HOST and REGISTRY_NAMESPACE from local deploy secrets or CI." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -24,13 +24,14 @@ droboticflow_resolve_image_reference() {
|
||||
}
|
||||
|
||||
droboticflow_resolve_image_configuration() {
|
||||
if [[ -z "${API_IMAGE:-}" || -z "${WORKER_IMAGE:-}" || -z "${OPERATOR_RUNNER_IMAGE:-}" ]]; then
|
||||
if [[ -z "${API_IMAGE:-}" || -z "${WORKER_IMAGE:-}" || -z "${DROSYNC_COORDINATOR_IMAGE:-}" || -z "${OPERATOR_RUNNER_IMAGE:-}" ]]; then
|
||||
IMAGE_TAG="${IMAGE_TAG:-$(droboticflow_default_image_tag)}"
|
||||
else
|
||||
IMAGE_TAG="${IMAGE_TAG:-}"
|
||||
fi
|
||||
API_IMAGE="$(droboticflow_resolve_image_reference API_IMAGE droboticflow-api)"
|
||||
WORKER_IMAGE="$(droboticflow_resolve_image_reference WORKER_IMAGE droboticflow-worker)"
|
||||
DROSYNC_COORDINATOR_IMAGE="$(droboticflow_resolve_image_reference DROSYNC_COORDINATOR_IMAGE drosync-coordinator)"
|
||||
OPERATOR_RUNNER_IMAGE="$(droboticflow_resolve_image_reference OPERATOR_RUNNER_IMAGE droboticflow-operator-runner)"
|
||||
export IMAGE_TAG API_IMAGE WORKER_IMAGE OPERATOR_RUNNER_IMAGE
|
||||
export IMAGE_TAG API_IMAGE WORKER_IMAGE DROSYNC_COORDINATOR_IMAGE OPERATOR_RUNNER_IMAGE
|
||||
}
|
||||
|
||||
22
scripts/render-drosync-observability.sh
Executable file
22
scripts/render-drosync-observability.sh
Executable file
@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
||||
|
||||
if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then
|
||||
DEPLOY_ENV="$1"
|
||||
fi
|
||||
|
||||
export DEPLOY_ENV
|
||||
# shellcheck source=scripts/load-deploy-env.sh
|
||||
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
||||
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||||
|
||||
K8S_NAMESPACE="$K8S_NAMESPACE" TEMPLATE="$ROOT_DIR/k8s/drosync-observability.template.yaml" python3 - <<'PY'
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
template = pathlib.Path(os.environ["TEMPLATE"]).read_text(encoding="utf-8")
|
||||
print(template.replace("__NAMESPACE__", os.environ["K8S_NAMESPACE"]), end="")
|
||||
PY
|
||||
@ -33,6 +33,8 @@ API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}"
|
||||
API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}"
|
||||
API_REPLICAS="${API_REPLICAS:-2}"
|
||||
WORKER_REPLICAS="${WORKER_REPLICAS:-2}"
|
||||
DROSYNC_COORDINATOR_REPLICAS="${DROSYNC_COORDINATOR_REPLICAS:-2}"
|
||||
DROBOTICFLOW_OBJECT_SYNC_ENABLED="${DROBOTICFLOW_OBJECT_SYNC_ENABLED:-false}"
|
||||
DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
|
||||
DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-}"
|
||||
OPERATOR_RUNNER_BUILD_COMMIT="${OPERATOR_RUNNER_BUILD_COMMIT:-$DROBOTICFLOW_BUILD_COMMIT}"
|
||||
@ -118,6 +120,8 @@ export \
|
||||
API_NODE_PORT_BLOCK \
|
||||
API_REPLICAS \
|
||||
WORKER_REPLICAS \
|
||||
DROSYNC_COORDINATOR_REPLICAS \
|
||||
DROBOTICFLOW_OBJECT_SYNC_ENABLED \
|
||||
DROBOTICFLOW_BUILD_COMMIT \
|
||||
DROBOTICFLOW_BUILD_TIME \
|
||||
OPERATOR_RUNNER_BUILD_COMMIT \
|
||||
@ -131,6 +135,7 @@ export \
|
||||
AI_MODEL \
|
||||
API_IMAGE \
|
||||
WORKER_IMAGE \
|
||||
DROSYNC_COORDINATOR_IMAGE \
|
||||
OPERATOR_RUNNER_IMAGE \
|
||||
OPERATOR_GPU_NODE_SELECTOR \
|
||||
OPERATOR_GPU_TOLERATIONS_JSON_BLOCK \
|
||||
@ -170,6 +175,8 @@ replacements = {
|
||||
"__API_NODE_PORT_BLOCK__": os.environ["API_NODE_PORT_BLOCK"],
|
||||
"__API_REPLICAS__": os.environ["API_REPLICAS"],
|
||||
"__WORKER_REPLICAS__": os.environ["WORKER_REPLICAS"],
|
||||
"__DROSYNC_COORDINATOR_REPLICAS__": os.environ["DROSYNC_COORDINATOR_REPLICAS"],
|
||||
"__DROBOTICFLOW_OBJECT_SYNC_ENABLED__": os.environ["DROBOTICFLOW_OBJECT_SYNC_ENABLED"],
|
||||
"__DROBOTICFLOW_BUILD_COMMIT__": os.environ["DROBOTICFLOW_BUILD_COMMIT"],
|
||||
"__DROBOTICFLOW_BUILD_TIME__": os.environ["DROBOTICFLOW_BUILD_TIME"],
|
||||
"__OPERATOR_RUNNER_BUILD_COMMIT__": os.environ["OPERATOR_RUNNER_BUILD_COMMIT"],
|
||||
@ -183,6 +190,7 @@ replacements = {
|
||||
"__AI_MODEL__": os.environ["AI_MODEL"],
|
||||
"__API_IMAGE__": os.environ["API_IMAGE"],
|
||||
"__WORKER_IMAGE__": os.environ["WORKER_IMAGE"],
|
||||
"__DROSYNC_COORDINATOR_IMAGE__": os.environ["DROSYNC_COORDINATOR_IMAGE"],
|
||||
"__OPERATOR_RUNNER_IMAGE__": os.environ["OPERATOR_RUNNER_IMAGE"],
|
||||
"__OPERATOR_GPU_NODE_SELECTOR__": os.environ["OPERATOR_GPU_NODE_SELECTOR"],
|
||||
"__OPERATOR_GPU_TOLERATIONS_JSON__": os.environ["OPERATOR_GPU_TOLERATIONS_JSON_BLOCK"],
|
||||
|
||||
@ -89,6 +89,7 @@ const DataAssets = lazyRoute(() => import("./pages/DataAssets"));
|
||||
const DataOperations = lazyRoute(() => import("./pages/DataOperations"));
|
||||
const DatabaseBrowser = lazyRoute(() => import("./pages/DatabaseBrowser"));
|
||||
const ObjectStorageBrowser = lazyRoute(() => import("./pages/ObjectStorageBrowser"));
|
||||
const ObjectSync = lazyRoute(() => import("./pages/ObjectSync"));
|
||||
const LegacyDataAssetsRedirect = lazyRoute(() => import("./pages/LegacyDataAssetsRedirect"));
|
||||
const Sampling = lazyRoute(() => import("./pages/Sampling"));
|
||||
|
||||
@ -114,6 +115,7 @@ export default function App() {
|
||||
<Route path="/data-operations" element={<DataOperations />} />
|
||||
<Route path="/database" element={<DatabaseBrowser />} />
|
||||
<Route path="/object-storage" element={<ObjectStorageBrowser />} />
|
||||
<Route path="/object-sync" element={<ObjectSync />} />
|
||||
<Route path="/source-data" element={<LegacyDataAssetsRedirect />} />
|
||||
<Route path="/packages" element={<LegacyDataAssetsRedirect />} />
|
||||
<Route path="/sampling" element={<Sampling />} />
|
||||
|
||||
@ -35,11 +35,25 @@ type ComponentStatus = {
|
||||
};
|
||||
|
||||
export function Layout({ children }: LayoutProps) {
|
||||
const [isCollapsed, setIsCollapsed] = useState(false);
|
||||
const [isCollapsed, setIsCollapsed] = useState(
|
||||
() => typeof window !== "undefined" && window.matchMedia("(max-width: 767px)").matches,
|
||||
);
|
||||
const [health, setHealth] = useState<HealthState>({ status: "loading" });
|
||||
const lastHealthErrorRef = useRef("");
|
||||
const { currentUser, isSuperAdmin, loading } = useUser();
|
||||
|
||||
useEffect(() => {
|
||||
const mobileViewport = window.matchMedia("(max-width: 767px)");
|
||||
const collapseForMobile = () => {
|
||||
if (mobileViewport.matches) {
|
||||
setIsCollapsed(true);
|
||||
}
|
||||
};
|
||||
collapseForMobile();
|
||||
mobileViewport.addEventListener("change", collapseForMobile);
|
||||
return () => mobileViewport.removeEventListener("change", collapseForMobile);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
@ -154,11 +168,13 @@ export function Layout({ children }: LayoutProps) {
|
||||
{currentUser && !isSuperAdmin ? (
|
||||
<Sidebar isCollapsed={isCollapsed} onToggle={() => setIsCollapsed(!isCollapsed)} />
|
||||
) : null}
|
||||
<main className="flex-1 flex flex-col overflow-hidden">
|
||||
<header className="h-16 border-b border-[#2a2b2f] bg-[#151619] flex items-center justify-between px-8">
|
||||
<main className="flex min-w-0 flex-1 flex-col overflow-hidden">
|
||||
<header className="flex h-16 items-center justify-between gap-3 border-b border-[#2a2b2f] bg-[#151619] px-4 md:px-8">
|
||||
<div className="flex items-center gap-4">
|
||||
<Logo size="sm" />
|
||||
<h2 className="text-sm font-medium text-gray-400 uppercase tracking-widest">Drobodata Platform</h2>
|
||||
<h2 className="hidden text-sm font-medium uppercase tracking-widest text-gray-400 sm:block">
|
||||
Drobodata Platform
|
||||
</h2>
|
||||
</div>
|
||||
<div className="flex items-center gap-4">
|
||||
<details className="group relative z-40">
|
||||
@ -192,7 +208,7 @@ export function Layout({ children }: LayoutProps) {
|
||||
<span
|
||||
aria-live="polite"
|
||||
className={[
|
||||
"text-[10px] font-mono uppercase tracking-wider",
|
||||
"hidden text-[10px] font-mono uppercase tracking-wider sm:inline",
|
||||
versions.mismatch
|
||||
? "text-orange-300"
|
||||
: componentStatusHealthy
|
||||
@ -267,7 +283,7 @@ export function Layout({ children }: LayoutProps) {
|
||||
</details>
|
||||
</div>
|
||||
</header>
|
||||
<div className="flex-1 overflow-auto p-8">
|
||||
<div className="flex-1 overflow-auto p-4 md:p-8">
|
||||
{loading ? (
|
||||
<div className="flex h-full items-center justify-center">
|
||||
<div className="flex items-center gap-3 rounded-2xl border border-[#2a2b2f] bg-[#151619] px-6 py-4 text-sm text-gray-400">
|
||||
|
||||
@ -20,6 +20,7 @@ import {
|
||||
Shield,
|
||||
Database,
|
||||
Cloudy,
|
||||
CloudUpload,
|
||||
DatabaseZap,
|
||||
BarChart3,
|
||||
LogOut,
|
||||
@ -57,6 +58,12 @@ const navItems = [
|
||||
{ icon: Cpu, label: "算子管理", path: "/operators", roles: ["admin", "pipeline_engineer", "operator"] },
|
||||
{ icon: Database, label: "业务数据库", path: "/database", roles: ["admin"] },
|
||||
{ icon: Cloudy, label: "对象存储", path: "/object-storage", roles: ["admin"] },
|
||||
{
|
||||
icon: CloudUpload,
|
||||
label: "drosync",
|
||||
path: "/object-sync",
|
||||
roles: ["admin", "pipeline_engineer", "data_operator", "compliance_auditor", "auditor", "operator"],
|
||||
},
|
||||
{
|
||||
icon: DatabaseZap,
|
||||
label: "数据资产",
|
||||
@ -141,13 +148,13 @@ export function Sidebar({ isCollapsed, onToggle }: SidebarProps) {
|
||||
return (
|
||||
<aside
|
||||
className={cn(
|
||||
"bg-[#151619] border-r border-[#2a2b2f] flex flex-col transition-all duration-300 relative",
|
||||
"relative flex h-screen shrink-0 flex-col border-r border-[#2a2b2f] bg-[#151619] transition-all duration-300",
|
||||
isCollapsed ? "w-20" : "w-64",
|
||||
)}
|
||||
>
|
||||
<button
|
||||
onClick={onToggle}
|
||||
className="absolute -right-3 top-20 w-6 h-6 bg-[#2a2b2f] border border-[#3a3b3f] rounded-full flex items-center justify-center text-gray-400 hover:text-white z-50"
|
||||
className="absolute -right-3 top-20 z-50 hidden h-6 w-6 items-center justify-center rounded-full border border-[#3a3b3f] bg-[#2a2b2f] text-gray-400 hover:text-white md:flex"
|
||||
>
|
||||
{isCollapsed ? <ChevronRight className="w-4 h-4" /> : <ChevronLeft className="w-4 h-4" />}
|
||||
</button>
|
||||
|
||||
1133
src/pages/ObjectSync.tsx
Normal file
1133
src/pages/ObjectSync.tsx
Normal file
File diff suppressed because it is too large
Load Diff
246
src/services/api/object-sync.ts
Normal file
246
src/services/api/object-sync.ts
Normal file
@ -0,0 +1,246 @@
|
||||
import { apiFetch, apiRequest } from "./client";
|
||||
|
||||
export type ObjectSyncCapabilityState = "unverified" | "create_only" | "update_enabled" | "drifted" | "unsupported";
|
||||
|
||||
export type ObjectSyncCapability = {
|
||||
state: ObjectSyncCapabilityState;
|
||||
conditionalCreate: boolean;
|
||||
conditionalUpdate: boolean;
|
||||
conditionalMultipart: boolean;
|
||||
multipartCopy: boolean;
|
||||
versionId: boolean;
|
||||
exclusivePrefixConfirmed: boolean;
|
||||
probedAt?: string;
|
||||
probeError?: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncSpace = {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
connectionId: string;
|
||||
bucket: string;
|
||||
rootPrefix: string;
|
||||
mode: string;
|
||||
enabled: boolean;
|
||||
policy: {
|
||||
allowConditionalUpdate: boolean;
|
||||
allowForceOverwrite: boolean;
|
||||
};
|
||||
capability: ObjectSyncCapability;
|
||||
updatedAt: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncSession = {
|
||||
id: string;
|
||||
syncSpaceId: string;
|
||||
deviceId: string;
|
||||
mappingId: string;
|
||||
clientVersion: string;
|
||||
status: string;
|
||||
totalBytes: number;
|
||||
counts: {
|
||||
manifest: number;
|
||||
unchanged: number;
|
||||
create: number;
|
||||
update: number;
|
||||
keptBoth: number;
|
||||
denied: number;
|
||||
failed: number;
|
||||
committed: number;
|
||||
};
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncItem = {
|
||||
id: string;
|
||||
sessionId: string;
|
||||
relativePath: string;
|
||||
targetKey: string;
|
||||
logicalTargetKey?: string;
|
||||
sizeBytes: number;
|
||||
sha256: string;
|
||||
action: string;
|
||||
status: string;
|
||||
lastError?: string;
|
||||
updatedAt: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncDevice = {
|
||||
id: string;
|
||||
name: string;
|
||||
platform?: string;
|
||||
architecture?: string;
|
||||
clientVersion?: string;
|
||||
status: string;
|
||||
lastSeenAt?: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncAuthorization = {
|
||||
id: string;
|
||||
deviceId: string;
|
||||
deviceName: string;
|
||||
platform?: string;
|
||||
architecture?: string;
|
||||
userCode: string;
|
||||
status: string;
|
||||
createdAt: string;
|
||||
expiresAt: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncCredential = {
|
||||
id: string;
|
||||
deviceId: string;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
tokenPreview: string;
|
||||
scopes: Array<{ syncSpaceId: string; allowedRelativePrefixes: string[]; actions: string[] }>;
|
||||
lastUsedAt?: string;
|
||||
revokedAt?: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncRelease = {
|
||||
id: string;
|
||||
syncSpaceId: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
sessionIds: string[];
|
||||
manifestKey?: string;
|
||||
manifestSha256?: string;
|
||||
objectCount: number;
|
||||
totalBytes: number;
|
||||
status: string;
|
||||
createdAt: string;
|
||||
publishedAt?: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncAuditEvent = {
|
||||
id: string;
|
||||
occurredAt: string;
|
||||
actorType: string;
|
||||
actorId: string;
|
||||
action: string;
|
||||
result: string;
|
||||
sessionId?: string;
|
||||
itemId?: string;
|
||||
eventHash: string;
|
||||
previousHash?: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncClientRelease = {
|
||||
id: string;
|
||||
version: string;
|
||||
channel: string;
|
||||
protocolVersion: string;
|
||||
minimumServerVersion?: string;
|
||||
rolloutPercent: number;
|
||||
paused: boolean;
|
||||
artifacts: Record<string, string>;
|
||||
sha256: Record<string, string>;
|
||||
signature: string;
|
||||
createdAt: string;
|
||||
stageChangedAt: string;
|
||||
};
|
||||
|
||||
export type ObjectSyncReconcile = {
|
||||
id: string;
|
||||
operationId: string;
|
||||
syncSpaceId: string;
|
||||
status: "pending" | "running" | "completed" | "failed";
|
||||
scanned: number;
|
||||
drifted: number;
|
||||
};
|
||||
|
||||
const mutationHeaders = () => ({
|
||||
"Idempotency-Key": globalThis.crypto?.randomUUID?.() || `web-${Date.now()}-${Math.random().toString(16).slice(2)}`,
|
||||
});
|
||||
|
||||
const mutation = <T>(path: string, method: "POST" | "PUT", body: Record<string, unknown>) =>
|
||||
apiRequest<T>(path, { method, headers: mutationHeaders() }, body);
|
||||
|
||||
const list = async <T>(path: string) => (await apiRequest<{ items: T[] }>(path)).items;
|
||||
|
||||
export const objectSyncApi = {
|
||||
listSpaces: () => list<ObjectSyncSpace>("/api/object-sync/spaces"),
|
||||
createSpace: (body: {
|
||||
name: string;
|
||||
description: string;
|
||||
connectionId: string;
|
||||
bucket: string;
|
||||
rootPrefix: string;
|
||||
}) => mutation<ObjectSyncSpace>("/api/object-sync/spaces", "POST", body),
|
||||
updateSpace: (
|
||||
id: string,
|
||||
body: {
|
||||
enabled?: boolean;
|
||||
exclusivePrefixConfirmed?: boolean;
|
||||
allowForceOverwrite?: boolean;
|
||||
name?: string;
|
||||
description?: string;
|
||||
},
|
||||
) => mutation<ObjectSyncSpace>(`/api/object-sync/spaces/${encodeURIComponent(id)}`, "PUT", body),
|
||||
probeSpace: (id: string) =>
|
||||
mutation<ObjectSyncSpace>(`/api/object-sync/spaces/${encodeURIComponent(id)}/capability-probe`, "POST", {}),
|
||||
reconcileSpace: (id: string) =>
|
||||
mutation<ObjectSyncReconcile>(`/api/object-sync/spaces/${encodeURIComponent(id)}/reconcile`, "POST", {}),
|
||||
listSessions: () => list<ObjectSyncSession>("/api/object-sync/sessions?limit=200"),
|
||||
getSessionItems: (id: string, status?: string) =>
|
||||
apiRequest<{ items: ObjectSyncItem[]; nextCursor?: string }>(
|
||||
`/api/object-sync/sessions/${encodeURIComponent(id)}/items?limit=1000${status ? `&status=${status}` : ""}`,
|
||||
),
|
||||
abortSession: (id: string) =>
|
||||
mutation<ObjectSyncSession>(`/api/object-sync/sessions/${encodeURIComponent(id)}/admin-abort`, "POST", {}),
|
||||
forceOverwriteConflict: (id: string) =>
|
||||
mutation<{ operationId: string; item: ObjectSyncItem }>(
|
||||
`/api/object-sync/items/${encodeURIComponent(id)}/force-overwrite`,
|
||||
"POST",
|
||||
{},
|
||||
),
|
||||
listDevices: () => list<ObjectSyncDevice>("/api/object-sync/devices"),
|
||||
listAuthorizations: () => list<ObjectSyncAuthorization>("/api/object-sync/device-authorizations"),
|
||||
findAuthorizationByCode: (userCode: string) =>
|
||||
apiRequest<ObjectSyncAuthorization>(
|
||||
`/api/object-sync/device-authorizations/by-code?userCode=${encodeURIComponent(userCode)}`,
|
||||
),
|
||||
decideAuthorization: (
|
||||
id: string,
|
||||
decision: "approve" | "deny",
|
||||
scopes: Array<{ syncSpaceId: string; allowedRelativePrefixes: string[]; actions: string[] }>,
|
||||
) =>
|
||||
mutation<ObjectSyncAuthorization>(
|
||||
`/api/object-sync/device-authorizations/${encodeURIComponent(id)}/${decision}`,
|
||||
"POST",
|
||||
{ scopes },
|
||||
),
|
||||
listCredentials: () => list<ObjectSyncCredential>("/api/object-sync/credentials"),
|
||||
mutateCredential: (id: string, action: "rotate" | "revoke") =>
|
||||
mutation<{ credential: ObjectSyncCredential; token?: string }>(
|
||||
`/api/object-sync/credentials/${encodeURIComponent(id)}/${action}`,
|
||||
"POST",
|
||||
{},
|
||||
),
|
||||
listReleases: () => list<ObjectSyncRelease>("/api/object-sync/releases"),
|
||||
createRelease: (body: { name: string; description: string; sessionIds: string[] }) =>
|
||||
mutation<{ operationId: string; release: ObjectSyncRelease }>("/api/object-sync/releases", "POST", body),
|
||||
listAudit: () => list<ObjectSyncAuditEvent>("/api/object-sync/audit-events?limit=500"),
|
||||
auditExportURL: (format: "csv" | "ndjson") => `/api/object-sync/audit-events/export?format=${format}`,
|
||||
listClientReleases: () => list<ObjectSyncClientRelease>("/api/object-sync/client-releases"),
|
||||
saveClientRelease: (body: ObjectSyncClientRelease) =>
|
||||
mutation<ObjectSyncClientRelease>(
|
||||
"/api/object-sync/client-releases",
|
||||
"POST",
|
||||
body as unknown as Record<string, unknown>,
|
||||
),
|
||||
downloadAudit: async (format: "csv" | "ndjson") => {
|
||||
const response = await apiFetch(`/api/object-sync/audit-events/export?format=${format}`);
|
||||
if (!response.ok) throw new Error(`审计导出失败:${response.status}`);
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = url;
|
||||
anchor.download = `drosync-audit.${format === "csv" ? "csv" : "ndjson"}`;
|
||||
anchor.click();
|
||||
URL.revokeObjectURL(url);
|
||||
},
|
||||
};
|
||||
64
tests/frontend/object-sync.test.mjs
Normal file
64
tests/frontend/object-sync.test.mjs
Normal file
@ -0,0 +1,64 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
|
||||
const pageURL = new URL("../../src/pages/ObjectSync.tsx", import.meta.url);
|
||||
const serviceURL = new URL("../../src/services/api/object-sync.ts", import.meta.url);
|
||||
const appURL = new URL("../../src/App.tsx", import.meta.url);
|
||||
const sidebarURL = new URL("../../src/components/Sidebar.tsx", import.meta.url);
|
||||
|
||||
test("drosync page exposes capability, session, conflict, release, audit and update surfaces", async () => {
|
||||
const source = await readFile(pageURL, "utf8");
|
||||
for (const label of [
|
||||
"能力门禁",
|
||||
"同步空间",
|
||||
"设备与授权",
|
||||
"Session",
|
||||
"Keep-both 冲突",
|
||||
"不可变 Release",
|
||||
"审计哈希链",
|
||||
"客户端更新策略",
|
||||
]) {
|
||||
assert.match(source, new RegExp(label));
|
||||
}
|
||||
assert.match(source, /条件创建/);
|
||||
assert.match(source, /发现漂移/);
|
||||
assert.match(source, /已确认无外部直写/);
|
||||
assert.match(source, /条件覆盖/);
|
||||
assert.match(source, /window\.confirm/);
|
||||
assert.match(source, /配对短码/);
|
||||
assert.match(source, /findAuthorizationByCode/);
|
||||
});
|
||||
|
||||
test("drosync page applies role-specific views", async () => {
|
||||
const source = await readFile(pageURL, "utf8");
|
||||
assert.match(source, /role === "admin"/);
|
||||
assert.match(source, /role === "pipeline_engineer"/);
|
||||
assert.match(source, /role === "compliance_auditor"/);
|
||||
assert.match(source, /visible: isAdmin/);
|
||||
assert.match(source, /visible: isAdmin \|\| isAuditor/);
|
||||
assert.match(source, /visible: isAdmin \|\| isPipeline/);
|
||||
});
|
||||
|
||||
test("drosync API client adds idempotency keys to every mutation", async () => {
|
||||
const source = await readFile(serviceURL, "utf8");
|
||||
assert.match(source, /"Idempotency-Key"/);
|
||||
assert.match(source, /crypto\?\.randomUUID/);
|
||||
assert.match(source, /const mutation =/);
|
||||
assert.match(source, /\/api\/object-sync\/spaces/);
|
||||
assert.match(source, /capability-probe/);
|
||||
assert.match(source, /device-authorizations/);
|
||||
assert.match(source, /audit-events\/export/);
|
||||
assert.match(source, /force-overwrite/);
|
||||
assert.match(source, /device-authorizations\/by-code\?userCode=/);
|
||||
});
|
||||
|
||||
test("drosync route is lazy-loaded and visible to all supported business roles", async () => {
|
||||
const [app, sidebar] = await Promise.all([readFile(appURL, "utf8"), readFile(sidebarURL, "utf8")]);
|
||||
assert.match(app, /lazyRoute\(\(\) => import\("\.\/pages\/ObjectSync"\)\)/);
|
||||
assert.match(app, /path="\/object-sync"/);
|
||||
assert.match(sidebar, /label: "drosync"/);
|
||||
for (const role of ["admin", "pipeline_engineer", "data_operator", "compliance_auditor", "auditor", "operator"]) {
|
||||
assert.match(sidebar, new RegExp(`"${role}"`));
|
||||
}
|
||||
});
|
||||
4
tools/drosync/.gitignore
vendored
Normal file
4
tools/drosync/.gitignore
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
/target/
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
6228
tools/drosync/Cargo.lock
generated
Normal file
6228
tools/drosync/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
60
tools/drosync/Cargo.toml
Normal file
60
tools/drosync/Cargo.toml
Normal file
@ -0,0 +1,60 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"crates/core",
|
||||
"crates/protocol",
|
||||
"crates/state",
|
||||
"crates/transfer",
|
||||
"crates/platform",
|
||||
"crates/cli",
|
||||
"crates/daemon",
|
||||
"crates/gui",
|
||||
"crates/updater",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.0.0"
|
||||
edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
rust-version = "1.89"
|
||||
|
||||
[workspace.dependencies]
|
||||
anyhow = "1.0"
|
||||
base64 = "0.22"
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
clap = { version = "4.5", features = ["derive", "env"] }
|
||||
directories = "6.0"
|
||||
ed25519-dalek = "2.2"
|
||||
eframe = "0.32"
|
||||
hex = "0.4"
|
||||
keyring = "3.6"
|
||||
md-5 = "0.10"
|
||||
notify = "8.2"
|
||||
rayon = "1.11"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
|
||||
rusqlite = { version = "0.38", features = ["bundled"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
sha2 = "0.10"
|
||||
tempfile = "3.23"
|
||||
thiserror = "2.0"
|
||||
tokio = { version = "1.48", features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] }
|
||||
tokio-util = { version = "0.7", features = ["io"] }
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
tray-icon = "0.21"
|
||||
uuid = { version = "1.18", features = ["v4", "serde"] }
|
||||
walkdir = "2.5"
|
||||
|
||||
[workspace.lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
|
||||
[workspace.lints.clippy]
|
||||
all = { level = "warn", priority = -1 }
|
||||
pedantic = { level = "warn", priority = -1 }
|
||||
missing_errors_doc = "allow"
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 1
|
||||
strip = "symbols"
|
||||
23
tools/drosync/README.md
Normal file
23
tools/drosync/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# drosync Rust workspace
|
||||
|
||||
Workspace crates:
|
||||
|
||||
- `core`: mapping policy, bounded stable-file hashing, manifest inputs.
|
||||
- `protocol`: OpenAPI-derived state enums and wire DTOs.
|
||||
- `state`: WAL SQLite migrations and crash/multipart journal.
|
||||
- `transfer`: rustls HTTP client, 16-request semaphore, integrity-bound streaming upload.
|
||||
- `platform`: directories, OS credential store, and headless secret policy.
|
||||
- `cli`: fixed `drosync` command surface.
|
||||
- `daemon`: IPC owner and scan/plan/upload/commit orchestration.
|
||||
- `gui`: tray control surface.
|
||||
- `updater`: Ed25519 and artifact verification.
|
||||
|
||||
Development:
|
||||
|
||||
```bash
|
||||
cargo fmt --all -- --check
|
||||
cargo clippy --workspace --all-targets -- -D warnings
|
||||
cargo test --workspace
|
||||
```
|
||||
|
||||
Release packaging lives under `packaging/`; signing material is required from CI secrets.
|
||||
25
tools/drosync/crates/cli/Cargo.toml
Normal file
25
tools/drosync/crates/cli/Cargo.toml
Normal file
@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "drosync-cli"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "drosync"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
drosync-core = { path = "../core" }
|
||||
drosync-daemon = { path = "../daemon" }
|
||||
drosync-platform = { path = "../platform" }
|
||||
drosync-protocol = { path = "../protocol" }
|
||||
drosync-transfer = { path = "../transfer" }
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
343
tools/drosync/crates/cli/src/main.rs
Normal file
343
tools/drosync/crates/cli/src/main.rs
Normal file
@ -0,0 +1,343 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use clap::{Args, Parser, Subcommand};
|
||||
use drosync_core::Mapping;
|
||||
use drosync_daemon::{IpcRequest, request};
|
||||
use drosync_protocol::{DeviceAuthorization, PROTOCOL_VERSION};
|
||||
use serde_json::{Value, json};
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(
|
||||
name = "drosync",
|
||||
version,
|
||||
about = "Secure one-way local-to-BOS synchronization"
|
||||
)]
|
||||
struct App {
|
||||
#[arg(long, global = true)]
|
||||
json: bool,
|
||||
#[arg(long, global = true)]
|
||||
non_interactive: bool,
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Command {
|
||||
Configure(Configure),
|
||||
Auth(Auth),
|
||||
Mapping(MappingCommand),
|
||||
Plan(MappingRun),
|
||||
Sync(MappingRun),
|
||||
Status,
|
||||
Conflicts,
|
||||
Retry(IdArgument),
|
||||
Abort(IdArgument),
|
||||
Doctor,
|
||||
Daemon(DaemonCommand),
|
||||
Update(UpdateCommand),
|
||||
Version,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct Configure {
|
||||
#[arg(long)]
|
||||
server: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct Auth {
|
||||
#[arg(long)]
|
||||
server: String,
|
||||
#[arg(long, default_value = "default")]
|
||||
profile: String,
|
||||
#[arg(long, default_value_t = default_device_name())]
|
||||
device_name: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct MappingCommand {
|
||||
#[command(subcommand)]
|
||||
command: MappingSubcommand,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum MappingSubcommand {
|
||||
List,
|
||||
Add {
|
||||
local_root: PathBuf,
|
||||
#[arg(long)]
|
||||
space: String,
|
||||
#[arg(long)]
|
||||
remote_prefix: Option<String>,
|
||||
#[arg(long)]
|
||||
include_hidden: bool,
|
||||
#[arg(long)]
|
||||
require_done: bool,
|
||||
},
|
||||
Remove {
|
||||
id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct MappingRun {
|
||||
mapping_id: String,
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct IdArgument {
|
||||
id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct DaemonCommand {
|
||||
#[arg(long)]
|
||||
foreground: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct UpdateCommand {
|
||||
#[arg(long)]
|
||||
check: bool,
|
||||
#[arg(long)]
|
||||
pin: Option<String>,
|
||||
#[arg(long)]
|
||||
pause: bool,
|
||||
#[arg(long)]
|
||||
rollback: bool,
|
||||
}
|
||||
|
||||
fn default_device_name() -> String {
|
||||
std::env::var("COMPUTERNAME")
|
||||
.or_else(|_| std::env::var("HOSTNAME"))
|
||||
.unwrap_or_else(|_| "drosync-device".to_owned())
|
||||
}
|
||||
|
||||
async fn ipc(command: &str, arguments: Value) -> Result<Value> {
|
||||
let paths = drosync_platform::paths()?;
|
||||
let response = request(
|
||||
&paths.ipc_endpoint,
|
||||
&IpcRequest {
|
||||
command: command.to_owned(),
|
||||
arguments,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if response.ok {
|
||||
Ok(response.result)
|
||||
} else {
|
||||
Err(anyhow!(response.error))
|
||||
}
|
||||
}
|
||||
|
||||
fn print_value(value: &Value, as_json: bool) {
|
||||
if as_json {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string(value).expect("serializable output")
|
||||
);
|
||||
} else {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(value).expect("serializable output")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn exit_code_for(error: &anyhow::Error) -> u8 {
|
||||
if let Some(drosync_transfer::TransferError::Api { status, error }) =
|
||||
error.downcast_ref::<drosync_transfer::TransferError>()
|
||||
{
|
||||
if status.as_u16() == 401
|
||||
|| error.code.contains("credential")
|
||||
|| error.code.contains("authorization")
|
||||
{
|
||||
return 10;
|
||||
}
|
||||
if status.as_u16() == 403
|
||||
|| error.code.contains("scope")
|
||||
|| error.code.contains("capability")
|
||||
|| error.code.contains("space_not_ready")
|
||||
{
|
||||
return 20;
|
||||
}
|
||||
if status.as_u16() == 409 || error.code.contains("lease") || error.code.contains("conflict")
|
||||
{
|
||||
return 30;
|
||||
}
|
||||
}
|
||||
let message = error.to_string().to_ascii_lowercase();
|
||||
if message.contains("credential") || message.contains("authorization") {
|
||||
10
|
||||
} else if message.contains("scope")
|
||||
|| message.contains("capability")
|
||||
|| message.contains("space is")
|
||||
{
|
||||
20
|
||||
} else if message.contains("lease") || message.contains("conflict") {
|
||||
30
|
||||
} else if message.contains("sqlite")
|
||||
|| message.contains("mapping root")
|
||||
|| message.contains("local path")
|
||||
|| message.contains("file changed")
|
||||
{
|
||||
40
|
||||
} else {
|
||||
50
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
async fn execute(app: &App) -> Result<Value> {
|
||||
match &app.command {
|
||||
Command::Configure(input) => ipc("configure", json!({"serverUrl": input.server})).await,
|
||||
Command::Auth(input) => {
|
||||
let client = drosync_transfer::Client::new(&input.server, None)?;
|
||||
let authorization = client
|
||||
.create_device_authorization(&DeviceAuthorization {
|
||||
device_id: Uuid::new_v4().to_string(),
|
||||
device_name: input.device_name.clone(),
|
||||
platform: drosync_platform::platform_name().to_owned(),
|
||||
architecture: drosync_platform::architecture().to_owned(),
|
||||
client_version: env!("CARGO_PKG_VERSION").to_owned(),
|
||||
})
|
||||
.await?;
|
||||
if app.non_interactive {
|
||||
return Ok(serde_json::to_value(authorization)?);
|
||||
}
|
||||
eprintln!(
|
||||
"Approve device code {} at {}",
|
||||
authorization.user_code, authorization.verification_uri
|
||||
);
|
||||
let deadline =
|
||||
tokio::time::Instant::now() + Duration::from_secs(authorization.expires_in);
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(authorization.interval.max(1))).await;
|
||||
match client
|
||||
.poll_device_authorization(&authorization.device_code)
|
||||
.await
|
||||
{
|
||||
Ok(value) => {
|
||||
let token = value
|
||||
.get("credential")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| anyhow!("authorization response omitted credential"))?;
|
||||
drosync_platform::save_credential(&input.profile, token)?;
|
||||
let _ = ipc(
|
||||
"configure",
|
||||
json!({
|
||||
"serverUrl": input.server,
|
||||
"credentialProfile": input.profile,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
return Ok(json!({"authorized": true, "profile": input.profile}));
|
||||
}
|
||||
Err(drosync_transfer::TransferError::Api { error, .. })
|
||||
if error.code == "authorization_pending"
|
||||
&& tokio::time::Instant::now() < deadline => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
Command::Mapping(input) => match &input.command {
|
||||
MappingSubcommand::List => ipc("mapping.list", json!({})).await,
|
||||
MappingSubcommand::Add {
|
||||
local_root,
|
||||
space,
|
||||
remote_prefix,
|
||||
include_hidden,
|
||||
require_done,
|
||||
} => {
|
||||
let canonical = local_root
|
||||
.canonicalize()
|
||||
.with_context(|| format!("invalid mapping root {}", local_root.display()))?;
|
||||
let mut mapping = Mapping::new(canonical, space.clone());
|
||||
mapping.remote_prefix = remote_prefix.clone().unwrap_or_default();
|
||||
mapping.include_hidden = *include_hidden;
|
||||
mapping.require_done_marker = *require_done;
|
||||
ipc("mapping.add", serde_json::to_value(mapping)?).await
|
||||
}
|
||||
MappingSubcommand::Remove { id } => ipc("mapping.remove", json!({"id": id})).await,
|
||||
},
|
||||
Command::Plan(input) => {
|
||||
ipc(
|
||||
"plan",
|
||||
json!({"mappingId": input.mapping_id, "dryRun": true}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
Command::Sync(input) => {
|
||||
ipc(
|
||||
"sync",
|
||||
json!({"mappingId": input.mapping_id, "dryRun": input.dry_run}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
Command::Status => ipc("status", json!({})).await,
|
||||
Command::Conflicts => ipc("conflicts", json!({})).await,
|
||||
Command::Retry(input) => ipc("retry", json!({"id": input.id})).await,
|
||||
Command::Abort(input) => ipc("abort", json!({"id": input.id})).await,
|
||||
Command::Doctor => ipc("doctor", json!({})).await,
|
||||
Command::Daemon(input) => {
|
||||
if input.foreground {
|
||||
let paths = drosync_platform::paths()?;
|
||||
drosync_daemon::run(&paths).await?;
|
||||
Ok(json!({"stopped": true}))
|
||||
} else {
|
||||
Err(anyhow!(
|
||||
"use `drosync daemon --foreground` or the installed service"
|
||||
))
|
||||
}
|
||||
}
|
||||
Command::Update(input) => {
|
||||
ipc(
|
||||
"update",
|
||||
json!({
|
||||
"check": input.check,
|
||||
"pin": input.pin,
|
||||
"pause": input.pause,
|
||||
"rollback": input.rollback,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
Command::Version => Ok(json!({
|
||||
"name": "drosync",
|
||||
"version": env!("CARGO_PKG_VERSION"),
|
||||
"protocolVersion": PROTOCOL_VERSION,
|
||||
"platform": drosync_platform::platform_name(),
|
||||
"architecture": drosync_platform::architecture(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> ExitCode {
|
||||
let app = App::parse();
|
||||
match execute(&app).await {
|
||||
Ok(value) => {
|
||||
print_value(&value, app.json);
|
||||
ExitCode::SUCCESS
|
||||
}
|
||||
Err(error) => {
|
||||
let exit_code = exit_code_for(&error);
|
||||
if app.json {
|
||||
println!(
|
||||
"{}",
|
||||
json!({"error": error.to_string(), "retryable": false, "exitCode": exit_code})
|
||||
);
|
||||
} else {
|
||||
eprintln!("drosync: {error:#}");
|
||||
}
|
||||
ExitCode::from(exit_code)
|
||||
}
|
||||
}
|
||||
}
|
||||
23
tools/drosync/crates/core/Cargo.toml
Normal file
23
tools/drosync/crates/core/Cargo.toml
Normal file
@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "drosync-core"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
chrono.workspace = true
|
||||
drosync-protocol = { path = "../protocol" }
|
||||
notify.workspace = true
|
||||
rayon.workspace = true
|
||||
serde.workspace = true
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
uuid.workspace = true
|
||||
walkdir.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
372
tools/drosync/crates/core/src/lib.rs
Normal file
372
tools/drosync/crates/core/src/lib.rs
Normal file
@ -0,0 +1,372 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use drosync_protocol::ManifestItem;
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, BufReader, Read};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use thiserror::Error;
|
||||
use uuid::Uuid;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
pub const DEFAULT_STABLE_FOR: Duration = Duration::from_secs(60);
|
||||
pub const DEFAULT_RENAME_SETTLE: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CoreError {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("path is outside the mapping root")]
|
||||
OutsideRoot,
|
||||
#[error("path contains unsupported components")]
|
||||
InvalidPath,
|
||||
#[error("file changed while hashing")]
|
||||
ChangedDuringHash,
|
||||
#[error("scan page size must be greater than zero")]
|
||||
InvalidPageSize,
|
||||
#[error("scan page sink failed: {0}")]
|
||||
Sink(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Mapping {
|
||||
pub id: String,
|
||||
pub local_root: PathBuf,
|
||||
pub sync_space_id: String,
|
||||
#[serde(default)]
|
||||
pub remote_prefix: String,
|
||||
#[serde(default)]
|
||||
pub include_hidden: bool,
|
||||
#[serde(default)]
|
||||
pub require_done_marker: bool,
|
||||
}
|
||||
|
||||
impl Mapping {
|
||||
#[must_use]
|
||||
pub fn new(local_root: PathBuf, sync_space_id: String) -> Self {
|
||||
Self {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
local_root,
|
||||
sync_space_id,
|
||||
remote_prefix: String::new(),
|
||||
include_hidden: false,
|
||||
require_done_marker: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScanOptions {
|
||||
pub include_hidden: bool,
|
||||
pub require_done_marker: bool,
|
||||
pub stable_for: Duration,
|
||||
}
|
||||
|
||||
impl Default for ScanOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
include_hidden: false,
|
||||
require_done_marker: false,
|
||||
stable_for: DEFAULT_STABLE_FOR,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScannedFile {
|
||||
pub absolute_path: PathBuf,
|
||||
pub manifest: ManifestItem,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct ScanSummary {
|
||||
pub file_count: u64,
|
||||
pub total_bytes: i64,
|
||||
}
|
||||
|
||||
const TEMP_SUFFIXES: &[&str] = &["tmp", "part", "partial", "crdownload", "swp", "download"];
|
||||
|
||||
fn is_hidden(path: &Path) -> bool {
|
||||
path.file_name()
|
||||
.and_then(OsStr::to_str)
|
||||
.is_some_and(|name| name.starts_with('.'))
|
||||
}
|
||||
|
||||
fn is_temporary(path: &Path) -> bool {
|
||||
path.extension()
|
||||
.and_then(OsStr::to_str)
|
||||
.is_some_and(|value| {
|
||||
TEMP_SUFFIXES
|
||||
.iter()
|
||||
.any(|suffix| value.eq_ignore_ascii_case(suffix))
|
||||
})
|
||||
}
|
||||
|
||||
fn normalized_relative(root: &Path, path: &Path) -> Result<String, CoreError> {
|
||||
let relative = path
|
||||
.strip_prefix(root)
|
||||
.map_err(|_| CoreError::OutsideRoot)?;
|
||||
let mut parts = Vec::new();
|
||||
for component in relative.components() {
|
||||
match component {
|
||||
Component::Normal(value) => {
|
||||
let value = value.to_str().ok_or(CoreError::InvalidPath)?;
|
||||
if value.is_empty()
|
||||
|| value == "."
|
||||
|| value == ".."
|
||||
|| value.chars().any(char::is_control)
|
||||
{
|
||||
return Err(CoreError::InvalidPath);
|
||||
}
|
||||
parts.push(value);
|
||||
}
|
||||
_ => return Err(CoreError::InvalidPath),
|
||||
}
|
||||
}
|
||||
if parts.is_empty() {
|
||||
return Err(CoreError::InvalidPath);
|
||||
}
|
||||
Ok(parts.join("/"))
|
||||
}
|
||||
|
||||
fn hash_stable_file(path: &Path) -> Result<(String, fs::Metadata), CoreError> {
|
||||
let before = fs::metadata(path)?;
|
||||
let mut reader = BufReader::new(File::open(path)?);
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buffer = vec![0_u8; 1024 * 1024];
|
||||
loop {
|
||||
let read = reader.read(&mut buffer)?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
}
|
||||
let after = fs::metadata(path)?;
|
||||
if before.len() != after.len() || before.modified()? != after.modified()? {
|
||||
return Err(CoreError::ChangedDuringHash);
|
||||
}
|
||||
Ok((hex_digest(hasher.finalize().as_slice()), after))
|
||||
}
|
||||
|
||||
fn hex_digest(bytes: &[u8]) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut output = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
output.push(char::from(HEX[usize::from(byte >> 4)]));
|
||||
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn hash_scan_batch(
|
||||
pool: &rayon::ThreadPool,
|
||||
canonical_root: &Path,
|
||||
candidates: &[PathBuf],
|
||||
) -> Result<Vec<ScannedFile>, CoreError> {
|
||||
let mut output = pool.install(|| {
|
||||
candidates
|
||||
.par_iter()
|
||||
.map(|path| {
|
||||
let relative_path = normalized_relative(canonical_root, path)?;
|
||||
let (sha256, stable_metadata) = hash_stable_file(path)?;
|
||||
let modified_at: DateTime<Utc> = DateTime::from(stable_metadata.modified()?);
|
||||
Ok(ScannedFile {
|
||||
absolute_path: path.clone(),
|
||||
manifest: ManifestItem {
|
||||
client_item_id: Uuid::new_v4().to_string(),
|
||||
relative_path,
|
||||
kind: "file".to_owned(),
|
||||
size_bytes: i64::try_from(stable_metadata.len()).unwrap_or(i64::MAX),
|
||||
modified_at: modified_at.to_rfc3339(),
|
||||
sha256,
|
||||
content_type: "application/octet-stream".to_owned(),
|
||||
last_known_remote: drosync_protocol::RemoteBase::default(),
|
||||
},
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, CoreError>>()
|
||||
})?;
|
||||
output.sort_by(|left, right| {
|
||||
left.manifest
|
||||
.relative_path
|
||||
.cmp(&right.manifest.relative_path)
|
||||
});
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
pub fn scan_mapping_paged<F>(
|
||||
mapping: &Mapping,
|
||||
options: &ScanOptions,
|
||||
page_size: usize,
|
||||
mut emit: F,
|
||||
) -> Result<ScanSummary, CoreError>
|
||||
where
|
||||
F: FnMut(Vec<ScannedFile>) -> Result<(), CoreError>,
|
||||
{
|
||||
if page_size == 0 {
|
||||
return Err(CoreError::InvalidPageSize);
|
||||
}
|
||||
let canonical_root = mapping.local_root.canonicalize()?;
|
||||
let worker_count = std::thread::available_parallelism()
|
||||
.map_or(2, usize::from)
|
||||
.clamp(1, 4);
|
||||
let pool = rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(worker_count)
|
||||
.thread_name(|index| format!("drosync-hash-{index}"))
|
||||
.build()
|
||||
.map_err(io::Error::other)?;
|
||||
let mut candidates = Vec::with_capacity(page_size);
|
||||
let mut summary = ScanSummary::default();
|
||||
for entry in WalkDir::new(&canonical_root)
|
||||
.follow_links(false)
|
||||
.into_iter()
|
||||
.filter_entry(|entry| {
|
||||
options.include_hidden || entry.path() == canonical_root || !is_hidden(entry.path())
|
||||
})
|
||||
.filter_map(Result::ok)
|
||||
{
|
||||
let path = entry.path();
|
||||
if path == canonical_root {
|
||||
continue;
|
||||
}
|
||||
if (!options.include_hidden && is_hidden(path)) || entry.file_type().is_symlink() {
|
||||
continue;
|
||||
}
|
||||
if !entry.file_type().is_file() || is_temporary(path) {
|
||||
continue;
|
||||
}
|
||||
if options.require_done_marker {
|
||||
let marker = PathBuf::from(format!("{}.done", path.to_string_lossy()));
|
||||
if !marker.is_file() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let metadata = entry.metadata().map_err(io::Error::other)?;
|
||||
let modified = metadata.modified()?;
|
||||
if SystemTime::now()
|
||||
.duration_since(modified)
|
||||
.unwrap_or_default()
|
||||
< options.stable_for
|
||||
{
|
||||
continue;
|
||||
}
|
||||
candidates.push(path.to_path_buf());
|
||||
if candidates.len() == page_size {
|
||||
let page = hash_scan_batch(&pool, &canonical_root, &candidates)?;
|
||||
summary.file_count = summary
|
||||
.file_count
|
||||
.saturating_add(u64::try_from(page.len()).unwrap_or(u64::MAX));
|
||||
summary.total_bytes = page.iter().fold(summary.total_bytes, |total, file| {
|
||||
total.saturating_add(file.manifest.size_bytes)
|
||||
});
|
||||
emit(page)?;
|
||||
candidates.clear();
|
||||
}
|
||||
}
|
||||
if !candidates.is_empty() {
|
||||
let page = hash_scan_batch(&pool, &canonical_root, &candidates)?;
|
||||
summary.file_count = summary
|
||||
.file_count
|
||||
.saturating_add(u64::try_from(page.len()).unwrap_or(u64::MAX));
|
||||
summary.total_bytes = page.iter().fold(summary.total_bytes, |total, file| {
|
||||
total.saturating_add(file.manifest.size_bytes)
|
||||
});
|
||||
emit(page)?;
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
pub fn scan_mapping(
|
||||
mapping: &Mapping,
|
||||
options: &ScanOptions,
|
||||
) -> Result<Vec<ScannedFile>, CoreError> {
|
||||
let mut output = Vec::new();
|
||||
scan_mapping_paged(mapping, options, 1_000, |mut page| {
|
||||
output.append(&mut page);
|
||||
Ok(())
|
||||
})?;
|
||||
output.sort_by(|left, right| {
|
||||
left.manifest
|
||||
.relative_path
|
||||
.cmp(&right.manifest.relative_path)
|
||||
});
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
CoreError, Mapping, ScanOptions, normalized_relative, scan_mapping, scan_mapping_paged,
|
||||
};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn scan_skips_hidden_symlink_and_temporary_files() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
fs::write(root.path().join("accepted.bin"), b"data").unwrap();
|
||||
fs::write(root.path().join(".hidden"), b"hidden").unwrap();
|
||||
fs::write(root.path().join("upload.part"), b"partial").unwrap();
|
||||
let mapping = Mapping::new(root.path().to_path_buf(), "space".to_owned());
|
||||
let files = scan_mapping(
|
||||
&mapping,
|
||||
&ScanOptions {
|
||||
stable_for: Duration::ZERO,
|
||||
..ScanOptions::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(files.len(), 1);
|
||||
assert_eq!(files[0].manifest.relative_path, "accepted.bin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paged_scan_bounds_each_emitted_batch() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for index in 0..5 {
|
||||
fs::write(root.path().join(format!("{index}.bin")), b"data").unwrap();
|
||||
}
|
||||
let mapping = Mapping::new(root.path().to_path_buf(), "space".to_owned());
|
||||
let mut page_lengths = Vec::new();
|
||||
let summary = scan_mapping_paged(
|
||||
&mapping,
|
||||
&ScanOptions {
|
||||
stable_for: Duration::ZERO,
|
||||
..ScanOptions::default()
|
||||
},
|
||||
2,
|
||||
|page| {
|
||||
page_lengths.push(page.len());
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(page_lengths, vec![2, 2, 1]);
|
||||
assert_eq!(summary.file_count, 5);
|
||||
assert_eq!(summary.total_bytes, 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_path_rejects_escape_and_control_characters() {
|
||||
let root = Path::new("/mapping");
|
||||
assert!(matches!(
|
||||
normalized_relative(root, Path::new("/mapping/../outside.bin")),
|
||||
Err(CoreError::InvalidPath)
|
||||
));
|
||||
assert!(matches!(
|
||||
normalized_relative(root, Path::new("/mapping/bad\nname.bin")),
|
||||
Err(CoreError::InvalidPath)
|
||||
));
|
||||
assert_eq!(
|
||||
normalized_relative(root, Path::new("/mapping/nested/good.bin")).unwrap(),
|
||||
"nested/good.bin"
|
||||
);
|
||||
}
|
||||
}
|
||||
28
tools/drosync/crates/daemon/Cargo.toml
Normal file
28
tools/drosync/crates/daemon/Cargo.toml
Normal file
@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "drosync-daemon"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
drosync-core = { path = "../core" }
|
||||
drosync-platform = { path = "../platform" }
|
||||
drosync-protocol = { path = "../protocol" }
|
||||
drosync-state = { path = "../state" }
|
||||
drosync-transfer = { path = "../transfer" }
|
||||
drosync-updater = { path = "../updater" }
|
||||
notify.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-service = "0.8"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
1200
tools/drosync/crates/daemon/src/lib.rs
Normal file
1200
tools/drosync/crates/daemon/src/lib.rs
Normal file
File diff suppressed because it is too large
Load Diff
115
tools/drosync/crates/daemon/src/main.rs
Normal file
115
tools/drosync/crates/daemon/src/main.rs
Normal file
@ -0,0 +1,115 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use anyhow::Result;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
fn init_tracing() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
|
||||
)
|
||||
.init();
|
||||
}
|
||||
|
||||
async fn run_console() -> Result<()> {
|
||||
let paths = drosync_platform::paths()?;
|
||||
drosync_daemon::run(&paths).await
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
async fn run_system_console() -> Result<()> {
|
||||
let paths = drosync_platform::paths()?;
|
||||
drosync_daemon::run_system(&paths).await
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
run_console().await
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
windows_service::define_windows_service!(ffi_service_main, windows_service_main);
|
||||
|
||||
#[cfg(windows)]
|
||||
fn windows_service_main(_arguments: Vec<std::ffi::OsString>) {
|
||||
let _ = run_windows_service();
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn run_windows_service() -> Result<()> {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use windows_service::service::{
|
||||
ServiceControl, ServiceControlAccept, ServiceExitCode, ServiceState, ServiceStatus,
|
||||
ServiceType,
|
||||
};
|
||||
use windows_service::service_control_handler::{self, ServiceControlHandlerResult};
|
||||
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let stop_for_handler = stop.clone();
|
||||
let status_handle =
|
||||
service_control_handler::register("drosync", move |control| match control {
|
||||
ServiceControl::Stop | ServiceControl::Shutdown => {
|
||||
stop_for_handler.store(true, Ordering::SeqCst);
|
||||
ServiceControlHandlerResult::NoError
|
||||
}
|
||||
ServiceControl::Interrogate => ServiceControlHandlerResult::NoError,
|
||||
_ => ServiceControlHandlerResult::NotImplemented,
|
||||
})?;
|
||||
status_handle.set_service_status(ServiceStatus {
|
||||
service_type: ServiceType::OWN_PROCESS,
|
||||
current_state: ServiceState::Running,
|
||||
controls_accepted: ServiceControlAccept::STOP | ServiceControlAccept::SHUTDOWN,
|
||||
exit_code: ServiceExitCode::Win32(0),
|
||||
checkpoint: 0,
|
||||
wait_hint: std::time::Duration::ZERO,
|
||||
process_id: None,
|
||||
})?;
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?;
|
||||
let result = runtime.block_on(async {
|
||||
let mut daemon = tokio::spawn(run_system_console());
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut daemon => return result.map_err(anyhow::Error::from)?,
|
||||
() = tokio::time::sleep(std::time::Duration::from_millis(250)) => {
|
||||
if stop.load(Ordering::SeqCst) {
|
||||
daemon.abort();
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
status_handle.set_service_status(ServiceStatus {
|
||||
service_type: ServiceType::OWN_PROCESS,
|
||||
current_state: ServiceState::Stopped,
|
||||
controls_accepted: ServiceControlAccept::empty(),
|
||||
exit_code: if result.is_ok() {
|
||||
ServiceExitCode::Win32(0)
|
||||
} else {
|
||||
ServiceExitCode::ServiceSpecific(1)
|
||||
},
|
||||
checkpoint: 0,
|
||||
wait_hint: std::time::Duration::ZERO,
|
||||
process_id: None,
|
||||
})?;
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
if std::env::args().any(|argument| argument == "--system-service") {
|
||||
windows_service::service_dispatcher::start("drosync", ffi_service_main)?;
|
||||
Ok(())
|
||||
} else {
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()?
|
||||
.block_on(run_console())
|
||||
}
|
||||
}
|
||||
17
tools/drosync/crates/gui/Cargo.toml
Normal file
17
tools/drosync/crates/gui/Cargo.toml
Normal file
@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "drosync-gui"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
drosync-daemon = { path = "../daemon" }
|
||||
drosync-platform = { path = "../platform" }
|
||||
eframe.workspace = true
|
||||
serde_json.workspace = true
|
||||
tokio.workspace = true
|
||||
tray-icon.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
119
tools/drosync/crates/gui/src/main.rs
Normal file
119
tools/drosync/crates/gui/src/main.rs
Normal file
@ -0,0 +1,119 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use eframe::egui;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::mpsc::{Receiver, TryRecvError};
|
||||
|
||||
struct DrosyncApp {
|
||||
paused: bool,
|
||||
message: String,
|
||||
pending: Option<Receiver<Result<Value, String>>>,
|
||||
}
|
||||
|
||||
impl Default for DrosyncApp {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
paused: false,
|
||||
message: "正在读取 daemon 状态…".to_owned(),
|
||||
pending: Some(run_ipc("status")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_ipc(command: &'static str) -> Receiver<Result<Value, String>> {
|
||||
let (sender, receiver) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let result = (|| {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|error| error.to_string())?;
|
||||
runtime.block_on(async move {
|
||||
let paths = drosync_platform::paths().map_err(|error| error.to_string())?;
|
||||
let response = drosync_daemon::request(
|
||||
&paths.ipc_endpoint,
|
||||
&drosync_daemon::IpcRequest {
|
||||
command: command.to_owned(),
|
||||
arguments: json!({}),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
if response.ok {
|
||||
Ok(response.result)
|
||||
} else {
|
||||
Err(response.error)
|
||||
}
|
||||
})
|
||||
})();
|
||||
let _ = sender.send(result);
|
||||
});
|
||||
receiver
|
||||
}
|
||||
|
||||
impl eframe::App for DrosyncApp {
|
||||
fn update(&mut self, context: &egui::Context, _frame: &mut eframe::Frame) {
|
||||
if let Some(receiver) = &self.pending {
|
||||
match receiver.try_recv() {
|
||||
Ok(Ok(value)) => {
|
||||
if let Some(paused) = value.get("paused").and_then(Value::as_bool) {
|
||||
self.paused = paused;
|
||||
}
|
||||
self.message = if self.paused {
|
||||
"新任务已暂停;当前对象完成后停止调度。".to_owned()
|
||||
} else {
|
||||
"Daemon 在线并接受新任务。".to_owned()
|
||||
};
|
||||
self.pending = None;
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
self.message = format!("Daemon 操作失败:{error}");
|
||||
self.pending = None;
|
||||
}
|
||||
Err(TryRecvError::Disconnected) => {
|
||||
"Daemon 状态通道已断开。".clone_into(&mut self.message);
|
||||
self.pending = None;
|
||||
}
|
||||
Err(TryRecvError::Empty) => {
|
||||
context.request_repaint_after(std::time::Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
}
|
||||
egui::CentralPanel::default().show(context, |ui| {
|
||||
ui.heading("drosync");
|
||||
ui.label("Local → BOS secure synchronization");
|
||||
ui.separator();
|
||||
ui.horizontal(|ui| {
|
||||
ui.label("Daemon:");
|
||||
ui.colored_label(egui::Color32::from_rgb(24, 160, 88), "managed by IPC");
|
||||
});
|
||||
if ui
|
||||
.add_enabled(
|
||||
self.pending.is_none(),
|
||||
egui::Button::new(if self.paused {
|
||||
"Resume uploads"
|
||||
} else {
|
||||
"Pause uploads"
|
||||
}),
|
||||
)
|
||||
.clicked()
|
||||
{
|
||||
self.pending = Some(run_ipc(if self.paused { "resume" } else { "pause" }));
|
||||
"正在更新 daemon 调度状态…".clone_into(&mut self.message);
|
||||
}
|
||||
ui.label(&self.message);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> eframe::Result {
|
||||
let _tray = tray_icon::TrayIconBuilder::new()
|
||||
.with_tooltip("drosync")
|
||||
.build()
|
||||
.ok();
|
||||
eframe::run_native(
|
||||
"drosync",
|
||||
eframe::NativeOptions::default(),
|
||||
Box::new(|_creation_context| Ok(Box::<DrosyncApp>::default())),
|
||||
)
|
||||
}
|
||||
18
tools/drosync/crates/platform/Cargo.toml
Normal file
18
tools/drosync/crates/platform/Cargo.toml
Normal file
@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "drosync-platform"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
directories.workspace = true
|
||||
keyring.workspace = true
|
||||
serde.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
145
tools/drosync/crates/platform/src/lib.rs
Normal file
145
tools/drosync/crates/platform/src/lib.rs
Normal file
@ -0,0 +1,145 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use directories::ProjectDirs;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum PlatformError {
|
||||
#[error("operating-system project directories are unavailable")]
|
||||
DirectoriesUnavailable,
|
||||
#[error("credential storage failed: {0}")]
|
||||
Credential(#[from] keyring::Error),
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("secret file permissions must be 0600")]
|
||||
InsecureSecretPermissions,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Paths {
|
||||
pub config_dir: PathBuf,
|
||||
pub data_dir: PathBuf,
|
||||
pub state_db: PathBuf,
|
||||
pub ipc_endpoint: PathBuf,
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn platform_name() -> &'static str {
|
||||
std::env::consts::OS
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn architecture() -> &'static str {
|
||||
match std::env::consts::ARCH {
|
||||
"aarch64" => "arm64",
|
||||
value => value,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn paths() -> Result<Paths, PlatformError> {
|
||||
let dirs = ProjectDirs::from("com", "drobotics", "drosync")
|
||||
.ok_or(PlatformError::DirectoriesUnavailable)?;
|
||||
let config_dir = dirs.config_dir().to_path_buf();
|
||||
let data_dir = dirs.data_local_dir().to_path_buf();
|
||||
fs::create_dir_all(&config_dir)?;
|
||||
fs::create_dir_all(&data_dir)?;
|
||||
Ok(Paths {
|
||||
state_db: data_dir.join("state.db"),
|
||||
ipc_endpoint: data_dir.join("drosync.sock"),
|
||||
config_dir,
|
||||
data_dir,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn save_credential(profile: &str, token: &str) -> Result<(), PlatformError> {
|
||||
keyring::Entry::new("drosync", profile)?.set_password(token)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn load_credential(profile: &str) -> Result<String, PlatformError> {
|
||||
Ok(keyring::Entry::new("drosync", profile)?.get_password()?)
|
||||
}
|
||||
|
||||
pub fn load_runtime_credential(profile: &str) -> Result<String, PlatformError> {
|
||||
if let Ok(value) = std::env::var("DROSYNC_TOKEN")
|
||||
&& !value.trim().is_empty()
|
||||
{
|
||||
return Ok(value.trim().to_owned());
|
||||
}
|
||||
if let Ok(path) = std::env::var("DROSYNC_SECRET_FILE")
|
||||
&& !path.trim().is_empty()
|
||||
{
|
||||
return read_headless_secret(Path::new(path.trim()));
|
||||
}
|
||||
load_credential(profile)
|
||||
}
|
||||
|
||||
pub fn delete_credential(profile: &str) -> Result<(), PlatformError> {
|
||||
keyring::Entry::new("drosync", profile)?.delete_credential()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read_headless_secret(path: &Path) -> Result<String, PlatformError> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if fs::metadata(path)?.permissions().mode() & 0o077 != 0 {
|
||||
return Err(PlatformError::InsecureSecretPermissions);
|
||||
}
|
||||
}
|
||||
Ok(fs::read_to_string(path)?.trim().to_owned())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub fn secure_runtime_file(path: &Path) -> Result<(), PlatformError> {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(path, fs::Permissions::from_mode(0o600))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub fn secure_runtime_file(_path: &Path) -> Result<(), PlatformError> {
|
||||
// Windows named-pipe ACLs are applied by the daemon when it creates the
|
||||
// current-user pipe. Files are not used for IPC on Windows.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::{PlatformError, read_headless_secret, secure_runtime_file};
|
||||
use std::fs;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::os::unix::net::UnixListener;
|
||||
|
||||
#[test]
|
||||
fn headless_secret_requires_private_permissions() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let secret = directory.path().join("token");
|
||||
fs::write(&secret, "device-token\n").unwrap();
|
||||
fs::set_permissions(&secret, fs::Permissions::from_mode(0o644)).unwrap();
|
||||
assert!(matches!(
|
||||
read_headless_secret(&secret),
|
||||
Err(PlatformError::InsecureSecretPermissions)
|
||||
));
|
||||
|
||||
fs::set_permissions(&secret, fs::Permissions::from_mode(0o600)).unwrap();
|
||||
assert_eq!(read_headless_secret(&secret).unwrap(), "device-token");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unix_socket_is_restricted_to_current_user() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let socket = directory.path().join("drosync.sock");
|
||||
let _listener = UnixListener::bind(&socket).unwrap();
|
||||
secure_runtime_file(&socket).unwrap();
|
||||
assert_eq!(
|
||||
fs::metadata(&socket).unwrap().permissions().mode() & 0o777,
|
||||
0o600
|
||||
);
|
||||
}
|
||||
}
|
||||
14
tools/drosync/crates/protocol/Cargo.toml
Normal file
14
tools/drosync/crates/protocol/Cargo.toml
Normal file
@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "drosync-protocol"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
chrono.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
39
tools/drosync/crates/protocol/src/generated.rs
Normal file
39
tools/drosync/crates/protocol/src/generated.rs
Normal file
@ -0,0 +1,39 @@
|
||||
// Code generated by scripts/generate_drosync_contract.py; DO NOT EDIT.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const PROTOCOL_VERSION: &str = "v1";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SessionStatus {
|
||||
CollectingManifest,
|
||||
Planning,
|
||||
Planned,
|
||||
Running,
|
||||
Completed,
|
||||
Partial,
|
||||
Conflicted,
|
||||
Failed,
|
||||
Aborted,
|
||||
Expired,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ItemStatus {
|
||||
Discovered,
|
||||
Planned,
|
||||
Reserved,
|
||||
Uploading,
|
||||
Uploaded,
|
||||
Committing,
|
||||
Committed,
|
||||
KeptBoth,
|
||||
Unchanged,
|
||||
Skipped,
|
||||
Denied,
|
||||
Failed,
|
||||
Aborted,
|
||||
Expired,
|
||||
}
|
||||
281
tools/drosync/crates/protocol/src/lib.rs
Normal file
281
tools/drosync/crates/protocol/src/lib.rs
Normal file
@ -0,0 +1,281 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
mod generated;
|
||||
pub use generated::{ItemStatus, PROTOCOL_VERSION, SessionStatus};
|
||||
|
||||
pub const MANIFEST_PAGE_ITEMS: usize = 1_000;
|
||||
pub const MAX_CONCURRENT_REQUESTS: usize = 16;
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct RemoteBase {
|
||||
#[serde(default)]
|
||||
pub exists: bool,
|
||||
#[serde(default)]
|
||||
pub etag: String,
|
||||
#[serde(default)]
|
||||
pub size_bytes: i64,
|
||||
#[serde(default)]
|
||||
pub last_modified: String,
|
||||
#[serde(default)]
|
||||
pub platform_version: i64,
|
||||
#[serde(default)]
|
||||
pub version_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ManifestItem {
|
||||
pub client_item_id: String,
|
||||
pub relative_path: String,
|
||||
pub kind: String,
|
||||
pub size_bytes: i64,
|
||||
pub modified_at: String,
|
||||
pub sha256: String,
|
||||
#[serde(default)]
|
||||
pub content_type: String,
|
||||
#[serde(default)]
|
||||
pub last_known_remote: RemoteBase,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ManifestPage {
|
||||
pub page_number: u32,
|
||||
#[serde(rename = "final")]
|
||||
pub final_page: bool,
|
||||
pub items: Vec<ManifestItem>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ErrorEnvelope {
|
||||
pub code: String,
|
||||
pub message: String,
|
||||
pub retryable: bool,
|
||||
pub request_id: String,
|
||||
#[serde(default)]
|
||||
pub details: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DeviceAuthorization {
|
||||
pub device_id: String,
|
||||
pub device_name: String,
|
||||
pub platform: String,
|
||||
pub architecture: String,
|
||||
pub client_version: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DeviceAuthorizationResponse {
|
||||
pub device_code: String,
|
||||
pub user_code: String,
|
||||
pub verification_uri: String,
|
||||
pub expires_in: u64,
|
||||
pub interval: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SessionCreate {
|
||||
pub sync_space_id: String,
|
||||
pub mapping_id: String,
|
||||
pub client_version: String,
|
||||
pub protocol_version: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SessionCounts {
|
||||
pub manifest: i64,
|
||||
pub unchanged: i64,
|
||||
pub create: i64,
|
||||
pub update: i64,
|
||||
pub kept_both: i64,
|
||||
pub skipped: i64,
|
||||
pub denied: i64,
|
||||
pub failed: i64,
|
||||
pub committed: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SessionRecord {
|
||||
pub id: String,
|
||||
pub sync_space_id: String,
|
||||
pub mapping_id: String,
|
||||
pub status: SessionStatus,
|
||||
#[serde(default)]
|
||||
pub counts: SessionCounts,
|
||||
#[serde(default)]
|
||||
pub total_bytes: i64,
|
||||
#[serde(default)]
|
||||
pub operation_id: String,
|
||||
#[serde(default)]
|
||||
pub error_summary: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ItemRecord {
|
||||
pub id: String,
|
||||
pub session_id: String,
|
||||
pub client_item_id: String,
|
||||
pub relative_path: String,
|
||||
pub target_key: String,
|
||||
#[serde(default)]
|
||||
pub logical_target_key: String,
|
||||
pub size_bytes: i64,
|
||||
pub sha256: String,
|
||||
#[serde(default)]
|
||||
pub action: String,
|
||||
pub status: ItemStatus,
|
||||
#[serde(default)]
|
||||
pub upload_mode: String,
|
||||
#[serde(default)]
|
||||
pub upload_id: String,
|
||||
#[serde(default)]
|
||||
pub last_error: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ItemPage {
|
||||
pub items: Vec<ItemRecord>,
|
||||
#[serde(default)]
|
||||
pub next_cursor: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct OperationAccepted {
|
||||
pub operation_id: String,
|
||||
#[serde(flatten)]
|
||||
pub payload: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PresignedUpload {
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub method: String,
|
||||
pub url: String,
|
||||
#[serde(default)]
|
||||
pub headers: BTreeMap<String, String>,
|
||||
pub expires_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DirectUploadPreparation {
|
||||
pub upload: PresignedUpload,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MultipartUpload {
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub upload_id: String,
|
||||
pub expires_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MultipartPartURL {
|
||||
pub bucket: String,
|
||||
pub key: String,
|
||||
pub upload_id: String,
|
||||
pub part_number: u32,
|
||||
pub method: String,
|
||||
pub url: String,
|
||||
#[serde(default)]
|
||||
pub headers: BTreeMap<String, String>,
|
||||
pub expires_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MultipartPart {
|
||||
pub part_number: u32,
|
||||
pub e_tag: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ClientRelease {
|
||||
pub id: String,
|
||||
pub version: String,
|
||||
pub channel: String,
|
||||
pub protocol_version: String,
|
||||
#[serde(default)]
|
||||
pub minimum_server_version: String,
|
||||
pub artifacts: BTreeMap<String, String>,
|
||||
pub sha256: BTreeMap<String, String>,
|
||||
pub signature: String,
|
||||
pub rollout_percent: u8,
|
||||
pub paused: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ItemStatus, ManifestItem, ManifestPage, RemoteBase, SessionStatus};
|
||||
|
||||
#[test]
|
||||
fn status_contract_is_snake_case() {
|
||||
assert_eq!(
|
||||
serde_json::to_string(&SessionStatus::CollectingManifest).unwrap(),
|
||||
"\"collecting_manifest\""
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&ItemStatus::KeptBoth).unwrap(),
|
||||
"\"kept_both\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_page_fixture_round_trips() {
|
||||
let page = ManifestPage {
|
||||
page_number: 7,
|
||||
final_page: true,
|
||||
items: vec![ManifestItem {
|
||||
client_item_id: "fixture-1".to_owned(),
|
||||
relative_path: "nested/data.bin".to_owned(),
|
||||
kind: "file".to_owned(),
|
||||
size_bytes: 42,
|
||||
modified_at: "2026-07-28T00:00:00Z".to_owned(),
|
||||
sha256: "0123456789abcdef".repeat(4),
|
||||
content_type: "application/octet-stream".to_owned(),
|
||||
last_known_remote: RemoteBase::default(),
|
||||
}],
|
||||
};
|
||||
let encoded = serde_json::to_vec(&page).unwrap();
|
||||
let decoded: ManifestPage = serde_json::from_slice(&encoded).unwrap();
|
||||
assert_eq!(decoded.page_number, 7);
|
||||
assert!(decoded.final_page);
|
||||
assert_eq!(decoded.items[0].relative_path, "nested/data.bin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_parser_fuzz_corpus_never_panics() {
|
||||
let mut seed = 0x5eed_d105_u64;
|
||||
for length in 0..=256 {
|
||||
let mut input = Vec::with_capacity(length);
|
||||
for _ in 0..length {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 7;
|
||||
seed ^= seed << 17;
|
||||
input.push(seed.to_le_bytes()[0]);
|
||||
}
|
||||
let _ = serde_json::from_slice::<ManifestPage>(&input);
|
||||
}
|
||||
}
|
||||
}
|
||||
20
tools/drosync/crates/state/Cargo.toml
Normal file
20
tools/drosync/crates/state/Cargo.toml
Normal file
@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "drosync-state"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
drosync-core = { path = "../core" }
|
||||
drosync-protocol = { path = "../protocol" }
|
||||
rusqlite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
627
tools/drosync/crates/state/src/lib.rs
Normal file
627
tools/drosync/crates/state/src/lib.rs
Normal file
@ -0,0 +1,627 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use drosync_core::{Mapping, ScannedFile};
|
||||
use drosync_protocol::{ItemRecord, ManifestItem};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::path::{Path, PathBuf};
|
||||
use thiserror::Error;
|
||||
|
||||
const SCHEMA_VERSION: i64 = 4;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct StoredSession {
|
||||
pub id: String,
|
||||
pub mapping_id: String,
|
||||
pub status: String,
|
||||
pub operation_id: String,
|
||||
pub document: Value,
|
||||
pub updated_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct StoredPart {
|
||||
pub item_id: String,
|
||||
pub part_number: u32,
|
||||
pub etag: String,
|
||||
pub size_bytes: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct StoredScanFile {
|
||||
pub local_run_id: String,
|
||||
pub path: PathBuf,
|
||||
pub manifest: ManifestItem,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum StateError {
|
||||
#[error("SQLite error: {0}")]
|
||||
Sqlite(#[from] rusqlite::Error),
|
||||
#[error("state serialization error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("state schema {0} is newer than this client")]
|
||||
SchemaTooNew(i64),
|
||||
}
|
||||
|
||||
pub struct StateStore {
|
||||
connection: Connection,
|
||||
path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl StateStore {
|
||||
pub fn open(path: &Path) -> Result<Self, StateError> {
|
||||
let connection = Connection::open(path)?;
|
||||
connection.pragma_update(None, "journal_mode", "WAL")?;
|
||||
connection.pragma_update(None, "synchronous", "FULL")?;
|
||||
connection.pragma_update(None, "foreign_keys", "ON")?;
|
||||
let mut store = Self {
|
||||
connection,
|
||||
path: Some(path.to_path_buf()),
|
||||
};
|
||||
store.migrate()?;
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub fn open_in_memory() -> Result<Self, StateError> {
|
||||
let connection = Connection::open_in_memory()?;
|
||||
let mut store = Self {
|
||||
connection,
|
||||
path: None,
|
||||
};
|
||||
store.migrate()?;
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn path(&self) -> Option<&Path> {
|
||||
self.path.as_deref()
|
||||
}
|
||||
|
||||
fn migrate(&mut self) -> Result<(), StateError> {
|
||||
let transaction = self.connection.transaction()?;
|
||||
transaction.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS schema_meta (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
version INTEGER NOT NULL
|
||||
);
|
||||
INSERT OR IGNORE INTO schema_meta(singleton, version) VALUES (1, 0);",
|
||||
)?;
|
||||
let current: i64 = transaction.query_row(
|
||||
"SELECT version FROM schema_meta WHERE singleton = 1",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if current > SCHEMA_VERSION {
|
||||
return Err(StateError::SchemaTooNew(current));
|
||||
}
|
||||
if current < 1 {
|
||||
transaction.execute_batch(
|
||||
"CREATE TABLE mappings (
|
||||
id TEXT PRIMARY KEY,
|
||||
document TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
mapping_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
operation_id TEXT NOT NULL DEFAULT '',
|
||||
document TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE INDEX sessions_status_updated_idx ON sessions(status, updated_at);",
|
||||
)?;
|
||||
}
|
||||
if current < 2 {
|
||||
transaction.execute_batch(
|
||||
"CREATE TABLE transfer_parts (
|
||||
item_id TEXT NOT NULL,
|
||||
part_number INTEGER NOT NULL,
|
||||
etag TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
PRIMARY KEY(item_id, part_number)
|
||||
);
|
||||
CREATE TABLE daemon_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
}
|
||||
if current < 3 {
|
||||
transaction.execute_batch(
|
||||
"CREATE INDEX IF NOT EXISTS sessions_mapping_status_updated_idx
|
||||
ON sessions(mapping_id, status, updated_at);
|
||||
CREATE INDEX IF NOT EXISTS transfer_parts_item_idx
|
||||
ON transfer_parts(item_id, part_number);",
|
||||
)?;
|
||||
}
|
||||
if current < 4 {
|
||||
transaction.execute_batch(
|
||||
"CREATE TABLE scan_files (
|
||||
local_run_id TEXT NOT NULL,
|
||||
client_item_id TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
manifest TEXT NOT NULL,
|
||||
PRIMARY KEY(local_run_id, client_item_id)
|
||||
);
|
||||
CREATE INDEX scan_files_run_item_idx
|
||||
ON scan_files(local_run_id, client_item_id);
|
||||
CREATE TABLE sync_items (
|
||||
session_id TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
document TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id, item_id)
|
||||
);
|
||||
CREATE INDEX sync_items_session_status_idx
|
||||
ON sync_items(session_id, status, item_id);
|
||||
CREATE INDEX sync_items_status_item_idx
|
||||
ON sync_items(status, item_id);",
|
||||
)?;
|
||||
}
|
||||
transaction.execute(
|
||||
"UPDATE schema_meta SET version = ?1 WHERE singleton = 1",
|
||||
[SCHEMA_VERSION],
|
||||
)?;
|
||||
transaction.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_mapping(&self, mapping: &Mapping) -> Result<(), StateError> {
|
||||
self.connection.execute(
|
||||
"INSERT INTO mappings(id, document, updated_at)
|
||||
VALUES (?1, ?2, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(id) DO UPDATE SET document = excluded.document, updated_at = CURRENT_TIMESTAMP",
|
||||
params![mapping.id, serde_json::to_string(mapping)?],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn remove_mapping(&self, id: &str) -> Result<bool, StateError> {
|
||||
Ok(self
|
||||
.connection
|
||||
.execute("DELETE FROM mappings WHERE id = ?1", [id])?
|
||||
> 0)
|
||||
}
|
||||
|
||||
pub fn mappings(&self) -> Result<Vec<Mapping>, StateError> {
|
||||
let mut statement = self
|
||||
.connection
|
||||
.prepare("SELECT document FROM mappings ORDER BY updated_at, id")?;
|
||||
let rows = statement.query_map([], |row| row.get::<_, String>(0))?;
|
||||
rows.map(|row| Ok(serde_json::from_str(&row?)?)).collect()
|
||||
}
|
||||
|
||||
pub fn mapping(&self, id: &str) -> Result<Option<Mapping>, StateError> {
|
||||
let document = self
|
||||
.connection
|
||||
.query_row("SELECT document FROM mappings WHERE id = ?1", [id], |row| {
|
||||
row.get::<_, String>(0)
|
||||
})
|
||||
.optional()?;
|
||||
document
|
||||
.map(|value| serde_json::from_str(&value).map_err(StateError::from))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub fn set_meta(&self, key: &str, value: &str) -> Result<(), StateError> {
|
||||
self.connection.execute(
|
||||
"INSERT INTO daemon_meta(key, value) VALUES (?1, ?2)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
params![key, value],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get_meta(&self, key: &str) -> Result<Option<String>, StateError> {
|
||||
Ok(self
|
||||
.connection
|
||||
.query_row(
|
||||
"SELECT value FROM daemon_meta WHERE key = ?1",
|
||||
[key],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub fn put_session(
|
||||
&self,
|
||||
id: &str,
|
||||
mapping_id: &str,
|
||||
status: &str,
|
||||
operation_id: &str,
|
||||
document: &Value,
|
||||
) -> Result<(), StateError> {
|
||||
self.connection.execute(
|
||||
"INSERT INTO sessions(id, mapping_id, status, operation_id, document, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
mapping_id = excluded.mapping_id,
|
||||
status = excluded.status,
|
||||
operation_id = excluded.operation_id,
|
||||
document = excluded.document,
|
||||
updated_at = CURRENT_TIMESTAMP",
|
||||
params![
|
||||
id,
|
||||
mapping_id,
|
||||
status,
|
||||
operation_id,
|
||||
serde_json::to_string(document)?
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn session(&self, id: &str) -> Result<Option<StoredSession>, StateError> {
|
||||
let row = self
|
||||
.connection
|
||||
.query_row(
|
||||
"SELECT id, mapping_id, status, operation_id, document, updated_at
|
||||
FROM sessions WHERE id = ?1",
|
||||
[id],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, String>(3)?,
|
||||
row.get::<_, String>(4)?,
|
||||
row.get::<_, String>(5)?,
|
||||
))
|
||||
},
|
||||
)
|
||||
.optional()?;
|
||||
row.map(
|
||||
|(id, mapping_id, status, operation_id, document, updated_at)| {
|
||||
Ok(StoredSession {
|
||||
id,
|
||||
mapping_id,
|
||||
status,
|
||||
operation_id,
|
||||
document: serde_json::from_str(&document)?,
|
||||
updated_at,
|
||||
})
|
||||
},
|
||||
)
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub fn latest_active_session(
|
||||
&self,
|
||||
mapping_id: &str,
|
||||
) -> Result<Option<StoredSession>, StateError> {
|
||||
let id = self
|
||||
.connection
|
||||
.query_row(
|
||||
"SELECT id FROM sessions
|
||||
WHERE mapping_id = ?1
|
||||
AND status NOT IN ('completed', 'partial', 'conflicted', 'failed', 'aborted', 'expired')
|
||||
ORDER BY updated_at DESC LIMIT 1",
|
||||
[mapping_id],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.optional()?;
|
||||
id.map(|value| self.session(&value))
|
||||
.transpose()
|
||||
.map(Option::flatten)
|
||||
}
|
||||
|
||||
pub fn sessions(&self) -> Result<Vec<StoredSession>, StateError> {
|
||||
let mut statement = self.connection.prepare(
|
||||
"SELECT id, mapping_id, status, operation_id, document, updated_at
|
||||
FROM sessions ORDER BY updated_at DESC, id DESC",
|
||||
)?;
|
||||
let rows = statement.query_map([], |row| {
|
||||
Ok((
|
||||
row.get::<_, String>(0)?,
|
||||
row.get::<_, String>(1)?,
|
||||
row.get::<_, String>(2)?,
|
||||
row.get::<_, String>(3)?,
|
||||
row.get::<_, String>(4)?,
|
||||
row.get::<_, String>(5)?,
|
||||
))
|
||||
})?;
|
||||
rows.map(|row| {
|
||||
let (id, mapping_id, status, operation_id, document, updated_at) = row?;
|
||||
Ok(StoredSession {
|
||||
id,
|
||||
mapping_id,
|
||||
status,
|
||||
operation_id,
|
||||
document: serde_json::from_str(&document)?,
|
||||
updated_at,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn put_part(&self, part: &StoredPart) -> Result<(), StateError> {
|
||||
self.connection.execute(
|
||||
"INSERT INTO transfer_parts(item_id, part_number, etag, size_bytes)
|
||||
VALUES (?1, ?2, ?3, ?4)
|
||||
ON CONFLICT(item_id, part_number) DO UPDATE SET
|
||||
etag = excluded.etag, size_bytes = excluded.size_bytes",
|
||||
params![part.item_id, part.part_number, part.etag, part.size_bytes],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn parts(&self, item_id: &str) -> Result<Vec<StoredPart>, StateError> {
|
||||
let mut statement = self.connection.prepare(
|
||||
"SELECT item_id, part_number, etag, size_bytes
|
||||
FROM transfer_parts WHERE item_id = ?1 ORDER BY part_number",
|
||||
)?;
|
||||
let rows = statement.query_map([item_id], |row| {
|
||||
Ok(StoredPart {
|
||||
item_id: row.get(0)?,
|
||||
part_number: row.get(1)?,
|
||||
etag: row.get(2)?,
|
||||
size_bytes: row.get(3)?,
|
||||
})
|
||||
})?;
|
||||
rows.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(StateError::from)
|
||||
}
|
||||
|
||||
pub fn clear_parts(&self, item_id: &str) -> Result<(), StateError> {
|
||||
self.connection
|
||||
.execute("DELETE FROM transfer_parts WHERE item_id = ?1", [item_id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear_scan_files(&self, local_run_id: &str) -> Result<(), StateError> {
|
||||
self.connection.execute(
|
||||
"DELETE FROM scan_files WHERE local_run_id = ?1",
|
||||
[local_run_id],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_scan_files(
|
||||
&mut self,
|
||||
local_run_id: &str,
|
||||
files: &[ScannedFile],
|
||||
) -> Result<(), StateError> {
|
||||
let transaction = self.connection.transaction()?;
|
||||
{
|
||||
let mut statement = transaction.prepare(
|
||||
"INSERT INTO scan_files(local_run_id, client_item_id, path, manifest)
|
||||
VALUES (?1, ?2, ?3, ?4)
|
||||
ON CONFLICT(local_run_id, client_item_id) DO UPDATE SET
|
||||
path = excluded.path, manifest = excluded.manifest",
|
||||
)?;
|
||||
for file in files {
|
||||
statement.execute(params![
|
||||
local_run_id,
|
||||
file.manifest.client_item_id,
|
||||
file.absolute_path.to_string_lossy(),
|
||||
serde_json::to_string(&file.manifest)?
|
||||
])?;
|
||||
}
|
||||
}
|
||||
transaction.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn scan_files_page(
|
||||
&self,
|
||||
local_run_id: &str,
|
||||
offset: u64,
|
||||
limit: usize,
|
||||
) -> Result<Vec<StoredScanFile>, StateError> {
|
||||
let mut statement = self.connection.prepare(
|
||||
"SELECT path, manifest FROM scan_files
|
||||
WHERE local_run_id = ?1
|
||||
ORDER BY client_item_id
|
||||
LIMIT ?2 OFFSET ?3",
|
||||
)?;
|
||||
let rows = statement.query_map(
|
||||
params![
|
||||
local_run_id,
|
||||
i64::try_from(limit).unwrap_or(i64::MAX),
|
||||
i64::try_from(offset).unwrap_or(i64::MAX)
|
||||
],
|
||||
|row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
|
||||
)?;
|
||||
rows.map(|row| {
|
||||
let (path, manifest) = row?;
|
||||
Ok(StoredScanFile {
|
||||
local_run_id: local_run_id.to_owned(),
|
||||
path: PathBuf::from(path),
|
||||
manifest: serde_json::from_str(&manifest)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn scan_file_path(
|
||||
&self,
|
||||
local_run_id: &str,
|
||||
client_item_id: &str,
|
||||
) -> Result<Option<PathBuf>, StateError> {
|
||||
Ok(self
|
||||
.connection
|
||||
.query_row(
|
||||
"SELECT path FROM scan_files
|
||||
WHERE local_run_id = ?1 AND client_item_id = ?2",
|
||||
params![local_run_id, client_item_id],
|
||||
|row| row.get::<_, String>(0),
|
||||
)
|
||||
.optional()?
|
||||
.map(PathBuf::from))
|
||||
}
|
||||
|
||||
pub fn clear_sync_items(&self, session_id: &str) -> Result<(), StateError> {
|
||||
self.connection
|
||||
.execute("DELETE FROM sync_items WHERE session_id = ?1", [session_id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn put_sync_items(
|
||||
&mut self,
|
||||
session_id: &str,
|
||||
items: &[ItemRecord],
|
||||
) -> Result<(), StateError> {
|
||||
let transaction = self.connection.transaction()?;
|
||||
{
|
||||
let mut statement = transaction.prepare(
|
||||
"INSERT INTO sync_items(session_id, item_id, status, document)
|
||||
VALUES (?1, ?2, ?3, ?4)
|
||||
ON CONFLICT(session_id, item_id) DO UPDATE SET
|
||||
status = excluded.status, document = excluded.document",
|
||||
)?;
|
||||
for item in items {
|
||||
let status = serde_json::to_value(&item.status)?
|
||||
.as_str()
|
||||
.unwrap_or("failed")
|
||||
.to_owned();
|
||||
statement.execute(params![
|
||||
session_id,
|
||||
item.id,
|
||||
status,
|
||||
serde_json::to_string(item)?
|
||||
])?;
|
||||
}
|
||||
}
|
||||
transaction.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn sync_items_by_status(
|
||||
&self,
|
||||
status: &str,
|
||||
limit: usize,
|
||||
) -> Result<Vec<ItemRecord>, StateError> {
|
||||
let mut statement = self.connection.prepare(
|
||||
"SELECT document FROM sync_items
|
||||
WHERE status = ?1 ORDER BY session_id DESC, item_id
|
||||
LIMIT ?2",
|
||||
)?;
|
||||
let rows = statement.query_map(
|
||||
params![status, i64::try_from(limit).unwrap_or(i64::MAX)],
|
||||
|row| row.get::<_, String>(0),
|
||||
)?;
|
||||
rows.map(|row| Ok(serde_json::from_str(&row?)?)).collect()
|
||||
}
|
||||
|
||||
pub fn integrity_check(&self) -> Result<String, StateError> {
|
||||
Ok(self
|
||||
.connection
|
||||
.query_row("PRAGMA integrity_check", [], |row| row.get(0))?)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::StateStore;
|
||||
use drosync_core::{Mapping, ScannedFile};
|
||||
use drosync_protocol::{ItemRecord, ItemStatus, ManifestItem, RemoteBase};
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn migrations_and_mapping_round_trip() {
|
||||
let mut store = StateStore::open_in_memory().unwrap();
|
||||
assert_eq!(store.integrity_check().unwrap(), "ok");
|
||||
let mapping = Mapping::new("/tmp".into(), "space".to_owned());
|
||||
store.put_mapping(&mapping).unwrap();
|
||||
assert_eq!(
|
||||
store.mapping(&mapping.id).unwrap().unwrap().sync_space_id,
|
||||
"space"
|
||||
);
|
||||
assert!(store.remove_mapping(&mapping.id).unwrap());
|
||||
|
||||
let file = ScannedFile {
|
||||
absolute_path: "/tmp/a.bin".into(),
|
||||
manifest: ManifestItem {
|
||||
client_item_id: "client-1".to_owned(),
|
||||
relative_path: "a.bin".to_owned(),
|
||||
kind: "file".to_owned(),
|
||||
size_bytes: 4,
|
||||
modified_at: "2026-01-01T00:00:00Z".to_owned(),
|
||||
sha256: "abcd".to_owned(),
|
||||
content_type: "application/octet-stream".to_owned(),
|
||||
last_known_remote: RemoteBase::default(),
|
||||
},
|
||||
};
|
||||
store.put_scan_files("run-1", &[file]).unwrap();
|
||||
let files = store.scan_files_page("run-1", 0, 10).unwrap();
|
||||
assert_eq!(files.len(), 1);
|
||||
assert_eq!(
|
||||
store.scan_file_path("run-1", "client-1").unwrap().unwrap(),
|
||||
PathBuf::from("/tmp/a.bin")
|
||||
);
|
||||
|
||||
let item = ItemRecord {
|
||||
id: "item-1".to_owned(),
|
||||
session_id: "session-1".to_owned(),
|
||||
client_item_id: "client-1".to_owned(),
|
||||
relative_path: "a.bin".to_owned(),
|
||||
target_key: "a.bin".to_owned(),
|
||||
logical_target_key: "a.bin".to_owned(),
|
||||
size_bytes: 4,
|
||||
sha256: "abcd".to_owned(),
|
||||
action: "create".to_owned(),
|
||||
status: ItemStatus::KeptBoth,
|
||||
upload_mode: String::new(),
|
||||
upload_id: String::new(),
|
||||
last_error: String::new(),
|
||||
};
|
||||
store.put_sync_items("session-1", &[item]).unwrap();
|
||||
assert_eq!(
|
||||
store.sync_items_by_status("kept_both", 10).unwrap().len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_survives_process_restart_with_pending_work() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("state.db");
|
||||
{
|
||||
let mut store = StateStore::open(&path).unwrap();
|
||||
store
|
||||
.set_meta("pending_journal:mapping-1", r#"{"sessionId":"session-1"}"#)
|
||||
.unwrap();
|
||||
store
|
||||
.put_scan_files(
|
||||
"run-1",
|
||||
&[ScannedFile {
|
||||
absolute_path: "/tmp/resumable.bin".into(),
|
||||
manifest: ManifestItem {
|
||||
client_item_id: "client-resume".to_owned(),
|
||||
relative_path: "resumable.bin".to_owned(),
|
||||
kind: "file".to_owned(),
|
||||
size_bytes: 8,
|
||||
modified_at: "2026-01-01T00:00:00Z".to_owned(),
|
||||
sha256: "0123456789abcdef".repeat(4),
|
||||
content_type: "application/octet-stream".to_owned(),
|
||||
last_known_remote: RemoteBase::default(),
|
||||
},
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let reopened = StateStore::open(&path).unwrap();
|
||||
assert_eq!(reopened.integrity_check().unwrap(), "ok");
|
||||
assert_eq!(
|
||||
reopened
|
||||
.get_meta("pending_journal:mapping-1")
|
||||
.unwrap()
|
||||
.as_deref(),
|
||||
Some(r#"{"sessionId":"session-1"}"#)
|
||||
);
|
||||
assert_eq!(
|
||||
reopened
|
||||
.scan_file_path("run-1", "client-resume")
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
PathBuf::from("/tmp/resumable.bin")
|
||||
);
|
||||
}
|
||||
}
|
||||
21
tools/drosync/crates/transfer/Cargo.toml
Normal file
21
tools/drosync/crates/transfer/Cargo.toml
Normal file
@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "drosync-transfer"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
drosync-protocol = { path = "../protocol" }
|
||||
md-5.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
495
tools/drosync/crates/transfer/src/lib.rs
Normal file
495
tools/drosync/crates/transfer/src/lib.rs
Normal file
@ -0,0 +1,495 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use base64::Engine;
|
||||
use drosync_protocol::{
|
||||
DeviceAuthorization, DeviceAuthorizationResponse, DirectUploadPreparation, ErrorEnvelope,
|
||||
ItemPage, ItemRecord, ManifestPage, MultipartPart, MultipartPartURL, MultipartUpload,
|
||||
OperationAccepted, SessionCreate, SessionRecord,
|
||||
};
|
||||
use md5::{Digest, Md5};
|
||||
use reqwest::{Method, StatusCode};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::Value;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use thiserror::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncSeekExt};
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
use tokio_util::io::ReaderStream;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum TransferError {
|
||||
#[error("HTTP transport failed: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[error("drosync API error {status}: {error:?}")]
|
||||
Api {
|
||||
status: StatusCode,
|
||||
error: ErrorEnvelope,
|
||||
},
|
||||
#[error("unexpected API response {status}: {body}")]
|
||||
Unexpected { status: StatusCode, body: String },
|
||||
#[error("upload source I/O failed: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("request JSON serialization failed: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("insecure or invalid endpoint: {0}")]
|
||||
InvalidEndpoint(String),
|
||||
#[error("request semaphore was closed")]
|
||||
Closed,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Client {
|
||||
base_url: String,
|
||||
credential: Option<String>,
|
||||
http: reqwest::Client,
|
||||
permits: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
pub fn new(
|
||||
base_url: impl Into<String>,
|
||||
credential: Option<String>,
|
||||
) -> Result<Self, TransferError> {
|
||||
let base_url = base_url.into().trim_end_matches('/').to_owned();
|
||||
validate_transport_url(&base_url)?;
|
||||
Ok(Self {
|
||||
base_url,
|
||||
credential,
|
||||
http: reqwest::Client::builder()
|
||||
.user_agent(concat!("drosync/", env!("CARGO_PKG_VERSION")))
|
||||
.https_only(false)
|
||||
.build()?,
|
||||
permits: Arc::new(Semaphore::new(drosync_protocol::MAX_CONCURRENT_REQUESTS)),
|
||||
})
|
||||
}
|
||||
|
||||
async fn permit(&self) -> Result<OwnedSemaphorePermit, TransferError> {
|
||||
self.permits
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|_| TransferError::Closed)
|
||||
}
|
||||
|
||||
async fn send<T: DeserializeOwned>(
|
||||
&self,
|
||||
method: Method,
|
||||
path: &str,
|
||||
body: Option<Value>,
|
||||
) -> Result<T, TransferError> {
|
||||
self.send_with_key(method, path, body, &Uuid::new_v4().to_string())
|
||||
.await
|
||||
}
|
||||
|
||||
async fn send_with_key<T: DeserializeOwned>(
|
||||
&self,
|
||||
method: Method,
|
||||
path: &str,
|
||||
body: Option<Value>,
|
||||
idempotency_key: &str,
|
||||
) -> Result<T, TransferError> {
|
||||
let _permit = self.permit().await?;
|
||||
let mut request = self
|
||||
.http
|
||||
.request(method, format!("{}{}", self.base_url, path))
|
||||
.header("Idempotency-Key", idempotency_key);
|
||||
if let Some(token) = &self.credential {
|
||||
request = request.bearer_auth(token);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.json(&body);
|
||||
}
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
let bytes = response.bytes().await?;
|
||||
if status.is_success() {
|
||||
return serde_json::from_slice(&bytes).map_err(|source| TransferError::Unexpected {
|
||||
status,
|
||||
body: format!("invalid JSON: {source}"),
|
||||
});
|
||||
}
|
||||
if let Ok(error) = serde_json::from_slice::<ErrorEnvelope>(&bytes) {
|
||||
return Err(TransferError::Api { status, error });
|
||||
}
|
||||
Err(TransferError::Unexpected {
|
||||
status,
|
||||
body: String::from_utf8_lossy(&bytes).into_owned(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn create_device_authorization(
|
||||
&self,
|
||||
input: &DeviceAuthorization,
|
||||
) -> Result<DeviceAuthorizationResponse, TransferError> {
|
||||
self.send(
|
||||
Method::POST,
|
||||
"/api/external/object-sync/device-authorizations",
|
||||
Some(serde_json::to_value(input)?),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn poll_device_authorization(
|
||||
&self,
|
||||
device_code: &str,
|
||||
) -> Result<Value, TransferError> {
|
||||
self.send(
|
||||
Method::POST,
|
||||
"/api/external/object-sync/device-authorizations/token",
|
||||
Some(serde_json::json!({"deviceCode": device_code})),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn capabilities(&self) -> Result<Value, TransferError> {
|
||||
self.send(Method::GET, "/api/external/object-sync/capabilities", None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn update_check(&self, current_version: &str) -> Result<Value, TransferError> {
|
||||
self.send(Method::GET, &update_check_path(current_version)?, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn create_session(
|
||||
&self,
|
||||
input: &SessionCreate,
|
||||
) -> Result<SessionRecord, TransferError> {
|
||||
self.create_session_with_key(input, &Uuid::new_v4().to_string())
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn create_session_with_key(
|
||||
&self,
|
||||
input: &SessionCreate,
|
||||
idempotency_key: &str,
|
||||
) -> Result<SessionRecord, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
"/api/external/object-sync/sessions",
|
||||
Some(serde_json::to_value(input)?),
|
||||
idempotency_key,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn upload_manifest_page(
|
||||
&self,
|
||||
session_id: &str,
|
||||
page: &ManifestPage,
|
||||
) -> Result<Value, TransferError> {
|
||||
self.send(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/sessions/{session_id}/manifest-pages"),
|
||||
Some(serde_json::to_value(page)?),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn upload_manifest_page_with_key(
|
||||
&self,
|
||||
session_id: &str,
|
||||
page: &ManifestPage,
|
||||
idempotency_key: &str,
|
||||
) -> Result<Value, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/sessions/{session_id}/manifest-pages"),
|
||||
Some(serde_json::to_value(page)?),
|
||||
idempotency_key,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn plan(&self, session_id: &str) -> Result<OperationAccepted, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/sessions/{session_id}/plan"),
|
||||
Some(serde_json::json!({})),
|
||||
&format!("{session_id}-plan"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn status(&self, session_id: &str) -> Result<SessionRecord, TransferError> {
|
||||
self.send(
|
||||
Method::GET,
|
||||
&format!("/api/external/object-sync/sessions/{session_id}/status"),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn list_items(
|
||||
&self,
|
||||
session_id: &str,
|
||||
cursor: &str,
|
||||
) -> Result<ItemPage, TransferError> {
|
||||
let suffix = if cursor.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("&cursor={cursor}")
|
||||
};
|
||||
self.send(
|
||||
Method::GET,
|
||||
&format!("/api/external/object-sync/sessions/{session_id}/items?limit=1000{suffix}"),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn reserve_item(&self, item_id: &str) -> Result<ItemRecord, TransferError> {
|
||||
let value: Value = self
|
||||
.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/reserve"),
|
||||
Some(serde_json::json!({})),
|
||||
&format!("{item_id}-reserve"),
|
||||
)
|
||||
.await?;
|
||||
serde_json::from_value(value.get("item").cloned().unwrap_or(value)).map_err(|source| {
|
||||
TransferError::Unexpected {
|
||||
status: StatusCode::OK,
|
||||
body: format!("invalid reserve response: {source}"),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn prepare_direct(
|
||||
&self,
|
||||
item_id: &str,
|
||||
content_md5: &str,
|
||||
url_nonce: &str,
|
||||
) -> Result<DirectUploadPreparation, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/uploads/direct"),
|
||||
Some(serde_json::json!({"contentMd5": content_md5})),
|
||||
&format!("{item_id}-direct-{content_md5}-{url_nonce}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn complete_direct(&self, item_id: &str) -> Result<Value, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/uploads/direct/complete"),
|
||||
Some(serde_json::json!({})),
|
||||
&format!("{item_id}-direct-complete"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn initiate_multipart(
|
||||
&self,
|
||||
item_id: &str,
|
||||
) -> Result<MultipartUpload, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/uploads/multipart"),
|
||||
Some(serde_json::json!({})),
|
||||
&format!("{item_id}-multipart-initiate"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn multipart_part_url(
|
||||
&self,
|
||||
item_id: &str,
|
||||
part_number: u32,
|
||||
content_md5: &str,
|
||||
size_bytes: u64,
|
||||
url_nonce: &str,
|
||||
) -> Result<MultipartPartURL, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/uploads/multipart/part-url"),
|
||||
Some(serde_json::json!({
|
||||
"partNumber": part_number,
|
||||
"contentMd5": content_md5,
|
||||
"sizeBytes": size_bytes,
|
||||
})),
|
||||
&format!("{item_id}-part-{part_number}-{content_md5}-{url_nonce}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn complete_multipart(
|
||||
&self,
|
||||
item_id: &str,
|
||||
parts: &[MultipartPart],
|
||||
) -> Result<Value, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/uploads/multipart/complete"),
|
||||
Some(serde_json::json!({"parts": parts})),
|
||||
&format!("{item_id}-multipart-complete"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn commit_item(&self, item_id: &str) -> Result<OperationAccepted, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/commit"),
|
||||
Some(serde_json::json!({})),
|
||||
&format!("{item_id}-commit"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn heartbeat(&self, item_id: &str) -> Result<Value, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/items/{item_id}/heartbeat"),
|
||||
Some(serde_json::json!({})),
|
||||
&format!("{item_id}-heartbeat-{}", Uuid::new_v4()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn finish_session(
|
||||
&self,
|
||||
session_id: &str,
|
||||
action: &str,
|
||||
) -> Result<SessionRecord, TransferError> {
|
||||
self.send_with_key(
|
||||
Method::POST,
|
||||
&format!("/api/external/object-sync/sessions/{session_id}/{action}"),
|
||||
Some(serde_json::json!({})),
|
||||
&format!("{session_id}-{action}"),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn put_presigned(
|
||||
&self,
|
||||
url: &str,
|
||||
path: &Path,
|
||||
headers: &[(String, String)],
|
||||
) -> Result<(), TransferError> {
|
||||
let size = tokio::fs::metadata(path).await?.len();
|
||||
self.put_presigned_range(url, path, 0, size, headers)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn put_presigned_range(
|
||||
&self,
|
||||
url: &str,
|
||||
path: &Path,
|
||||
offset: u64,
|
||||
size: u64,
|
||||
headers: &[(String, String)],
|
||||
) -> Result<String, TransferError> {
|
||||
validate_transport_url(url)?;
|
||||
let _permit = self.permit().await?;
|
||||
let mut file = tokio::fs::File::open(path).await?;
|
||||
file.seek(std::io::SeekFrom::Start(offset)).await?;
|
||||
let stream = ReaderStream::new(file.take(size));
|
||||
let mut request = self.http.put(url).body(reqwest::Body::wrap_stream(stream));
|
||||
if !headers
|
||||
.iter()
|
||||
.any(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
{
|
||||
request = request.header(reqwest::header::CONTENT_LENGTH, size);
|
||||
}
|
||||
for (name, value) in headers {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
let response = request.send().await?;
|
||||
if response.status().is_success() {
|
||||
Ok(response
|
||||
.headers()
|
||||
.get(reqwest::header::ETAG)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.trim_matches('"')
|
||||
.to_owned())
|
||||
} else {
|
||||
Err(TransferError::Unexpected {
|
||||
status: response.status(),
|
||||
body: response.text().await.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_transport_url(value: &str) -> Result<(), TransferError> {
|
||||
let parsed = reqwest::Url::parse(value)
|
||||
.map_err(|error| TransferError::InvalidEndpoint(error.to_string()))?;
|
||||
if parsed.scheme() == "https" {
|
||||
return Ok(());
|
||||
}
|
||||
let host = parsed.host_str().unwrap_or_default();
|
||||
let loopback = host.eq_ignore_ascii_case("localhost")
|
||||
|| host
|
||||
.parse::<std::net::IpAddr>()
|
||||
.is_ok_and(|address| address.is_loopback());
|
||||
if parsed.scheme() == "http" && loopback {
|
||||
return Ok(());
|
||||
}
|
||||
Err(TransferError::InvalidEndpoint(
|
||||
"HTTPS is required except for loopback development endpoints".to_owned(),
|
||||
))
|
||||
}
|
||||
|
||||
fn update_check_path(current_version: &str) -> Result<String, TransferError> {
|
||||
let mut url = reqwest::Url::parse("http://localhost/api/external/object-sync/client-updates")
|
||||
.map_err(|error| TransferError::InvalidEndpoint(error.to_string()))?;
|
||||
url.query_pairs_mut()
|
||||
.append_pair("currentVersion", current_version);
|
||||
Ok(format!(
|
||||
"{}?{}",
|
||||
url.path(),
|
||||
url.query().unwrap_or_default()
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn content_md5(path: &Path, offset: u64, size: u64) -> Result<String, TransferError> {
|
||||
let mut file = tokio::fs::File::open(path).await?;
|
||||
file.seek(std::io::SeekFrom::Start(offset)).await?;
|
||||
let mut limited = file.take(size);
|
||||
let mut hasher = Md5::new();
|
||||
let mut buffer = vec![0_u8; 1024 * 1024];
|
||||
let mut remaining = size;
|
||||
while remaining > 0 {
|
||||
let read = limited.read(&mut buffer).await?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
remaining = remaining.saturating_sub(u64::try_from(read).unwrap_or(u64::MAX));
|
||||
}
|
||||
if remaining != 0 {
|
||||
return Err(TransferError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"file ended while calculating Content-MD5",
|
||||
)));
|
||||
}
|
||||
Ok(base64::engine::general_purpose::STANDARD.encode(hasher.finalize()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Client, update_check_path};
|
||||
|
||||
#[test]
|
||||
fn client_requires_tls_except_for_loopback_development() {
|
||||
assert!(Client::new("https://sync.example", None).is_ok());
|
||||
assert!(Client::new("http://127.0.0.1:3001", None).is_ok());
|
||||
assert!(Client::new("http://localhost:3001", None).is_ok());
|
||||
assert!(Client::new("http://sync.example", None).is_err());
|
||||
assert!(Client::new("not-a-url", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_check_percent_encodes_semver_build_metadata() {
|
||||
assert_eq!(
|
||||
update_check_path("1.2.3+capture.7").expect("valid query"),
|
||||
"/api/external/object-sync/client-updates?currentVersion=1.2.3%2Bcapture.7"
|
||||
);
|
||||
}
|
||||
}
|
||||
20
tools/drosync/crates/updater/Cargo.toml
Normal file
20
tools/drosync/crates/updater/Cargo.toml
Normal file
@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "drosync-updater"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
drosync-protocol = { path = "../protocol" }
|
||||
ed25519-dalek.workspace = true
|
||||
hex.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@ -0,0 +1,62 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use base64::Engine;
|
||||
use drosync_protocol::ClientRelease;
|
||||
use ed25519_dalek::{Signer, SigningKey};
|
||||
use std::error::Error;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn signing_key() -> Result<SigningKey, Box<dyn Error>> {
|
||||
let encoded = std::env::var("DROSYNC_UPDATE_SIGNING_PRIVATE_KEY")?;
|
||||
let raw = base64::engine::general_purpose::STANDARD.decode(encoded.trim())?;
|
||||
match raw.len() {
|
||||
32 => {
|
||||
let seed: [u8; 32] = raw.try_into().map_err(|_| "invalid Ed25519 seed")?;
|
||||
Ok(SigningKey::from_bytes(&seed))
|
||||
}
|
||||
64 => {
|
||||
let keypair: [u8; 64] = raw.try_into().map_err(|_| "invalid Ed25519 keypair")?;
|
||||
Ok(SigningKey::from_keypair_bytes(&keypair)?)
|
||||
}
|
||||
_ => Err("DROSYNC_UPDATE_SIGNING_PRIVATE_KEY must decode to 32 or 64 bytes".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(release: &ClientRelease) -> Result<(), Box<dyn Error>> {
|
||||
let expected_percent = match release.channel.as_str() {
|
||||
"canary" => 5,
|
||||
"pilot" => 20,
|
||||
"stable" => 100,
|
||||
_ => return Err("channel must be canary, pilot or stable".into()),
|
||||
};
|
||||
if !release.paused && release.rollout_percent != expected_percent {
|
||||
return Err("active rollout percentage does not match its channel".into());
|
||||
}
|
||||
for platform in ["windows-x86_64", "linux-x86_64", "macos-arm64"] {
|
||||
if !release.artifacts.contains_key(platform) || !release.sha256.contains_key(platform) {
|
||||
return Err(format!("manifest is missing the {platform} artifact or digest").into());
|
||||
}
|
||||
let digest = &release.sha256[platform];
|
||||
if digest.len() != 64 || !digest.bytes().all(|value| value.is_ascii_hexdigit()) {
|
||||
return Err(format!("invalid SHA-256 for {platform}").into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn Error>> {
|
||||
let manifest_path = std::env::args_os()
|
||||
.nth(1)
|
||||
.map(PathBuf::from)
|
||||
.ok_or("usage: drosync-sign-manifest MANIFEST.json")?;
|
||||
let mut release: ClientRelease = serde_json::from_slice(&fs::read(&manifest_path)?)?;
|
||||
validate(&release)?;
|
||||
release.signature.clear();
|
||||
let signature = signing_key()?.sign(&serde_json::to_vec(&release)?);
|
||||
release.signature = base64::engine::general_purpose::STANDARD.encode(signature.to_bytes());
|
||||
let mut output = serde_json::to_vec_pretty(&release)?;
|
||||
output.push(b'\n');
|
||||
fs::write(&manifest_path, output)?;
|
||||
Ok(())
|
||||
}
|
||||
222
tools/drosync/crates/updater/src/lib.rs
Normal file
222
tools/drosync/crates/updater/src/lib.rs
Normal file
@ -0,0 +1,222 @@
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use base64::Engine;
|
||||
use drosync_protocol::ClientRelease;
|
||||
use ed25519_dalek::{Signature, Verifier, VerifyingKey};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::path::Path;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum UpdateError {
|
||||
#[error("invalid Ed25519 public key")]
|
||||
InvalidKey,
|
||||
#[error("invalid release signature")]
|
||||
InvalidSignature,
|
||||
#[error("artifact SHA-256 mismatch")]
|
||||
DigestMismatch,
|
||||
#[error("manifest serialization failed: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("update download failed: {0}")]
|
||||
Http(#[from] reqwest::Error),
|
||||
#[error("update file I/O failed: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("native package signature verification failed")]
|
||||
NativeSignature,
|
||||
#[error("native package installer failed")]
|
||||
Install,
|
||||
#[error("HTTPS is required for update downloads except on loopback")]
|
||||
InsecureURL,
|
||||
}
|
||||
|
||||
pub fn decode_public_key(value: &str) -> Result<[u8; 32], UpdateError> {
|
||||
let decoded = base64::engine::general_purpose::STANDARD
|
||||
.decode(value.trim())
|
||||
.map_err(|_| UpdateError::InvalidKey)?;
|
||||
decoded.try_into().map_err(|_| UpdateError::InvalidKey)
|
||||
}
|
||||
|
||||
pub async fn download_verified(
|
||||
url: &str,
|
||||
destination: &Path,
|
||||
expected_sha256: &str,
|
||||
) -> Result<(), UpdateError> {
|
||||
let parsed = reqwest::Url::parse(url).map_err(|_| UpdateError::InsecureURL)?;
|
||||
let host = parsed.host_str().unwrap_or_default();
|
||||
let loopback = host.eq_ignore_ascii_case("localhost")
|
||||
|| host
|
||||
.parse::<std::net::IpAddr>()
|
||||
.is_ok_and(|address| address.is_loopback());
|
||||
if parsed.scheme() != "https" && !(parsed.scheme() == "http" && loopback) {
|
||||
return Err(UpdateError::InsecureURL);
|
||||
}
|
||||
if let Some(parent) = destination.parent() {
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
}
|
||||
let temporary = destination.with_extension("download");
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent(concat!("drosync-updater/", env!("CARGO_PKG_VERSION")))
|
||||
.build()?;
|
||||
let mut response = client.get(url).send().await?.error_for_status()?;
|
||||
let mut file = tokio::fs::File::create(&temporary).await?;
|
||||
let mut hasher = Sha256::new();
|
||||
while let Some(chunk) = response.chunk().await? {
|
||||
hasher.update(&chunk);
|
||||
tokio::io::AsyncWriteExt::write_all(&mut file, &chunk).await?;
|
||||
}
|
||||
tokio::io::AsyncWriteExt::flush(&mut file).await?;
|
||||
let actual = hex::encode(hasher.finalize());
|
||||
if !actual.eq_ignore_ascii_case(expected_sha256) {
|
||||
let _ = tokio::fs::remove_file(&temporary).await;
|
||||
return Err(UpdateError::DigestMismatch);
|
||||
}
|
||||
tokio::fs::rename(temporary, destination).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn verify_native_signature(package: &Path) -> Result<(), UpdateError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
let status = {
|
||||
let package_signature = tokio::process::Command::new("/usr/sbin/pkgutil")
|
||||
.arg("--check-signature")
|
||||
.arg(package)
|
||||
.status()
|
||||
.await?;
|
||||
if !package_signature.success() {
|
||||
return Err(UpdateError::NativeSignature);
|
||||
}
|
||||
tokio::process::Command::new("/usr/sbin/spctl")
|
||||
.args(["--assess", "--type", "install"])
|
||||
.arg(package)
|
||||
.status()
|
||||
.await?
|
||||
};
|
||||
#[cfg(target_os = "linux")]
|
||||
let status = tokio::process::Command::new("/usr/bin/debsig-verify")
|
||||
.arg(package)
|
||||
.status()
|
||||
.await?;
|
||||
#[cfg(target_os = "windows")]
|
||||
let status = tokio::process::Command::new("powershell.exe")
|
||||
.args([
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
"if ((Get-AuthenticodeSignature -LiteralPath $args[0]).Status -ne 'Valid') { exit 1 }",
|
||||
"--",
|
||||
])
|
||||
.arg(package)
|
||||
.status()
|
||||
.await?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(UpdateError::NativeSignature)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn install_native_package(package: &Path) -> Result<(), UpdateError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
let status = tokio::process::Command::new("/usr/sbin/installer")
|
||||
.args(["-pkg"])
|
||||
.arg(package)
|
||||
.args(["-target", "/"])
|
||||
.status()
|
||||
.await?;
|
||||
#[cfg(target_os = "linux")]
|
||||
let status = tokio::process::Command::new("/usr/bin/dpkg")
|
||||
.arg("-i")
|
||||
.arg(package)
|
||||
.status()
|
||||
.await?;
|
||||
#[cfg(target_os = "windows")]
|
||||
let status = tokio::process::Command::new("msiexec.exe")
|
||||
.args(["/i"])
|
||||
.arg(package)
|
||||
.args(["/qn", "/norestart"])
|
||||
.status()
|
||||
.await?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(UpdateError::Install)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn verify_release(release: &ClientRelease, public_key: &[u8; 32]) -> Result<(), UpdateError> {
|
||||
let mut unsigned = release.clone();
|
||||
unsigned.signature.clear();
|
||||
let payload = serde_json::to_vec(&unsigned)?;
|
||||
let key = VerifyingKey::from_bytes(public_key).map_err(|_| UpdateError::InvalidKey)?;
|
||||
let signature_bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(&release.signature)
|
||||
.map_err(|_| UpdateError::InvalidSignature)?;
|
||||
let signature =
|
||||
Signature::from_slice(&signature_bytes).map_err(|_| UpdateError::InvalidSignature)?;
|
||||
key.verify(&payload, &signature)
|
||||
.map_err(|_| UpdateError::InvalidSignature)
|
||||
}
|
||||
|
||||
pub fn verify_artifact(bytes: &[u8], expected_sha256: &str) -> Result<(), UpdateError> {
|
||||
let actual = hex::encode(Sha256::digest(bytes));
|
||||
if actual.eq_ignore_ascii_case(expected_sha256) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(UpdateError::DigestMismatch)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{verify_artifact, verify_release};
|
||||
use base64::Engine;
|
||||
use drosync_protocol::ClientRelease;
|
||||
use ed25519_dalek::{Signer, SigningKey};
|
||||
use sha2::Digest;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
fn signed_release() -> (ClientRelease, [u8; 32]) {
|
||||
let signing_key = SigningKey::from_bytes(&[7_u8; 32]);
|
||||
let mut release = ClientRelease {
|
||||
id: "drosync-1.0.0".to_owned(),
|
||||
version: "1.0.0".to_owned(),
|
||||
channel: "canary".to_owned(),
|
||||
protocol_version: "v1".to_owned(),
|
||||
minimum_server_version: "1.0.0".to_owned(),
|
||||
artifacts: BTreeMap::from([(
|
||||
"linux-x86_64".to_owned(),
|
||||
"https://downloads.example/drosync.deb".to_owned(),
|
||||
)]),
|
||||
sha256: BTreeMap::from([(
|
||||
"linux-x86_64".to_owned(),
|
||||
hex::encode(sha2::Sha256::digest(b"package")),
|
||||
)]),
|
||||
signature: String::new(),
|
||||
rollout_percent: 5,
|
||||
paused: false,
|
||||
};
|
||||
let payload = serde_json::to_vec(&release).expect("release serializes");
|
||||
release.signature =
|
||||
base64::engine::general_purpose::STANDARD.encode(signing_key.sign(&payload).to_bytes());
|
||||
(release, signing_key.verifying_key().to_bytes())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signed_manifest_and_artifact_verify() {
|
||||
let (release, public_key) = signed_release();
|
||||
verify_release(&release, &public_key).expect("valid release signature");
|
||||
verify_artifact(b"package", &release.sha256["linux-x86_64"])
|
||||
.expect("valid artifact digest");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signed_manifest_rejects_tampering() {
|
||||
let (mut release, public_key) = signed_release();
|
||||
release.artifacts.insert(
|
||||
"linux-x86_64".to_owned(),
|
||||
"https://attacker.example/replaced.deb".to_owned(),
|
||||
);
|
||||
assert!(verify_release(&release, &public_key).is_err());
|
||||
assert!(verify_artifact(b"tampered", &release.sha256["linux-x86_64"]).is_err());
|
||||
}
|
||||
}
|
||||
26
tools/drosync/packaging/build-release.sh
Executable file
26
tools/drosync/packaging/build-release.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
: "${DROSYNC_UPDATE_PUBLIC_KEY_BASE64:?set DROSYNC_UPDATE_PUBLIC_KEY_BASE64 from CI secrets}"
|
||||
TARGET="${1:-}"
|
||||
case "$TARGET" in
|
||||
linux-x86_64)
|
||||
cargo build --manifest-path "$ROOT_DIR/Cargo.toml" --release --workspace
|
||||
"$ROOT_DIR/packaging/linux/build-deb.sh"
|
||||
"$ROOT_DIR/packaging/linux/build-portable.sh"
|
||||
;;
|
||||
macos-arm64)
|
||||
[[ "$(uname -s)-$(uname -m)" == "Darwin-arm64" ]] || { echo "macos-arm64 must be built on an Apple Silicon runner" >&2; exit 1; }
|
||||
cargo build --manifest-path "$ROOT_DIR/Cargo.toml" --release --workspace
|
||||
"$ROOT_DIR/packaging/macos/build-pkg.sh"
|
||||
;;
|
||||
windows-x86_64)
|
||||
echo "Run packaging/windows/build-msi.ps1 on a Windows x86_64 runner." >&2
|
||||
exit 2
|
||||
;;
|
||||
*)
|
||||
echo "usage: $0 linux-x86_64|macos-arm64|windows-x86_64" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
26
tools/drosync/packaging/linux/build-deb.sh
Executable file
26
tools/drosync/packaging/linux/build-deb.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
VERSION="${DROSYNC_VERSION:-$(sed -n 's/^version = "\(.*\)"/\1/p' "$ROOT_DIR/Cargo.toml" | head -1)}"
|
||||
ARCH="${DROSYNC_DEB_ARCH:-amd64}"
|
||||
OUTPUT_DIR="${DROSYNC_OUTPUT_DIR:-$ROOT_DIR/dist}"
|
||||
PACKAGE_ROOT="$(mktemp -d)"
|
||||
trap 'rm -rf "$PACKAGE_ROOT"' EXIT
|
||||
|
||||
mkdir -p "$PACKAGE_ROOT/DEBIAN" "$PACKAGE_ROOT/usr/bin" "$PACKAGE_ROOT/lib/systemd/system" "$PACKAGE_ROOT/usr/lib/systemd/user"
|
||||
cp "$ROOT_DIR/target/release/drosync" "$PACKAGE_ROOT/usr/bin/drosync"
|
||||
cp "$ROOT_DIR/target/release/drosync-daemon" "$PACKAGE_ROOT/usr/bin/drosync-daemon"
|
||||
cp "$ROOT_DIR/target/release/drosync-gui" "$PACKAGE_ROOT/usr/bin/drosync-gui"
|
||||
sed -e "s/^Version:.*/Version: $VERSION/" -e "s/^Architecture:.*/Architecture: $ARCH/" \
|
||||
"$ROOT_DIR/packaging/linux/debian/control" >"$PACKAGE_ROOT/DEBIAN/control"
|
||||
cp "$ROOT_DIR/packaging/linux/debian/postinst" "$PACKAGE_ROOT/DEBIAN/postinst"
|
||||
cp "$ROOT_DIR/packaging/linux/debian/prerm" "$PACKAGE_ROOT/DEBIAN/prerm"
|
||||
chmod 0755 "$PACKAGE_ROOT/DEBIAN/postinst" "$PACKAGE_ROOT/DEBIAN/prerm"
|
||||
cp "$ROOT_DIR/packaging/linux/drosync.service" "$PACKAGE_ROOT/lib/systemd/system/drosync.service"
|
||||
cp "$ROOT_DIR/packaging/linux/drosync-user.service" "$PACKAGE_ROOT/usr/lib/systemd/user/drosync.service"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
dpkg-deb --root-owner-group --build "$PACKAGE_ROOT" "$OUTPUT_DIR/drosync_${VERSION}_${ARCH}.deb"
|
||||
if [[ -n "${DROSYNC_DEB_SIGN_KEY:-}" ]]; then
|
||||
dpkg-sig --sign builder -k "$DROSYNC_DEB_SIGN_KEY" "$OUTPUT_DIR/drosync_${VERSION}_${ARCH}.deb"
|
||||
fi
|
||||
17
tools/drosync/packaging/linux/build-portable.sh
Executable file
17
tools/drosync/packaging/linux/build-portable.sh
Executable file
@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
VERSION="${DROSYNC_VERSION:-1.0.0}"
|
||||
OUTPUT_DIR="${DROSYNC_OUTPUT_DIR:-$ROOT_DIR/dist}"
|
||||
STAGE="$(mktemp -d)"
|
||||
trap 'rm -rf "$STAGE"' EXIT
|
||||
|
||||
mkdir -p "$STAGE/drosync-$VERSION" "$OUTPUT_DIR"
|
||||
cp "$ROOT_DIR/target/release/drosync" "$ROOT_DIR/target/release/drosync-daemon" "$ROOT_DIR/target/release/drosync-gui" "$STAGE/drosync-$VERSION/"
|
||||
cp "$ROOT_DIR/packaging/linux/drosync-user.service" "$STAGE/drosync-$VERSION/"
|
||||
tar -C "$STAGE" -czf "$OUTPUT_DIR/drosync-${VERSION}-linux-x86_64.tar.gz" "drosync-$VERSION"
|
||||
sha256sum "$OUTPUT_DIR/drosync-${VERSION}-linux-x86_64.tar.gz" >"$OUTPUT_DIR/drosync-${VERSION}-linux-x86_64.tar.gz.sha256"
|
||||
if [[ -n "${DROSYNC_GPG_SIGN_KEY:-}" ]]; then
|
||||
gpg --batch --yes --local-user "$DROSYNC_GPG_SIGN_KEY" --detach-sign --armor "$OUTPUT_DIR/drosync-${VERSION}-linux-x86_64.tar.gz"
|
||||
fi
|
||||
9
tools/drosync/packaging/linux/debian/control
Normal file
9
tools/drosync/packaging/linux/debian/control
Normal file
@ -0,0 +1,9 @@
|
||||
Package: drosync
|
||||
Version: 1.0.0
|
||||
Section: utils
|
||||
Priority: optional
|
||||
Architecture: amd64
|
||||
Maintainer: D-Robotics <support@example.invalid>
|
||||
Depends: ca-certificates, debsig-verify, libsecret-1-0
|
||||
Description: Secure one-way local-to-BOS synchronization client
|
||||
drosync provides a CLI, daemon, tray GUI, crash recovery, and signed updates.
|
||||
11
tools/drosync/packaging/linux/debian/postinst
Executable file
11
tools/drosync/packaging/linux/debian/postinst
Executable file
@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
if ! getent group drosync >/dev/null 2>&1; then
|
||||
addgroup --system drosync
|
||||
fi
|
||||
if ! getent passwd drosync >/dev/null 2>&1; then
|
||||
adduser --system --ingroup drosync --home /var/lib/drosync --no-create-home drosync
|
||||
fi
|
||||
install -d -o drosync -g drosync -m 0700 /var/lib/drosync
|
||||
systemctl daemon-reload
|
||||
6
tools/drosync/packaging/linux/debian/prerm
Executable file
6
tools/drosync/packaging/linux/debian/prerm
Executable file
@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
if [ "${1:-}" = "remove" ]; then
|
||||
systemctl disable --now drosync.service >/dev/null 2>&1 || true
|
||||
fi
|
||||
16
tools/drosync/packaging/linux/drosync-user.service
Normal file
16
tools/drosync/packaging/linux/drosync-user.service
Normal file
@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=drosync per-user synchronization daemon
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/drosync-daemon
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
24
tools/drosync/packaging/linux/drosync.service
Normal file
24
tools/drosync/packaging/linux/drosync.service
Normal file
@ -0,0 +1,24 @@
|
||||
[Unit]
|
||||
Description=drosync system synchronization daemon
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/drosync-daemon
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
User=drosync
|
||||
Group=drosync
|
||||
Environment=HOME=/var/lib/drosync
|
||||
Environment=XDG_DATA_HOME=/var/lib/drosync
|
||||
Environment=XDG_CONFIG_HOME=/etc/drosync
|
||||
EnvironmentFile=-/etc/drosync/environment
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=read-only
|
||||
ReadWritePaths=/var/lib/drosync
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
13
tools/drosync/packaging/macos/Info.plist
Normal file
13
tools/drosync/packaging/macos/Info.plist
Normal file
@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleIdentifier</key><string>com.drobotics.drosync</string>
|
||||
<key>CFBundleName</key><string>drosync</string>
|
||||
<key>CFBundleExecutable</key><string>drosync-gui</string>
|
||||
<key>CFBundlePackageType</key><string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key><string>1.0.0</string>
|
||||
<key>LSMinimumSystemVersion</key><string>13.0</string>
|
||||
<key>LSUIElement</key><true/>
|
||||
</dict>
|
||||
</plist>
|
||||
32
tools/drosync/packaging/macos/build-pkg.sh
Executable file
32
tools/drosync/packaging/macos/build-pkg.sh
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
: "${DROSYNC_APPLE_APPLICATION_IDENTITY:?set DROSYNC_APPLE_APPLICATION_IDENTITY from CI secrets}"
|
||||
: "${DROSYNC_APPLE_INSTALLER_IDENTITY:?set DROSYNC_APPLE_INSTALLER_IDENTITY from CI secrets}"
|
||||
: "${DROSYNC_APPLE_NOTARY_PROFILE:?set DROSYNC_APPLE_NOTARY_PROFILE from CI keychain}"
|
||||
VERSION="${DROSYNC_VERSION:-1.0.0}"
|
||||
OUTPUT_DIR="${DROSYNC_OUTPUT_DIR:-$ROOT_DIR/dist}"
|
||||
STAGE="$(mktemp -d)"
|
||||
trap 'rm -rf "$STAGE"' EXIT
|
||||
APP="$STAGE/Applications/drosync.app"
|
||||
|
||||
mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Library/LaunchServices" \
|
||||
"$STAGE/Library/LaunchAgents" "$STAGE/Library/LaunchDaemons" "$STAGE/usr/local/bin" "$OUTPUT_DIR"
|
||||
cp "$ROOT_DIR/packaging/macos/Info.plist" "$APP/Contents/Info.plist"
|
||||
cp "$ROOT_DIR/target/release/drosync-gui" "$APP/Contents/MacOS/drosync-gui"
|
||||
cp "$ROOT_DIR/target/release/drosync" "$APP/Contents/MacOS/drosync"
|
||||
cp "$ROOT_DIR/target/release/drosync-daemon" "$APP/Contents/MacOS/drosync-daemon"
|
||||
cp "$ROOT_DIR/packaging/macos/com.drobotics.drosync.agent.plist" "$STAGE/Library/LaunchAgents/"
|
||||
cp "$ROOT_DIR/packaging/macos/com.drobotics.drosync.daemon.plist" "$STAGE/Library/LaunchDaemons/"
|
||||
ln -s /Applications/drosync.app/Contents/MacOS/drosync "$STAGE/usr/local/bin/drosync"
|
||||
for binary in drosync drosync-daemon drosync-gui; do
|
||||
codesign --force --timestamp --options runtime --entitlements "$ROOT_DIR/packaging/macos/entitlements.plist" \
|
||||
--sign "$DROSYNC_APPLE_APPLICATION_IDENTITY" "$APP/Contents/MacOS/$binary"
|
||||
done
|
||||
codesign --force --timestamp --options runtime --sign "$DROSYNC_APPLE_APPLICATION_IDENTITY" "$APP"
|
||||
pkgbuild --root "$STAGE" --identifier com.drobotics.drosync --version "$VERSION" \
|
||||
--install-location / --sign "$DROSYNC_APPLE_INSTALLER_IDENTITY" "$OUTPUT_DIR/drosync-${VERSION}-macos-arm64.pkg"
|
||||
xcrun notarytool submit "$OUTPUT_DIR/drosync-${VERSION}-macos-arm64.pkg" --keychain-profile "$DROSYNC_APPLE_NOTARY_PROFILE" --wait
|
||||
xcrun stapler staple "$OUTPUT_DIR/drosync-${VERSION}-macos-arm64.pkg"
|
||||
spctl --assess --type install --verbose "$OUTPUT_DIR/drosync-${VERSION}-macos-arm64.pkg"
|
||||
@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key><string>com.drobotics.drosync.agent</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array><string>/Applications/drosync.app/Contents/MacOS/drosync-daemon</string></array>
|
||||
<key>RunAtLoad</key><true/>
|
||||
<key>KeepAlive</key><true/>
|
||||
<key>ProcessType</key><string>Background</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@ -0,0 +1,11 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key><string>com.drobotics.drosync.daemon</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array><string>/Applications/drosync.app/Contents/MacOS/drosync-daemon</string></array>
|
||||
<key>RunAtLoad</key><true/>
|
||||
<key>KeepAlive</key><true/>
|
||||
</dict>
|
||||
</plist>
|
||||
8
tools/drosync/packaging/macos/entitlements.plist
Normal file
8
tools/drosync/packaging/macos/entitlements.plist
Normal file
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>com.apple.security.app-sandbox</key><false/>
|
||||
<key>com.apple.security.network.client</key><true/>
|
||||
</dict>
|
||||
</plist>
|
||||
27
tools/drosync/packaging/update-manifest.schema.json
Normal file
27
tools/drosync/packaging/update-manifest.schema.json
Normal file
@ -0,0 +1,27 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://drobotics.example/schemas/drosync-update-manifest-v1.json",
|
||||
"title": "drosync signed update manifest",
|
||||
"type": "object",
|
||||
"required": ["id", "version", "channel", "protocolVersion", "artifacts", "sha256", "signature", "rolloutPercent", "paused"],
|
||||
"properties": {
|
||||
"id": { "type": "string" },
|
||||
"version": { "type": "string" },
|
||||
"channel": { "enum": ["canary", "pilot", "stable"] },
|
||||
"protocolVersion": { "const": "v1" },
|
||||
"minimumServerVersion": { "type": "string" },
|
||||
"rolloutPercent": { "type": "integer", "minimum": 0, "maximum": 100 },
|
||||
"paused": { "type": "boolean" },
|
||||
"artifacts": {
|
||||
"type": "object",
|
||||
"required": ["windows-x86_64", "linux-x86_64", "macos-arm64"],
|
||||
"additionalProperties": { "type": "string", "format": "uri" }
|
||||
},
|
||||
"sha256": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
|
||||
},
|
||||
"signature": { "type": "string", "contentEncoding": "base64" }
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
20
tools/drosync/packaging/windows/build-msi.ps1
Normal file
20
tools/drosync/packaging/windows/build-msi.ps1
Normal file
@ -0,0 +1,20 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if (-not $env:DROSYNC_AUTHENTICODE_CERT_THUMBPRINT) {
|
||||
throw "DROSYNC_AUTHENTICODE_CERT_THUMBPRINT must be injected by CI"
|
||||
}
|
||||
if (-not $env:DROSYNC_UPDATE_PUBLIC_KEY_BASE64) {
|
||||
throw "DROSYNC_UPDATE_PUBLIC_KEY_BASE64 must be injected by CI before compiling the binaries"
|
||||
}
|
||||
$Root = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
|
||||
$Version = if ($env:DROSYNC_VERSION) { $env:DROSYNC_VERSION } else { "1.0.0" }
|
||||
$Output = if ($env:DROSYNC_OUTPUT_DIR) { $env:DROSYNC_OUTPUT_DIR } else { Join-Path $Root "dist" }
|
||||
$BinaryDir = Join-Path $Root "target\x86_64-pc-windows-msvc\release"
|
||||
New-Item -ItemType Directory -Force -Path $Output | Out-Null
|
||||
|
||||
foreach ($Name in @("drosync.exe", "drosync-daemon.exe", "drosync-gui.exe")) {
|
||||
& signtool sign /sha1 $env:DROSYNC_AUTHENTICODE_CERT_THUMBPRINT /fd SHA256 /tr http://timestamp.digicert.com /td SHA256 (Join-Path $BinaryDir $Name)
|
||||
}
|
||||
& wix build (Join-Path $PSScriptRoot "drosync.wxs") -arch x64 -d "Version=$Version" -d "BinaryDir=$BinaryDir" -o (Join-Path $Output "drosync-$Version-windows-x86_64.msi")
|
||||
& signtool sign /sha1 $env:DROSYNC_AUTHENTICODE_CERT_THUMBPRINT /fd SHA256 /tr http://timestamp.digicert.com /td SHA256 (Join-Path $Output "drosync-$Version-windows-x86_64.msi")
|
||||
& signtool verify /pa /all (Join-Path $Output "drosync-$Version-windows-x86_64.msi")
|
||||
22
tools/drosync/packaging/windows/drosync.wxs
Normal file
22
tools/drosync/packaging/windows/drosync.wxs
Normal file
@ -0,0 +1,22 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Wix xmlns="http://wixtoolset.org/schemas/v4/wxs">
|
||||
<Package Name="drosync" Manufacturer="D-Robotics" Version="$(var.Version)" UpgradeCode="A0B60461-862A-4D93-A638-AEE6E2EC620A" Scope="perMachine">
|
||||
<MajorUpgrade DowngradeErrorMessage="A newer drosync version is already installed." />
|
||||
<MediaTemplate EmbedCab="yes" />
|
||||
<StandardDirectory Id="ProgramFiles64Folder">
|
||||
<Directory Id="INSTALLFOLDER" Name="drosync">
|
||||
<Component Id="Binaries" Guid="B896837B-0BD8-4D80-80AA-7111BA3C3450">
|
||||
<File Id="DrosyncCli" Source="$(var.BinaryDir)\drosync.exe" KeyPath="yes" />
|
||||
<File Id="DrosyncDaemon" Source="$(var.BinaryDir)\drosync-daemon.exe" />
|
||||
<File Id="DrosyncGui" Source="$(var.BinaryDir)\drosync-gui.exe" />
|
||||
<ServiceInstall Id="DrosyncService" Name="drosync" DisplayName="drosync" Description="Secure one-way BOS synchronization" Start="auto" Type="ownProcess" ErrorControl="normal" Account="LocalSystem" Arguments="--system-service" />
|
||||
<ServiceControl Id="DrosyncServiceControl" Name="drosync" Start="install" Stop="both" Remove="uninstall" Wait="yes" />
|
||||
<Environment Id="PathEntry" Name="PATH" Value="[INSTALLFOLDER]" Part="last" Action="set" System="yes" />
|
||||
</Component>
|
||||
</Directory>
|
||||
</StandardDirectory>
|
||||
<Feature Id="MainFeature">
|
||||
<ComponentRef Id="Binaries" />
|
||||
</Feature>
|
||||
</Package>
|
||||
</Wix>
|
||||
10
tools/drosync/packaging/windows/install-user.ps1
Normal file
10
tools/drosync/packaging/windows/install-user.ps1
Normal file
@ -0,0 +1,10 @@
|
||||
$ErrorActionPreference = "Stop"
|
||||
$InstallDir = Join-Path $env:LOCALAPPDATA "Programs\drosync"
|
||||
$Startup = [Environment]::GetFolderPath("Startup")
|
||||
New-Item -ItemType Directory -Force -Path $InstallDir | Out-Null
|
||||
Copy-Item (Join-Path $PSScriptRoot "..\..\target\x86_64-pc-windows-msvc\release\drosync*.exe") $InstallDir
|
||||
$Shell = New-Object -ComObject WScript.Shell
|
||||
$Shortcut = $Shell.CreateShortcut((Join-Path $Startup "drosync.lnk"))
|
||||
$Shortcut.TargetPath = Join-Path $InstallDir "drosync-gui.exe"
|
||||
$Shortcut.WorkingDirectory = $InstallDir
|
||||
$Shortcut.Save()
|
||||
4
tools/drosync/rust-toolchain.toml
Normal file
4
tools/drosync/rust-toolchain.toml
Normal file
@ -0,0 +1,4 @@
|
||||
[toolchain]
|
||||
channel = "1.89.0"
|
||||
components = ["clippy", "rustfmt"]
|
||||
profile = "minimal"
|
||||
Loading…
x
Reference in New Issue
Block a user