forked from longtao.wu/droboticflow
382 lines
13 KiB
Go
382 lines
13 KiB
Go
// query-replica is a rate-limited, independently deployed projector. It never
|
|
// claims tasks, repairs annotations, or starts business background workers.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"droboticflow/internal/store"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
"go.mongodb.org/mongo-driver/mongo/readconcern"
|
|
"go.mongodb.org/mongo-driver/mongo/readpref"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
func main() {
|
|
bootstrap := flag.Bool("bootstrap", false, "explicitly allow bounded resumable historical copy")
|
|
ensure := flag.Bool("ensure-source-indexes", false, "explicit source index writes; requires production authorization")
|
|
writers := flag.Bool("writers-covered", false, "attest all source writers use atomic query-source/v1 markers")
|
|
once := flag.Bool("once", false, "one bounded pass only")
|
|
interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes")
|
|
batchSize := flag.Int("batch-size", 200, "maximum source records per collection batch (1-200)")
|
|
minimumPass := flag.Duration("min-pass-interval", 0, "optional minimum start-to-start pass interval, including backfill (0-1m)")
|
|
safetyRun := flag.String("safety-run", "", "explicit durable shadow run identity; requires global transport and health budgets")
|
|
maxRecords := flag.Int64("max-source-records", 1000, "durable total read reservations;0 explicitly permits full backfill")
|
|
rowsRate := flag.Int("source-records-per-second", 10, "global source row pacing")
|
|
bytesRate := flag.Int("source-bytes-per-second", 65536, "shared actual source transport read budget")
|
|
commandsRate := flag.Int("source-commands-per-second", 2, "shared source socket-write budget, includes control overhead")
|
|
maintenanceUntil := flag.String("maintenance-until", "", "explicit RFC3339 deadline for the initialization-only CPU observation policy")
|
|
maintenance := flag.Bool("maintenance-initialization", false, "explicit finite initialization run: CPU is observed, not an admission/stop gate; resource and durability guards remain")
|
|
baseline := flag.Float64("source-cpu-baseline", 0, "fixed reviewed4core source CPU baseline, never increased on resume")
|
|
resume := flag.String("resume-pause", "", "exact current pause nonce after operator review")
|
|
initializeTarget := flag.Bool("initialize-target", false, "verify independent topology and bind target only; no source scans or indexes")
|
|
status := flag.Bool("safety-status", false, "target-only private safety status")
|
|
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
|
|
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
|
|
sourceCompress := flag.Bool("source-compress", false, "negotiate snappy/zstd wire compression on unguarded source reads")
|
|
flag.Parse()
|
|
var maintenanceDeadline *time.Time
|
|
if *maintenance {
|
|
value, err := time.Parse(time.RFC3339, *maintenanceUntil)
|
|
if err != nil || !value.After(time.Now()) {
|
|
log.Fatal("explicit future maintenance deadline required")
|
|
}
|
|
maintenanceDeadline = &value
|
|
}
|
|
if *maintenance && (*safetyRun == "" || *maxRecords <= 0) {
|
|
log.Fatal("maintenance initialization requires a guarded finite budget")
|
|
}
|
|
if *sampleSource {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if os.Getenv("MONGO_URI") == "" {
|
|
log.Fatal("source binding required")
|
|
}
|
|
c, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")).SetMaxPoolSize(1))
|
|
if e != nil {
|
|
log.Fatal("source unavailable")
|
|
}
|
|
defer c.Disconnect(context.Background())
|
|
var info struct {
|
|
Uptime int64 `bson:"uptimeMillis"`
|
|
Memory struct {
|
|
Resident int64 `bson:"resident"`
|
|
Supported bool `bson:"supported"`
|
|
} `bson:"mem"`
|
|
Extra struct {
|
|
User int64 `bson:"user_time_us"`
|
|
System int64 `bson:"system_time_us"`
|
|
} `bson:"extra_info"`
|
|
}
|
|
if c.Database("admin").RunCommand(ctx, map[string]any{"serverStatus": 1}).Decode(&info) != nil {
|
|
log.Fatal("source counters unavailable")
|
|
}
|
|
var hello struct {
|
|
Topology struct {
|
|
Process any `bson:"processId"`
|
|
} `bson:"topologyVersion"`
|
|
}
|
|
if c.Database("admin").RunCommand(ctx, map[string]any{"hello": 1}).Decode(&hello) != nil {
|
|
log.Fatal("source identity unavailable")
|
|
}
|
|
identity := sha256.Sum256([]byte(fmt.Sprint(hello.Topology.Process)))
|
|
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano), "residentMiB": info.Memory.Resident, "memorySupported": info.Memory.Supported})
|
|
return
|
|
}
|
|
if *observation != "" || *status {
|
|
var value store.QueryReplicaSafetyObservation
|
|
if !*status {
|
|
input := io.Reader(os.Stdin)
|
|
if *observation != "-" {
|
|
f, e := os.Open(*observation)
|
|
if e != nil {
|
|
log.Fatal("observation unavailable")
|
|
}
|
|
defer f.Close()
|
|
input = f
|
|
}
|
|
decoder := json.NewDecoder(io.LimitReader(input, 16384))
|
|
decoder.DisallowUnknownFields()
|
|
if decoder.Decode(&value) != nil {
|
|
log.Fatal("invalid observation")
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
uri, db := os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB")
|
|
if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
|
|
log.Fatal("explicit independent target required")
|
|
}
|
|
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(1))
|
|
if e != nil {
|
|
log.Fatal("target unavailable")
|
|
}
|
|
defer c.Disconnect(context.Background())
|
|
if *status {
|
|
value, e := store.NewMongoStore(c.Database(db)).QueryReplicaSafetyStatus(ctx)
|
|
if e != nil {
|
|
log.Fatal("safety state unavailable")
|
|
}
|
|
_ = json.NewEncoder(os.Stdout).Encode(value)
|
|
return
|
|
}
|
|
if store.NewMongoStore(c.Database(db)).RecordQueryReplicaObservation(ctx, value) != nil {
|
|
log.Fatal("observation rejected")
|
|
}
|
|
return
|
|
}
|
|
var transport *store.QueryReplicaTransport
|
|
if *safetyRun != "" {
|
|
var e error
|
|
transport, e = store.NewQueryReplicaTransport(*bytesRate, *commandsRate)
|
|
if e != nil || *ensure {
|
|
log.Fatal("invalid guarded source configuration")
|
|
}
|
|
} else if *resume != "" {
|
|
log.Fatal("resume requires durable safety")
|
|
}
|
|
|
|
if !validPacing(*batchSize, *minimumPass) {
|
|
log.Fatal("invalid bounded projector pacing")
|
|
}
|
|
mode := os.Getenv("QUERY_REPLICA_REFRESH_MODE")
|
|
if mode != "" && mode != "continuous" && mode != "periodic" {
|
|
log.Fatal("invalid query refresh mode")
|
|
}
|
|
if mode == "periodic" {
|
|
explicit := false
|
|
flag.Visit(func(f *flag.Flag) {
|
|
if f.Name == "interval" {
|
|
explicit = true
|
|
}
|
|
})
|
|
if !explicit {
|
|
*interval = store.QueryPeriodicSyncInterval
|
|
}
|
|
}
|
|
if *initializeTarget && *ensure {
|
|
log.Fatal("source index writes forbidden during target binding")
|
|
}
|
|
if !*writers || *interval < 100*time.Millisecond || *interval > 5*time.Minute || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
|
|
log.Fatal("writer coverage, namespace and bounded polling interval required")
|
|
}
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
connect := func(uri, db string, target bool) (*mongo.Client, *store.MongoStore) {
|
|
if uri == "" || db == "" {
|
|
log.Fatal("explicit source/query connection required")
|
|
}
|
|
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
|
|
if !target && transport == nil {
|
|
opts.SetMaxPoolSize(2)
|
|
if *sourceCompress {
|
|
opts.SetCompressors([]string{"snappy", "zstd"})
|
|
}
|
|
}
|
|
if !target && transport != nil {
|
|
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors(sourceCompressors(*maintenance)).SetRetryReads(false).SetTimeout(5 * time.Second)
|
|
}
|
|
if target {
|
|
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
|
|
}
|
|
c, e := mongo.Connect(ctx, opts)
|
|
if e != nil {
|
|
log.Fatal("database connection unavailable")
|
|
}
|
|
return c, store.NewMongoStore(c.Database(db))
|
|
}
|
|
src, source := connect(os.Getenv("MONGO_URI"), os.Getenv("MONGO_DB"), false)
|
|
defer src.Disconnect(context.Background())
|
|
dst, target := connect(os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB"), true)
|
|
defer dst.Disconnect(context.Background())
|
|
q, e := store.NewQueryReplica(source, target)
|
|
if e != nil {
|
|
log.Fatal("invalid projector configuration")
|
|
}
|
|
q.BatchSize = *batchSize
|
|
if q.VerifyTopology(ctx) != nil {
|
|
log.Fatal("source/query databases are not independently usable")
|
|
}
|
|
if *ensure && q.EnsureSourceIndexes(ctx) != nil {
|
|
log.Fatal("source index preparation failed")
|
|
}
|
|
for attempts := 0; ; attempts++ {
|
|
err := q.Acquire(ctx)
|
|
if err == nil {
|
|
break
|
|
}
|
|
if *once || !errors.Is(err, store.ErrQueryReplicaLease) {
|
|
log.Fatal("projector lease unavailable")
|
|
}
|
|
if *safetyRun == "" && attempts >= 35 {
|
|
log.Fatal("projector lease unavailable")
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(time.Second):
|
|
}
|
|
}
|
|
if *initializeTarget {
|
|
return
|
|
}
|
|
if *safetyRun != "" {
|
|
mode := ""
|
|
if *maintenance {
|
|
mode = store.QueryReplicaMaintenanceInitialization
|
|
}
|
|
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{Mode: mode, MaintenanceUntil: maintenanceDeadline, BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
|
|
log.Print("shadow safety paused; inspect durable state and explicitly resume")
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
}
|
|
workCtx := ctx
|
|
if *safetyRun != "" {
|
|
guarded, cancel := context.WithCancel(ctx)
|
|
defer cancel()
|
|
workCtx = guarded
|
|
go func() {
|
|
tick := time.NewTicker(5 * time.Second)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-guarded.Done():
|
|
return
|
|
case <-tick.C:
|
|
short, stop := context.WithTimeout(guarded, 3*time.Second)
|
|
err := q.CheckSafety(short)
|
|
stop()
|
|
if err != nil {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
pause := func(reason string) {
|
|
short, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if q.PauseSafety(short, reason) != nil {
|
|
log.Print("pause persistence unavailable; startup fencing remains required")
|
|
}
|
|
}
|
|
defer func() {
|
|
if ctx.Err() != nil {
|
|
pause("operator_stop")
|
|
}
|
|
}()
|
|
if q.EnsureTargetIndexes(workCtx) != nil {
|
|
pause("target_error")
|
|
log.Print("query index preparation paused")
|
|
if *safetyRun == "" {
|
|
os.Exit(1)
|
|
}
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
consecutiveFailures := 0
|
|
for workCtx.Err() == nil {
|
|
started := time.Now()
|
|
ready, e := q.RunOnce(workCtx, *bootstrap)
|
|
if e != nil {
|
|
if ctx.Err() != nil {
|
|
return // The deferred operator_stop records the intentional shutdown.
|
|
}
|
|
d := store.QueryReplicaDiagnostic(e)
|
|
log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode)
|
|
if *safetyRun == "" && !*once {
|
|
if !daemonRetryable(e) {
|
|
log.Print("projector stopped on hard failure; checkpoint retained")
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
wait := daemonRetryDelay(consecutiveFailures)
|
|
consecutiveFailures++
|
|
timer := time.NewTimer(wait)
|
|
select {
|
|
case <-workCtx.Done():
|
|
timer.Stop()
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
if errors.Is(e, store.ErrQueryReplicaLease) {
|
|
_ = q.Acquire(workCtx)
|
|
}
|
|
continue
|
|
}
|
|
short, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
|
if q.PauseSafetyFailure(short, e) != nil {
|
|
log.Print("pause diagnostics persistence unavailable; startup fencing remains required")
|
|
}
|
|
stop()
|
|
log.Print("projector paused; checkpoint retained (no customer payload logged)")
|
|
if *safetyRun != "" {
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
os.Exit(1)
|
|
}
|
|
consecutiveFailures = 0
|
|
if *once {
|
|
if !ready {
|
|
pause("incomplete_pilot")
|
|
log.Print("projection incomplete; resume required")
|
|
os.Exit(2)
|
|
}
|
|
return
|
|
}
|
|
// Optional pacing covers backfill too. Idle lease maintenance only
|
|
// touches the independent query DB, never scans authority for keepalive.
|
|
delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass)
|
|
if *safetyRun == "" {
|
|
if cool := adaptiveBackfillCooldown(ready, time.Since(started)); cool > delay {
|
|
delay = cool
|
|
}
|
|
}
|
|
if delay > 0 {
|
|
if e := q.WaitNextPass(workCtx, delay); e != nil {
|
|
if ctx.Err() == nil {
|
|
if *safetyRun == "" && !*once {
|
|
log.Print("projector stopped on lease/persistence failure; checkpoint retained")
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
pause("target_error")
|
|
log.Print("projector lease unavailable; checkpoint retained")
|
|
if *safetyRun != "" {
|
|
<-ctx.Done()
|
|
return
|
|
}
|
|
os.Exit(1)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
if *safetyRun != "" && ctx.Err() == nil {
|
|
pause("operation_error")
|
|
<-ctx.Done()
|
|
}
|
|
}
|
|
|
|
// Source compression changes only the finite maintenance transport. The driver
|
|
// negotiates supported codecs and falls back to uncompressed messages.
|
|
func sourceCompressors(maintenance bool) []string {
|
|
if maintenance {
|
|
return []string{"snappy", "zstd"}
|
|
}
|
|
return []string{}
|
|
}
|