382 lines
13 KiB
Go

// query-replica is a rate-limited, independently deployed projector. It never
// claims tasks, repairs annotations, or starts business background workers.
package main
import (
"context"
"crypto/sha256"
"droboticflow/internal/store"
"encoding/json"
"errors"
"flag"
"fmt"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readconcern"
"go.mongodb.org/mongo-driver/mongo/readpref"
"io"
"log"
"os"
"os/signal"
"syscall"
"time"
)
func main() {
bootstrap := flag.Bool("bootstrap", false, "explicitly allow bounded resumable historical copy")
ensure := flag.Bool("ensure-source-indexes", false, "explicit source index writes; requires production authorization")
writers := flag.Bool("writers-covered", false, "attest all source writers use atomic query-source/v1 markers")
once := flag.Bool("once", false, "one bounded pass only")
interval := flag.Duration("interval", time.Second, "minimum delay between bounded passes")
batchSize := flag.Int("batch-size", 200, "maximum source records per collection batch (1-200)")
minimumPass := flag.Duration("min-pass-interval", 0, "optional minimum start-to-start pass interval, including backfill (0-1m)")
safetyRun := flag.String("safety-run", "", "explicit durable shadow run identity; requires global transport and health budgets")
maxRecords := flag.Int64("max-source-records", 1000, "durable total read reservations;0 explicitly permits full backfill")
rowsRate := flag.Int("source-records-per-second", 10, "global source row pacing")
bytesRate := flag.Int("source-bytes-per-second", 65536, "shared actual source transport read budget")
commandsRate := flag.Int("source-commands-per-second", 2, "shared source socket-write budget, includes control overhead")
maintenanceUntil := flag.String("maintenance-until", "", "explicit RFC3339 deadline for the initialization-only CPU observation policy")
maintenance := flag.Bool("maintenance-initialization", false, "explicit finite initialization run: CPU is observed, not an admission/stop gate; resource and durability guards remain")
baseline := flag.Float64("source-cpu-baseline", 0, "fixed reviewed4core source CPU baseline, never increased on resume")
resume := flag.String("resume-pause", "", "exact current pause nonce after operator review")
initializeTarget := flag.Bool("initialize-target", false, "verify independent topology and bind target only; no source scans or indexes")
status := flag.Bool("safety-status", false, "target-only private safety status")
sampleSource := flag.Bool("sample-source", false, "readonly server counters for trusted observer, no customer records")
observation := flag.String("record-observation", "", "trusted observer JSON file or stdin '-'; target-only, never connects to source")
sourceCompress := flag.Bool("source-compress", false, "negotiate snappy/zstd wire compression on unguarded source reads")
flag.Parse()
var maintenanceDeadline *time.Time
if *maintenance {
value, err := time.Parse(time.RFC3339, *maintenanceUntil)
if err != nil || !value.After(time.Now()) {
log.Fatal("explicit future maintenance deadline required")
}
maintenanceDeadline = &value
}
if *maintenance && (*safetyRun == "" || *maxRecords <= 0) {
log.Fatal("maintenance initialization requires a guarded finite budget")
}
if *sampleSource {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if os.Getenv("MONGO_URI") == "" {
log.Fatal("source binding required")
}
c, e := mongo.Connect(ctx, options.Client().ApplyURI(os.Getenv("MONGO_URI")).SetMaxPoolSize(1))
if e != nil {
log.Fatal("source unavailable")
}
defer c.Disconnect(context.Background())
var info struct {
Uptime int64 `bson:"uptimeMillis"`
Memory struct {
Resident int64 `bson:"resident"`
Supported bool `bson:"supported"`
} `bson:"mem"`
Extra struct {
User int64 `bson:"user_time_us"`
System int64 `bson:"system_time_us"`
} `bson:"extra_info"`
}
if c.Database("admin").RunCommand(ctx, map[string]any{"serverStatus": 1}).Decode(&info) != nil {
log.Fatal("source counters unavailable")
}
var hello struct {
Topology struct {
Process any `bson:"processId"`
} `bson:"topologyVersion"`
}
if c.Database("admin").RunCommand(ctx, map[string]any{"hello": 1}).Decode(&hello) != nil {
log.Fatal("source identity unavailable")
}
identity := sha256.Sum256([]byte(fmt.Sprint(hello.Topology.Process)))
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"uptimeMillis": info.Uptime, "cpuMicroseconds": info.Extra.User + info.Extra.System, "sourceProcess": fmt.Sprintf("%x", identity), "sampledAt": time.Now().UTC().Format(time.RFC3339Nano), "residentMiB": info.Memory.Resident, "memorySupported": info.Memory.Supported})
return
}
if *observation != "" || *status {
var value store.QueryReplicaSafetyObservation
if !*status {
input := io.Reader(os.Stdin)
if *observation != "-" {
f, e := os.Open(*observation)
if e != nil {
log.Fatal("observation unavailable")
}
defer f.Close()
input = f
}
decoder := json.NewDecoder(io.LimitReader(input, 16384))
decoder.DisallowUnknownFields()
if decoder.Decode(&value) != nil {
log.Fatal("invalid observation")
}
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
uri, db := os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB")
if uri == "" || db == "" || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
log.Fatal("explicit independent target required")
}
c, e := mongo.Connect(ctx, options.Client().ApplyURI(uri).SetMaxPoolSize(1))
if e != nil {
log.Fatal("target unavailable")
}
defer c.Disconnect(context.Background())
if *status {
value, e := store.NewMongoStore(c.Database(db)).QueryReplicaSafetyStatus(ctx)
if e != nil {
log.Fatal("safety state unavailable")
}
_ = json.NewEncoder(os.Stdout).Encode(value)
return
}
if store.NewMongoStore(c.Database(db)).RecordQueryReplicaObservation(ctx, value) != nil {
log.Fatal("observation rejected")
}
return
}
var transport *store.QueryReplicaTransport
if *safetyRun != "" {
var e error
transport, e = store.NewQueryReplicaTransport(*bytesRate, *commandsRate)
if e != nil || *ensure {
log.Fatal("invalid guarded source configuration")
}
} else if *resume != "" {
log.Fatal("resume requires durable safety")
}
if !validPacing(*batchSize, *minimumPass) {
log.Fatal("invalid bounded projector pacing")
}
mode := os.Getenv("QUERY_REPLICA_REFRESH_MODE")
if mode != "" && mode != "continuous" && mode != "periodic" {
log.Fatal("invalid query refresh mode")
}
if mode == "periodic" {
explicit := false
flag.Visit(func(f *flag.Flag) {
if f.Name == "interval" {
explicit = true
}
})
if !explicit {
*interval = store.QueryPeriodicSyncInterval
}
}
if *initializeTarget && *ensure {
log.Fatal("source index writes forbidden during target binding")
}
if !*writers || *interval < 100*time.Millisecond || *interval > 5*time.Minute || os.Getenv("QUERY_SERVING_NAMESPACE") == "" {
log.Fatal("writer coverage, namespace and bounded polling interval required")
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
connect := func(uri, db string, target bool) (*mongo.Client, *store.MongoStore) {
if uri == "" || db == "" {
log.Fatal("explicit source/query connection required")
}
opts := options.Client().ApplyURI(uri).SetMaxPoolSize(4).SetServerSelectionTimeout(5 * time.Second)
if !target && transport == nil {
opts.SetMaxPoolSize(2)
if *sourceCompress {
opts.SetCompressors([]string{"snappy", "zstd"})
}
}
if !target && transport != nil {
opts.SetMaxPoolSize(1).SetDialer(transport).SetCompressors(sourceCompressors(*maintenance)).SetRetryReads(false).SetTimeout(5 * time.Second)
}
if target {
opts.SetReadPreference(readpref.Primary()).SetReadConcern(readconcern.Majority())
}
c, e := mongo.Connect(ctx, opts)
if e != nil {
log.Fatal("database connection unavailable")
}
return c, store.NewMongoStore(c.Database(db))
}
src, source := connect(os.Getenv("MONGO_URI"), os.Getenv("MONGO_DB"), false)
defer src.Disconnect(context.Background())
dst, target := connect(os.Getenv("QUERY_READ_MONGO_URI"), os.Getenv("QUERY_READ_MONGO_DB"), true)
defer dst.Disconnect(context.Background())
q, e := store.NewQueryReplica(source, target)
if e != nil {
log.Fatal("invalid projector configuration")
}
q.BatchSize = *batchSize
if q.VerifyTopology(ctx) != nil {
log.Fatal("source/query databases are not independently usable")
}
if *ensure && q.EnsureSourceIndexes(ctx) != nil {
log.Fatal("source index preparation failed")
}
for attempts := 0; ; attempts++ {
err := q.Acquire(ctx)
if err == nil {
break
}
if *once || !errors.Is(err, store.ErrQueryReplicaLease) {
log.Fatal("projector lease unavailable")
}
if *safetyRun == "" && attempts >= 35 {
log.Fatal("projector lease unavailable")
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
if *initializeTarget {
return
}
if *safetyRun != "" {
mode := ""
if *maintenance {
mode = store.QueryReplicaMaintenanceInitialization
}
if q.ConfigureSafety(ctx, store.QueryReplicaSafetyPolicy{Mode: mode, MaintenanceUntil: maintenanceDeadline, BytesPerSecond: *bytesRate, CommandsPerSecond: *commandsRate, MaxBatchRows: *batchSize, RunID: *safetyRun, MaxRecords: *maxRecords, RecordsPerSecond: *rowsRate, BaselineCPU: *baseline}, *resume) != nil {
log.Print("shadow safety paused; inspect durable state and explicitly resume")
<-ctx.Done()
return
}
}
workCtx := ctx
if *safetyRun != "" {
guarded, cancel := context.WithCancel(ctx)
defer cancel()
workCtx = guarded
go func() {
tick := time.NewTicker(5 * time.Second)
defer tick.Stop()
for {
select {
case <-guarded.Done():
return
case <-tick.C:
short, stop := context.WithTimeout(guarded, 3*time.Second)
err := q.CheckSafety(short)
stop()
if err != nil {
cancel()
return
}
}
}
}()
}
pause := func(reason string) {
short, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if q.PauseSafety(short, reason) != nil {
log.Print("pause persistence unavailable; startup fencing remains required")
}
}
defer func() {
if ctx.Err() != nil {
pause("operator_stop")
}
}()
if q.EnsureTargetIndexes(workCtx) != nil {
pause("target_error")
log.Print("query index preparation paused")
if *safetyRun == "" {
os.Exit(1)
}
<-ctx.Done()
return
}
consecutiveFailures := 0
for workCtx.Err() == nil {
started := time.Now()
ready, e := q.RunOnce(workCtx, *bootstrap)
if e != nil {
if ctx.Err() != nil {
return // The deferred operator_stop records the intentional shutdown.
}
d := store.QueryReplicaDiagnostic(e)
log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode)
if *safetyRun == "" && !*once {
if !daemonRetryable(e) {
log.Print("projector stopped on hard failure; checkpoint retained")
<-ctx.Done()
return
}
wait := daemonRetryDelay(consecutiveFailures)
consecutiveFailures++
timer := time.NewTimer(wait)
select {
case <-workCtx.Done():
timer.Stop()
return
case <-timer.C:
}
if errors.Is(e, store.ErrQueryReplicaLease) {
_ = q.Acquire(workCtx)
}
continue
}
short, stop := context.WithTimeout(context.Background(), 5*time.Second)
if q.PauseSafetyFailure(short, e) != nil {
log.Print("pause diagnostics persistence unavailable; startup fencing remains required")
}
stop()
log.Print("projector paused; checkpoint retained (no customer payload logged)")
if *safetyRun != "" {
<-ctx.Done()
return
}
os.Exit(1)
}
consecutiveFailures = 0
if *once {
if !ready {
pause("incomplete_pilot")
log.Print("projection incomplete; resume required")
os.Exit(2)
}
return
}
// Optional pacing covers backfill too. Idle lease maintenance only
// touches the independent query DB, never scans authority for keepalive.
delay := passDelay(mode, ready, time.Since(started), *interval, *minimumPass)
if *safetyRun == "" {
if cool := adaptiveBackfillCooldown(ready, time.Since(started)); cool > delay {
delay = cool
}
}
if delay > 0 {
if e := q.WaitNextPass(workCtx, delay); e != nil {
if ctx.Err() == nil {
if *safetyRun == "" && !*once {
log.Print("projector stopped on lease/persistence failure; checkpoint retained")
<-ctx.Done()
return
}
pause("target_error")
log.Print("projector lease unavailable; checkpoint retained")
if *safetyRun != "" {
<-ctx.Done()
return
}
os.Exit(1)
}
return
}
}
}
if *safetyRun != "" && ctx.Err() == nil {
pause("operation_error")
<-ctx.Done()
}
}
// Source compression changes only the finite maintenance transport. The driver
// negotiates supported codecs and falls back to uncompressed messages.
func sourceCompressors(maintenance bool) []string {
if maintenance {
return []string{"snappy", "zstd"}
}
return []string{}
}