🐛 Trim run input reads before throttled projection

This commit is contained in:
d-robotics 2026-11-14 22:07:22 -08:00
parent 1082d0b527
commit b64d950f64
10 changed files with 380 additions and 16 deletions

View File

@ -280,6 +280,11 @@ func main() {
started := time.Now()
ready, e := q.RunOnce(workCtx, *bootstrap)
if e != nil {
if ctx.Err() != nil {
return // The deferred operator_stop records the intentional shutdown.
}
d := store.QueryReplicaDiagnostic(e)
log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode)
pause("operation_error")
log.Print("projector paused; checkpoint retained (no customer payload logged)")
if *safetyRun != "" {

View File

@ -12,6 +12,8 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
)
@ -138,6 +140,49 @@ func TestProjectorGuardedProcessRestartLocal(t *testing.T) {
if json.Unmarshal(out, &v) != nil || v["paused"] != true {
t.Fatal("operator status invalid")
}
// Resume the same guarded run, then stop it while it is actively reading.
// SIGTERM must persist operator_stop rather than a misleading operation_error.
if e = d.RecordQueryReplicaObservation(ctx, store.QueryReplicaSafetyObservation{
At: time.Now().UTC(), CPU: 99, Ready: true, WritersCovered: true,
HostFreeBytes: 120 << 30, TargetUsedFraction: .1,
SafetySignalsVersion: 1, SourceAvailable: true,
}); e != nil {
t.Fatal("maintenance health unavailable")
}
_, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
for i, arg := range args {
if strings.HasPrefix(arg, "--max-source-records=") {
args[i] = "--max-source-records=10000"
}
}
args = append(args, "--resume-pause="+after["resumeToken"].(string), "--maintenance-initialization", "--maintenance-until="+time.Now().UTC().Add(10*time.Minute).Format(time.RFC3339))
active, stopped := start()
deadline = time.Now().Add(15 * time.Second)
for time.Now().Before(deadline) {
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
if after["paused"] == false && after["reserved"].(int64) > 2 {
break
}
time.Sleep(50 * time.Millisecond)
}
if after["paused"] != false || after["reserved"].(int64) <= 2 {
t.Fatal("same-run maintenance resume did not preserve and advance budget")
}
if e = active.Process.Signal(syscall.SIGTERM); e != nil {
t.Fatal("could not signal isolated process")
}
select {
case e = <-stopped:
if e != nil {
t.Fatal("graceful stop failed")
}
case <-time.After(6 * time.Second):
t.Fatal("graceful stop did not finish")
}
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
if after["paused"] != true || after["reason"] != "operator_stop" || after["reserved"].(int64) < 2 {
t.Fatal("intentional stop misclassified or reset budget")
}
count, e := source.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"privateDraft": "not projected"})
if e != nil || count != 3 {
t.Fatal("source payload changed")

View File

@ -244,7 +244,7 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
parent := ctx
limit, e := q.sourcePermit(ctx, q.limit())
if e != nil {
return nil, nil, e
return nil, nil, queryReplicaFailure(n, "source_permit", e)
}
ctx, cancel := q.sourceContext(ctx)
defer cancel()
@ -256,7 +256,7 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
}
cur, e := q.Source.db.Collection(n).Find(ctx, filter, opts)
if e != nil {
return nil, nil, e
return nil, nil, queryReplicaFailure(n, "source_find", e)
}
defer cur.Close(ctx)
records := []queryReplicaRecord{}
@ -264,20 +264,20 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
for cur.Next(ctx) {
var d bson.M
if e = cur.Decode(&d); e != nil {
return nil, nil, e
return nil, nil, queryReplicaFailure(n, "source_decode", e)
}
last = d["_id"]
r, e := queryExportRecord(n, d)
if e != nil {
return nil, nil, e
return nil, nil, queryReplicaFailure(n, "source_export", e)
}
records = append(records, r)
}
if e := cur.Err(); e != nil {
return nil, nil, e
return nil, nil, queryReplicaFailure(n, "source_cursor", e)
}
if e := q.paceRows(parent, len(records)); e != nil {
return nil, nil, e
return nil, nil, queryReplicaFailure(n, "row_pacing", e)
}
return records, last, nil
}
@ -297,7 +297,7 @@ func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []quer
if len(models) == 0 {
return nil
}
return q.sourceMetadataBulk(ctx, n, models)
return queryReplicaFailure(n, "source_ack", q.sourceMetadataBulk(ctx, n, models))
}
// BootstrapStep is bounded and resumable. $expr uses BSON total ordering across
@ -309,7 +309,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int,
}
cp, e := q.checkpoint(ctx, n)
if e != nil || cp.Complete {
return cp.Complete, 0, e
return cp.Complete, 0, queryReplicaFailure(n, "checkpoint", e)
}
filter := bson.M{}
if cp.HasLast || cp.Last != nil {
@ -333,7 +333,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int,
return e
})
if e != nil {
return false, 0, e
return false, 0, queryReplicaFailure(n, "target_apply", e)
}
return done, len(rows), q.acknowledge(ctx, n, rows)
}
@ -367,7 +367,7 @@ func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) {
}
e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) })
if e != nil {
return 0, e
return 0, queryReplicaFailure(n, "target_apply", e)
}
return len(rows), q.acknowledge(ctx, n, rows)
}

View File

@ -0,0 +1,95 @@
package store
import (
"context"
"errors"
"go.mongodb.org/mongo-driver/bson/bsoncodec"
"go.mongodb.org/mongo-driver/mongo"
)
// QueryReplicaFailureDiagnostic deliberately excludes error text, filters,
// identifiers and values: driver errors may contain credentials or documents.
type QueryReplicaFailureDiagnostic struct {
Collection string `json:"collection"`
Phase string `json:"phase"`
Class string `json:"class"`
MongoCode int32 `json:"mongoCode"`
}
type queryReplicaStepError struct {
collection, phase string
cause error
}
func (e *queryReplicaStepError) Unwrap() error { return e.cause }
func (e *queryReplicaStepError) Error() string {
d := QueryReplicaDiagnostic(e)
return "query replica " + d.Collection + "/" + d.Phase + ": " + d.Class
}
func queryReplicaFailure(collection, phase string, err error) error {
if err == nil {
return nil
}
var existing *queryReplicaStepError
if errors.As(err, &existing) {
return err
}
if queryReplicaFields[collection] == "" && collection != querySyncDeleteCollection && collection != queryReplicaControl {
collection = "unknown"
}
switch phase {
case "source_permit", "source_find", "source_decode", "source_export", "source_cursor", "row_pacing", "source_ack", "target_apply", "checkpoint", "backlog", "schedule", "publication", "delete_reconcile":
default:
phase = "operation"
}
return &queryReplicaStepError{collection, phase, err}
}
func QueryReplicaDiagnostic(err error) QueryReplicaFailureDiagnostic {
d := QueryReplicaFailureDiagnostic{Collection: "unknown", Phase: "operation", Class: "other"}
var step *queryReplicaStepError
if errors.As(err, &step) {
d.Collection, d.Phase = step.collection, step.phase
}
var command mongo.CommandError
var write mongo.WriteException
var bulk mongo.BulkWriteException
var decode *bsoncodec.DecodeError
switch {
case errors.Is(err, context.Canceled):
d.Class = "cancelled"
case errors.Is(err, context.DeadlineExceeded):
d.Class = "deadline"
case errors.Is(err, ErrQueryReplicaPaused):
d.Class = "safety_pause"
case errors.Is(err, ErrQueryReplicaLease):
d.Class = "lease"
case errors.Is(err, ErrQueryReplicaUnsupported):
d.Class = "unsupported_fact"
case errors.As(err, &command):
d.Class, d.MongoCode = "mongo_command", command.Code
case errors.As(err, &write):
d.Class = "mongo_write"
if len(write.WriteErrors) > 0 {
d.MongoCode = int32(write.WriteErrors[0].Code)
} else if write.WriteConcernError != nil {
d.MongoCode = int32(write.WriteConcernError.Code)
}
case errors.As(err, &bulk):
d.Class = "mongo_bulk_write"
if len(bulk.WriteErrors) > 0 {
d.MongoCode = int32(bulk.WriteErrors[0].Code)
} else if bulk.WriteConcernError != nil {
d.MongoCode = int32(bulk.WriteConcernError.Code)
}
case errors.As(err, &decode):
d.Class = "bson_decode"
case mongo.IsNetworkError(err):
d.Class = "network"
case mongo.IsTimeout(err):
d.Class = "timeout"
}
return d
}

View File

@ -0,0 +1,68 @@
package store
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
)
func TestQueryReplicaDiagnosticPreservesCauseWithoutLeaking(t *testing.T) {
const private = "mongodb://private-user:private-password@private-host/customer?signature=private-token"
var run domain.RunRecord
b, _ := bson.Marshal(bson.M{"inputs": private})
decodeErr := bson.Unmarshal(b, &run)
if decodeErr == nil {
t.Fatal("missing BSON decoder fixture")
}
cases := []struct {
cause error
class string
code int32
}{
{mongo.CommandError{Message: private, Wrapped: context.DeadlineExceeded}, "deadline", 0},
{context.Canceled, "cancelled", 0},
{fmt.Errorf("%s: %w", private, ErrQueryReplicaPaused), "safety_pause", 0},
{ErrQueryReplicaLease, "lease", 0},
{fmt.Errorf("%s: %w", private, ErrQueryReplicaUnsupported), "unsupported_fact", 0},
{mongo.CommandError{Code: 50, Message: private}, "mongo_command", 50},
{mongo.WriteException{WriteErrors: mongo.WriteErrors{{Code: 11000, Message: private}}}, "mongo_write", 11000},
{mongo.BulkWriteException{WriteErrors: []mongo.BulkWriteError{{WriteError: mongo.WriteError{Code: 11000, Message: private}}}}, "mongo_bulk_write", 11000},
{decodeErr, "bson_decode", 0},
{errors.New(private), "other", 0},
}
for _, test := range cases {
err := queryReplicaFailure("runs", "source_find", test.cause)
if !reflect.DeepEqual(errors.Unwrap(err), test.cause) {
t.Fatal("error identity lost")
}
if test.class == "deadline" && !errors.Is(err, context.DeadlineExceeded) {
t.Fatal("deadline unwrapping lost")
}
err = queryReplicaFailure("runs", "target_apply", err)
d := QueryReplicaDiagnostic(err)
if d.Collection != "runs" || d.Phase != "source_find" || d.Class != test.class || d.MongoCode != test.code {
t.Fatalf("unexpected safe classification: %+v", d)
}
raw, _ := json.Marshal(d)
for _, value := range []string{private, "private-password", "private-host", "signature"} {
if strings.Contains(string(raw), value) || strings.Contains(err.Error(), value) {
t.Fatal("raw cause leaked")
}
}
}
if queryReplicaFailure("runs", "source_find", nil) != nil {
t.Fatal("nil became failure")
}
d := QueryReplicaDiagnostic(queryReplicaFailure(private, private, errors.New(private)))
if d.Collection != "unknown" || d.Phase != "operation" {
t.Fatal("uncontrolled diagnostic labels")
}
}

View File

@ -25,11 +25,38 @@ type queryReplicaRecord struct {
func querySourceProjection(collection string) bson.M {
p := queryReplicaProjection(collection)
if collection == "runs" {
// Only these references participate in legacy search names. Selecting the
// whole input also transfers inline payloads which export discards later;
// at a bounded wire rate that can exhaust the fixed source read deadline.
p["inputs"] = queryRunInputReferencesExpression()
}
if collection == "humanItems" {
p["__legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression(collection), bson.M{}, "$subject.payload"}}
}
return p
}
func queryRunInputReferencesExpression() bson.M {
object := func(value string) bson.M {
return bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": value}, "object"}}, value, bson.M{}}}
}
selected := bson.M{"$arrayToObject": bson.M{"$filter": bson.M{
"input": bson.M{"$objectToArray": object("$$input.v.ref")},
"as": "field",
"cond": bson.M{"$in": bson.A{"$$field.k", bson.A{"captureUnit", "prefix", "key", "manifestKey"}}},
}}}
ref := bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$$input.v.ref"}, "object"}}, selected, bson.M{"$ifNull": bson.A{"$$input.v.ref", bson.M{}}}}}
value := bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$$input.v"}, "object"}}, bson.M{"ref": ref}, "$$input.v"}}
inputs := bson.M{"$arrayToObject": bson.M{"$map": bson.M{
"input": bson.M{"$objectToArray": object("$inputs")},
"as": "input",
"in": bson.M{"k": "$$input.k", "v": value},
}}}
// Preserve absent/null/non-object outer values; do not silently invent a
// valid record for an unsupported historical input-container shape.
return bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$inputs"}, "object"}}, inputs, "$inputs"}}
}
func safeReplicaNames(d any) bool {
switch value := d.(type) {
case string:

View File

@ -0,0 +1,121 @@
package store
import (
"context"
"errors"
"fmt"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"reflect"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestQueryReplicaRunsProjectionFitsThrottledDeadlineLocal(t *testing.T) {
source := newQuerySyncSource(t)
payload := strings.Repeat("synthetic-inline-private-not-query-fact", 1024)
docs := []any{}
for i := 0; i < 50; i++ {
docs = append(docs, bson.M{"_id": fmt.Sprintf("physical-%03d", i), "id": fmt.Sprintf("run-%03d", i), "organizationId": "org", "projectId": "project", "status": "completed", "captureDirectories": bson.A{"capture/part0001"}, "inputs": bson.M{"source": bson.M{"mode": "reference", "inlineValue": payload, "ref": bson.M{"captureUnit": "capture/part0001", "key": "capture/part0001/left.mp4", "unrelated": "not needed"}}}})
}
if _, err := source.db.Collection("runs").InsertMany(t.Context(), docs); err != nil {
t.Fatal(err)
}
transport, err := NewQueryReplicaTransport(128*1024, 5)
if err != nil {
t.Fatal(err)
}
var replyBytes atomic.Int64
monitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
if e.CommandName == "find" {
replyBytes.Add(int64(len(e.Reply)))
}
}}
client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5*time.Second).SetMonitor(monitor))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = client.Disconnect(ctx)
})
q := &QueryReplica{Source: NewMongoStore(client.Database(source.db.Name())), BatchSize: 50}
start := time.Now()
rows, _, err := q.read(t.Context(), "runs", bson.M{})
elapsed := time.Since(start)
if err != nil {
var command mongo.CommandError
errors.As(err, &command)
t.Fatalf("unneeded inline input blocks fixed source deadline at128KiB/s: class=%T mongoCode=%d contextDeadline=%t timeout=%t", err, command.Code, errors.Is(err, context.DeadlineExceeded), mongo.IsTimeout(err))
}
if len(rows) != 50 {
t.Fatalf("rows=%d, wanted50", len(rows))
}
if elapsed >= 5*time.Second || replyBytes.Load() > 100*1024 {
t.Fatalf("response should contain references only: elapsed=%s replyBytes=%d", elapsed, replyBytes.Load())
}
for _, r := range rows {
raw, _ := bson.Marshal(r.Document)
if strings.Contains(string(raw), "synthetic-inline-private") || strings.Contains(string(raw), "not needed") {
t.Fatal("non-query payload leaked")
}
}
var unchanged bson.M
if err := source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "physical-000"}).Decode(&unchanged); err != nil {
t.Fatal(err)
}
if deletionDoc(deletionDoc(unchanged["inputs"])["source"])["inlineValue"] != payload {
t.Fatal("source content changed")
}
t.Logf("same50 rows at128KiB/s, fixed5s: elapsed=%s responseBytes=%d", elapsed, replyBytes.Load())
}
func TestQueryReplicaRunReferenceProjectionCompatibilityLocal(t *testing.T) {
source := newQuerySyncSource(t)
cases := []struct {
name string
inputs any
absent bool
}{
{name: "absent", absent: true},
{name: "null", inputs: nil},
{name: "empty", inputs: bson.M{}},
{name: "references-and-inline", inputs: bson.M{"source": bson.M{"inlineValue": "https://private.invalid/data?signature=do-not-export", "ref": bson.M{"captureUnit": "capture/part0001", "prefix": nil, "key": "capture/part0001/left.mp4", "manifestKey": "capture/manifest.json", "unrelated": "do-not-export"}}}},
{name: "no-reference", inputs: bson.M{"source": bson.M{"inlineValue": "legacy-query-irrelevant"}}},
{name: "null-reference", inputs: bson.M{"source": bson.M{"ref": nil}}},
{name: "unsupported-container", inputs: bson.A{"legacy"}},
{name: "unsupported-input-value", inputs: bson.M{"source": "legacy"}},
{name: "unsupported-ref", inputs: bson.M{"source": bson.M{"ref": "legacy"}}},
}
for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
doc := bson.M{"_id": test.name, "id": test.name, "status": "failed", "error": "meta 与 PTS", "artifacts": bson.A{bson.M{"id": "artifact", "name": "capture/part0001"}}}
if !test.absent {
doc["inputs"] = test.inputs
}
if _, err := source.db.Collection("runs").InsertOne(t.Context(), doc); err != nil {
t.Fatal(err)
}
read := func(projection bson.M) bson.M {
var d bson.M
if err := source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": test.name}, options.FindOne().SetProjection(projection)).Decode(&d); err != nil {
t.Fatal(err)
}
return d
}
before, oldErr := queryExportRecord("runs", read(queryReplicaProjection("runs")))
after, newErr := queryExportRecord("runs", read(querySourceProjection("runs")))
if (oldErr == nil) != (newErr == nil) {
t.Fatalf("changed support of historical shape: before=%T after=%T", oldErr, newErr)
}
if oldErr == nil && !reflect.DeepEqual(before.Document, after.Document) {
t.Fatal("legacy search names, metadata or exception category changed")
}
})
}
}

View File

@ -51,7 +51,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
}
e := q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor)
if e != nil && e != mongo.ErrNoDocuments {
return false, e
return false, queryReplicaFailure(queryReplicaControl, "schedule", e)
}
start := cursor.Next % len(names)
if start < 0 {
@ -66,7 +66,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
}
age, e := q.backlogAge(ctx, name, n)
if e != nil {
return false, e
return false, queryReplicaFailure(name, "backlog", e)
}
if age > 5*time.Minute {
for extra := 0; extra < 3; extra++ {
@ -84,7 +84,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
}
cp, e := q.checkpoint(ctx, name)
if e != nil {
return false, e
return false, queryReplicaFailure(name, "checkpoint", e)
}
if !cp.Complete {
if !bootstrap {
@ -98,7 +98,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
_, e := q.control().UpdateOne(sc, bson.M{"_id": "schedule"}, bson.M{"$set": bson.M{"next": (index + 1) % len(names)}}, options.Update().SetUpsert(true))
return e
}); e != nil {
return false, e
return false, queryReplicaFailure(name, "schedule", e)
}
}
complete := true
@ -120,7 +120,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady {
return false, nil
} else if e != nil {
return false, e
return false, queryReplicaFailure(queryReplicaControl, "publication", e)
}
return true, nil
}

View File

@ -20,6 +20,8 @@ CASES = ('cmd/query-replica/main_local_test.go',
'internal/store/query_replica_safety_local_test.go',
'internal/store/query_replica_maintenance_flow_local_test.go',
'internal/store/query_replica_safety_test.go',
'internal/store/query_replica_diagnostic_test.go',
'internal/store/query_replica_projection_local_test.go',
'internal/store/query_replica_local_test.go',
'internal/store/query_sync_local_test.go',
'internal/store/query_replica_count_cache_test.go',

View File

@ -1065,7 +1065,8 @@
"^TestQueryReplicaCandidateCache",
"^TestQueryReplicaCountOnly",
"^TestQueryReplicaMissingConfiguration",
"^TestQueryReplicaRejectsSigned"
"^TestQueryReplicaRejectsSigned",
"^TestQueryReplicaDiagnostic"
],
"./cmd/query-cdc-probe": ["^TestClassification"],
"./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],