From b64d950f64329c2710813a8d7eeeb16e55a48ce6 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Sat, 14 Nov 2026 22:07:22 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Trim=20run=20input=20reads=20bef?= =?UTF-8?q?ore=20throttled=20projection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/query-replica/main.go | 5 + cmd/query-replica/main_local_test.go | 45 +++++++ internal/store/query_replica.go | 20 +-- internal/store/query_replica_diagnostic.go | 95 ++++++++++++++ .../store/query_replica_diagnostic_test.go | 68 ++++++++++ internal/store/query_replica_projection.go | 27 ++++ .../query_replica_projection_local_test.go | 121 ++++++++++++++++++ internal/store/query_replica_schedule.go | 10 +- scripts/test_query_replica.py | 2 + scripts/verification-scopes.json | 3 +- 10 files changed, 380 insertions(+), 16 deletions(-) create mode 100644 internal/store/query_replica_diagnostic.go create mode 100644 internal/store/query_replica_diagnostic_test.go create mode 100644 internal/store/query_replica_projection_local_test.go diff --git a/cmd/query-replica/main.go b/cmd/query-replica/main.go index bf8ef5c3..d1d6a2f6 100644 --- a/cmd/query-replica/main.go +++ b/cmd/query-replica/main.go @@ -280,6 +280,11 @@ func main() { started := time.Now() ready, e := q.RunOnce(workCtx, *bootstrap) if e != nil { + if ctx.Err() != nil { + return // The deferred operator_stop records the intentional shutdown. + } + d := store.QueryReplicaDiagnostic(e) + log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode) pause("operation_error") log.Print("projector paused; checkpoint retained (no customer payload logged)") if *safetyRun != "" { diff --git a/cmd/query-replica/main_local_test.go b/cmd/query-replica/main_local_test.go index cce8bf82..9830bd31 100644 --- a/cmd/query-replica/main_local_test.go +++ b/cmd/query-replica/main_local_test.go @@ -12,6 +12,8 @@ import ( "os" "os/exec" "path/filepath" + "strings" + "syscall" "testing" "time" ) @@ -138,6 +140,49 @@ func TestProjectorGuardedProcessRestartLocal(t *testing.T) { if json.Unmarshal(out, &v) != nil || v["paused"] != true { t.Fatal("operator status invalid") } + // Resume the same guarded run, then stop it while it is actively reading. + // SIGTERM must persist operator_stop rather than a misleading operation_error. + if e = d.RecordQueryReplicaObservation(ctx, store.QueryReplicaSafetyObservation{ + At: time.Now().UTC(), CPU: 99, Ready: true, WritersCovered: true, + HostFreeBytes: 120 << 30, TargetUsedFraction: .1, + SafetySignalsVersion: 1, SourceAvailable: true, + }); e != nil { + t.Fatal("maintenance health unavailable") + } + _, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}}) + for i, arg := range args { + if strings.HasPrefix(arg, "--max-source-records=") { + args[i] = "--max-source-records=10000" + } + } + args = append(args, "--resume-pause="+after["resumeToken"].(string), "--maintenance-initialization", "--maintenance-until="+time.Now().UTC().Add(10*time.Minute).Format(time.RFC3339)) + active, stopped := start() + deadline = time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + _ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after) + if after["paused"] == false && after["reserved"].(int64) > 2 { + break + } + time.Sleep(50 * time.Millisecond) + } + if after["paused"] != false || after["reserved"].(int64) <= 2 { + t.Fatal("same-run maintenance resume did not preserve and advance budget") + } + if e = active.Process.Signal(syscall.SIGTERM); e != nil { + t.Fatal("could not signal isolated process") + } + select { + case e = <-stopped: + if e != nil { + t.Fatal("graceful stop failed") + } + case <-time.After(6 * time.Second): + t.Fatal("graceful stop did not finish") + } + _ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after) + if after["paused"] != true || after["reason"] != "operator_stop" || after["reserved"].(int64) < 2 { + t.Fatal("intentional stop misclassified or reset budget") + } count, e := source.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"privateDraft": "not projected"}) if e != nil || count != 3 { t.Fatal("source payload changed") diff --git a/internal/store/query_replica.go b/internal/store/query_replica.go index 437dc23a..b3418f12 100644 --- a/internal/store/query_replica.go +++ b/internal/store/query_replica.go @@ -244,7 +244,7 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que parent := ctx limit, e := q.sourcePermit(ctx, q.limit()) if e != nil { - return nil, nil, e + return nil, nil, queryReplicaFailure(n, "source_permit", e) } ctx, cancel := q.sourceContext(ctx) defer cancel() @@ -256,7 +256,7 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que } cur, e := q.Source.db.Collection(n).Find(ctx, filter, opts) if e != nil { - return nil, nil, e + return nil, nil, queryReplicaFailure(n, "source_find", e) } defer cur.Close(ctx) records := []queryReplicaRecord{} @@ -264,20 +264,20 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que for cur.Next(ctx) { var d bson.M if e = cur.Decode(&d); e != nil { - return nil, nil, e + return nil, nil, queryReplicaFailure(n, "source_decode", e) } last = d["_id"] r, e := queryExportRecord(n, d) if e != nil { - return nil, nil, e + return nil, nil, queryReplicaFailure(n, "source_export", e) } records = append(records, r) } if e := cur.Err(); e != nil { - return nil, nil, e + return nil, nil, queryReplicaFailure(n, "source_cursor", e) } if e := q.paceRows(parent, len(records)); e != nil { - return nil, nil, e + return nil, nil, queryReplicaFailure(n, "row_pacing", e) } return records, last, nil } @@ -297,7 +297,7 @@ func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []quer if len(models) == 0 { return nil } - return q.sourceMetadataBulk(ctx, n, models) + return queryReplicaFailure(n, "source_ack", q.sourceMetadataBulk(ctx, n, models)) } // BootstrapStep is bounded and resumable. $expr uses BSON total ordering across @@ -309,7 +309,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int, } cp, e := q.checkpoint(ctx, n) if e != nil || cp.Complete { - return cp.Complete, 0, e + return cp.Complete, 0, queryReplicaFailure(n, "checkpoint", e) } filter := bson.M{} if cp.HasLast || cp.Last != nil { @@ -333,7 +333,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int, return e }) if e != nil { - return false, 0, e + return false, 0, queryReplicaFailure(n, "target_apply", e) } return done, len(rows), q.acknowledge(ctx, n, rows) } @@ -367,7 +367,7 @@ func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) { } e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) }) if e != nil { - return 0, e + return 0, queryReplicaFailure(n, "target_apply", e) } return len(rows), q.acknowledge(ctx, n, rows) } diff --git a/internal/store/query_replica_diagnostic.go b/internal/store/query_replica_diagnostic.go new file mode 100644 index 00000000..319953c4 --- /dev/null +++ b/internal/store/query_replica_diagnostic.go @@ -0,0 +1,95 @@ +package store + +import ( + "context" + "errors" + + "go.mongodb.org/mongo-driver/bson/bsoncodec" + "go.mongodb.org/mongo-driver/mongo" +) + +// QueryReplicaFailureDiagnostic deliberately excludes error text, filters, +// identifiers and values: driver errors may contain credentials or documents. +type QueryReplicaFailureDiagnostic struct { + Collection string `json:"collection"` + Phase string `json:"phase"` + Class string `json:"class"` + MongoCode int32 `json:"mongoCode"` +} + +type queryReplicaStepError struct { + collection, phase string + cause error +} + +func (e *queryReplicaStepError) Unwrap() error { return e.cause } +func (e *queryReplicaStepError) Error() string { + d := QueryReplicaDiagnostic(e) + return "query replica " + d.Collection + "/" + d.Phase + ": " + d.Class +} + +func queryReplicaFailure(collection, phase string, err error) error { + if err == nil { + return nil + } + var existing *queryReplicaStepError + if errors.As(err, &existing) { + return err + } + if queryReplicaFields[collection] == "" && collection != querySyncDeleteCollection && collection != queryReplicaControl { + collection = "unknown" + } + switch phase { + case "source_permit", "source_find", "source_decode", "source_export", "source_cursor", "row_pacing", "source_ack", "target_apply", "checkpoint", "backlog", "schedule", "publication", "delete_reconcile": + default: + phase = "operation" + } + return &queryReplicaStepError{collection, phase, err} +} + +func QueryReplicaDiagnostic(err error) QueryReplicaFailureDiagnostic { + d := QueryReplicaFailureDiagnostic{Collection: "unknown", Phase: "operation", Class: "other"} + var step *queryReplicaStepError + if errors.As(err, &step) { + d.Collection, d.Phase = step.collection, step.phase + } + var command mongo.CommandError + var write mongo.WriteException + var bulk mongo.BulkWriteException + var decode *bsoncodec.DecodeError + switch { + case errors.Is(err, context.Canceled): + d.Class = "cancelled" + case errors.Is(err, context.DeadlineExceeded): + d.Class = "deadline" + case errors.Is(err, ErrQueryReplicaPaused): + d.Class = "safety_pause" + case errors.Is(err, ErrQueryReplicaLease): + d.Class = "lease" + case errors.Is(err, ErrQueryReplicaUnsupported): + d.Class = "unsupported_fact" + case errors.As(err, &command): + d.Class, d.MongoCode = "mongo_command", command.Code + case errors.As(err, &write): + d.Class = "mongo_write" + if len(write.WriteErrors) > 0 { + d.MongoCode = int32(write.WriteErrors[0].Code) + } else if write.WriteConcernError != nil { + d.MongoCode = int32(write.WriteConcernError.Code) + } + case errors.As(err, &bulk): + d.Class = "mongo_bulk_write" + if len(bulk.WriteErrors) > 0 { + d.MongoCode = int32(bulk.WriteErrors[0].Code) + } else if bulk.WriteConcernError != nil { + d.MongoCode = int32(bulk.WriteConcernError.Code) + } + case errors.As(err, &decode): + d.Class = "bson_decode" + case mongo.IsNetworkError(err): + d.Class = "network" + case mongo.IsTimeout(err): + d.Class = "timeout" + } + return d +} diff --git a/internal/store/query_replica_diagnostic_test.go b/internal/store/query_replica_diagnostic_test.go new file mode 100644 index 00000000..28ec0daa --- /dev/null +++ b/internal/store/query_replica_diagnostic_test.go @@ -0,0 +1,68 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "droboticflow/internal/domain" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +func TestQueryReplicaDiagnosticPreservesCauseWithoutLeaking(t *testing.T) { + const private = "mongodb://private-user:private-password@private-host/customer?signature=private-token" + var run domain.RunRecord + b, _ := bson.Marshal(bson.M{"inputs": private}) + decodeErr := bson.Unmarshal(b, &run) + if decodeErr == nil { + t.Fatal("missing BSON decoder fixture") + } + cases := []struct { + cause error + class string + code int32 + }{ + {mongo.CommandError{Message: private, Wrapped: context.DeadlineExceeded}, "deadline", 0}, + {context.Canceled, "cancelled", 0}, + {fmt.Errorf("%s: %w", private, ErrQueryReplicaPaused), "safety_pause", 0}, + {ErrQueryReplicaLease, "lease", 0}, + {fmt.Errorf("%s: %w", private, ErrQueryReplicaUnsupported), "unsupported_fact", 0}, + {mongo.CommandError{Code: 50, Message: private}, "mongo_command", 50}, + {mongo.WriteException{WriteErrors: mongo.WriteErrors{{Code: 11000, Message: private}}}, "mongo_write", 11000}, + {mongo.BulkWriteException{WriteErrors: []mongo.BulkWriteError{{WriteError: mongo.WriteError{Code: 11000, Message: private}}}}, "mongo_bulk_write", 11000}, + {decodeErr, "bson_decode", 0}, + {errors.New(private), "other", 0}, + } + for _, test := range cases { + err := queryReplicaFailure("runs", "source_find", test.cause) + if !reflect.DeepEqual(errors.Unwrap(err), test.cause) { + t.Fatal("error identity lost") + } + if test.class == "deadline" && !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("deadline unwrapping lost") + } + err = queryReplicaFailure("runs", "target_apply", err) + d := QueryReplicaDiagnostic(err) + if d.Collection != "runs" || d.Phase != "source_find" || d.Class != test.class || d.MongoCode != test.code { + t.Fatalf("unexpected safe classification: %+v", d) + } + raw, _ := json.Marshal(d) + for _, value := range []string{private, "private-password", "private-host", "signature"} { + if strings.Contains(string(raw), value) || strings.Contains(err.Error(), value) { + t.Fatal("raw cause leaked") + } + } + } + if queryReplicaFailure("runs", "source_find", nil) != nil { + t.Fatal("nil became failure") + } + d := QueryReplicaDiagnostic(queryReplicaFailure(private, private, errors.New(private))) + if d.Collection != "unknown" || d.Phase != "operation" { + t.Fatal("uncontrolled diagnostic labels") + } +} diff --git a/internal/store/query_replica_projection.go b/internal/store/query_replica_projection.go index d16b2203..38e2d495 100644 --- a/internal/store/query_replica_projection.go +++ b/internal/store/query_replica_projection.go @@ -25,11 +25,38 @@ type queryReplicaRecord struct { func querySourceProjection(collection string) bson.M { p := queryReplicaProjection(collection) + if collection == "runs" { + // Only these references participate in legacy search names. Selecting the + // whole input also transfers inline payloads which export discards later; + // at a bounded wire rate that can exhaust the fixed source read deadline. + p["inputs"] = queryRunInputReferencesExpression() + } if collection == "humanItems" { p["__legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression(collection), bson.M{}, "$subject.payload"}} } return p } + +func queryRunInputReferencesExpression() bson.M { + object := func(value string) bson.M { + return bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": value}, "object"}}, value, bson.M{}}} + } + selected := bson.M{"$arrayToObject": bson.M{"$filter": bson.M{ + "input": bson.M{"$objectToArray": object("$$input.v.ref")}, + "as": "field", + "cond": bson.M{"$in": bson.A{"$$field.k", bson.A{"captureUnit", "prefix", "key", "manifestKey"}}}, + }}} + ref := bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$$input.v.ref"}, "object"}}, selected, bson.M{"$ifNull": bson.A{"$$input.v.ref", bson.M{}}}}} + value := bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$$input.v"}, "object"}}, bson.M{"ref": ref}, "$$input.v"}} + inputs := bson.M{"$arrayToObject": bson.M{"$map": bson.M{ + "input": bson.M{"$objectToArray": object("$inputs")}, + "as": "input", + "in": bson.M{"k": "$$input.k", "v": value}, + }}} + // Preserve absent/null/non-object outer values; do not silently invent a + // valid record for an unsupported historical input-container shape. + return bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$inputs"}, "object"}}, inputs, "$inputs"}} +} func safeReplicaNames(d any) bool { switch value := d.(type) { case string: diff --git a/internal/store/query_replica_projection_local_test.go b/internal/store/query_replica_projection_local_test.go new file mode 100644 index 00000000..590e6997 --- /dev/null +++ b/internal/store/query_replica_projection_local_test.go @@ -0,0 +1,121 @@ +package store + +import ( + "context" + "errors" + "fmt" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestQueryReplicaRunsProjectionFitsThrottledDeadlineLocal(t *testing.T) { + source := newQuerySyncSource(t) + payload := strings.Repeat("synthetic-inline-private-not-query-fact", 1024) + docs := []any{} + for i := 0; i < 50; i++ { + docs = append(docs, bson.M{"_id": fmt.Sprintf("physical-%03d", i), "id": fmt.Sprintf("run-%03d", i), "organizationId": "org", "projectId": "project", "status": "completed", "captureDirectories": bson.A{"capture/part0001"}, "inputs": bson.M{"source": bson.M{"mode": "reference", "inlineValue": payload, "ref": bson.M{"captureUnit": "capture/part0001", "key": "capture/part0001/left.mp4", "unrelated": "not needed"}}}}) + } + if _, err := source.db.Collection("runs").InsertMany(t.Context(), docs); err != nil { + t.Fatal(err) + } + transport, err := NewQueryReplicaTransport(128*1024, 5) + if err != nil { + t.Fatal(err) + } + var replyBytes atomic.Int64 + monitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) { + if e.CommandName == "find" { + replyBytes.Add(int64(len(e.Reply))) + } + }} + client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5*time.Second).SetMonitor(monitor)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = client.Disconnect(ctx) + }) + q := &QueryReplica{Source: NewMongoStore(client.Database(source.db.Name())), BatchSize: 50} + start := time.Now() + rows, _, err := q.read(t.Context(), "runs", bson.M{}) + elapsed := time.Since(start) + if err != nil { + var command mongo.CommandError + errors.As(err, &command) + t.Fatalf("unneeded inline input blocks fixed source deadline at128KiB/s: class=%T mongoCode=%d contextDeadline=%t timeout=%t", err, command.Code, errors.Is(err, context.DeadlineExceeded), mongo.IsTimeout(err)) + } + if len(rows) != 50 { + t.Fatalf("rows=%d, wanted50", len(rows)) + } + if elapsed >= 5*time.Second || replyBytes.Load() > 100*1024 { + t.Fatalf("response should contain references only: elapsed=%s replyBytes=%d", elapsed, replyBytes.Load()) + } + for _, r := range rows { + raw, _ := bson.Marshal(r.Document) + if strings.Contains(string(raw), "synthetic-inline-private") || strings.Contains(string(raw), "not needed") { + t.Fatal("non-query payload leaked") + } + } + var unchanged bson.M + if err := source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "physical-000"}).Decode(&unchanged); err != nil { + t.Fatal(err) + } + if deletionDoc(deletionDoc(unchanged["inputs"])["source"])["inlineValue"] != payload { + t.Fatal("source content changed") + } + t.Logf("same50 rows at128KiB/s, fixed5s: elapsed=%s responseBytes=%d", elapsed, replyBytes.Load()) +} + +func TestQueryReplicaRunReferenceProjectionCompatibilityLocal(t *testing.T) { + source := newQuerySyncSource(t) + cases := []struct { + name string + inputs any + absent bool + }{ + {name: "absent", absent: true}, + {name: "null", inputs: nil}, + {name: "empty", inputs: bson.M{}}, + {name: "references-and-inline", inputs: bson.M{"source": bson.M{"inlineValue": "https://private.invalid/data?signature=do-not-export", "ref": bson.M{"captureUnit": "capture/part0001", "prefix": nil, "key": "capture/part0001/left.mp4", "manifestKey": "capture/manifest.json", "unrelated": "do-not-export"}}}}, + {name: "no-reference", inputs: bson.M{"source": bson.M{"inlineValue": "legacy-query-irrelevant"}}}, + {name: "null-reference", inputs: bson.M{"source": bson.M{"ref": nil}}}, + {name: "unsupported-container", inputs: bson.A{"legacy"}}, + {name: "unsupported-input-value", inputs: bson.M{"source": "legacy"}}, + {name: "unsupported-ref", inputs: bson.M{"source": bson.M{"ref": "legacy"}}}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + doc := bson.M{"_id": test.name, "id": test.name, "status": "failed", "error": "meta 与 PTS", "artifacts": bson.A{bson.M{"id": "artifact", "name": "capture/part0001"}}} + if !test.absent { + doc["inputs"] = test.inputs + } + if _, err := source.db.Collection("runs").InsertOne(t.Context(), doc); err != nil { + t.Fatal(err) + } + read := func(projection bson.M) bson.M { + var d bson.M + if err := source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": test.name}, options.FindOne().SetProjection(projection)).Decode(&d); err != nil { + t.Fatal(err) + } + return d + } + before, oldErr := queryExportRecord("runs", read(queryReplicaProjection("runs"))) + after, newErr := queryExportRecord("runs", read(querySourceProjection("runs"))) + if (oldErr == nil) != (newErr == nil) { + t.Fatalf("changed support of historical shape: before=%T after=%T", oldErr, newErr) + } + if oldErr == nil && !reflect.DeepEqual(before.Document, after.Document) { + t.Fatal("legacy search names, metadata or exception category changed") + } + }) + } +} diff --git a/internal/store/query_replica_schedule.go b/internal/store/query_replica_schedule.go index 5ed0dbf3..cc50db8e 100644 --- a/internal/store/query_replica_schedule.go +++ b/internal/store/query_replica_schedule.go @@ -51,7 +51,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool } e := q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor) if e != nil && e != mongo.ErrNoDocuments { - return false, e + return false, queryReplicaFailure(queryReplicaControl, "schedule", e) } start := cursor.Next % len(names) if start < 0 { @@ -66,7 +66,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool } age, e := q.backlogAge(ctx, name, n) if e != nil { - return false, e + return false, queryReplicaFailure(name, "backlog", e) } if age > 5*time.Minute { for extra := 0; extra < 3; extra++ { @@ -84,7 +84,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool } cp, e := q.checkpoint(ctx, name) if e != nil { - return false, e + return false, queryReplicaFailure(name, "checkpoint", e) } if !cp.Complete { if !bootstrap { @@ -98,7 +98,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool _, e := q.control().UpdateOne(sc, bson.M{"_id": "schedule"}, bson.M{"$set": bson.M{"next": (index + 1) % len(names)}}, options.Update().SetUpsert(true)) return e }); e != nil { - return false, e + return false, queryReplicaFailure(name, "schedule", e) } } complete := true @@ -120,7 +120,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady { return false, nil } else if e != nil { - return false, e + return false, queryReplicaFailure(queryReplicaControl, "publication", e) } return true, nil } diff --git a/scripts/test_query_replica.py b/scripts/test_query_replica.py index ddb1e037..098b3e9b 100644 --- a/scripts/test_query_replica.py +++ b/scripts/test_query_replica.py @@ -20,6 +20,8 @@ CASES = ('cmd/query-replica/main_local_test.go', 'internal/store/query_replica_safety_local_test.go', 'internal/store/query_replica_maintenance_flow_local_test.go', 'internal/store/query_replica_safety_test.go', + 'internal/store/query_replica_diagnostic_test.go', + 'internal/store/query_replica_projection_local_test.go', 'internal/store/query_replica_local_test.go', 'internal/store/query_sync_local_test.go', 'internal/store/query_replica_count_cache_test.go', diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index ffaf6b73..7af13f58 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1065,7 +1065,8 @@ "^TestQueryReplicaCandidateCache", "^TestQueryReplicaCountOnly", "^TestQueryReplicaMissingConfiguration", - "^TestQueryReplicaRejectsSigned" + "^TestQueryReplicaRejectsSigned", + "^TestQueryReplicaDiagnostic" ], "./cmd/query-cdc-probe": ["^TestClassification"], "./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],