🐛 Trim run input reads before throttled projection
This commit is contained in:
parent
1082d0b527
commit
b64d950f64
@ -280,6 +280,11 @@ func main() {
|
||||
started := time.Now()
|
||||
ready, e := q.RunOnce(workCtx, *bootstrap)
|
||||
if e != nil {
|
||||
if ctx.Err() != nil {
|
||||
return // The deferred operator_stop records the intentional shutdown.
|
||||
}
|
||||
d := store.QueryReplicaDiagnostic(e)
|
||||
log.Printf("projector operation failed collection=%s phase=%s class=%s mongoCode=%d", d.Collection, d.Phase, d.Class, d.MongoCode)
|
||||
pause("operation_error")
|
||||
log.Print("projector paused; checkpoint retained (no customer payload logged)")
|
||||
if *safetyRun != "" {
|
||||
|
||||
@ -12,6 +12,8 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@ -138,6 +140,49 @@ func TestProjectorGuardedProcessRestartLocal(t *testing.T) {
|
||||
if json.Unmarshal(out, &v) != nil || v["paused"] != true {
|
||||
t.Fatal("operator status invalid")
|
||||
}
|
||||
// Resume the same guarded run, then stop it while it is actively reading.
|
||||
// SIGTERM must persist operator_stop rather than a misleading operation_error.
|
||||
if e = d.RecordQueryReplicaObservation(ctx, store.QueryReplicaSafetyObservation{
|
||||
At: time.Now().UTC(), CPU: 99, Ready: true, WritersCovered: true,
|
||||
HostFreeBytes: 120 << 30, TargetUsedFraction: .1,
|
||||
SafetySignalsVersion: 1, SourceAvailable: true,
|
||||
}); e != nil {
|
||||
t.Fatal("maintenance health unavailable")
|
||||
}
|
||||
_, _ = target.Collection("queryReplicaControl").UpdateOne(ctx, bson.M{"_id": "lease"}, bson.M{"$set": bson.M{"until": time.Time{}}})
|
||||
for i, arg := range args {
|
||||
if strings.HasPrefix(arg, "--max-source-records=") {
|
||||
args[i] = "--max-source-records=10000"
|
||||
}
|
||||
}
|
||||
args = append(args, "--resume-pause="+after["resumeToken"].(string), "--maintenance-initialization", "--maintenance-until="+time.Now().UTC().Add(10*time.Minute).Format(time.RFC3339))
|
||||
active, stopped := start()
|
||||
deadline = time.Now().Add(15 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
|
||||
if after["paused"] == false && after["reserved"].(int64) > 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
if after["paused"] != false || after["reserved"].(int64) <= 2 {
|
||||
t.Fatal("same-run maintenance resume did not preserve and advance budget")
|
||||
}
|
||||
if e = active.Process.Signal(syscall.SIGTERM); e != nil {
|
||||
t.Fatal("could not signal isolated process")
|
||||
}
|
||||
select {
|
||||
case e = <-stopped:
|
||||
if e != nil {
|
||||
t.Fatal("graceful stop failed")
|
||||
}
|
||||
case <-time.After(6 * time.Second):
|
||||
t.Fatal("graceful stop did not finish")
|
||||
}
|
||||
_ = target.Collection("queryReplicaControl").FindOne(ctx, bson.M{"_id": "safety"}).Decode(&after)
|
||||
if after["paused"] != true || after["reason"] != "operator_stop" || after["reserved"].(int64) < 2 {
|
||||
t.Fatal("intentional stop misclassified or reset budget")
|
||||
}
|
||||
count, e := source.Collection("annotationIndexJobs").CountDocuments(ctx, bson.M{"privateDraft": "not projected"})
|
||||
if e != nil || count != 3 {
|
||||
t.Fatal("source payload changed")
|
||||
|
||||
@ -244,7 +244,7 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
|
||||
parent := ctx
|
||||
limit, e := q.sourcePermit(ctx, q.limit())
|
||||
if e != nil {
|
||||
return nil, nil, e
|
||||
return nil, nil, queryReplicaFailure(n, "source_permit", e)
|
||||
}
|
||||
ctx, cancel := q.sourceContext(ctx)
|
||||
defer cancel()
|
||||
@ -256,7 +256,7 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
|
||||
}
|
||||
cur, e := q.Source.db.Collection(n).Find(ctx, filter, opts)
|
||||
if e != nil {
|
||||
return nil, nil, e
|
||||
return nil, nil, queryReplicaFailure(n, "source_find", e)
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
records := []queryReplicaRecord{}
|
||||
@ -264,20 +264,20 @@ func (q *QueryReplica) read(ctx context.Context, n string, filter bson.M) ([]que
|
||||
for cur.Next(ctx) {
|
||||
var d bson.M
|
||||
if e = cur.Decode(&d); e != nil {
|
||||
return nil, nil, e
|
||||
return nil, nil, queryReplicaFailure(n, "source_decode", e)
|
||||
}
|
||||
last = d["_id"]
|
||||
r, e := queryExportRecord(n, d)
|
||||
if e != nil {
|
||||
return nil, nil, e
|
||||
return nil, nil, queryReplicaFailure(n, "source_export", e)
|
||||
}
|
||||
records = append(records, r)
|
||||
}
|
||||
if e := cur.Err(); e != nil {
|
||||
return nil, nil, e
|
||||
return nil, nil, queryReplicaFailure(n, "source_cursor", e)
|
||||
}
|
||||
if e := q.paceRows(parent, len(records)); e != nil {
|
||||
return nil, nil, e
|
||||
return nil, nil, queryReplicaFailure(n, "row_pacing", e)
|
||||
}
|
||||
return records, last, nil
|
||||
}
|
||||
@ -297,7 +297,7 @@ func (q *QueryReplica) acknowledge(ctx context.Context, n string, records []quer
|
||||
if len(models) == 0 {
|
||||
return nil
|
||||
}
|
||||
return q.sourceMetadataBulk(ctx, n, models)
|
||||
return queryReplicaFailure(n, "source_ack", q.sourceMetadataBulk(ctx, n, models))
|
||||
}
|
||||
|
||||
// BootstrapStep is bounded and resumable. $expr uses BSON total ordering across
|
||||
@ -309,7 +309,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int,
|
||||
}
|
||||
cp, e := q.checkpoint(ctx, n)
|
||||
if e != nil || cp.Complete {
|
||||
return cp.Complete, 0, e
|
||||
return cp.Complete, 0, queryReplicaFailure(n, "checkpoint", e)
|
||||
}
|
||||
filter := bson.M{}
|
||||
if cp.HasLast || cp.Last != nil {
|
||||
@ -333,7 +333,7 @@ func (q *QueryReplica) BootstrapStep(ctx context.Context, n string) (bool, int,
|
||||
return e
|
||||
})
|
||||
if e != nil {
|
||||
return false, 0, e
|
||||
return false, 0, queryReplicaFailure(n, "target_apply", e)
|
||||
}
|
||||
return done, len(rows), q.acknowledge(ctx, n, rows)
|
||||
}
|
||||
@ -367,7 +367,7 @@ func (q *QueryReplica) DrainStep(ctx context.Context, n string) (int, error) {
|
||||
}
|
||||
e = q.transaction(ctx, func(sc mongo.SessionContext) error { return q.apply(sc, n, rows) })
|
||||
if e != nil {
|
||||
return 0, e
|
||||
return 0, queryReplicaFailure(n, "target_apply", e)
|
||||
}
|
||||
return len(rows), q.acknowledge(ctx, n, rows)
|
||||
}
|
||||
|
||||
95
internal/store/query_replica_diagnostic.go
Normal file
95
internal/store/query_replica_diagnostic.go
Normal file
@ -0,0 +1,95 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/bsoncodec"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
// QueryReplicaFailureDiagnostic deliberately excludes error text, filters,
|
||||
// identifiers and values: driver errors may contain credentials or documents.
|
||||
type QueryReplicaFailureDiagnostic struct {
|
||||
Collection string `json:"collection"`
|
||||
Phase string `json:"phase"`
|
||||
Class string `json:"class"`
|
||||
MongoCode int32 `json:"mongoCode"`
|
||||
}
|
||||
|
||||
type queryReplicaStepError struct {
|
||||
collection, phase string
|
||||
cause error
|
||||
}
|
||||
|
||||
func (e *queryReplicaStepError) Unwrap() error { return e.cause }
|
||||
func (e *queryReplicaStepError) Error() string {
|
||||
d := QueryReplicaDiagnostic(e)
|
||||
return "query replica " + d.Collection + "/" + d.Phase + ": " + d.Class
|
||||
}
|
||||
|
||||
func queryReplicaFailure(collection, phase string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var existing *queryReplicaStepError
|
||||
if errors.As(err, &existing) {
|
||||
return err
|
||||
}
|
||||
if queryReplicaFields[collection] == "" && collection != querySyncDeleteCollection && collection != queryReplicaControl {
|
||||
collection = "unknown"
|
||||
}
|
||||
switch phase {
|
||||
case "source_permit", "source_find", "source_decode", "source_export", "source_cursor", "row_pacing", "source_ack", "target_apply", "checkpoint", "backlog", "schedule", "publication", "delete_reconcile":
|
||||
default:
|
||||
phase = "operation"
|
||||
}
|
||||
return &queryReplicaStepError{collection, phase, err}
|
||||
}
|
||||
|
||||
func QueryReplicaDiagnostic(err error) QueryReplicaFailureDiagnostic {
|
||||
d := QueryReplicaFailureDiagnostic{Collection: "unknown", Phase: "operation", Class: "other"}
|
||||
var step *queryReplicaStepError
|
||||
if errors.As(err, &step) {
|
||||
d.Collection, d.Phase = step.collection, step.phase
|
||||
}
|
||||
var command mongo.CommandError
|
||||
var write mongo.WriteException
|
||||
var bulk mongo.BulkWriteException
|
||||
var decode *bsoncodec.DecodeError
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
d.Class = "cancelled"
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
d.Class = "deadline"
|
||||
case errors.Is(err, ErrQueryReplicaPaused):
|
||||
d.Class = "safety_pause"
|
||||
case errors.Is(err, ErrQueryReplicaLease):
|
||||
d.Class = "lease"
|
||||
case errors.Is(err, ErrQueryReplicaUnsupported):
|
||||
d.Class = "unsupported_fact"
|
||||
case errors.As(err, &command):
|
||||
d.Class, d.MongoCode = "mongo_command", command.Code
|
||||
case errors.As(err, &write):
|
||||
d.Class = "mongo_write"
|
||||
if len(write.WriteErrors) > 0 {
|
||||
d.MongoCode = int32(write.WriteErrors[0].Code)
|
||||
} else if write.WriteConcernError != nil {
|
||||
d.MongoCode = int32(write.WriteConcernError.Code)
|
||||
}
|
||||
case errors.As(err, &bulk):
|
||||
d.Class = "mongo_bulk_write"
|
||||
if len(bulk.WriteErrors) > 0 {
|
||||
d.MongoCode = int32(bulk.WriteErrors[0].Code)
|
||||
} else if bulk.WriteConcernError != nil {
|
||||
d.MongoCode = int32(bulk.WriteConcernError.Code)
|
||||
}
|
||||
case errors.As(err, &decode):
|
||||
d.Class = "bson_decode"
|
||||
case mongo.IsNetworkError(err):
|
||||
d.Class = "network"
|
||||
case mongo.IsTimeout(err):
|
||||
d.Class = "timeout"
|
||||
}
|
||||
return d
|
||||
}
|
||||
68
internal/store/query_replica_diagnostic_test.go
Normal file
68
internal/store/query_replica_diagnostic_test.go
Normal file
@ -0,0 +1,68 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
func TestQueryReplicaDiagnosticPreservesCauseWithoutLeaking(t *testing.T) {
|
||||
const private = "mongodb://private-user:private-password@private-host/customer?signature=private-token"
|
||||
var run domain.RunRecord
|
||||
b, _ := bson.Marshal(bson.M{"inputs": private})
|
||||
decodeErr := bson.Unmarshal(b, &run)
|
||||
if decodeErr == nil {
|
||||
t.Fatal("missing BSON decoder fixture")
|
||||
}
|
||||
cases := []struct {
|
||||
cause error
|
||||
class string
|
||||
code int32
|
||||
}{
|
||||
{mongo.CommandError{Message: private, Wrapped: context.DeadlineExceeded}, "deadline", 0},
|
||||
{context.Canceled, "cancelled", 0},
|
||||
{fmt.Errorf("%s: %w", private, ErrQueryReplicaPaused), "safety_pause", 0},
|
||||
{ErrQueryReplicaLease, "lease", 0},
|
||||
{fmt.Errorf("%s: %w", private, ErrQueryReplicaUnsupported), "unsupported_fact", 0},
|
||||
{mongo.CommandError{Code: 50, Message: private}, "mongo_command", 50},
|
||||
{mongo.WriteException{WriteErrors: mongo.WriteErrors{{Code: 11000, Message: private}}}, "mongo_write", 11000},
|
||||
{mongo.BulkWriteException{WriteErrors: []mongo.BulkWriteError{{WriteError: mongo.WriteError{Code: 11000, Message: private}}}}, "mongo_bulk_write", 11000},
|
||||
{decodeErr, "bson_decode", 0},
|
||||
{errors.New(private), "other", 0},
|
||||
}
|
||||
for _, test := range cases {
|
||||
err := queryReplicaFailure("runs", "source_find", test.cause)
|
||||
if !reflect.DeepEqual(errors.Unwrap(err), test.cause) {
|
||||
t.Fatal("error identity lost")
|
||||
}
|
||||
if test.class == "deadline" && !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatal("deadline unwrapping lost")
|
||||
}
|
||||
err = queryReplicaFailure("runs", "target_apply", err)
|
||||
d := QueryReplicaDiagnostic(err)
|
||||
if d.Collection != "runs" || d.Phase != "source_find" || d.Class != test.class || d.MongoCode != test.code {
|
||||
t.Fatalf("unexpected safe classification: %+v", d)
|
||||
}
|
||||
raw, _ := json.Marshal(d)
|
||||
for _, value := range []string{private, "private-password", "private-host", "signature"} {
|
||||
if strings.Contains(string(raw), value) || strings.Contains(err.Error(), value) {
|
||||
t.Fatal("raw cause leaked")
|
||||
}
|
||||
}
|
||||
}
|
||||
if queryReplicaFailure("runs", "source_find", nil) != nil {
|
||||
t.Fatal("nil became failure")
|
||||
}
|
||||
d := QueryReplicaDiagnostic(queryReplicaFailure(private, private, errors.New(private)))
|
||||
if d.Collection != "unknown" || d.Phase != "operation" {
|
||||
t.Fatal("uncontrolled diagnostic labels")
|
||||
}
|
||||
}
|
||||
@ -25,11 +25,38 @@ type queryReplicaRecord struct {
|
||||
|
||||
func querySourceProjection(collection string) bson.M {
|
||||
p := queryReplicaProjection(collection)
|
||||
if collection == "runs" {
|
||||
// Only these references participate in legacy search names. Selecting the
|
||||
// whole input also transfers inline payloads which export discards later;
|
||||
// at a bounded wire rate that can exhaust the fixed source read deadline.
|
||||
p["inputs"] = queryRunInputReferencesExpression()
|
||||
}
|
||||
if collection == "humanItems" {
|
||||
p["__legacyPayload"] = bson.M{"$cond": bson.A{poolSearchValidExpression(collection), bson.M{}, "$subject.payload"}}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func queryRunInputReferencesExpression() bson.M {
|
||||
object := func(value string) bson.M {
|
||||
return bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": value}, "object"}}, value, bson.M{}}}
|
||||
}
|
||||
selected := bson.M{"$arrayToObject": bson.M{"$filter": bson.M{
|
||||
"input": bson.M{"$objectToArray": object("$$input.v.ref")},
|
||||
"as": "field",
|
||||
"cond": bson.M{"$in": bson.A{"$$field.k", bson.A{"captureUnit", "prefix", "key", "manifestKey"}}},
|
||||
}}}
|
||||
ref := bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$$input.v.ref"}, "object"}}, selected, bson.M{"$ifNull": bson.A{"$$input.v.ref", bson.M{}}}}}
|
||||
value := bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$$input.v"}, "object"}}, bson.M{"ref": ref}, "$$input.v"}}
|
||||
inputs := bson.M{"$arrayToObject": bson.M{"$map": bson.M{
|
||||
"input": bson.M{"$objectToArray": object("$inputs")},
|
||||
"as": "input",
|
||||
"in": bson.M{"k": "$$input.k", "v": value},
|
||||
}}}
|
||||
// Preserve absent/null/non-object outer values; do not silently invent a
|
||||
// valid record for an unsupported historical input-container shape.
|
||||
return bson.M{"$cond": bson.A{bson.M{"$eq": bson.A{bson.M{"$type": "$inputs"}, "object"}}, inputs, "$inputs"}}
|
||||
}
|
||||
func safeReplicaNames(d any) bool {
|
||||
switch value := d.(type) {
|
||||
case string:
|
||||
|
||||
121
internal/store/query_replica_projection_local_test.go
Normal file
121
internal/store/query_replica_projection_local_test.go
Normal file
@ -0,0 +1,121 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryReplicaRunsProjectionFitsThrottledDeadlineLocal(t *testing.T) {
|
||||
source := newQuerySyncSource(t)
|
||||
payload := strings.Repeat("synthetic-inline-private-not-query-fact", 1024)
|
||||
docs := []any{}
|
||||
for i := 0; i < 50; i++ {
|
||||
docs = append(docs, bson.M{"_id": fmt.Sprintf("physical-%03d", i), "id": fmt.Sprintf("run-%03d", i), "organizationId": "org", "projectId": "project", "status": "completed", "captureDirectories": bson.A{"capture/part0001"}, "inputs": bson.M{"source": bson.M{"mode": "reference", "inlineValue": payload, "ref": bson.M{"captureUnit": "capture/part0001", "key": "capture/part0001/left.mp4", "unrelated": "not needed"}}}})
|
||||
}
|
||||
if _, err := source.db.Collection("runs").InsertMany(t.Context(), docs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transport, err := NewQueryReplicaTransport(128*1024, 5)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var replyBytes atomic.Int64
|
||||
monitor := &event.CommandMonitor{Succeeded: func(_ context.Context, e *event.CommandSucceededEvent) {
|
||||
if e.CommandName == "find" {
|
||||
replyBytes.Add(int64(len(e.Reply)))
|
||||
}
|
||||
}}
|
||||
client, err := mongo.Connect(t.Context(), options.Client().ApplyURI("mongodb://127.0.0.1:27119/?directConnection=true").SetMaxPoolSize(1).SetDialer(transport).SetCompressors([]string{}).SetTimeout(5*time.Second).SetMonitor(monitor))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = client.Disconnect(ctx)
|
||||
})
|
||||
q := &QueryReplica{Source: NewMongoStore(client.Database(source.db.Name())), BatchSize: 50}
|
||||
start := time.Now()
|
||||
rows, _, err := q.read(t.Context(), "runs", bson.M{})
|
||||
elapsed := time.Since(start)
|
||||
if err != nil {
|
||||
var command mongo.CommandError
|
||||
errors.As(err, &command)
|
||||
t.Fatalf("unneeded inline input blocks fixed source deadline at128KiB/s: class=%T mongoCode=%d contextDeadline=%t timeout=%t", err, command.Code, errors.Is(err, context.DeadlineExceeded), mongo.IsTimeout(err))
|
||||
}
|
||||
if len(rows) != 50 {
|
||||
t.Fatalf("rows=%d, wanted50", len(rows))
|
||||
}
|
||||
if elapsed >= 5*time.Second || replyBytes.Load() > 100*1024 {
|
||||
t.Fatalf("response should contain references only: elapsed=%s replyBytes=%d", elapsed, replyBytes.Load())
|
||||
}
|
||||
for _, r := range rows {
|
||||
raw, _ := bson.Marshal(r.Document)
|
||||
if strings.Contains(string(raw), "synthetic-inline-private") || strings.Contains(string(raw), "not needed") {
|
||||
t.Fatal("non-query payload leaked")
|
||||
}
|
||||
}
|
||||
var unchanged bson.M
|
||||
if err := source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": "physical-000"}).Decode(&unchanged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deletionDoc(deletionDoc(unchanged["inputs"])["source"])["inlineValue"] != payload {
|
||||
t.Fatal("source content changed")
|
||||
}
|
||||
t.Logf("same50 rows at128KiB/s, fixed5s: elapsed=%s responseBytes=%d", elapsed, replyBytes.Load())
|
||||
}
|
||||
|
||||
func TestQueryReplicaRunReferenceProjectionCompatibilityLocal(t *testing.T) {
|
||||
source := newQuerySyncSource(t)
|
||||
cases := []struct {
|
||||
name string
|
||||
inputs any
|
||||
absent bool
|
||||
}{
|
||||
{name: "absent", absent: true},
|
||||
{name: "null", inputs: nil},
|
||||
{name: "empty", inputs: bson.M{}},
|
||||
{name: "references-and-inline", inputs: bson.M{"source": bson.M{"inlineValue": "https://private.invalid/data?signature=do-not-export", "ref": bson.M{"captureUnit": "capture/part0001", "prefix": nil, "key": "capture/part0001/left.mp4", "manifestKey": "capture/manifest.json", "unrelated": "do-not-export"}}}},
|
||||
{name: "no-reference", inputs: bson.M{"source": bson.M{"inlineValue": "legacy-query-irrelevant"}}},
|
||||
{name: "null-reference", inputs: bson.M{"source": bson.M{"ref": nil}}},
|
||||
{name: "unsupported-container", inputs: bson.A{"legacy"}},
|
||||
{name: "unsupported-input-value", inputs: bson.M{"source": "legacy"}},
|
||||
{name: "unsupported-ref", inputs: bson.M{"source": bson.M{"ref": "legacy"}}},
|
||||
}
|
||||
for _, test := range cases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
doc := bson.M{"_id": test.name, "id": test.name, "status": "failed", "error": "meta 与 PTS", "artifacts": bson.A{bson.M{"id": "artifact", "name": "capture/part0001"}}}
|
||||
if !test.absent {
|
||||
doc["inputs"] = test.inputs
|
||||
}
|
||||
if _, err := source.db.Collection("runs").InsertOne(t.Context(), doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read := func(projection bson.M) bson.M {
|
||||
var d bson.M
|
||||
if err := source.db.Collection("runs").FindOne(t.Context(), bson.M{"_id": test.name}, options.FindOne().SetProjection(projection)).Decode(&d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
before, oldErr := queryExportRecord("runs", read(queryReplicaProjection("runs")))
|
||||
after, newErr := queryExportRecord("runs", read(querySourceProjection("runs")))
|
||||
if (oldErr == nil) != (newErr == nil) {
|
||||
t.Fatalf("changed support of historical shape: before=%T after=%T", oldErr, newErr)
|
||||
}
|
||||
if oldErr == nil && !reflect.DeepEqual(before.Document, after.Document) {
|
||||
t.Fatal("legacy search names, metadata or exception category changed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -51,7 +51,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
|
||||
}
|
||||
e := q.control().FindOne(ctx, bson.M{"_id": "schedule"}).Decode(&cursor)
|
||||
if e != nil && e != mongo.ErrNoDocuments {
|
||||
return false, e
|
||||
return false, queryReplicaFailure(queryReplicaControl, "schedule", e)
|
||||
}
|
||||
start := cursor.Next % len(names)
|
||||
if start < 0 {
|
||||
@ -66,7 +66,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
|
||||
}
|
||||
age, e := q.backlogAge(ctx, name, n)
|
||||
if e != nil {
|
||||
return false, e
|
||||
return false, queryReplicaFailure(name, "backlog", e)
|
||||
}
|
||||
if age > 5*time.Minute {
|
||||
for extra := 0; extra < 3; extra++ {
|
||||
@ -84,7 +84,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
|
||||
}
|
||||
cp, e := q.checkpoint(ctx, name)
|
||||
if e != nil {
|
||||
return false, e
|
||||
return false, queryReplicaFailure(name, "checkpoint", e)
|
||||
}
|
||||
if !cp.Complete {
|
||||
if !bootstrap {
|
||||
@ -98,7 +98,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
|
||||
_, e := q.control().UpdateOne(sc, bson.M{"_id": "schedule"}, bson.M{"$set": bson.M{"next": (index + 1) % len(names)}}, options.Update().SetUpsert(true))
|
||||
return e
|
||||
}); e != nil {
|
||||
return false, e
|
||||
return false, queryReplicaFailure(name, "schedule", e)
|
||||
}
|
||||
}
|
||||
complete := true
|
||||
@ -120,7 +120,7 @@ func (q *QueryReplica) runGuardedOnce(ctx context.Context, bootstrap bool) (bool
|
||||
if e := q.PublishReady(ctx); e == ErrQueryReplicaNotReady {
|
||||
return false, nil
|
||||
} else if e != nil {
|
||||
return false, e
|
||||
return false, queryReplicaFailure(queryReplicaControl, "publication", e)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@ -20,6 +20,8 @@ CASES = ('cmd/query-replica/main_local_test.go',
|
||||
'internal/store/query_replica_safety_local_test.go',
|
||||
'internal/store/query_replica_maintenance_flow_local_test.go',
|
||||
'internal/store/query_replica_safety_test.go',
|
||||
'internal/store/query_replica_diagnostic_test.go',
|
||||
'internal/store/query_replica_projection_local_test.go',
|
||||
'internal/store/query_replica_local_test.go',
|
||||
'internal/store/query_sync_local_test.go',
|
||||
'internal/store/query_replica_count_cache_test.go',
|
||||
|
||||
@ -1065,7 +1065,8 @@
|
||||
"^TestQueryReplicaCandidateCache",
|
||||
"^TestQueryReplicaCountOnly",
|
||||
"^TestQueryReplicaMissingConfiguration",
|
||||
"^TestQueryReplicaRejectsSigned"
|
||||
"^TestQueryReplicaRejectsSigned",
|
||||
"^TestQueryReplicaDiagnostic"
|
||||
],
|
||||
"./cmd/query-cdc-probe": ["^TestClassification"],
|
||||
"./cmd/query-replica": ["^TestPassDelay", "^TestPacing"],
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user