🛑 Preserve query build failure and retirement fences
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions

This commit is contained in:
d-robotics 2026-11-15 09:12:57 -08:00
parent ffe186473c
commit 86e9fac8c7
10 changed files with 542 additions and 50 deletions

View File

@ -64,3 +64,19 @@
从零构建确认写入148,610,000 BSON字节,限速理论下界约283秒,实际约319秒;不需要通过增加全程超时或削弱持久性才能完成。采样查询容器CPU最高37.41%,无OOM;Docker VM的IO full avg10最高0.91%,该值不是单库IO等待。新增键索引在14万事实时约14.44MB。内存包含Mongo文件缓存压力,本地1GiB测试不能替代生产8GiB长时验证。
快速检查通过。首次全任务执行的浏览器、统计稳定性、权限和共享缓存检查通过;隔离副本组在总编排900秒耗尽后被停止,原报告`verify-1794758125279658000`保留为incomplete,不能写成通过。后续对失败落盘分支完成故障注入后,集中补跑受影响的隔离组;已通过且输入未变的浏览器证据保留。
### 最终功能补验与容量边界
- 故障注入已复现并修复:结果/失败原因落盘失败不能丢失停止标记;恢复时断点库不可用也必须停止。红绿证据为`query-build-outcome-red-confirmed.log`及`query-build-outcome-green.log`。
- 长期复用的旧完整载荷在切换后增加15分钟退休宽限期,随新头在同一事务确认;跨无变化刷新保留,到期回收,避免在途读取被清理。红绿证据为`query-build-retirement-red.log`、`query-build-retirement-green.log`。
- 真实本机Mongo中途重启验证通过:同一容器、同一卷、1核1Gi、实际构建游标`0/79`恢复,408条事实完整发布,重复原生输入读取为0。命令:`QUERY_REPLICA_LOCAL_TEST=1 QUERY_BUILD_PHYSICAL_RESTART=1 go test ./internal/store -run '^TestQueryBuildPhysicalMongoRestartMidApplyLocal$' -count=1 -race -v`。该用例需要独占本机实验库,不放进普通并行任务;证据`query-build-physical-restart.log`。
- 完整注册功能用例110项在6批中全部通过,missing=[]、skipped=false、sourceChanged=false;6次同卷重启的majority+journal断点校验均通过。证据`query-build-batched-final.log`,原始完整日志`check-6dddc83cc2f54acbba24785a686e1a4f.log`。
- 单进程连续执行完整套用例曾两次导致1Gi实验Mongo OOM(exit137),原始失败保留,不能改写为容量通过。Docker VM总内存约5.8Gi并运行其他开发任务,未增加限制或停止其他容器。`--bounded-query`功能验证改为每批最多20项,批间验证没有其他客户端、原卷及资源不变后重启;不跳过用例,不在单个用例途中释放资源,不降低持久性。此模式不证明连续1Gi或生产8Gi容量。
源码、Go夹具、依赖和数据库资源未变的浏览器/统计/缓存证据继续复用;工具单元测试的诊断枚举断言另行补验。正式发布仍须绑定同步dev的最终源码与不可变镜像。
### 加急影子试运行建议(尚未批准执行)
不再以首次构建5分钟、1Gi实验库连续全套运行、全部查询500ms或200/500人长时压测阻塞影子部署。先更新影子构建器,在有效新窗口中追平同步,确认一个完整发布及关键生产对账,再灰度接入少量看板/作业记录。保留现有3分钟后台刷新、15分钟最大展示年龄;首轮以首屏约2–3秒、无超时、结果和权限正确为目标。全量入口和长期容量须后续独立验收,不能把分批功能通过换算成容量承诺。
不可放宽项:majority+journal、权限/供应商隔离、版本竞争、客户标注保护、完整代次原子发布、硬故障停止与回退。客户现有标注、切片、送检、质检和验收继续使用现有生产链路,不等待新查询链路完成。原运行窗口已结束,新的启动截止、镜像及目标仍须按项目规则确认。

View File

@ -305,6 +305,30 @@ func (s *MongoStore) queryBuildPublish(ctx context.Context, l queryModelLease, j
if state.Heads[workforceFactKey(j.Organization)].FactEpoch != j.OldEpoch {
return errors.New("query build published epoch changed")
}
// A payload can be reused far longer than the orphan grace. Pin its
// retirement in the same commit as the replacement heads, rather than
// interpreting the payload's original createdAt as retirement time.
now := time.Now().UTC()
oldProduction := state.Heads[queryModelKey("production", j.Organization, j.Projects)]
next.Retired = nil
for _, pin := range j.Retired {
if now.Before(pin.Until) && pin.Generation != j.Production.Generation {
next.Retired = append(next.Retired, pin)
}
}
if oldProduction.Generation != "" && oldProduction.Generation != j.Production.Generation {
until := now.Add(15 * time.Minute)
found := false
for i := range next.Retired {
if next.Retired[i].Generation == oldProduction.Generation {
next.Retired[i].Until = until
found = true
}
}
if !found {
next.Retired = append(next.Retired, queryBuildRetiredGeneration{Generation: oldProduction.Generation, Until: until})
}
}
wf := QueryModelHead{Generation: workforceFactSchema, Collection: workforceFactCollection, FactEpoch: epoch, Rows: j.PlanRows, AsOf: j.AsOf, SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
result, e := s.queryControl().UpdateOne(sc, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + workforceFactKey(j.Organization): wf, "heads." + queryModelKey("production", j.Organization, j.Projects): j.Production}})
if e != nil {

View File

@ -18,45 +18,51 @@ const queryBuildJobsCollection = "queryBuildJobs"
const queryBuildPartsCollection = "queryBuildParts"
const queryBuildJobSchema = "query-build-job/v1"
type queryBuildRetiredGeneration struct {
Generation string `bson:"generation"`
Until time.Time `bson:"until"`
}
type queryBuildJob struct {
ID string `bson:"_id"`
Schema string `bson:"schema"`
InputGeneration string `bson:"inputGeneration"`
Generation string `bson:"generation"`
Organization string `bson:"organizationId"`
Projects []string `bson:"projects"`
Cold bool `bson:"cold"`
Phase string `bson:"phase"`
Snapshot primitive.Timestamp `bson:"snapshot"`
AsOf time.Time `bson:"asOf"`
SourceRevision int64 `bson:"sourceRevision"`
GlobalRevision int64 `bson:"globalRevision"`
OldEpoch int64 `bson:"oldEpoch"`
Epoch int64 `bson:"epoch"`
Production QueryModelHead `bson:"production"`
PlanParts int `bson:"planParts"`
PlanRows int `bson:"planRows"`
PlanDigest string `bson:"planDigest"`
PlanSaved int `bson:"planSaved"`
PlanBytes int64 `bson:"planBytes"`
ProductionSaved int `bson:"productionSaved"`
NextPart int `bson:"nextPart"`
NextRow int `bson:"nextRow"`
Changed bool `bson:"changed"`
CleanupHasAfter bool `bson:"cleanupHasAfter"`
CleanupPlanPart int `bson:"cleanupPlanPart"`
CleanupPlanRow int `bson:"cleanupPlanRow"`
CleanupAfter string `bson:"cleanupAfter"`
CleanupFuture bool `bson:"cleanupFuture"`
ConfirmedRows int64 `bson:"confirmedRows"`
ConfirmedBytes int64 `bson:"confirmedBytes"`
ReusedRows int64 `bson:"reusedRows"`
ActiveWithoutProgress time.Duration `bson:"activeWithoutProgress"`
ProgressAt time.Time `bson:"progressAt"`
FailurePhase string `bson:"failurePhase,omitempty"`
FailureCode int32 `bson:"failureCode,omitempty"`
FailureClass string `bson:"failureClass,omitempty"`
UpdatedAt time.Time `bson:"updatedAt"`
Retired []queryBuildRetiredGeneration `bson:"retiredGenerations,omitempty"`
ID string `bson:"_id"`
Schema string `bson:"schema"`
InputGeneration string `bson:"inputGeneration"`
Generation string `bson:"generation"`
Organization string `bson:"organizationId"`
Projects []string `bson:"projects"`
Cold bool `bson:"cold"`
Phase string `bson:"phase"`
Snapshot primitive.Timestamp `bson:"snapshot"`
AsOf time.Time `bson:"asOf"`
SourceRevision int64 `bson:"sourceRevision"`
GlobalRevision int64 `bson:"globalRevision"`
OldEpoch int64 `bson:"oldEpoch"`
Epoch int64 `bson:"epoch"`
Production QueryModelHead `bson:"production"`
PlanParts int `bson:"planParts"`
PlanRows int `bson:"planRows"`
PlanDigest string `bson:"planDigest"`
PlanSaved int `bson:"planSaved"`
PlanBytes int64 `bson:"planBytes"`
ProductionSaved int `bson:"productionSaved"`
NextPart int `bson:"nextPart"`
NextRow int `bson:"nextRow"`
Changed bool `bson:"changed"`
CleanupHasAfter bool `bson:"cleanupHasAfter"`
CleanupPlanPart int `bson:"cleanupPlanPart"`
CleanupPlanRow int `bson:"cleanupPlanRow"`
CleanupAfter string `bson:"cleanupAfter"`
CleanupFuture bool `bson:"cleanupFuture"`
ConfirmedRows int64 `bson:"confirmedRows"`
ConfirmedBytes int64 `bson:"confirmedBytes"`
ReusedRows int64 `bson:"reusedRows"`
ActiveWithoutProgress time.Duration `bson:"activeWithoutProgress"`
ProgressAt time.Time `bson:"progressAt"`
FailurePhase string `bson:"failurePhase,omitempty"`
FailureCode int32 `bson:"failureCode,omitempty"`
FailureClass string `bson:"failureClass,omitempty"`
UpdatedAt time.Time `bson:"updatedAt"`
}
func queryBuildJobKey(database, org string) string {

View File

@ -0,0 +1,75 @@
package store
import (
"droboticflow/internal/testsupport"
"errors"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"testing"
"time"
)
func TestQueryBuildOutcomePersistenceFailureIsTerminalLocal(t *testing.T) {
for _, kind := range []string{"budget", "phase"} {
t.Run(kind, func(t *testing.T) {
_, s, _ := newLocalReplica(t)
ctx := t.Context()
s.queryReplicaFactStore = true
s.queryPeriodic = true
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
j, e := s.queryBuildStart(ctx, l, "outcome-fixture", nil)
if e != nil {
t.Fatal(e)
}
if kind == "phase" {
j.Phase = "unsupported"
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
t.Fatal(e)
}
}
validator := bson.M{"$and": bson.A{bson.M{"activeWithoutProgress": bson.M{"$lte": int64(0)}}, bson.M{"failureClass": bson.M{"$exists": false}}}}
if e = s.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: queryBuildJobsCollection}, {Key: "validator", Value: validator}, {Key: "validationLevel", Value: "strict"}, {Key: "validationAction", Value: "error"}}).Err(); e != nil {
t.Fatal(e)
}
round := newQueryBuildRound(true)
if kind == "budget" {
now := time.Now()
round = &queryBuildRound{until: now, now: func() time.Time { return now }}
}
e = s.queryBuildRunRound(ctx, l, j.Organization, nil, round)
if diagnostic := QueryReplicaDiagnostic(e); diagnostic.MongoCode != 121 {
t.Fatal("failed to exercise actual checkpoint validation failure", e)
}
if !errors.Is(e, ErrQueryBuildStopped) || errors.Is(e, ErrQueryBuildYielded) {
t.Fatal("durability failure became retry/yield", e)
}
preserved, ok, e := s.queryBuildJob(ctx, j.Organization)
if e != nil || !ok || preserved.Generation != j.Generation || preserved.NextPart != 0 {
t.Fatal("checkpoint lost", e)
}
})
}
}
func TestQueryBuildResumeReadFailureIsTerminalLocal(t *testing.T) {
_, s, _ := newLocalReplica(t)
ctx := t.Context()
client, e := mongo.Connect(ctx, options.Client().ApplyURI(testsupport.QueryReplicaTargetURI(t)))
if e != nil {
t.Fatal(e)
}
disconnected := NewMongoStore(client.Database(s.db.Name()))
disconnected.queryReplicaFactStore = true
disconnected.queryPeriodic = true
if e = client.Disconnect(ctx); e != nil {
t.Fatal(e)
}
e = disconnected.queryBuildRunRound(ctx, queryModelLease{}, "resume-fixture", nil, nil)
if !errors.Is(e, ErrQueryBuildStopped) || !errors.Is(e, mongo.ErrClientDisconnected) {
t.Fatal("unavailable checkpoint storage automatically retried", e)
}
}

View File

@ -0,0 +1,181 @@
package store
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"sync/atomic"
"testing"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/testsupport"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/event"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
)
// Explicit opt-in: this stops only the verified owned loopback query container.
// Run alone, outside other Mongo/browser tests; never on a company cluster.
func TestQueryBuildPhysicalMongoRestartMidApplyLocal(t *testing.T) {
if os.Getenv("QUERY_BUILD_PHYSICAL_RESTART") != "1" {
t.Skip("explicit owned lab restart required")
}
ctx := t.Context()
const container = "fanyu-query-replica-local-query-1"
command := func(args ...string) []byte {
t.Helper()
op, stop := context.WithTimeout(ctx, 40*time.Second)
defer stop()
out, e := exec.CommandContext(op, "docker", args...).Output()
if e != nil {
t.Fatal("owned lab command failed", e)
}
return out
}
var before []struct {
ID string `json:"Id"`
Name string
Config struct{ Labels map[string]string }
HostConfig struct{ Memory, NanoCpus int64 }
Mounts []struct{ Type, Name, Destination string }
}
if e := json.Unmarshal(command("inspect", container), &before); e != nil || len(before) != 1 {
t.Fatal(e)
}
cfg := before[0]
if cfg.Name != "/"+container || cfg.HostConfig.Memory != 1<<30 || cfg.HostConfig.NanoCpus != 1_000_000_000 || cfg.Config.Labels["com.docker.compose.project"] != "fanyu-query-replica-local" {
t.Fatal("different fixture identity")
}
hasVolume := false
for _, v := range cfg.Mounts {
hasVolume = hasVolume || v.Destination == "/data/db"
}
if !hasVolume {
t.Fatal("persistent volume missing")
}
var native atomic.Int64
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
if e.CommandName == "find" {
name, _ := e.Command.Lookup("find").StringValueOK()
if querySourceCollection(name) {
native.Add(1)
}
}
}}
_, s, _ := newLocalReplica(t, nil, monitor)
org := "workforce-test"
seedWorkforceStats(t, s)
events, tasks := []any{}, []any{}
seconds := 1.0
for i := 0; i < 100; i++ {
id := fmt.Sprintf("physical-task-%03d", i)
tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": "p", "supplierId": "s", "runId": "run"})
events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("physical-event-%03d", i), OrganizationID: org, TaskID: id, ProjectID: "p", SupplierID: "s", UserID: "u", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: time.Date(2026, 9, 28, 13, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
}
if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil {
t.Fatal(e)
}
if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil {
t.Fatal(e)
}
s.queryReplicaFactStore = true
s.queryPeriodic = true
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
t.Fatal(e)
}
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
if e != nil {
t.Fatal(e)
}
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
t.Fatal(e)
}
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
t.Fatal(e)
}
calls := 0
now := time.Now()
round := &queryBuildRound{until: now.Add(time.Second), now: func() time.Time {
calls++
if calls > 2 {
return now.Add(2 * time.Second)
}
return now
}}
if e = s.queryBuildApply(ctx, l, &j, round); !errors.Is(e, ErrQueryBuildYielded) {
t.Fatal(e)
}
checkpoint, ok, e := s.queryBuildJob(ctx, org)
if e != nil || !ok || checkpoint.NextPart+checkpoint.NextRow == 0 {
t.Fatal("no durable mid-apply checkpoint", e)
}
beforeReads := native.Load()
// Gracefully releasing the old process lease is independent of the database
// restart; its new holder must still fence the old epoch afterwards.
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); e != nil {
t.Fatal(e)
}
command("restart", "-t", "30", container)
var after []struct {
ID string `json:"Id"`
HostConfig struct{ Memory, NanoCpus int64 }
Mounts []struct{ Type, Name, Destination string }
}
if e = json.Unmarshal(command("inspect", container), &after); e != nil || len(after) != 1 {
t.Fatal(e)
}
a, _ := json.Marshal(cfg.Mounts)
b, _ := json.Marshal(after[0].Mounts)
if after[0].ID != cfg.ID || after[0].HostConfig.Memory != cfg.HostConfig.Memory || after[0].HostConfig.NanoCpus != cfg.HostConfig.NanoCpus || string(a) != string(b) {
t.Fatal("restart changed identity, resources or volume")
}
client, e := mongo.Connect(ctx, options.Client().ApplyURI(testsupport.QueryReplicaTargetURI(t)).SetMonitor(monitor))
if e != nil {
t.Fatal(e)
}
defer client.Disconnect(ctx)
ready := false
deadline := time.Now().Add(25 * time.Second)
for time.Now().Before(deadline) {
op, stop := context.WithTimeout(ctx, time.Second)
e = client.Ping(op, readpref.Primary())
stop()
if e == nil {
ready = true
break
}
time.Sleep(200 * time.Millisecond)
}
if !ready {
t.Fatal("same-volume primary did not recover", e)
}
resumed := NewMongoStore(client.Database(s.db.Name()))
resumed.queryReplicaFactStore = true
resumed.queryPeriodic = true
recovered, ok, e := resumed.queryBuildJob(ctx, org)
if e != nil || !ok || recovered.Generation != checkpoint.Generation || recovered.NextPart != checkpoint.NextPart || recovered.NextRow != checkpoint.NextRow {
t.Fatal("physical restart lost confirmed checkpoint", e)
}
next, ok, e := resumed.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
if e = resumed.queryBuildRunRound(ctx, next, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
completed, _, e := resumed.queryBuildJob(ctx, org)
if e != nil || completed.Phase != "complete" || completed.PlanRows != 408 || native.Load() != beforeReads {
t.Fatal("physical restart repeated input or lost facts", completed.PlanRows, native.Load()-beforeReads, e)
}
t.Logf("physicalRestart=true sameVolume=true sameResources=true confirmedCursor=%d/%d finalFacts=%d repeatedNativeReads=0 majorityJournal=true", checkpoint.NextPart, checkpoint.NextRow, completed.PlanRows)
}

View File

@ -358,3 +358,70 @@ func TestQueryBuildSealedInputSurvivesExpiredSnapshotAndOrphanGCLocal(t *testing
t.Fatal("GC deleted unfinished durable blob", n, e)
}
}
func TestQueryBuildRetiredPayloadKeepsInflightReaderGraceLocal(t *testing.T) {
_, s, _ := newLocalReplica(t)
ctx := t.Context()
org := "workforce-test"
seedWorkforceStats(t, s)
s.queryReplicaFactStore = true
s.queryPeriodic = true
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
t.Fatal(e)
}
l, ok, e := s.acquireQueryModelLease(ctx)
if e != nil || !ok {
t.Fatal(e)
}
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
first, _, e := s.queryBuildJob(ctx, org)
if e != nil {
t.Fatal(e)
}
old := first.Production.Generation
if _, e = s.queryData().UpdateMany(ctx, bson.M{"generation": old}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-24 * time.Hour)}}); e != nil {
t.Fatal(e)
}
// A changed generation retires a payload that may have been reused for days.
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"productionRevision": 1}}); e != nil {
t.Fatal(e)
}
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
if e = s.cleanQueryModelOrphans(ctx); e != nil {
t.Fatal(e)
}
n, e := s.queryData().CountDocuments(ctx, bson.M{"generation": old})
if e != nil || n != int64(first.Production.Chunks) {
t.Fatal("retired payload disappeared while in-flight reader may still hydrate it", n, e)
}
retained, _, e := s.queryBuildJob(ctx, org)
if e != nil || len(retained.Retired) != 1 {
t.Fatal("missing retirement checkpoint", e)
}
until := retained.Retired[0].Until
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
t.Fatal(e)
}
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
t.Fatal(e)
}
retained, _, e = s.queryBuildJob(ctx, org)
if e != nil || len(retained.Retired) != 1 || !retained.Retired[0].Until.Equal(until) {
t.Fatal("unchanged refresh lost or extended retirement grace", e)
}
retained.Retired[0].Until = time.Now().Add(-time.Minute)
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, retained) }); e != nil {
t.Fatal(e)
}
if e = s.cleanQueryModelOrphans(ctx); e != nil {
t.Fatal(e)
}
n, e = s.queryData().CountDocuments(ctx, bson.M{"generation": old})
if e != nil || n != 0 {
t.Fatal("expired payload pinned forever", n, e)
}
}

View File

@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"log"
"reflect"
"sort"
"time"
@ -56,6 +57,7 @@ func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org
}
head := state.Heads[workforceFactKey(org)]
j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()}
j.Retired = append([]queryBuildRetiredGeneration{}, old.Retired...)
j.InputGeneration = j.Generation
if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second {
j.AsOf = state.Observed
@ -86,7 +88,12 @@ func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org
func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, org string, projects []string, r *queryBuildRound) (err error) {
j, err := s.queryBuildStart(ctx, l, org, projects)
if err != nil {
return err
if ctx.Err() != nil {
return ctx.Err()
}
d := QueryReplicaDiagnostic(err)
log.Printf("query_model_build result=stopped stage=resume category=%s mongoCode=%d", d.Class, d.MongoCode)
return fmt.Errorf("%w: resume: %w", ErrQueryBuildStopped, err)
}
if r == nil {
r = newQueryBuildRound(j.Cold)
@ -123,7 +130,9 @@ func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease,
err = fmt.Errorf("%w: %w", ErrQueryBuildStopped, err)
}
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, current) }); e != nil {
err = fmt.Errorf("query build could not persist outcome: %w", e)
d := QueryReplicaDiagnostic(e)
log.Printf("query_model_build result=stopped stage=checkpoint category=%s mongoCode=%d", d.Class, d.MongoCode)
err = fmt.Errorf("%w: query build could not persist outcome: %w", ErrQueryBuildStopped, e)
}
}()
for {

View File

@ -42,7 +42,7 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
keep = append(keep, h.Generation)
}
if s.queryReplicaFactStore && s.periodicQueryStore() {
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema, "phase": bson.M{"$ne": "complete"}}, options.Find().SetProjection(bson.M{"production.generation": 1}))
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"production.generation": 1, "phase": 1, "retiredGenerations": 1}))
if err != nil {
return err
}
@ -53,9 +53,14 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
return err
}
for _, j := range jobs {
if j.Production.Generation != "" {
if j.Phase != "complete" && j.Production.Generation != "" {
keep = append(keep, j.Production.Generation)
}
for _, pin := range j.Retired {
if time.Now().Before(pin.Until) {
keep = append(keep, pin.Generation)
}
}
}
}
filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}}

View File

@ -35,5 +35,31 @@ class ReplicaRunnerSafety(unittest.TestCase):
self.assertEqual(call.call_count, 1)
class ReplicaRunnerBatchSafety(unittest.TestCase):
def test_bounded_batches_cover_every_registered_case_once(self):
required = {f'TestCase{i:03d}' for i in range(53)}
batches = runner.case_batches(required, True, False)
self.assertEqual([len(x) for x in batches], [20, 20, 13])
self.assertEqual([x for group in batches for x in group], sorted(required))
self.assertEqual(len(runner.case_batches(required, False, False)), 1)
self.assertEqual(len(runner.case_batches(required, True, True)), 1)
with self.assertRaisesRegex(RuntimeError, 'no registered'):
runner.case_batches(set(), True, False)
def test_restart_refuses_other_clients_without_mutating_fixture(self):
import json
fixture = {'Name': '/fanyu-query-replica-local-query-1',
'HostConfig': {'Memory': 1024 ** 3, 'NanoCpus': 1_000_000_000},
'Config': {'Labels': {'com.docker.compose.project': 'fanyu-query-replica-local'}},
'Mounts': [{'Destination': '/data/db'}]}
with patch.object(runner.subprocess, 'check_output', return_value=json.dumps([fixture])), \
patch.object(runner, 'query_probe', return_value=(3, 'other clients')), \
patch.object(runner.time, 'sleep'), patch.object(runner, 'compose') as change:
with self.assertRaisesRegex(RuntimeError, 'other_clients'):
runner.restart_query_fixture({}, 'synthetic-probe', 7)
change.assert_not_called()
if __name__ == "__main__":
unittest.main()

View File

@ -10,6 +10,8 @@ import signal
import socket
import subprocess
import time
import uuid
import sys
from lib.release_checks import check_environment, source_fingerprint, write_log
ROOT = Path(__file__).resolve().parents[1]
@ -33,6 +35,7 @@ CASES = ('cmd/query-replica/main_local_test.go',
'internal/store/workforce_facts_local_test.go',
'internal/store/query_build_capture_local_test.go',
'internal/store/query_build_resume_local_test.go',
'internal/store/query_build_outcome_local_test.go',
'internal/store/workforce_publication_local_test.go',
'internal/store/query_replica_page_local_test.go',
'internal/store/supplier_authorization_batch_local_test.go',
@ -128,6 +131,70 @@ def start(env):
raise RuntimeError('query replica did not become primary')
def case_batches(required, bounded, native):
names = sorted(required)
if not names:
raise RuntimeError('no registered replica cases')
size = 20 if bounded and not native else len(names)
return [names[i:i + size] for i in range(0, len(names), size)]
def query_probe(env, script):
# Fixed loopback, no credentials, four-second process bound. This is not
# an arbitrary database command entry point or a production probe.
return run(['docker', 'exec', 'fanyu-query-replica-local-query-1',
'mongosh', '--quiet',
'mongodb://127.0.0.1:27129/?serverSelectionTimeoutMS=1000&connectTimeoutMS=1000&appName=droboticflow-query-fixture-probe&maxPoolSize=1&maxConnecting=1',
'--eval', script], env, 4)
def restart_query_fixture(env, probe, cursor):
name = 'fanyu-query-replica-local-query-1'
before = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
if (before['Name'] != '/' + name or before['HostConfig']['Memory'] != 1024 ** 3
or before['HostConfig']['NanoCpus'] != 1_000_000_000
or before['Config']['Labels'].get('com.docker.compose.project') != 'fanyu-query-replica-local'
or not any(m['Destination'] == '/data/db' for m in before['Mounts'])):
raise RuntimeError('refusing to restart a different query fixture')
# A direct client from another checkout must not be interrupted. Exclude only this named probe and the replica set internal client;
# wait briefly for completed test clients and in-progress handshakes to close.
for attempt in range(3):
code, _ = query_probe(env,
'sleep(200);'
'const clients=db.getSiblingDB("admin").aggregate(['
'{$currentOp:{allUsers:true,idleConnections:true}},'
'{$match:{client:{$exists:true},"clientMetadata.application.name":{$ne:"droboticflow-query-fixture-probe"},'
'"clientMetadata.driver.name":{$not:/^NetworkInterfaceTL/}}},'
'{$limit:1}]).toArray();if(clients.length) quit(3)')
if code == 0:
break
if attempt == 2:
raise RuntimeError('query restart admission failed: ' + ('other_clients' if code == 3 else 'probe_exit_' + str(code)))
time.sleep(.5)
owner = json.dumps(probe)
code, _ = query_probe(env, 'db.getSiblingDB(' + owner + ').checkpoint.updateOne({_id:"progress"},'
'{$set:{cursor:' + str(cursor) + ',fingerprint:"synthetic-majority-journal"}},'
'{upsert:true,writeConcern:{w:"majority",j:true}})')
if code:
raise RuntimeError('restart canary persistence failed')
compose(env, 'restart', '-t', '30', 'query')
after = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
if (after['Id'] != before['Id'] or after['Mounts'] != before['Mounts']
or after['HostConfig']['Memory'] != before['HostConfig']['Memory']
or after['HostConfig']['NanoCpus'] != before['HostConfig']['NanoCpus']):
raise RuntimeError('query restart changed volume or resource identity')
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
code, _ = query_probe(env, 'if(!db.hello().isWritablePrimary||db.hello().setName!=="query-read-rs") quit(2);'
'const p=db.getSiblingDB(' + owner + ').checkpoint.findOne({_id:"progress"});'
'if(!p||p.cursor!==' + str(cursor) + '||p.fingerprint!=="synthetic-majority-journal") quit(4)')
if code == 0:
return {'container': before['Id'], 'cursor': cursor, 'majorityJournalRecovered': True,
'sameVolume': True, 'sameResources': True}
time.sleep(.5)
raise RuntimeError('query restart readiness or durable canary recovery failed')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--native', action='store_true')
@ -171,21 +238,37 @@ def main():
if args.scale:
required = {'TestQueryReplicaScaleLocal'}
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
regex = '^(' + '|'.join(sorted(required)) + ')$'
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
'-run', regex, '-count=1', '-timeout=10m', '-v']
if args.race:
argv.append('-race')
code, output = run(argv, env)
batches = case_batches(required, args.bounded_query, native)
outputs, restart_proofs, commands = [], [], []
code = 0
probe = 'query_replica_restart_probe_' + uuid.uuid4().hex
for index, names in enumerate(batches):
if len(batches) > 1:
restart_proofs.append(restart_query_fixture(env, probe, index))
regex = '^(' + '|'.join(names) + ')$'
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
'-run', regex, '-count=1', '-timeout=10m', '-v']
if args.race:
argv.append('-race')
commands.append(argv)
code, output = run(argv, env)
outputs.append(output)
if code:
break
output = '\n'.join(outputs)
passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M))
missing = sorted(required - passed)
changed = not before or source_fingerprint(ROOT) != before
skipped = bool(re.search(r'^--- SKIP:', output, re.M))
success = code == 0 and not missing and not changed and not skipped
proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed')
proof = write_log(PRIVATE / 'logs', [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]], output, 'passed' if success else 'pending_or_failed')
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
productionCapacityAccepted=False, reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
productionCapacityAccepted=False, boundedBatches=len(batches),
continuousCapacityAccepted=False, restartProofs=restart_proofs, requiredCount=len(required),
passedCount=len(passed), batchCommands=commands, completedBatches=len(outputs),
reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
print(json.dumps(proof, ensure_ascii=False))
return 0 if success else (code or 2)