🛑 Preserve query build failure and retirement fences
This commit is contained in:
parent
ffe186473c
commit
86e9fac8c7
@ -64,3 +64,19 @@
|
||||
从零构建确认写入148,610,000 BSON字节,限速理论下界约283秒,实际约319秒;不需要通过增加全程超时或削弱持久性才能完成。采样查询容器CPU最高37.41%,无OOM;Docker VM的IO full avg10最高0.91%,该值不是单库IO等待。新增键索引在14万事实时约14.44MB。内存包含Mongo文件缓存压力,本地1GiB测试不能替代生产8GiB长时验证。
|
||||
|
||||
快速检查通过。首次全任务执行的浏览器、统计稳定性、权限和共享缓存检查通过;隔离副本组在总编排900秒耗尽后被停止,原报告`verify-1794758125279658000`保留为incomplete,不能写成通过。后续对失败落盘分支完成故障注入后,集中补跑受影响的隔离组;已通过且输入未变的浏览器证据保留。
|
||||
|
||||
### 最终功能补验与容量边界
|
||||
|
||||
- 故障注入已复现并修复:结果/失败原因落盘失败不能丢失停止标记;恢复时断点库不可用也必须停止。红绿证据为`query-build-outcome-red-confirmed.log`及`query-build-outcome-green.log`。
|
||||
- 长期复用的旧完整载荷在切换后增加15分钟退休宽限期,随新头在同一事务确认;跨无变化刷新保留,到期回收,避免在途读取被清理。红绿证据为`query-build-retirement-red.log`、`query-build-retirement-green.log`。
|
||||
- 真实本机Mongo中途重启验证通过:同一容器、同一卷、1核1Gi、实际构建游标`0/79`恢复,408条事实完整发布,重复原生输入读取为0。命令:`QUERY_REPLICA_LOCAL_TEST=1 QUERY_BUILD_PHYSICAL_RESTART=1 go test ./internal/store -run '^TestQueryBuildPhysicalMongoRestartMidApplyLocal$' -count=1 -race -v`。该用例需要独占本机实验库,不放进普通并行任务;证据`query-build-physical-restart.log`。
|
||||
- 完整注册功能用例110项在6批中全部通过,missing=[]、skipped=false、sourceChanged=false;6次同卷重启的majority+journal断点校验均通过。证据`query-build-batched-final.log`,原始完整日志`check-6dddc83cc2f54acbba24785a686e1a4f.log`。
|
||||
- 单进程连续执行完整套用例曾两次导致1Gi实验Mongo OOM(exit137),原始失败保留,不能改写为容量通过。Docker VM总内存约5.8Gi并运行其他开发任务,未增加限制或停止其他容器。`--bounded-query`功能验证改为每批最多20项,批间验证没有其他客户端、原卷及资源不变后重启;不跳过用例,不在单个用例途中释放资源,不降低持久性。此模式不证明连续1Gi或生产8Gi容量。
|
||||
|
||||
源码、Go夹具、依赖和数据库资源未变的浏览器/统计/缓存证据继续复用;工具单元测试的诊断枚举断言另行补验。正式发布仍须绑定同步dev的最终源码与不可变镜像。
|
||||
|
||||
### 加急影子试运行建议(尚未批准执行)
|
||||
|
||||
不再以首次构建5分钟、1Gi实验库连续全套运行、全部查询500ms或200/500人长时压测阻塞影子部署。先更新影子构建器,在有效新窗口中追平同步,确认一个完整发布及关键生产对账,再灰度接入少量看板/作业记录。保留现有3分钟后台刷新、15分钟最大展示年龄;首轮以首屏约2–3秒、无超时、结果和权限正确为目标。全量入口和长期容量须后续独立验收,不能把分批功能通过换算成容量承诺。
|
||||
|
||||
不可放宽项:majority+journal、权限/供应商隔离、版本竞争、客户标注保护、完整代次原子发布、硬故障停止与回退。客户现有标注、切片、送检、质检和验收继续使用现有生产链路,不等待新查询链路完成。原运行窗口已结束,新的启动截止、镜像及目标仍须按项目规则确认。
|
||||
|
||||
@ -305,6 +305,30 @@ func (s *MongoStore) queryBuildPublish(ctx context.Context, l queryModelLease, j
|
||||
if state.Heads[workforceFactKey(j.Organization)].FactEpoch != j.OldEpoch {
|
||||
return errors.New("query build published epoch changed")
|
||||
}
|
||||
// A payload can be reused far longer than the orphan grace. Pin its
|
||||
// retirement in the same commit as the replacement heads, rather than
|
||||
// interpreting the payload's original createdAt as retirement time.
|
||||
now := time.Now().UTC()
|
||||
oldProduction := state.Heads[queryModelKey("production", j.Organization, j.Projects)]
|
||||
next.Retired = nil
|
||||
for _, pin := range j.Retired {
|
||||
if now.Before(pin.Until) && pin.Generation != j.Production.Generation {
|
||||
next.Retired = append(next.Retired, pin)
|
||||
}
|
||||
}
|
||||
if oldProduction.Generation != "" && oldProduction.Generation != j.Production.Generation {
|
||||
until := now.Add(15 * time.Minute)
|
||||
found := false
|
||||
for i := range next.Retired {
|
||||
if next.Retired[i].Generation == oldProduction.Generation {
|
||||
next.Retired[i].Until = until
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
next.Retired = append(next.Retired, queryBuildRetiredGeneration{Generation: oldProduction.Generation, Until: until})
|
||||
}
|
||||
}
|
||||
wf := QueryModelHead{Generation: workforceFactSchema, Collection: workforceFactCollection, FactEpoch: epoch, Rows: j.PlanRows, AsOf: j.AsOf, SourceRevision: j.SourceRevision, GlobalRevision: j.GlobalRevision}
|
||||
result, e := s.queryControl().UpdateOne(sc, queryLeaseFilter(l), bson.M{"$set": bson.M{"heads." + workforceFactKey(j.Organization): wf, "heads." + queryModelKey("production", j.Organization, j.Projects): j.Production}})
|
||||
if e != nil {
|
||||
|
||||
@ -18,45 +18,51 @@ const queryBuildJobsCollection = "queryBuildJobs"
|
||||
const queryBuildPartsCollection = "queryBuildParts"
|
||||
const queryBuildJobSchema = "query-build-job/v1"
|
||||
|
||||
type queryBuildRetiredGeneration struct {
|
||||
Generation string `bson:"generation"`
|
||||
Until time.Time `bson:"until"`
|
||||
}
|
||||
|
||||
type queryBuildJob struct {
|
||||
ID string `bson:"_id"`
|
||||
Schema string `bson:"schema"`
|
||||
InputGeneration string `bson:"inputGeneration"`
|
||||
Generation string `bson:"generation"`
|
||||
Organization string `bson:"organizationId"`
|
||||
Projects []string `bson:"projects"`
|
||||
Cold bool `bson:"cold"`
|
||||
Phase string `bson:"phase"`
|
||||
Snapshot primitive.Timestamp `bson:"snapshot"`
|
||||
AsOf time.Time `bson:"asOf"`
|
||||
SourceRevision int64 `bson:"sourceRevision"`
|
||||
GlobalRevision int64 `bson:"globalRevision"`
|
||||
OldEpoch int64 `bson:"oldEpoch"`
|
||||
Epoch int64 `bson:"epoch"`
|
||||
Production QueryModelHead `bson:"production"`
|
||||
PlanParts int `bson:"planParts"`
|
||||
PlanRows int `bson:"planRows"`
|
||||
PlanDigest string `bson:"planDigest"`
|
||||
PlanSaved int `bson:"planSaved"`
|
||||
PlanBytes int64 `bson:"planBytes"`
|
||||
ProductionSaved int `bson:"productionSaved"`
|
||||
NextPart int `bson:"nextPart"`
|
||||
NextRow int `bson:"nextRow"`
|
||||
Changed bool `bson:"changed"`
|
||||
CleanupHasAfter bool `bson:"cleanupHasAfter"`
|
||||
CleanupPlanPart int `bson:"cleanupPlanPart"`
|
||||
CleanupPlanRow int `bson:"cleanupPlanRow"`
|
||||
CleanupAfter string `bson:"cleanupAfter"`
|
||||
CleanupFuture bool `bson:"cleanupFuture"`
|
||||
ConfirmedRows int64 `bson:"confirmedRows"`
|
||||
ConfirmedBytes int64 `bson:"confirmedBytes"`
|
||||
ReusedRows int64 `bson:"reusedRows"`
|
||||
ActiveWithoutProgress time.Duration `bson:"activeWithoutProgress"`
|
||||
ProgressAt time.Time `bson:"progressAt"`
|
||||
FailurePhase string `bson:"failurePhase,omitempty"`
|
||||
FailureCode int32 `bson:"failureCode,omitempty"`
|
||||
FailureClass string `bson:"failureClass,omitempty"`
|
||||
UpdatedAt time.Time `bson:"updatedAt"`
|
||||
Retired []queryBuildRetiredGeneration `bson:"retiredGenerations,omitempty"`
|
||||
ID string `bson:"_id"`
|
||||
Schema string `bson:"schema"`
|
||||
InputGeneration string `bson:"inputGeneration"`
|
||||
Generation string `bson:"generation"`
|
||||
Organization string `bson:"organizationId"`
|
||||
Projects []string `bson:"projects"`
|
||||
Cold bool `bson:"cold"`
|
||||
Phase string `bson:"phase"`
|
||||
Snapshot primitive.Timestamp `bson:"snapshot"`
|
||||
AsOf time.Time `bson:"asOf"`
|
||||
SourceRevision int64 `bson:"sourceRevision"`
|
||||
GlobalRevision int64 `bson:"globalRevision"`
|
||||
OldEpoch int64 `bson:"oldEpoch"`
|
||||
Epoch int64 `bson:"epoch"`
|
||||
Production QueryModelHead `bson:"production"`
|
||||
PlanParts int `bson:"planParts"`
|
||||
PlanRows int `bson:"planRows"`
|
||||
PlanDigest string `bson:"planDigest"`
|
||||
PlanSaved int `bson:"planSaved"`
|
||||
PlanBytes int64 `bson:"planBytes"`
|
||||
ProductionSaved int `bson:"productionSaved"`
|
||||
NextPart int `bson:"nextPart"`
|
||||
NextRow int `bson:"nextRow"`
|
||||
Changed bool `bson:"changed"`
|
||||
CleanupHasAfter bool `bson:"cleanupHasAfter"`
|
||||
CleanupPlanPart int `bson:"cleanupPlanPart"`
|
||||
CleanupPlanRow int `bson:"cleanupPlanRow"`
|
||||
CleanupAfter string `bson:"cleanupAfter"`
|
||||
CleanupFuture bool `bson:"cleanupFuture"`
|
||||
ConfirmedRows int64 `bson:"confirmedRows"`
|
||||
ConfirmedBytes int64 `bson:"confirmedBytes"`
|
||||
ReusedRows int64 `bson:"reusedRows"`
|
||||
ActiveWithoutProgress time.Duration `bson:"activeWithoutProgress"`
|
||||
ProgressAt time.Time `bson:"progressAt"`
|
||||
FailurePhase string `bson:"failurePhase,omitempty"`
|
||||
FailureCode int32 `bson:"failureCode,omitempty"`
|
||||
FailureClass string `bson:"failureClass,omitempty"`
|
||||
UpdatedAt time.Time `bson:"updatedAt"`
|
||||
}
|
||||
|
||||
func queryBuildJobKey(database, org string) string {
|
||||
|
||||
75
internal/store/query_build_outcome_local_test.go
Normal file
75
internal/store/query_build_outcome_local_test.go
Normal file
@ -0,0 +1,75 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"droboticflow/internal/testsupport"
|
||||
"errors"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestQueryBuildOutcomePersistenceFailureIsTerminalLocal(t *testing.T) {
|
||||
for _, kind := range []string{"budget", "phase"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
_, s, _ := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
j, e := s.queryBuildStart(ctx, l, "outcome-fixture", nil)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if kind == "phase" {
|
||||
j.Phase = "unsupported"
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, j) }); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
validator := bson.M{"$and": bson.A{bson.M{"activeWithoutProgress": bson.M{"$lte": int64(0)}}, bson.M{"failureClass": bson.M{"$exists": false}}}}
|
||||
if e = s.db.RunCommand(ctx, bson.D{{Key: "collMod", Value: queryBuildJobsCollection}, {Key: "validator", Value: validator}, {Key: "validationLevel", Value: "strict"}, {Key: "validationAction", Value: "error"}}).Err(); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
round := newQueryBuildRound(true)
|
||||
if kind == "budget" {
|
||||
now := time.Now()
|
||||
round = &queryBuildRound{until: now, now: func() time.Time { return now }}
|
||||
}
|
||||
e = s.queryBuildRunRound(ctx, l, j.Organization, nil, round)
|
||||
if diagnostic := QueryReplicaDiagnostic(e); diagnostic.MongoCode != 121 {
|
||||
t.Fatal("failed to exercise actual checkpoint validation failure", e)
|
||||
}
|
||||
if !errors.Is(e, ErrQueryBuildStopped) || errors.Is(e, ErrQueryBuildYielded) {
|
||||
t.Fatal("durability failure became retry/yield", e)
|
||||
}
|
||||
preserved, ok, e := s.queryBuildJob(ctx, j.Organization)
|
||||
if e != nil || !ok || preserved.Generation != j.Generation || preserved.NextPart != 0 {
|
||||
t.Fatal("checkpoint lost", e)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildResumeReadFailureIsTerminalLocal(t *testing.T) {
|
||||
_, s, _ := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI(testsupport.QueryReplicaTargetURI(t)))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
disconnected := NewMongoStore(client.Database(s.db.Name()))
|
||||
disconnected.queryReplicaFactStore = true
|
||||
disconnected.queryPeriodic = true
|
||||
if e = client.Disconnect(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
e = disconnected.queryBuildRunRound(ctx, queryModelLease{}, "resume-fixture", nil, nil)
|
||||
if !errors.Is(e, ErrQueryBuildStopped) || !errors.Is(e, mongo.ErrClientDisconnected) {
|
||||
t.Fatal("unavailable checkpoint storage automatically retried", e)
|
||||
}
|
||||
}
|
||||
181
internal/store/query_build_process_restart_local_test.go
Normal file
181
internal/store/query_build_process_restart_local_test.go
Normal file
@ -0,0 +1,181 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/testsupport"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/event"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/mongo/readpref"
|
||||
)
|
||||
|
||||
// Explicit opt-in: this stops only the verified owned loopback query container.
|
||||
// Run alone, outside other Mongo/browser tests; never on a company cluster.
|
||||
func TestQueryBuildPhysicalMongoRestartMidApplyLocal(t *testing.T) {
|
||||
if os.Getenv("QUERY_BUILD_PHYSICAL_RESTART") != "1" {
|
||||
t.Skip("explicit owned lab restart required")
|
||||
}
|
||||
ctx := t.Context()
|
||||
const container = "fanyu-query-replica-local-query-1"
|
||||
command := func(args ...string) []byte {
|
||||
t.Helper()
|
||||
op, stop := context.WithTimeout(ctx, 40*time.Second)
|
||||
defer stop()
|
||||
out, e := exec.CommandContext(op, "docker", args...).Output()
|
||||
if e != nil {
|
||||
t.Fatal("owned lab command failed", e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
var before []struct {
|
||||
ID string `json:"Id"`
|
||||
Name string
|
||||
Config struct{ Labels map[string]string }
|
||||
HostConfig struct{ Memory, NanoCpus int64 }
|
||||
Mounts []struct{ Type, Name, Destination string }
|
||||
}
|
||||
if e := json.Unmarshal(command("inspect", container), &before); e != nil || len(before) != 1 {
|
||||
t.Fatal(e)
|
||||
}
|
||||
cfg := before[0]
|
||||
if cfg.Name != "/"+container || cfg.HostConfig.Memory != 1<<30 || cfg.HostConfig.NanoCpus != 1_000_000_000 || cfg.Config.Labels["com.docker.compose.project"] != "fanyu-query-replica-local" {
|
||||
t.Fatal("different fixture identity")
|
||||
}
|
||||
hasVolume := false
|
||||
for _, v := range cfg.Mounts {
|
||||
hasVolume = hasVolume || v.Destination == "/data/db"
|
||||
}
|
||||
if !hasVolume {
|
||||
t.Fatal("persistent volume missing")
|
||||
}
|
||||
var native atomic.Int64
|
||||
monitor := &event.CommandMonitor{Started: func(_ context.Context, e *event.CommandStartedEvent) {
|
||||
if e.CommandName == "find" {
|
||||
name, _ := e.Command.Lookup("find").StringValueOK()
|
||||
if querySourceCollection(name) {
|
||||
native.Add(1)
|
||||
}
|
||||
}
|
||||
}}
|
||||
_, s, _ := newLocalReplica(t, nil, monitor)
|
||||
org := "workforce-test"
|
||||
seedWorkforceStats(t, s)
|
||||
events, tasks := []any{}, []any{}
|
||||
seconds := 1.0
|
||||
for i := 0; i < 100; i++ {
|
||||
id := fmt.Sprintf("physical-task-%03d", i)
|
||||
tasks = append(tasks, bson.M{"_id": id, "id": id, "organizationId": org, "projectId": "p", "supplierId": "s", "runId": "run"})
|
||||
events = append(events, domain.AnnotationWorklogEventRecord{ID: fmt.Sprintf("physical-event-%03d", i), OrganizationID: org, TaskID: id, ProjectID: "p", SupplierID: "s", UserID: "u", EventType: "annotation_submitted", Mode: domain.RunModeProduction, Revision: 1, VideoSeconds: &seconds, OutputDelta: 1, CreatedAt: time.Date(2026, 9, 28, 13, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano)})
|
||||
}
|
||||
if _, e := s.annotationTasks.InsertMany(ctx, tasks); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e := s.annotationWorklogEvents.InsertMany(ctx, events); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
j, e := s.queryBuildStart(ctx, l, org, []string{"p"})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildProductionPlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildWorkforcePlan(ctx, l, &j, newQueryBuildRound(true)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
calls := 0
|
||||
now := time.Now()
|
||||
round := &queryBuildRound{until: now.Add(time.Second), now: func() time.Time {
|
||||
calls++
|
||||
if calls > 2 {
|
||||
return now.Add(2 * time.Second)
|
||||
}
|
||||
return now
|
||||
}}
|
||||
if e = s.queryBuildApply(ctx, l, &j, round); !errors.Is(e, ErrQueryBuildYielded) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
checkpoint, ok, e := s.queryBuildJob(ctx, org)
|
||||
if e != nil || !ok || checkpoint.NextPart+checkpoint.NextRow == 0 {
|
||||
t.Fatal("no durable mid-apply checkpoint", e)
|
||||
}
|
||||
beforeReads := native.Load()
|
||||
// Gracefully releasing the old process lease is independent of the database
|
||||
// restart; its new holder must still fence the old epoch afterwards.
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"until": time.Now().Add(-time.Second)}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
command("restart", "-t", "30", container)
|
||||
var after []struct {
|
||||
ID string `json:"Id"`
|
||||
HostConfig struct{ Memory, NanoCpus int64 }
|
||||
Mounts []struct{ Type, Name, Destination string }
|
||||
}
|
||||
if e = json.Unmarshal(command("inspect", container), &after); e != nil || len(after) != 1 {
|
||||
t.Fatal(e)
|
||||
}
|
||||
a, _ := json.Marshal(cfg.Mounts)
|
||||
b, _ := json.Marshal(after[0].Mounts)
|
||||
if after[0].ID != cfg.ID || after[0].HostConfig.Memory != cfg.HostConfig.Memory || after[0].HostConfig.NanoCpus != cfg.HostConfig.NanoCpus || string(a) != string(b) {
|
||||
t.Fatal("restart changed identity, resources or volume")
|
||||
}
|
||||
client, e := mongo.Connect(ctx, options.Client().ApplyURI(testsupport.QueryReplicaTargetURI(t)).SetMonitor(monitor))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer client.Disconnect(ctx)
|
||||
ready := false
|
||||
deadline := time.Now().Add(25 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
op, stop := context.WithTimeout(ctx, time.Second)
|
||||
e = client.Ping(op, readpref.Primary())
|
||||
stop()
|
||||
if e == nil {
|
||||
ready = true
|
||||
break
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
if !ready {
|
||||
t.Fatal("same-volume primary did not recover", e)
|
||||
}
|
||||
resumed := NewMongoStore(client.Database(s.db.Name()))
|
||||
resumed.queryReplicaFactStore = true
|
||||
resumed.queryPeriodic = true
|
||||
recovered, ok, e := resumed.queryBuildJob(ctx, org)
|
||||
if e != nil || !ok || recovered.Generation != checkpoint.Generation || recovered.NextPart != checkpoint.NextPart || recovered.NextRow != checkpoint.NextRow {
|
||||
t.Fatal("physical restart lost confirmed checkpoint", e)
|
||||
}
|
||||
next, ok, e := resumed.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = resumed.queryBuildRunRound(ctx, next, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
completed, _, e := resumed.queryBuildJob(ctx, org)
|
||||
if e != nil || completed.Phase != "complete" || completed.PlanRows != 408 || native.Load() != beforeReads {
|
||||
t.Fatal("physical restart repeated input or lost facts", completed.PlanRows, native.Load()-beforeReads, e)
|
||||
}
|
||||
t.Logf("physicalRestart=true sameVolume=true sameResources=true confirmedCursor=%d/%d finalFacts=%d repeatedNativeReads=0 majorityJournal=true", checkpoint.NextPart, checkpoint.NextRow, completed.PlanRows)
|
||||
}
|
||||
@ -358,3 +358,70 @@ func TestQueryBuildSealedInputSurvivesExpiredSnapshotAndOrphanGCLocal(t *testing
|
||||
t.Fatal("GC deleted unfinished durable blob", n, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryBuildRetiredPayloadKeepsInflightReaderGraceLocal(t *testing.T) {
|
||||
_, s, _ := newLocalReplica(t)
|
||||
ctx := t.Context()
|
||||
org := "workforce-test"
|
||||
seedWorkforceStats(t, s)
|
||||
s.queryReplicaFactStore = true
|
||||
s.queryPeriodic = true
|
||||
if e := s.ensureWorkforceFactIndexes(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
l, ok, e := s.acquireQueryModelLease(ctx)
|
||||
if e != nil || !ok {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
first, _, e := s.queryBuildJob(ctx, org)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
old := first.Production.Generation
|
||||
if _, e = s.queryData().UpdateMany(ctx, bson.M{"generation": old}, bson.M{"$set": bson.M{"createdAt": time.Now().Add(-24 * time.Hour)}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
// A changed generation retires a payload that may have been reused for days.
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$inc": bson.M{"productionRevision": 1}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.cleanQueryModelOrphans(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
n, e := s.queryData().CountDocuments(ctx, bson.M{"generation": old})
|
||||
if e != nil || n != int64(first.Production.Chunks) {
|
||||
t.Fatal("retired payload disappeared while in-flight reader may still hydrate it", n, e)
|
||||
}
|
||||
retained, _, e := s.queryBuildJob(ctx, org)
|
||||
if e != nil || len(retained.Retired) != 1 {
|
||||
t.Fatal("missing retirement checkpoint", e)
|
||||
}
|
||||
until := retained.Retired[0].Until
|
||||
if _, e = s.queryControl().UpdateOne(ctx, queryLeaseFilter(l), bson.M{"$set": bson.M{"observedAt": time.Now().UTC()}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.queryBuildRunRound(ctx, l, org, []string{"p"}, nil); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
retained, _, e = s.queryBuildJob(ctx, org)
|
||||
if e != nil || len(retained.Retired) != 1 || !retained.Retired[0].Until.Equal(until) {
|
||||
t.Fatal("unchanged refresh lost or extended retirement grace", e)
|
||||
}
|
||||
retained.Retired[0].Until = time.Now().Add(-time.Minute)
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, retained) }); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = s.cleanQueryModelOrphans(ctx); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
n, e = s.queryData().CountDocuments(ctx, bson.M{"generation": old})
|
||||
if e != nil || n != 0 {
|
||||
t.Fatal("expired payload pinned forever", n, e)
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"reflect"
|
||||
"sort"
|
||||
"time"
|
||||
@ -56,6 +57,7 @@ func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org
|
||||
}
|
||||
head := state.Heads[workforceFactKey(org)]
|
||||
j := queryBuildJob{ID: queryBuildJobKey(s.db.Name(), org), Schema: queryBuildJobSchema, Organization: org, Projects: projects, Generation: queryserving.Key(queryBuildJobSchema, org, at.T, at.I, time.Now().UnixNano()), Snapshot: at, AsOf: observedAt, Phase: "production_plan", OldEpoch: head.FactEpoch, Epoch: head.FactEpoch + 1, Cold: head.FactEpoch == 0, SourceRevision: state.Revision, GlobalRevision: state.Global, ProgressAt: time.Now().UTC()}
|
||||
j.Retired = append([]queryBuildRetiredGeneration{}, old.Retired...)
|
||||
j.InputGeneration = j.Generation
|
||||
if exists && old.Phase == "complete" && reflect.DeepEqual(old.Projects, projects) && old.SourceRevision == state.Revision && old.GlobalRevision == state.Global && !state.Observed.Before(old.AsOf) && time.Since(state.Observed) >= 0 && time.Since(state.Observed) < 15*time.Second {
|
||||
j.AsOf = state.Observed
|
||||
@ -86,7 +88,12 @@ func (s *MongoStore) queryBuildStart(ctx context.Context, l queryModelLease, org
|
||||
func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease, org string, projects []string, r *queryBuildRound) (err error) {
|
||||
j, err := s.queryBuildStart(ctx, l, org, projects)
|
||||
if err != nil {
|
||||
return err
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
d := QueryReplicaDiagnostic(err)
|
||||
log.Printf("query_model_build result=stopped stage=resume category=%s mongoCode=%d", d.Class, d.MongoCode)
|
||||
return fmt.Errorf("%w: resume: %w", ErrQueryBuildStopped, err)
|
||||
}
|
||||
if r == nil {
|
||||
r = newQueryBuildRound(j.Cold)
|
||||
@ -123,7 +130,9 @@ func (s *MongoStore) queryBuildRunRound(ctx context.Context, l queryModelLease,
|
||||
err = fmt.Errorf("%w: %w", ErrQueryBuildStopped, err)
|
||||
}
|
||||
if e = s.queryBuildTransaction(ctx, l, func(sc mongo.SessionContext) error { return s.saveQueryBuildJob(sc, current) }); e != nil {
|
||||
err = fmt.Errorf("query build could not persist outcome: %w", e)
|
||||
d := QueryReplicaDiagnostic(e)
|
||||
log.Printf("query_model_build result=stopped stage=checkpoint category=%s mongoCode=%d", d.Class, d.MongoCode)
|
||||
err = fmt.Errorf("%w: query build could not persist outcome: %w", ErrQueryBuildStopped, e)
|
||||
}
|
||||
}()
|
||||
for {
|
||||
|
||||
@ -42,7 +42,7 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
|
||||
keep = append(keep, h.Generation)
|
||||
}
|
||||
if s.queryReplicaFactStore && s.periodicQueryStore() {
|
||||
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema, "phase": bson.M{"$ne": "complete"}}, options.Find().SetProjection(bson.M{"production.generation": 1}))
|
||||
cur, err := s.db.Collection(queryBuildJobsCollection).Find(ctx, bson.M{"schema": queryBuildJobSchema}, options.Find().SetProjection(bson.M{"production.generation": 1, "phase": 1, "retiredGenerations": 1}))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -53,9 +53,14 @@ func (s *MongoStore) cleanQueryModelOrphans(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
for _, j := range jobs {
|
||||
if j.Production.Generation != "" {
|
||||
if j.Phase != "complete" && j.Production.Generation != "" {
|
||||
keep = append(keep, j.Production.Generation)
|
||||
}
|
||||
for _, pin := range j.Retired {
|
||||
if time.Now().Before(pin.Until) {
|
||||
keep = append(keep, pin.Generation)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
filter := bson.M{"generation": bson.M{"$nin": keep}, "createdAt": bson.M{"$lt": time.Now().UTC().Add(-15 * time.Minute)}}
|
||||
|
||||
@ -35,5 +35,31 @@ class ReplicaRunnerSafety(unittest.TestCase):
|
||||
self.assertEqual(call.call_count, 1)
|
||||
|
||||
|
||||
|
||||
class ReplicaRunnerBatchSafety(unittest.TestCase):
|
||||
def test_bounded_batches_cover_every_registered_case_once(self):
|
||||
required = {f'TestCase{i:03d}' for i in range(53)}
|
||||
batches = runner.case_batches(required, True, False)
|
||||
self.assertEqual([len(x) for x in batches], [20, 20, 13])
|
||||
self.assertEqual([x for group in batches for x in group], sorted(required))
|
||||
self.assertEqual(len(runner.case_batches(required, False, False)), 1)
|
||||
self.assertEqual(len(runner.case_batches(required, True, True)), 1)
|
||||
with self.assertRaisesRegex(RuntimeError, 'no registered'):
|
||||
runner.case_batches(set(), True, False)
|
||||
|
||||
def test_restart_refuses_other_clients_without_mutating_fixture(self):
|
||||
import json
|
||||
fixture = {'Name': '/fanyu-query-replica-local-query-1',
|
||||
'HostConfig': {'Memory': 1024 ** 3, 'NanoCpus': 1_000_000_000},
|
||||
'Config': {'Labels': {'com.docker.compose.project': 'fanyu-query-replica-local'}},
|
||||
'Mounts': [{'Destination': '/data/db'}]}
|
||||
with patch.object(runner.subprocess, 'check_output', return_value=json.dumps([fixture])), \
|
||||
patch.object(runner, 'query_probe', return_value=(3, 'other clients')), \
|
||||
patch.object(runner.time, 'sleep'), patch.object(runner, 'compose') as change:
|
||||
with self.assertRaisesRegex(RuntimeError, 'other_clients'):
|
||||
runner.restart_query_fixture({}, 'synthetic-probe', 7)
|
||||
change.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@ -10,6 +10,8 @@ import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
import sys
|
||||
from lib.release_checks import check_environment, source_fingerprint, write_log
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@ -33,6 +35,7 @@ CASES = ('cmd/query-replica/main_local_test.go',
|
||||
'internal/store/workforce_facts_local_test.go',
|
||||
'internal/store/query_build_capture_local_test.go',
|
||||
'internal/store/query_build_resume_local_test.go',
|
||||
'internal/store/query_build_outcome_local_test.go',
|
||||
'internal/store/workforce_publication_local_test.go',
|
||||
'internal/store/query_replica_page_local_test.go',
|
||||
'internal/store/supplier_authorization_batch_local_test.go',
|
||||
@ -128,6 +131,70 @@ def start(env):
|
||||
raise RuntimeError('query replica did not become primary')
|
||||
|
||||
|
||||
def case_batches(required, bounded, native):
|
||||
names = sorted(required)
|
||||
if not names:
|
||||
raise RuntimeError('no registered replica cases')
|
||||
size = 20 if bounded and not native else len(names)
|
||||
return [names[i:i + size] for i in range(0, len(names), size)]
|
||||
|
||||
|
||||
def query_probe(env, script):
|
||||
# Fixed loopback, no credentials, four-second process bound. This is not
|
||||
# an arbitrary database command entry point or a production probe.
|
||||
return run(['docker', 'exec', 'fanyu-query-replica-local-query-1',
|
||||
'mongosh', '--quiet',
|
||||
'mongodb://127.0.0.1:27129/?serverSelectionTimeoutMS=1000&connectTimeoutMS=1000&appName=droboticflow-query-fixture-probe&maxPoolSize=1&maxConnecting=1',
|
||||
'--eval', script], env, 4)
|
||||
|
||||
|
||||
def restart_query_fixture(env, probe, cursor):
|
||||
name = 'fanyu-query-replica-local-query-1'
|
||||
before = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
|
||||
if (before['Name'] != '/' + name or before['HostConfig']['Memory'] != 1024 ** 3
|
||||
or before['HostConfig']['NanoCpus'] != 1_000_000_000
|
||||
or before['Config']['Labels'].get('com.docker.compose.project') != 'fanyu-query-replica-local'
|
||||
or not any(m['Destination'] == '/data/db' for m in before['Mounts'])):
|
||||
raise RuntimeError('refusing to restart a different query fixture')
|
||||
# A direct client from another checkout must not be interrupted. Exclude only this named probe and the replica set internal client;
|
||||
# wait briefly for completed test clients and in-progress handshakes to close.
|
||||
for attempt in range(3):
|
||||
code, _ = query_probe(env,
|
||||
'sleep(200);'
|
||||
'const clients=db.getSiblingDB("admin").aggregate(['
|
||||
'{$currentOp:{allUsers:true,idleConnections:true}},'
|
||||
'{$match:{client:{$exists:true},"clientMetadata.application.name":{$ne:"droboticflow-query-fixture-probe"},'
|
||||
'"clientMetadata.driver.name":{$not:/^NetworkInterfaceTL/}}},'
|
||||
'{$limit:1}]).toArray();if(clients.length) quit(3)')
|
||||
if code == 0:
|
||||
break
|
||||
if attempt == 2:
|
||||
raise RuntimeError('query restart admission failed: ' + ('other_clients' if code == 3 else 'probe_exit_' + str(code)))
|
||||
time.sleep(.5)
|
||||
owner = json.dumps(probe)
|
||||
code, _ = query_probe(env, 'db.getSiblingDB(' + owner + ').checkpoint.updateOne({_id:"progress"},'
|
||||
'{$set:{cursor:' + str(cursor) + ',fingerprint:"synthetic-majority-journal"}},'
|
||||
'{upsert:true,writeConcern:{w:"majority",j:true}})')
|
||||
if code:
|
||||
raise RuntimeError('restart canary persistence failed')
|
||||
compose(env, 'restart', '-t', '30', 'query')
|
||||
after = json.loads(subprocess.check_output(['docker', 'inspect', name], text=True))[0]
|
||||
if (after['Id'] != before['Id'] or after['Mounts'] != before['Mounts']
|
||||
or after['HostConfig']['Memory'] != before['HostConfig']['Memory']
|
||||
or after['HostConfig']['NanoCpus'] != before['HostConfig']['NanoCpus']):
|
||||
raise RuntimeError('query restart changed volume or resource identity')
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
code, _ = query_probe(env, 'if(!db.hello().isWritablePrimary||db.hello().setName!=="query-read-rs") quit(2);'
|
||||
'const p=db.getSiblingDB(' + owner + ').checkpoint.findOne({_id:"progress"});'
|
||||
'if(!p||p.cursor!==' + str(cursor) + '||p.fingerprint!=="synthetic-majority-journal") quit(4)')
|
||||
if code == 0:
|
||||
return {'container': before['Id'], 'cursor': cursor, 'majorityJournalRecovered': True,
|
||||
'sameVolume': True, 'sameResources': True}
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('query restart readiness or durable canary recovery failed')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--native', action='store_true')
|
||||
@ -171,21 +238,37 @@ def main():
|
||||
if args.scale:
|
||||
required = {'TestQueryReplicaScaleLocal'}
|
||||
env['QUERY_REPLICA_SCALE_LOCAL'] = '1'
|
||||
regex = '^(' + '|'.join(sorted(required)) + ')$'
|
||||
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
|
||||
'-run', regex, '-count=1', '-timeout=10m', '-v']
|
||||
if args.race:
|
||||
argv.append('-race')
|
||||
code, output = run(argv, env)
|
||||
batches = case_batches(required, args.bounded_query, native)
|
||||
outputs, restart_proofs, commands = [], [], []
|
||||
code = 0
|
||||
probe = 'query_replica_restart_probe_' + uuid.uuid4().hex
|
||||
for index, names in enumerate(batches):
|
||||
if len(batches) > 1:
|
||||
restart_proofs.append(restart_query_fixture(env, probe, index))
|
||||
regex = '^(' + '|'.join(names) + ')$'
|
||||
argv = ['go', 'test', '-p=1', './internal/store', './internal/api', './cmd/query-cdc-probe', './cmd/query-replica',
|
||||
'-run', regex, '-count=1', '-timeout=10m', '-v']
|
||||
if args.race:
|
||||
argv.append('-race')
|
||||
commands.append(argv)
|
||||
code, output = run(argv, env)
|
||||
outputs.append(output)
|
||||
if code:
|
||||
break
|
||||
output = '\n'.join(outputs)
|
||||
|
||||
passed = set(re.findall(r'^--- PASS: (Test\w+)', output, re.M))
|
||||
missing = sorted(required - passed)
|
||||
changed = not before or source_fingerprint(ROOT) != before
|
||||
skipped = bool(re.search(r'^--- SKIP:', output, re.M))
|
||||
success = code == 0 and not missing and not changed and not skipped
|
||||
proof = write_log(PRIVATE / 'logs', argv, output, 'passed' if success else 'pending_or_failed')
|
||||
proof = write_log(PRIVATE / 'logs', [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]], output, 'passed' if success else 'pending_or_failed')
|
||||
proof.update(missing=missing, sourceChanged=changed, skipped=skipped,
|
||||
environment='native-uncapped' if native else ('docker-source-4cpu-query-1cpu-1Gi' if args.bounded_query else 'docker-source-4cpu-query-4cpu'),
|
||||
productionCapacityAccepted=False, reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
|
||||
productionCapacityAccepted=False, boundedBatches=len(batches),
|
||||
continuousCapacityAccepted=False, restartProofs=restart_proofs, requiredCount=len(required),
|
||||
passedCount=len(passed), batchCommands=commands, completedBatches=len(outputs),
|
||||
reusedResources=json.loads(env.get('QUERY_REPLICA_REUSED_RESOURCES', '[]')))
|
||||
print(json.dumps(proof, ensure_ascii=False))
|
||||
return 0 if success else (code or 2)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user