🐛 Bound legacy pool count joins to the active page

This commit is contained in:
Codex 2026-11-13 01:10:34 -08:00
parent e13d4bd6b6
commit c8d34faa85
4 changed files with 55 additions and 3 deletions

View File

@ -115,3 +115,9 @@
IO保护新增严格的FUSE归因分支,并有先失败后通过的回归:只在所有捕获的D状态栈均为FUSE、该Pod没有新增块设备读写、其他受保护Pod没有新增IO等待、物理盘等待≤1ms且忙碌≤5%时,将该窗口单列为网络文件系统等待。未知、空栈、混合栈、本地块IO、后端等待或盘变慢仍触发原保护;无新增PSI的衰减值不重复算持续恶化。API健康、节点压力、空间、CPU和盘等待的独立停测条件保留。每个超限窗口保留归因JSON和原值,不将过去失败追溯改成通过。
修正后的空载基线在首次采样完成后计时,至少120秒。此前`scale-81-4`实际采样首尾119.4秒的记录保留。12项离线安全回归和中断恢复后的快速检查通过,下一轮完整规模与任务级验证仍以新记录为准,尚未生产验收。
`scale-81-5`在修正临时目录后完成305.481秒回填及报表构建,四类接口的管理员业务JSON通过精确对照;进入旧路径81并发时,合成源/驱动Pod在4Gi限制下被OOMKilled(退出137),没有得到完整性能报告。不是生产Pod OOM,也不是性能通过。没有增加副本、内存或IO预算来掩盖失败。
进一步定位到旧路径计数的请求缓存跨页保留`itemsByTask`,内存随任务总量和并发数增长。新增实际缓存生命周期回归先失败(第二页32组、预期16组),修正为计数仅保留当前页的任务关联;批次/权限缓存仍为请求内,当前页旧任务关联与完整列表行为保持原样。`--race`计数/列表权限等价、多页、自审、跨用户隔离等回归通过,受影响任务级Mongo和浏览器检查全部通过。此缺口足以解释明显的内存放大,但仍须同规模实测确认它是否为本次OOM的唯一原因。
下一轮两条路径均包含这项内存稳定性修复,不能将其旧路径结果标成未修改生产版本的结果。原OOM证据与之前的失败均保留。驱动最终健康采样失败时也会记录原始退出码和覆盖缺项,不丢失失败原因。扩大为20轮重复的尝试被自动审批以共享节点持续负载风险拒绝,未执行;仍保持2轮,不用多次小跑绕过该限制。

View File

@ -83,6 +83,12 @@ func (server *server) withHumanListReads(ctx context.Context, org, project strin
// Counts never hydrate task details or compute response-only claim flags.
// Keep authorization batch/member caches, but do not retain a second copy
// of every task's history join across the exhaustive count scan.
if mode, _ := ctx.Value(humanPoolReadModeKey{}).(humanPoolReadMode); mode == humanPoolReadCount {
// Legacy authority counts also traverse the entire pool. Only joins for
// the active page are used below; keeping past pages grows with corpus
// size for every concurrent caller. Permission caches remain request-local.
clear(cache.itemsByTask)
}
if reader, ok := server.store.(interface{ UsesQueryReplicaCountRead(context.Context) bool }); ok && reader.UsesQueryReplicaCountRead(ctx) {
return context.WithValue(ctx, humanListReadsKey{}, cache), nil
}

View File

@ -17,6 +17,42 @@ type acceptanceSourceReadProbe struct {
failSecondBatch bool
}
func TestHumanCountReadCacheRetainsOnlyActivePage(t *testing.T) {
for _, count := range []bool{true, false} {
t.Run(fmt.Sprintf("count=%v", count), func(t *testing.T) {
s := &server{store: store.NewMemoryStore()}
ctx := context.Background()
if count {
ctx = context.WithValue(ctx, humanPoolReadModeKey{}, humanPoolReadCount)
}
for page := 0; page < 3; page++ {
items := make([]domain.HumanItemRecord, 16)
for i := range items {
id := fmt.Sprintf("page-%d-task-%d", page, i)
items[i] = domain.HumanItemRecord{ID: id, LegacyTaskID: id, OrganizationID: "org", ProjectID: "project", PluginID: "annotation-qa"}
}
var err error
ctx, err = s.withHumanListReads(ctx, "org", "project", items)
if err != nil {
t.Fatal(err)
}
cache := ctx.Value(humanListReadsKey{}).(*humanListReads)
want := len(items) * (page + 1)
if count {
want = len(items)
}
if len(cache.itemsByTask) != want {
t.Fatalf("retained %d task groups, want %d", len(cache.itemsByTask), want)
}
got, err := s.humanItemForLegacyTask(ctx, domain.AnnotationTaskRecord{ID: items[0].LegacyTaskID, OrganizationID: "org", ProjectID: "project"})
if err != nil || got == nil || got.ID != items[0].ID {
t.Fatalf("active-page legacy lookup changed: %v %v", got, err)
}
}
})
}
}
func (p *acceptanceSourceReadProbe) GetHumanBatch(ctx context.Context, id string) (*domain.HumanBatchRecord, error) {
p.singles++
return p.Store.GetHumanBatch(ctx, id)

View File

@ -307,20 +307,24 @@ def main():
child.wait(timeout=5)
(out / "stopped.json").write_text(json.dumps(stopped))
raise
sample()
final_health_error = None
try:
sample()
except Exception as error:
final_health_error = str(error)
if fixture:
result = subprocess.run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "cat", remote_report], capture_output=True, text=True, timeout=30)
if result.returncode == 0:
(out / "comparison.json").write_text(result.stdout)
changed = source_fingerprint() != manifest["sourceFingerprint"]
exit_record = {"returncode": child.returncode, "sourceChanged": changed, "coverageComplete": False}
exit_record = {"returncode": child.returncode, "sourceChanged": changed, "coverageComplete": False, "finalHealthError": final_health_error}
try:
validate_comparison(out / "comparison.json", args.tasks, args.waves, args.max_actors)
exit_record["coverageComplete"] = True
except (OSError, ValueError, KeyError, AssertionError) as error:
exit_record["coverageError"] = str(error)
(out / "exit.json").write_text(json.dumps(exit_record))
raise SystemExit(child.returncode or (2 if changed or not exit_record["coverageComplete"] else 0))
raise SystemExit(child.returncode or (2 if changed or final_health_error or not exit_record["coverageComplete"] else 0))
if __name__ == "__main__":