forked from longtao.wu/droboticflow
🐛 Bound legacy pool count joins to the active page
This commit is contained in:
parent
e13d4bd6b6
commit
c8d34faa85
@ -115,3 +115,9 @@
|
||||
IO保护新增严格的FUSE归因分支,并有先失败后通过的回归:只在所有捕获的D状态栈均为FUSE、该Pod没有新增块设备读写、其他受保护Pod没有新增IO等待、物理盘等待≤1ms且忙碌≤5%时,将该窗口单列为网络文件系统等待。未知、空栈、混合栈、本地块IO、后端等待或盘变慢仍触发原保护;无新增PSI的衰减值不重复算持续恶化。API健康、节点压力、空间、CPU和盘等待的独立停测条件保留。每个超限窗口保留归因JSON和原值,不将过去失败追溯改成通过。
|
||||
|
||||
修正后的空载基线在首次采样完成后计时,至少120秒。此前`scale-81-4`实际采样首尾119.4秒的记录保留。12项离线安全回归和中断恢复后的快速检查通过,下一轮完整规模与任务级验证仍以新记录为准,尚未生产验收。
|
||||
|
||||
`scale-81-5`在修正临时目录后完成305.481秒回填及报表构建,四类接口的管理员业务JSON通过精确对照;进入旧路径81并发时,合成源/驱动Pod在4Gi限制下被OOMKilled(退出137),没有得到完整性能报告。不是生产Pod OOM,也不是性能通过。没有增加副本、内存或IO预算来掩盖失败。
|
||||
|
||||
进一步定位到旧路径计数的请求缓存跨页保留`itemsByTask`,内存随任务总量和并发数增长。新增实际缓存生命周期回归先失败(第二页32组、预期16组),修正为计数仅保留当前页的任务关联;批次/权限缓存仍为请求内,当前页旧任务关联与完整列表行为保持原样。`--race`计数/列表权限等价、多页、自审、跨用户隔离等回归通过,受影响任务级Mongo和浏览器检查全部通过。此缺口足以解释明显的内存放大,但仍须同规模实测确认它是否为本次OOM的唯一原因。
|
||||
|
||||
下一轮两条路径均包含这项内存稳定性修复,不能将其旧路径结果标成未修改生产版本的结果。原OOM证据与之前的失败均保留。驱动最终健康采样失败时也会记录原始退出码和覆盖缺项,不丢失失败原因。扩大为20轮重复的尝试被自动审批以共享节点持续负载风险拒绝,未执行;仍保持2轮,不用多次小跑绕过该限制。
|
||||
|
||||
@ -83,6 +83,12 @@ func (server *server) withHumanListReads(ctx context.Context, org, project strin
|
||||
// Counts never hydrate task details or compute response-only claim flags.
|
||||
// Keep authorization batch/member caches, but do not retain a second copy
|
||||
// of every task's history join across the exhaustive count scan.
|
||||
if mode, _ := ctx.Value(humanPoolReadModeKey{}).(humanPoolReadMode); mode == humanPoolReadCount {
|
||||
// Legacy authority counts also traverse the entire pool. Only joins for
|
||||
// the active page are used below; keeping past pages grows with corpus
|
||||
// size for every concurrent caller. Permission caches remain request-local.
|
||||
clear(cache.itemsByTask)
|
||||
}
|
||||
if reader, ok := server.store.(interface{ UsesQueryReplicaCountRead(context.Context) bool }); ok && reader.UsesQueryReplicaCountRead(ctx) {
|
||||
return context.WithValue(ctx, humanListReadsKey{}, cache), nil
|
||||
}
|
||||
|
||||
@ -17,6 +17,42 @@ type acceptanceSourceReadProbe struct {
|
||||
failSecondBatch bool
|
||||
}
|
||||
|
||||
func TestHumanCountReadCacheRetainsOnlyActivePage(t *testing.T) {
|
||||
for _, count := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("count=%v", count), func(t *testing.T) {
|
||||
s := &server{store: store.NewMemoryStore()}
|
||||
ctx := context.Background()
|
||||
if count {
|
||||
ctx = context.WithValue(ctx, humanPoolReadModeKey{}, humanPoolReadCount)
|
||||
}
|
||||
for page := 0; page < 3; page++ {
|
||||
items := make([]domain.HumanItemRecord, 16)
|
||||
for i := range items {
|
||||
id := fmt.Sprintf("page-%d-task-%d", page, i)
|
||||
items[i] = domain.HumanItemRecord{ID: id, LegacyTaskID: id, OrganizationID: "org", ProjectID: "project", PluginID: "annotation-qa"}
|
||||
}
|
||||
var err error
|
||||
ctx, err = s.withHumanListReads(ctx, "org", "project", items)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cache := ctx.Value(humanListReadsKey{}).(*humanListReads)
|
||||
want := len(items) * (page + 1)
|
||||
if count {
|
||||
want = len(items)
|
||||
}
|
||||
if len(cache.itemsByTask) != want {
|
||||
t.Fatalf("retained %d task groups, want %d", len(cache.itemsByTask), want)
|
||||
}
|
||||
got, err := s.humanItemForLegacyTask(ctx, domain.AnnotationTaskRecord{ID: items[0].LegacyTaskID, OrganizationID: "org", ProjectID: "project"})
|
||||
if err != nil || got == nil || got.ID != items[0].ID {
|
||||
t.Fatalf("active-page legacy lookup changed: %v %v", got, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (p *acceptanceSourceReadProbe) GetHumanBatch(ctx context.Context, id string) (*domain.HumanBatchRecord, error) {
|
||||
p.singles++
|
||||
return p.Store.GetHumanBatch(ctx, id)
|
||||
|
||||
@ -307,20 +307,24 @@ def main():
|
||||
child.wait(timeout=5)
|
||||
(out / "stopped.json").write_text(json.dumps(stopped))
|
||||
raise
|
||||
sample()
|
||||
final_health_error = None
|
||||
try:
|
||||
sample()
|
||||
except Exception as error:
|
||||
final_health_error = str(error)
|
||||
if fixture:
|
||||
result = subprocess.run(k + ["-n", "droboticflow-query-lab", "exec", "node4-query-fixture", "-c", "lab", "--", "cat", remote_report], capture_output=True, text=True, timeout=30)
|
||||
if result.returncode == 0:
|
||||
(out / "comparison.json").write_text(result.stdout)
|
||||
changed = source_fingerprint() != manifest["sourceFingerprint"]
|
||||
exit_record = {"returncode": child.returncode, "sourceChanged": changed, "coverageComplete": False}
|
||||
exit_record = {"returncode": child.returncode, "sourceChanged": changed, "coverageComplete": False, "finalHealthError": final_health_error}
|
||||
try:
|
||||
validate_comparison(out / "comparison.json", args.tasks, args.waves, args.max_actors)
|
||||
exit_record["coverageComplete"] = True
|
||||
except (OSError, ValueError, KeyError, AssertionError) as error:
|
||||
exit_record["coverageError"] = str(error)
|
||||
(out / "exit.json").write_text(json.dumps(exit_record))
|
||||
raise SystemExit(child.returncode or (2 if changed or not exit_record["coverageComplete"] else 0))
|
||||
raise SystemExit(child.returncode or (2 if changed or final_health_error or not exit_record["coverageComplete"] else 0))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user