From 91be80191ec8703b0f971746120c5c0b5e2d2c5b Mon Sep 17 00:00:00 2001 From: d-robotics Date: Wed, 18 Nov 2026 22:27:04 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Budget=20nightly=20acceptance=20?= =?UTF-8?q?retries=20for=20unfinished=20organizations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/acceptance-supplier-priority.md | 4 +- internal/api/acceptance_priority_refresh.go | 64 ++++++++++------- .../api/acceptance_priority_stats_test.go | 71 +++++++++++++++++-- 3 files changed, 106 insertions(+), 33 deletions(-) diff --git a/docs/acceptance-supplier-priority.md b/docs/acceptance-supplier-priority.md index 155a7efa..326568ea 100644 --- a/docs/acceptance-supplier-priority.md +++ b/docs/acceptance-supplier-priority.md @@ -48,4 +48,6 @@ 管理员面板读取组织最近一次成功保存的汇总,按当前已保存配置的项目范围展示“待领取”和“归属待核对”,同时显示北京时间的统计完成时间。首次统计前返回 `status: pending`、两个计数为 null。配置变化可以直接用同一份分项目汇总计算对应范围。统计读取失败仍可重试。 -API 实例在北京时间 01:00 竞争当日刷新资格,单实例串行处理组织,每 100 条候选任务批量解析来源。一个组织完整处理后原子替换其汇总。01:20 再检查一次,只补当天未完成的组织。每个窗口最多 15 分钟。每个组织最多使用剩余时间除以待检查的组织数,超时或读取失败后继续下一个组织,提前完成时,未用的时间留给后续组织。窗口内重启可以参与未被领取的窗口。两个窗口都错过或未完成时保留最近成功值,次日再试。完成日志记录候选条数和用时。统计反映扫描期间看到的数据,业务领取和配置保存按原流程实时处理。 +API 实例在北京时间 01:00–01:30 首轮刷新,01:35–02:05 补跑。每轮由一个实例串行处理,每 100 条候选任务批量解析来源。每轮先筛出当天未完成的组织,再按剩余窗口时间和仍待处理的组织数分配预算,每组织最多 15 分钟。提前完成时,未用的时间留给后续组织。 + +一个组织完整处理后原子替换其汇总。读取失败或超时后继续处理下一个组织。窗口内重启可以参与尚未被领取的窗口,两轮均未完成时保留最近成功值,次日再试。完成日志记录候选条数和用时。统计反映扫描期间看到的数据,业务领取和配置保存按原流程实时处理。 diff --git a/internal/api/acceptance_priority_refresh.go b/internal/api/acceptance_priority_refresh.go index db78298b..4d42d3c7 100644 --- a/internal/api/acceptance_priority_refresh.go +++ b/internal/api/acceptance_priority_refresh.go @@ -12,14 +12,19 @@ import ( var acceptanceZone = time.FixedZone("Asia/Shanghai", 8*60*60) -const acceptanceWindow = 15 * time.Minute +const ( + acceptanceWindow = 30 * time.Minute + acceptanceRetryOffset = acceptanceWindow + 5*time.Minute + acceptanceOrganizationBudget = 15 * time.Minute + acceptanceSnapshotLookupBudget = 2 * time.Second +) -// Give a failed or interrupted run one more window at 01:20. Deploying +// Give a failed or interrupted run one more window at 01:35. Deploying // during the day never starts a catch-up scan during business hours. func acceptanceRefreshSlot(now time.Time) time.Time { now = now.In(acceptanceZone) first := time.Date(now.Year(), now.Month(), now.Day(), 1, 0, 0, 0, acceptanceZone) - for _, slot := range []time.Time{first, first.Add(20 * time.Minute)} { + for _, slot := range []time.Time{first, first.Add(acceptanceRetryOffset)} { if now.Before(slot.Add(acceptanceWindow)) { return slot } @@ -65,44 +70,49 @@ func refreshAcceptanceOrganizations(ctx context.Context, db store.Store, slot ti return err } day := slot.In(acceptanceZone).Format(time.DateOnly) - for i, org := range orgs { + pending := make([]string, 0, len(orgs)) + for _, org := range orgs { if err := ctx.Err(); err != nil { return err } - // Reserve an equal share of the remaining window for each organization. - // A fast or already-completed organization leaves its unused time to the rest. + lookup, cancel := context.WithTimeout(ctx, acceptanceSnapshotLookupBudget) + last, err := db.ReadAcceptanceSnapshot(lookup, org.ID) + cancel() + if err != nil { + log.Printf("acceptance statistics org=%q could not read previous snapshot: %v", org.ID, err) + continue + } + if last != nil { + completed, err := time.Parse(time.RFC3339Nano, last.CompletedAt) + if err == nil && completed.In(acceptanceZone).Format(time.DateOnly) == day { + continue + } + } + pending = append(pending, org.ID) + } + + for i, org := range pending { + if err := ctx.Err(); err != nil { + return err + } + // Share only among organizations that still need a scan this window. remaining := acceptanceWindow if deadline, ok := ctx.Deadline(); ok { remaining = time.Until(deadline) } - orgCtx, cancel := context.WithTimeout(ctx, remaining/time.Duration(len(orgs)-i)) - err := refreshAcceptanceOrganization(orgCtx, db, org.ID, day) + budget := min(acceptanceOrganizationBudget, remaining/time.Duration(len(pending)-i)) + orgCtx, cancel := context.WithTimeout(ctx, budget) + start := time.Now() + n, err := refreshAcceptanceSnapshot(orgCtx, db, org) cancel() + log.Printf("acceptance statistics org=%q candidates=%d elapsed=%s completed=%t", org, n, time.Since(start), err == nil) if err != nil { - log.Printf("acceptance statistics org=%q refresh failed: %v", org.ID, err) + log.Printf("acceptance statistics org=%q refresh failed: %v", org, err) } } return ctx.Err() } -func refreshAcceptanceOrganization(ctx context.Context, db store.Store, org, day string) error { - last, err := db.ReadAcceptanceSnapshot(ctx, org) - if err != nil { - return fmt.Errorf("read previous snapshot: %w", err) - } - if last != nil { - completed, err := time.Parse(time.RFC3339Nano, last.CompletedAt) - if err == nil && completed.In(acceptanceZone).Format(time.DateOnly) == day { - return nil - } - } - - start := time.Now() - n, err := refreshAcceptanceSnapshot(ctx, db, org) - log.Printf("acceptance statistics org=%q candidates=%d elapsed=%s completed=%t", org, n, time.Since(start), err == nil) - return err -} - func refreshAcceptanceSnapshot(ctx context.Context, db store.Store, org string) (int, error) { type key struct{ project, supplier string } counts := map[key]int{} diff --git a/internal/api/acceptance_priority_stats_test.go b/internal/api/acceptance_priority_stats_test.go index 17682f51..97720bfc 100644 --- a/internal/api/acceptance_priority_stats_test.go +++ b/internal/api/acceptance_priority_stats_test.go @@ -162,10 +162,19 @@ func TestAcceptancePrioritySaveWhileStatisticsIsPending(t *testing.T) { } func TestAcceptancePriorityRefreshWindow(t *testing.T) { - for _, tc := range []struct{ now, want string }{{"00:59", "01:00"}, {"01:02", "01:00"}, {"01:15", "01:20"}, {"01:32", "01:20"}, {"01:36", "01:00"}} { + for _, tc := range []struct{ now, want string }{ + {"00:59", "18 01:00"}, + {"01:00", "18 01:00"}, + {"01:29", "18 01:00"}, + {"01:30", "18 01:35"}, + {"01:34", "18 01:35"}, + {"01:35", "18 01:35"}, + {"02:04", "18 01:35"}, + {"02:05", "19 01:00"}, + } { now, _ := time.ParseInLocation("2006-01-02 15:04", "2026-11-18 "+tc.now, acceptanceZone) slot := acceptanceRefreshSlot(now.UTC()) - if slot.Format("15:04") != tc.want || (tc.now == "01:36" && slot.Day() != 19) { + if slot.Format("02 15:04") != tc.want { t.Fatal(tc, slot) } } @@ -192,6 +201,7 @@ func TestAcceptancePrioritySnapshotFirstRunAndFailure(t *testing.T) { type acceptanceRefreshTimeoutStore struct { store.Store attempts map[string]int + slowScan time.Duration } func (s *acceptanceRefreshTimeoutStore) ListOrganizations(context.Context) ([]domain.OrganizationRecord, error) { @@ -200,9 +210,19 @@ func (s *acceptanceRefreshTimeoutStore) ListOrganizations(context.Context) ([]do func (s *acceptanceRefreshTimeoutStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error { s.attempts[org]++ - if org == "first" && s.attempts[org] == 1 { - <-ctx.Done() - return ctx.Err() + if org == "first" { + if s.slowScan > 0 { + timer := time.NewTimer(s.slowScan) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + } else if s.attempts[org] == 1 { + <-ctx.Done() + return ctx.Err() + } } return s.Store.VisitAcceptanceCandidates(ctx, org, stage, visit) } @@ -250,3 +270,44 @@ func TestAcceptancePriorityRefreshTimeoutLetsLaterOrganizationFinish(t *testing. t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts) } } + +func TestAcceptancePriorityRefreshRetryGivesSlowOrganizationEnoughBudget(t *testing.T) { + const window = 2 * time.Second + db := &acceptanceRefreshTimeoutStore{ + Store: store.NewMemoryStore(), + attempts: make(map[string]int), + slowScan: 1400 * time.Millisecond, + } + + ctx, cancel := context.WithTimeout(t.Context(), window) + defer cancel() + if err := refreshAcceptanceOrganizations(ctx, db, time.Now()); err != nil { + t.Fatal(err) + } + first, err := db.ReadAcceptanceSnapshot(t.Context(), "first") + if err != nil || first != nil { + t.Fatalf("slow organization should time out in its initial share: %+v, %v", first, err) + } + second, err := db.ReadAcceptanceSnapshot(t.Context(), "second") + if err != nil || second == nil { + t.Fatalf("fast organization should finish in the first window: %+v, %v", second, err) + } + + // Use the successful snapshot's day so the retry is independent of wall-clock midnight. + retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt) + if err != nil { + t.Fatal(err) + } + retry, cancelRetry := context.WithTimeout(t.Context(), window) + defer cancelRetry() + if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil { + t.Fatal(err) + } + first, err = db.ReadAcceptanceSnapshot(t.Context(), "first") + if err != nil || first == nil { + t.Fatalf("slow organization should finish when it no longer shares the retry budget: %+v, %v", first, err) + } + if db.attempts["first"] != 2 || db.attempts["second"] != 1 { + t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts) + } +}