🐛 Budget nightly acceptance retries for unfinished organizations
Some checks are pending
web-ci / checks (pull_request) Blocked by required conditions
web-ci / pool-search-mongo (pull_request) Blocked by required conditions
web-ci / work-availability (pull_request) Blocked by required conditions
web-ci / query-serving (pull_request) Blocked by required conditions
Some checks are pending
web-ci / checks (pull_request) Blocked by required conditions
web-ci / pool-search-mongo (pull_request) Blocked by required conditions
web-ci / work-availability (pull_request) Blocked by required conditions
web-ci / query-serving (pull_request) Blocked by required conditions
This commit is contained in:
parent
3034a5a3cb
commit
91be80191e
@ -48,4 +48,6 @@
|
||||
|
||||
管理员面板读取组织最近一次成功保存的汇总,按当前已保存配置的项目范围展示“待领取”和“归属待核对”,同时显示北京时间的统计完成时间。首次统计前返回 `status: pending`、两个计数为 null。配置变化可以直接用同一份分项目汇总计算对应范围。统计读取失败仍可重试。
|
||||
|
||||
API 实例在北京时间 01:00 竞争当日刷新资格,单实例串行处理组织,每 100 条候选任务批量解析来源。一个组织完整处理后原子替换其汇总。01:20 再检查一次,只补当天未完成的组织。每个窗口最多 15 分钟。每个组织最多使用剩余时间除以待检查的组织数,超时或读取失败后继续下一个组织,提前完成时,未用的时间留给后续组织。窗口内重启可以参与未被领取的窗口。两个窗口都错过或未完成时保留最近成功值,次日再试。完成日志记录候选条数和用时。统计反映扫描期间看到的数据,业务领取和配置保存按原流程实时处理。
|
||||
API 实例在北京时间 01:00–01:30 首轮刷新,01:35–02:05 补跑。每轮由一个实例串行处理,每 100 条候选任务批量解析来源。每轮先筛出当天未完成的组织,再按剩余窗口时间和仍待处理的组织数分配预算,每组织最多 15 分钟。提前完成时,未用的时间留给后续组织。
|
||||
|
||||
一个组织完整处理后原子替换其汇总。读取失败或超时后继续处理下一个组织。窗口内重启可以参与尚未被领取的窗口,两轮均未完成时保留最近成功值,次日再试。完成日志记录候选条数和用时。统计反映扫描期间看到的数据,业务领取和配置保存按原流程实时处理。
|
||||
|
||||
@ -12,14 +12,19 @@ import (
|
||||
|
||||
var acceptanceZone = time.FixedZone("Asia/Shanghai", 8*60*60)
|
||||
|
||||
const acceptanceWindow = 15 * time.Minute
|
||||
const (
|
||||
acceptanceWindow = 30 * time.Minute
|
||||
acceptanceRetryOffset = acceptanceWindow + 5*time.Minute
|
||||
acceptanceOrganizationBudget = 15 * time.Minute
|
||||
acceptanceSnapshotLookupBudget = 2 * time.Second
|
||||
)
|
||||
|
||||
// Give a failed or interrupted run one more window at 01:20. Deploying
|
||||
// Give a failed or interrupted run one more window at 01:35. Deploying
|
||||
// during the day never starts a catch-up scan during business hours.
|
||||
func acceptanceRefreshSlot(now time.Time) time.Time {
|
||||
now = now.In(acceptanceZone)
|
||||
first := time.Date(now.Year(), now.Month(), now.Day(), 1, 0, 0, 0, acceptanceZone)
|
||||
for _, slot := range []time.Time{first, first.Add(20 * time.Minute)} {
|
||||
for _, slot := range []time.Time{first, first.Add(acceptanceRetryOffset)} {
|
||||
if now.Before(slot.Add(acceptanceWindow)) {
|
||||
return slot
|
||||
}
|
||||
@ -65,44 +70,49 @@ func refreshAcceptanceOrganizations(ctx context.Context, db store.Store, slot ti
|
||||
return err
|
||||
}
|
||||
day := slot.In(acceptanceZone).Format(time.DateOnly)
|
||||
for i, org := range orgs {
|
||||
pending := make([]string, 0, len(orgs))
|
||||
for _, org := range orgs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Reserve an equal share of the remaining window for each organization.
|
||||
// A fast or already-completed organization leaves its unused time to the rest.
|
||||
lookup, cancel := context.WithTimeout(ctx, acceptanceSnapshotLookupBudget)
|
||||
last, err := db.ReadAcceptanceSnapshot(lookup, org.ID)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("acceptance statistics org=%q could not read previous snapshot: %v", org.ID, err)
|
||||
continue
|
||||
}
|
||||
if last != nil {
|
||||
completed, err := time.Parse(time.RFC3339Nano, last.CompletedAt)
|
||||
if err == nil && completed.In(acceptanceZone).Format(time.DateOnly) == day {
|
||||
continue
|
||||
}
|
||||
}
|
||||
pending = append(pending, org.ID)
|
||||
}
|
||||
|
||||
for i, org := range pending {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Share only among organizations that still need a scan this window.
|
||||
remaining := acceptanceWindow
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
remaining = time.Until(deadline)
|
||||
}
|
||||
orgCtx, cancel := context.WithTimeout(ctx, remaining/time.Duration(len(orgs)-i))
|
||||
err := refreshAcceptanceOrganization(orgCtx, db, org.ID, day)
|
||||
budget := min(acceptanceOrganizationBudget, remaining/time.Duration(len(pending)-i))
|
||||
orgCtx, cancel := context.WithTimeout(ctx, budget)
|
||||
start := time.Now()
|
||||
n, err := refreshAcceptanceSnapshot(orgCtx, db, org)
|
||||
cancel()
|
||||
log.Printf("acceptance statistics org=%q candidates=%d elapsed=%s completed=%t", org, n, time.Since(start), err == nil)
|
||||
if err != nil {
|
||||
log.Printf("acceptance statistics org=%q refresh failed: %v", org.ID, err)
|
||||
log.Printf("acceptance statistics org=%q refresh failed: %v", org, err)
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func refreshAcceptanceOrganization(ctx context.Context, db store.Store, org, day string) error {
|
||||
last, err := db.ReadAcceptanceSnapshot(ctx, org)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read previous snapshot: %w", err)
|
||||
}
|
||||
if last != nil {
|
||||
completed, err := time.Parse(time.RFC3339Nano, last.CompletedAt)
|
||||
if err == nil && completed.In(acceptanceZone).Format(time.DateOnly) == day {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
n, err := refreshAcceptanceSnapshot(ctx, db, org)
|
||||
log.Printf("acceptance statistics org=%q candidates=%d elapsed=%s completed=%t", org, n, time.Since(start), err == nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func refreshAcceptanceSnapshot(ctx context.Context, db store.Store, org string) (int, error) {
|
||||
type key struct{ project, supplier string }
|
||||
counts := map[key]int{}
|
||||
|
||||
@ -162,10 +162,19 @@ func TestAcceptancePrioritySaveWhileStatisticsIsPending(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAcceptancePriorityRefreshWindow(t *testing.T) {
|
||||
for _, tc := range []struct{ now, want string }{{"00:59", "01:00"}, {"01:02", "01:00"}, {"01:15", "01:20"}, {"01:32", "01:20"}, {"01:36", "01:00"}} {
|
||||
for _, tc := range []struct{ now, want string }{
|
||||
{"00:59", "18 01:00"},
|
||||
{"01:00", "18 01:00"},
|
||||
{"01:29", "18 01:00"},
|
||||
{"01:30", "18 01:35"},
|
||||
{"01:34", "18 01:35"},
|
||||
{"01:35", "18 01:35"},
|
||||
{"02:04", "18 01:35"},
|
||||
{"02:05", "19 01:00"},
|
||||
} {
|
||||
now, _ := time.ParseInLocation("2006-01-02 15:04", "2026-11-18 "+tc.now, acceptanceZone)
|
||||
slot := acceptanceRefreshSlot(now.UTC())
|
||||
if slot.Format("15:04") != tc.want || (tc.now == "01:36" && slot.Day() != 19) {
|
||||
if slot.Format("02 15:04") != tc.want {
|
||||
t.Fatal(tc, slot)
|
||||
}
|
||||
}
|
||||
@ -192,6 +201,7 @@ func TestAcceptancePrioritySnapshotFirstRunAndFailure(t *testing.T) {
|
||||
type acceptanceRefreshTimeoutStore struct {
|
||||
store.Store
|
||||
attempts map[string]int
|
||||
slowScan time.Duration
|
||||
}
|
||||
|
||||
func (s *acceptanceRefreshTimeoutStore) ListOrganizations(context.Context) ([]domain.OrganizationRecord, error) {
|
||||
@ -200,9 +210,19 @@ func (s *acceptanceRefreshTimeoutStore) ListOrganizations(context.Context) ([]do
|
||||
|
||||
func (s *acceptanceRefreshTimeoutStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
|
||||
s.attempts[org]++
|
||||
if org == "first" && s.attempts[org] == 1 {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
if org == "first" {
|
||||
if s.slowScan > 0 {
|
||||
timer := time.NewTimer(s.slowScan)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
} else if s.attempts[org] == 1 {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
return s.Store.VisitAcceptanceCandidates(ctx, org, stage, visit)
|
||||
}
|
||||
@ -250,3 +270,44 @@ func TestAcceptancePriorityRefreshTimeoutLetsLaterOrganizationFinish(t *testing.
|
||||
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptancePriorityRefreshRetryGivesSlowOrganizationEnoughBudget(t *testing.T) {
|
||||
const window = 2 * time.Second
|
||||
db := &acceptanceRefreshTimeoutStore{
|
||||
Store: store.NewMemoryStore(),
|
||||
attempts: make(map[string]int),
|
||||
slowScan: 1400 * time.Millisecond,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), window)
|
||||
defer cancel()
|
||||
if err := refreshAcceptanceOrganizations(ctx, db, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := db.ReadAcceptanceSnapshot(t.Context(), "first")
|
||||
if err != nil || first != nil {
|
||||
t.Fatalf("slow organization should time out in its initial share: %+v, %v", first, err)
|
||||
}
|
||||
second, err := db.ReadAcceptanceSnapshot(t.Context(), "second")
|
||||
if err != nil || second == nil {
|
||||
t.Fatalf("fast organization should finish in the first window: %+v, %v", second, err)
|
||||
}
|
||||
|
||||
// Use the successful snapshot's day so the retry is independent of wall-clock midnight.
|
||||
retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retry, cancelRetry := context.WithTimeout(t.Context(), window)
|
||||
defer cancelRetry()
|
||||
if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err = db.ReadAcceptanceSnapshot(t.Context(), "first")
|
||||
if err != nil || first == nil {
|
||||
t.Fatalf("slow organization should finish when it no longer shares the retry budget: %+v, %v", first, err)
|
||||
}
|
||||
if db.attempts["first"] != 2 || db.attempts["second"] != 1 {
|
||||
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user