droboticflow/internal/api/acceptance_priority_stats_test.go
d-robotics 91be80191e
Some checks are pending
web-ci / checks (pull_request) Blocked by required conditions
web-ci / pool-search-mongo (pull_request) Blocked by required conditions
web-ci / work-availability (pull_request) Blocked by required conditions
web-ci / query-serving (pull_request) Blocked by required conditions
🐛 Budget nightly acceptance retries for unfinished organizations
2026-11-18 22:27:04 +08:00

314 lines
11 KiB
Go

package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"encoding/json"
"errors"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
)
type priorityStatsGuard struct {
store.Store
calls atomic.Int32
wait bool
}
func (s *priorityStatsGuard) ListReviewCandidates(ctx context.Context, org, project, user string, mine bool, offset, limit int, stages ...string) ([]domain.HumanItemRecord, error) {
s.calls.Add(1)
if s.wait {
<-ctx.Done()
return nil, ctx.Err()
}
return nil, errors.New("statistics unavailable")
}
func (s *priorityStatsGuard) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
s.calls.Add(1)
if s.wait {
<-ctx.Done()
return nil, ctx.Err()
}
return nil, errors.New("snapshot unavailable")
}
func buildPrioritySnapshot(t *testing.T, f combinedDraftFixture) {
t.Helper()
if _, err := refreshAcceptanceSnapshot(f.ctx, f.store, f.item.OrganizationID); err != nil {
t.Fatal(err)
}
}
func TestAcceptancePriorityConfigDoesNotWaitForStatistics(t *testing.T) {
f := priorityFixture(t)
g := &priorityStatsGuard{Store: f.store}
f.handler = NewHandler(g, nil)
// Representative backlog is irrelevant to metadata GET/save/cancel.
for i := 0; i < 5600; i++ {
v := f.item
v.ID = fmt.Sprintf("backlog-%d", i)
_ = f.store.UpsertHumanItem(f.ctx, v)
}
for _, op := range []struct {
method string
body any
}{{"GET", nil}, {"PUT", map[string]any{"enabled": true, "version": 0, "expiresAt": time.Now().Add(time.Hour).UTC().Format(time.RFC3339)}}, {"PUT", map[string]any{"enabled": false, "version": 1}}} {
at := time.Now()
r := f.request(op.method, "/api/suppliers/priority-production/acceptance-priority", f.admin, op.body)
if r.Code != 200 || time.Since(at) >= time.Second {
t.Fatalf("config blocked %d %s", r.Code, r.Body)
}
var v map[string]any
_ = json.Unmarshal(r.Body.Bytes(), &v)
if v["remainingCount"] != nil || v["unresolvedCount"] != nil {
t.Fatal("unknown statistics became zero")
}
}
if g.calls.Load() != 0 {
t.Fatal("config touched candidate scan")
}
rows, _ := f.store.ListAcceptancePriorities(f.ctx, f.item.OrganizationID)
if len(rows) != 1 || rows[0].Version != 2 || rows[0].Enabled || len(rows[0].History) != 2 {
t.Fatal("save/cancel not persisted")
}
}
func TestAcceptancePriorityStatisticsFailureAndDeadline(t *testing.T) {
for _, wait := range []bool{false, true} {
t.Run(fmt.Sprint(wait), func(t *testing.T) {
f := priorityFixture(t)
g := &priorityStatsGuard{Store: f.store, wait: wait}
f.handler = NewHandler(g, nil)
at := time.Now()
r := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=0", f.admin, nil)
if r.Code != 200 || !json.Valid(r.Body.Bytes()) {
t.Fatalf("stats %d %s", r.Code, r.Body)
}
var v map[string]any
_ = json.Unmarshal(r.Body.Bytes(), &v)
if v["status"] != "incomplete" || v["remainingCount"] != nil || v["unresolvedCount"] != nil {
t.Fatalf("partial stats mistaken for count %s", r.Body)
}
if g.calls.Load() != 1 || time.Since(at) > 3*time.Second {
t.Fatal("statistics deadline not enforced")
}
})
}
}
func TestAcceptancePriorityStatisticsVersionAndBoundary(t *testing.T) {
f := priorityFixture(t)
buildPrioritySnapshot(t, f)
path := "/api/suppliers/priority-production/acceptance-priority/statistics"
for _, actor := range []domain.UserRecord{f.annotator, f.reviewer} {
if r := f.request("GET", path+"?version=0", actor, nil); r.Code != 403 {
t.Fatal("stats identity leaked")
}
}
r := f.request("GET", path+"?version=0", f.admin, nil)
var v map[string]any
_ = json.Unmarshal(r.Body.Bytes(), &v)
if r.Code != 200 || v["status"] != "completed" || v["remainingCount"] != float64(0) {
t.Fatalf("real zero %d %s", r.Code, r.Body)
}
priorityPolicy(t, f, true, 0, time.Now().Add(time.Hour).UTC().Format(time.RFC3339), nil)
r = f.request("GET", path+"?version=0", f.admin, nil)
_ = json.Unmarshal(r.Body.Bytes(), &v)
if v["status"] != "incomplete" || v["remainingCount"] != nil {
t.Fatal("stale scope counted")
}
}
// Saving must be independent even while an earlier statistics request is waiting.
type priorityConcurrentStatsStore struct {
store.Store
started chan struct{}
release chan struct{}
}
func (s *priorityConcurrentStatsStore) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
select {
case s.started <- struct{}{}:
default:
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-s.release:
}
return s.Store.ReadAcceptanceSnapshot(ctx, org)
}
func TestAcceptancePrioritySaveWhileStatisticsIsPending(t *testing.T) {
f := priorityFixture(t)
g := &priorityConcurrentStatsStore{Store: f.store, started: make(chan struct{}, 1), release: make(chan struct{})}
f.handler = NewHandler(g, nil)
done := make(chan string, 1)
go func() {
r := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=0", f.admin, nil)
done <- r.Body.String()
}()
<-g.started
at := time.Now()
priorityPolicy(t, f, true, 0, time.Now().Add(time.Hour).UTC().Format(time.RFC3339), []string{f.item.ProjectID})
if time.Since(at) >= time.Second {
t.Fatal("save waited for statistics")
}
close(g.release)
var v map[string]any
_ = json.Unmarshal([]byte(<-done), &v)
if v["status"] != "incomplete" || v["remainingCount"] != nil {
t.Fatal("old version statistics published after save")
}
}
func TestAcceptancePriorityRefreshWindow(t *testing.T) {
for _, tc := range []struct{ now, want string }{
{"00:59", "18 01:00"},
{"01:00", "18 01:00"},
{"01:29", "18 01:00"},
{"01:30", "18 01:35"},
{"01:34", "18 01:35"},
{"01:35", "18 01:35"},
{"02:04", "18 01:35"},
{"02:05", "19 01:00"},
} {
now, _ := time.ParseInLocation("2006-01-02 15:04", "2026-11-18 "+tc.now, acceptanceZone)
slot := acceptanceRefreshSlot(now.UTC())
if slot.Format("02 15:04") != tc.want {
t.Fatal(tc, slot)
}
}
}
func TestAcceptancePrioritySnapshotFirstRunAndFailure(t *testing.T) {
f := priorityFixture(t)
path := "/api/suppliers/priority-production/acceptance-priority/statistics?version=0"
if r := f.request("GET", path, f.admin, nil); !strings.Contains(r.Body.String(), `"status":"pending"`) {
t.Fatal(r.Body)
}
buildPrioritySnapshot(t, f)
first := f.request("GET", path, f.admin, nil).Body.String()
ctx, cancel := context.WithCancel(f.ctx)
cancel()
if _, err := refreshAcceptanceSnapshot(ctx, f.store, f.item.OrganizationID); err == nil {
t.Fatal("published interrupted scan")
}
if after := f.request("GET", path, f.admin, nil).Body.String(); after != first {
t.Fatal(after, first)
}
}
type acceptanceRefreshTimeoutStore struct {
store.Store
attempts map[string]int
slowScan time.Duration
}
func (s *acceptanceRefreshTimeoutStore) ListOrganizations(context.Context) ([]domain.OrganizationRecord, error) {
return []domain.OrganizationRecord{{ID: "first"}, {ID: "second"}}, nil
}
func (s *acceptanceRefreshTimeoutStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
s.attempts[org]++
if org == "first" {
if s.slowScan > 0 {
timer := time.NewTimer(s.slowScan)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
}
} else if s.attempts[org] == 1 {
<-ctx.Done()
return ctx.Err()
}
}
return s.Store.VisitAcceptanceCandidates(ctx, org, stage, visit)
}
func TestAcceptancePriorityRefreshTimeoutLetsLaterOrganizationFinish(t *testing.T) {
db := &acceptanceRefreshTimeoutStore{Store: store.NewMemoryStore(), attempts: make(map[string]int)}
old := domain.AcceptanceSnapshot{
OrganizationID: "first",
CompletedAt: time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339Nano),
Counts: []domain.AcceptanceCount{{ProjectID: "project", Count: 12}},
}
if err := db.SaveAcceptanceSnapshot(t.Context(), old); err != nil {
t.Fatal(err)
}
slot := time.Now()
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
if err := refreshAcceptanceOrganizations(ctx, db, slot); err != nil {
t.Fatalf("slow organization exhausted the entire window: %v", err)
}
first, err := db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first == nil || first.CompletedAt != old.CompletedAt || len(first.Counts) != 1 || first.Counts[0] != old.Counts[0] {
t.Fatalf("timed-out refresh replaced the previous snapshot: %+v, %v", first, err)
}
second, err := db.ReadAcceptanceSnapshot(t.Context(), "second")
if err != nil || second == nil {
t.Fatalf("later organization did not publish its snapshot: %+v, %v", second, err)
}
// Use the successful snapshot's day so this test also works around midnight.
retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt)
if err != nil {
t.Fatal(err)
}
retry, cancelRetry := context.WithTimeout(t.Context(), time.Second)
defer cancelRetry()
if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil {
t.Fatalf("retry failed: %v", err)
}
first, err = db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first == nil || first.CompletedAt == old.CompletedAt {
t.Fatalf("retry did not publish a fresh snapshot: %+v, %v", first, err)
}
if db.attempts["first"] != 2 || db.attempts["second"] != 1 {
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
}
}
func TestAcceptancePriorityRefreshRetryGivesSlowOrganizationEnoughBudget(t *testing.T) {
const window = 2 * time.Second
db := &acceptanceRefreshTimeoutStore{
Store: store.NewMemoryStore(),
attempts: make(map[string]int),
slowScan: 1400 * time.Millisecond,
}
ctx, cancel := context.WithTimeout(t.Context(), window)
defer cancel()
if err := refreshAcceptanceOrganizations(ctx, db, time.Now()); err != nil {
t.Fatal(err)
}
first, err := db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first != nil {
t.Fatalf("slow organization should time out in its initial share: %+v, %v", first, err)
}
second, err := db.ReadAcceptanceSnapshot(t.Context(), "second")
if err != nil || second == nil {
t.Fatalf("fast organization should finish in the first window: %+v, %v", second, err)
}
// Use the successful snapshot's day so the retry is independent of wall-clock midnight.
retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt)
if err != nil {
t.Fatal(err)
}
retry, cancelRetry := context.WithTimeout(t.Context(), window)
defer cancelRetry()
if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil {
t.Fatal(err)
}
first, err = db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first == nil {
t.Fatalf("slow organization should finish when it no longer shares the retry budget: %+v, %v", first, err)
}
if db.attempts["first"] != 2 || db.attempts["second"] != 1 {
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
}
}