Merge pull request '验收优先统计改为每天凌晨 1 点刷新' (#80) from yao.wang/droboticflow:dev into dev
This commit is contained in:
commit
90336acc34
@ -235,6 +235,8 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
go api.RunAcceptanceStatisticsSchedule(ctx, dataStore)
|
||||
|
||||
aiService := ai.NewService(cfg.AIAPIBaseURL, cfg.AIAPIKey, cfg.AIModel)
|
||||
handler := spaHandler(api.NewHandlerWithConfig(dataStore, aiService, cfg), cfg.DistDir)
|
||||
server := &http.Server{
|
||||
|
||||
@ -43,3 +43,11 @@
|
||||
- 成功的 PUT 响应确认配置 CAS 已落库,不再附带可能失败的统计。传输中断仍需刷新核对结果,不能自动重放保存或把网络超时当作确定未保存。
|
||||
|
||||
随机领取、连续审查、服务端权限复核、匿名字段、任务版本 CAS 及原有列表/计数接口均不变。真实供应商、项目与有效期仍须管理员另行配置;管理入口修复与实际启用验收前,不得对外宣称优先验收已生效。
|
||||
|
||||
## 每日统计
|
||||
|
||||
管理员面板读取组织最近一次成功保存的汇总,按当前已保存配置的项目范围展示“待领取”和“归属待核对”,同时显示北京时间的统计完成时间。首次统计前返回 `status: pending`、两个计数为 null。配置变化可以直接用同一份分项目汇总计算对应范围。统计读取失败仍可重试。
|
||||
|
||||
API 实例在北京时间 01:00–01:30 首轮刷新,01:35–02:05 补跑。每轮由一个实例串行处理,每 100 条候选任务批量解析来源。每轮先筛出当天未完成的组织,再按剩余窗口时间和仍待处理的组织数分配预算,每组织最多 15 分钟。提前完成时,未用的时间留给后续组织。
|
||||
|
||||
一个组织完整处理后原子替换其汇总。读取失败或超时后继续处理下一个组织。窗口内重启可以参与尚未被领取的窗口,两轮均未完成时保留最近成功值,次日再试。完成日志记录候选条数和用时。统计反映扫描期间看到的数据,业务领取和配置保存按原流程实时处理。
|
||||
|
||||
@ -200,107 +200,44 @@ func (s *server) handleAcceptancePriorityStatistics(w http.ResponseWriter, r *ht
|
||||
renderJSON(w, 404, map[string]string{"error": "供应商不存在"})
|
||||
return
|
||||
}
|
||||
resolver, err := s.newAcceptancePriorityResolver(ctx, org)
|
||||
snapshot, err := s.store.ReadAcceptanceSnapshot(ctx, org)
|
||||
if err != nil {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
value := domain.AcceptancePriority{OrganizationID: org, SupplierID: id}
|
||||
for _, v := range resolver.rules {
|
||||
if v.SupplierID == id {
|
||||
value = v
|
||||
}
|
||||
}
|
||||
if value.Version != expected {
|
||||
rules, err := s.store.ListAcceptancePriorities(ctx, org)
|
||||
if err != nil {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
project := ""
|
||||
if len(value.ProjectIDs) == 1 {
|
||||
project = value.ProjectIDs[0]
|
||||
}
|
||||
remaining, unresolved := 0, 0
|
||||
for offset := 0; ; offset += 100 {
|
||||
if ctx.Err() != nil {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
rows, e := s.store.ListReviewCandidates(r.Context(), org, project, "", false, offset, 100, fanyuV5PrePackageAcceptanceStageID)
|
||||
if e != nil {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
pageSize := len(rows)
|
||||
if len(value.ProjectIDs) > 0 {
|
||||
scoped := []domain.HumanItemRecord{}
|
||||
for _, item := range rows {
|
||||
if slices.Contains(value.ProjectIDs, item.ProjectID) {
|
||||
scoped = append(scoped, item)
|
||||
}
|
||||
}
|
||||
rows = scoped
|
||||
}
|
||||
batchIDs := []string{}
|
||||
for _, item := range rows {
|
||||
batchIDs = append(batchIDs, item.BatchID)
|
||||
}
|
||||
batchRows, e := s.store.ListHumanBatchAccessRecords(r.Context(), org, batchIDs)
|
||||
if e != nil {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
batches := map[string]*domain.HumanBatchRecord{}
|
||||
for i := range batchRows {
|
||||
batches[batchRows[i].ID] = &batchRows[i]
|
||||
}
|
||||
if e := resolver.prime(rows); e != nil {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
for _, item := range rows {
|
||||
if len(value.ProjectIDs) > 0 && !slices.Contains(value.ProjectIDs, item.ProjectID) {
|
||||
continue
|
||||
}
|
||||
b := batches[item.BatchID]
|
||||
if !domain.SharedAcceptanceItem(item, b) {
|
||||
continue
|
||||
}
|
||||
sid, e := resolver.supplier(item)
|
||||
if e != nil {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
if sid == "" {
|
||||
unresolved++
|
||||
} else if sid == id {
|
||||
remaining++
|
||||
}
|
||||
}
|
||||
if pageSize < 100 {
|
||||
var policy domain.AcceptancePriority
|
||||
for _, rule := range rules {
|
||||
if rule.SupplierID == id {
|
||||
policy = rule
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
if policy.Version != expected {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
latest, e := s.store.ListAcceptancePriorities(ctx, org)
|
||||
if e != nil {
|
||||
incomplete()
|
||||
if snapshot == nil {
|
||||
renderJSON(w, 200, map[string]any{"status": "pending", "policyVersion": expected, "remainingCount": nil, "unresolvedCount": nil})
|
||||
return
|
||||
}
|
||||
version := 0
|
||||
for _, v := range latest {
|
||||
if v.SupplierID == id {
|
||||
version = v.Version
|
||||
remaining, unresolved := 0, 0
|
||||
for _, group := range snapshot.Counts {
|
||||
if len(policy.ProjectIDs) > 0 && !slices.Contains(policy.ProjectIDs, group.ProjectID) {
|
||||
continue
|
||||
}
|
||||
switch group.SupplierID {
|
||||
case "":
|
||||
unresolved += group.Count
|
||||
case id:
|
||||
remaining += group.Count
|
||||
}
|
||||
}
|
||||
if version != expected {
|
||||
incomplete()
|
||||
return
|
||||
}
|
||||
renderJSON(w, 200, map[string]any{"status": "completed", "policyVersion": expected, "remainingCount": remaining, "unresolvedCount": unresolved})
|
||||
renderJSON(w, 200, map[string]any{"status": "completed", "policyVersion": expected, "remainingCount": remaining, "unresolvedCount": unresolved, "completedAt": snapshot.CompletedAt})
|
||||
}
|
||||
|
||||
// Preserve the existing first-eligible-page random semantics inside each group.
|
||||
|
||||
@ -21,6 +21,7 @@ func TestAcceptancePriorityBrowser(t *testing.T) {
|
||||
item := f.item
|
||||
item.ID = "priority-browser"
|
||||
addPrioritySource(t, f, item, "priority-production")
|
||||
buildPrioritySnapshot(t, f)
|
||||
h := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Test-only authentication at the loopback fixture boundary.
|
||||
user := f.admin
|
||||
|
||||
@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"fmt"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
@ -9,6 +10,7 @@ import (
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@ -45,9 +47,31 @@ func TestAcceptancePriorityConfigMongo(t *testing.T) {
|
||||
if e = s.UpsertUser(ctx, f.admin); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
org := f.item.OrganizationID
|
||||
template, err := f.store.GetHumanBatch(ctx, f.item.BatchID)
|
||||
if err != nil || template == nil {
|
||||
t.Fatal("missing fixture batch", err)
|
||||
}
|
||||
for i := 0; i < 56; i++ {
|
||||
b := *template
|
||||
b.ID = fmt.Sprintf("target-%d", i)
|
||||
b.SourceBatchID = fmt.Sprintf("source-%d", i)
|
||||
if err := s.UpsertHumanBatch(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.ID = b.SourceBatchID
|
||||
b.SourceBatchID = ""
|
||||
if err := s.UpsertHumanBatch(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: b.ID, BatchID: b.ID, OrganizationID: org, ProjectID: b.ProjectID, SupplierID: supplier.ID, Status: domain.HumanItemAccepted}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
scans.Store(0)
|
||||
docs := make([]any, 5600)
|
||||
for i := range docs {
|
||||
docs[i] = bson.M{"id": fmt.Sprintf("candidate-%06d", i), "organizationId": f.item.OrganizationID, "currentStageId": "prepackage_acceptance", "status": "ready", "pluginId": "annotation-qa", "subject": bson.M{"payload": bson.M{"synthetic": "fixture"}}}
|
||||
docs[i] = bson.M{"id": fmt.Sprintf("candidate-%06d", i), "organizationId": f.item.OrganizationID, "currentStageId": "prepackage_acceptance", "status": "ready", "projectId": template.ProjectID, "batchId": fmt.Sprintf("target-%d", i/100), "pluginId": "annotation-qa", "subject": bson.M{"payload": bson.M{"synthetic": "fixture"}}}
|
||||
}
|
||||
if _, e = db.Collection("humanItems").InsertMany(ctx, docs); e != nil {
|
||||
t.Fatal(e)
|
||||
@ -72,4 +96,24 @@ func TestAcceptancePriorityConfigMongo(t *testing.T) {
|
||||
if e != nil || len(rows) != 1 || rows[0].Version != 2 || rows[0].Enabled || len(rows[0].History) != 2 {
|
||||
t.Fatal("confirmed writes did not survive new store")
|
||||
}
|
||||
at := time.Now()
|
||||
n, err := refreshAcceptanceSnapshot(ctx, s, org)
|
||||
if err != nil || n != len(docs) {
|
||||
t.Fatalf("refresh rows=%d err=%v", n, err)
|
||||
}
|
||||
t.Logf("nightly complete: %d candidates, 56 source batches, %s", n, time.Since(at))
|
||||
scans.Store(0)
|
||||
restarted := store.NewMongoStore(db)
|
||||
f.handler = NewHandler(restarted, nil)
|
||||
r := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=2", f.admin, nil)
|
||||
if !strings.Contains(r.Body.String(), `"remainingCount":5600`) || scans.Load() != 0 {
|
||||
t.Fatal("snapshot read", r.Body, scans.Load())
|
||||
}
|
||||
for i, replica := range []*store.MongoStore{s, restarted} {
|
||||
claimed, err := replica.ClaimAcceptanceRefresh(ctx, "2026-11-18T01:00:00+08:00")
|
||||
if claimed != (i == 0) || err != nil {
|
||||
t.Fatalf("replica %d claimed=%v err=%v", i, claimed, err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
163
internal/api/acceptance_priority_refresh.go
Normal file
163
internal/api/acceptance_priority_refresh.go
Normal file
@ -0,0 +1,163 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
var acceptanceZone = time.FixedZone("Asia/Shanghai", 8*60*60)
|
||||
|
||||
const (
|
||||
acceptanceWindow = 30 * time.Minute
|
||||
acceptanceRetryOffset = acceptanceWindow + 5*time.Minute
|
||||
acceptanceOrganizationBudget = 15 * time.Minute
|
||||
acceptanceSnapshotLookupBudget = 2 * time.Second
|
||||
)
|
||||
|
||||
// Give a failed or interrupted run one more window at 01:35. Deploying
|
||||
// during the day never starts a catch-up scan during business hours.
|
||||
func acceptanceRefreshSlot(now time.Time) time.Time {
|
||||
now = now.In(acceptanceZone)
|
||||
first := time.Date(now.Year(), now.Month(), now.Day(), 1, 0, 0, 0, acceptanceZone)
|
||||
for _, slot := range []time.Time{first, first.Add(acceptanceRetryOffset)} {
|
||||
if now.Before(slot.Add(acceptanceWindow)) {
|
||||
return slot
|
||||
}
|
||||
}
|
||||
return first.AddDate(0, 0, 1)
|
||||
}
|
||||
|
||||
func RunAcceptanceStatisticsSchedule(ctx context.Context, db *store.MongoStore) {
|
||||
for ctx.Err() == nil {
|
||||
slot := acceptanceRefreshSlot(time.Now())
|
||||
if !waitUntil(ctx, slot) {
|
||||
return
|
||||
}
|
||||
window, cancel := context.WithDeadline(ctx, slot.Add(acceptanceWindow))
|
||||
claimed, err := db.ClaimAcceptanceRefresh(window, slot.Format(time.RFC3339))
|
||||
if err == nil && claimed {
|
||||
err = refreshAcceptanceOrganizations(window, db, slot)
|
||||
}
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("acceptance statistics window failed: %v", err)
|
||||
}
|
||||
if !waitUntil(ctx, slot.Add(acceptanceWindow)) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitUntil(ctx context.Context, at time.Time) bool {
|
||||
t := time.NewTimer(time.Until(at))
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.C:
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
}
|
||||
|
||||
func refreshAcceptanceOrganizations(ctx context.Context, db store.Store, slot time.Time) error {
|
||||
orgs, err := db.ListOrganizations(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
day := slot.In(acceptanceZone).Format(time.DateOnly)
|
||||
pending := make([]string, 0, len(orgs))
|
||||
for _, org := range orgs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
lookup, cancel := context.WithTimeout(ctx, acceptanceSnapshotLookupBudget)
|
||||
last, err := db.ReadAcceptanceSnapshot(lookup, org.ID)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Printf("acceptance statistics org=%q could not read previous snapshot: %v", org.ID, err)
|
||||
continue
|
||||
}
|
||||
if last != nil {
|
||||
completed, err := time.Parse(time.RFC3339Nano, last.CompletedAt)
|
||||
if err == nil && completed.In(acceptanceZone).Format(time.DateOnly) == day {
|
||||
continue
|
||||
}
|
||||
}
|
||||
pending = append(pending, org.ID)
|
||||
}
|
||||
|
||||
for i, org := range pending {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Share only among organizations that still need a scan this window.
|
||||
remaining := acceptanceWindow
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
remaining = time.Until(deadline)
|
||||
}
|
||||
budget := min(acceptanceOrganizationBudget, remaining/time.Duration(len(pending)-i))
|
||||
orgCtx, cancel := context.WithTimeout(ctx, budget)
|
||||
start := time.Now()
|
||||
n, err := refreshAcceptanceSnapshot(orgCtx, db, org)
|
||||
cancel()
|
||||
log.Printf("acceptance statistics org=%q candidates=%d elapsed=%s completed=%t", org, n, time.Since(start), err == nil)
|
||||
if err != nil {
|
||||
log.Printf("acceptance statistics org=%q refresh failed: %v", org, err)
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func refreshAcceptanceSnapshot(ctx context.Context, db store.Store, org string) (int, error) {
|
||||
type key struct{ project, supplier string }
|
||||
counts := map[key]int{}
|
||||
total := 0
|
||||
err := db.VisitAcceptanceCandidates(ctx, org, fanyuV5PrePackageAcceptanceStageID, func(page []domain.HumanItemRecord) error {
|
||||
total += len(page)
|
||||
ids := make([]string, 0, len(page))
|
||||
seen := map[string]bool{}
|
||||
for _, row := range page {
|
||||
if row.BatchID != "" && !seen[row.BatchID] {
|
||||
ids = append(ids, row.BatchID)
|
||||
seen[row.BatchID] = true
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
batches, err := db.ListHumanBatchAccessRecords(ctx, org, ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byID := map[string]*domain.HumanBatchRecord{}
|
||||
for i := range batches {
|
||||
byID[batches[i].ID] = &batches[i]
|
||||
}
|
||||
owners, err := db.ResolveAcceptanceSourceSuppliers(ctx, org, ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, row := range page {
|
||||
if domain.SharedAcceptanceItem(row, byID[row.BatchID]) {
|
||||
counts[key{row.ProjectID, owners[row.BatchID]}]++
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("scan acceptance candidates: %w", err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return total, err
|
||||
}
|
||||
v := domain.AcceptanceSnapshot{OrganizationID: org, CompletedAt: time.Now().UTC().Format(time.RFC3339Nano)}
|
||||
for k, n := range counts {
|
||||
v.Counts = append(v.Counts, domain.AcceptanceCount{ProjectID: k.project, SupplierID: k.supplier, Count: n})
|
||||
}
|
||||
return total, db.SaveAcceptanceSnapshot(ctx, v)
|
||||
}
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@ -26,6 +27,20 @@ func (s *priorityStatsGuard) ListReviewCandidates(ctx context.Context, org, proj
|
||||
}
|
||||
return nil, errors.New("statistics unavailable")
|
||||
}
|
||||
func (s *priorityStatsGuard) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
|
||||
s.calls.Add(1)
|
||||
if s.wait {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return nil, errors.New("snapshot unavailable")
|
||||
}
|
||||
func buildPrioritySnapshot(t *testing.T, f combinedDraftFixture) {
|
||||
t.Helper()
|
||||
if _, err := refreshAcceptanceSnapshot(f.ctx, f.store, f.item.OrganizationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
func TestAcceptancePriorityConfigDoesNotWaitForStatistics(t *testing.T) {
|
||||
f := priorityFixture(t)
|
||||
g := &priorityStatsGuard{Store: f.store}
|
||||
@ -83,6 +98,7 @@ func TestAcceptancePriorityStatisticsFailureAndDeadline(t *testing.T) {
|
||||
}
|
||||
func TestAcceptancePriorityStatisticsVersionAndBoundary(t *testing.T) {
|
||||
f := priorityFixture(t)
|
||||
buildPrioritySnapshot(t, f)
|
||||
path := "/api/suppliers/priority-production/acceptance-priority/statistics"
|
||||
for _, actor := range []domain.UserRecord{f.annotator, f.reviewer} {
|
||||
if r := f.request("GET", path+"?version=0", actor, nil); r.Code != 403 {
|
||||
@ -110,7 +126,7 @@ type priorityConcurrentStatsStore struct {
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (s *priorityConcurrentStatsStore) ListReviewCandidates(ctx context.Context, org, project, user string, mine bool, offset, limit int, stages ...string) ([]domain.HumanItemRecord, error) {
|
||||
func (s *priorityConcurrentStatsStore) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
|
||||
select {
|
||||
case s.started <- struct{}{}:
|
||||
default:
|
||||
@ -120,7 +136,7 @@ func (s *priorityConcurrentStatsStore) ListReviewCandidates(ctx context.Context,
|
||||
return nil, ctx.Err()
|
||||
case <-s.release:
|
||||
}
|
||||
return s.Store.ListReviewCandidates(ctx, org, project, user, mine, offset, limit, stages...)
|
||||
return s.Store.ReadAcceptanceSnapshot(ctx, org)
|
||||
}
|
||||
func TestAcceptancePrioritySaveWhileStatisticsIsPending(t *testing.T) {
|
||||
f := priorityFixture(t)
|
||||
@ -144,3 +160,154 @@ func TestAcceptancePrioritySaveWhileStatisticsIsPending(t *testing.T) {
|
||||
t.Fatal("old version statistics published after save")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptancePriorityRefreshWindow(t *testing.T) {
|
||||
for _, tc := range []struct{ now, want string }{
|
||||
{"00:59", "18 01:00"},
|
||||
{"01:00", "18 01:00"},
|
||||
{"01:29", "18 01:00"},
|
||||
{"01:30", "18 01:35"},
|
||||
{"01:34", "18 01:35"},
|
||||
{"01:35", "18 01:35"},
|
||||
{"02:04", "18 01:35"},
|
||||
{"02:05", "19 01:00"},
|
||||
} {
|
||||
now, _ := time.ParseInLocation("2006-01-02 15:04", "2026-11-18 "+tc.now, acceptanceZone)
|
||||
slot := acceptanceRefreshSlot(now.UTC())
|
||||
if slot.Format("02 15:04") != tc.want {
|
||||
t.Fatal(tc, slot)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptancePrioritySnapshotFirstRunAndFailure(t *testing.T) {
|
||||
f := priorityFixture(t)
|
||||
path := "/api/suppliers/priority-production/acceptance-priority/statistics?version=0"
|
||||
if r := f.request("GET", path, f.admin, nil); !strings.Contains(r.Body.String(), `"status":"pending"`) {
|
||||
t.Fatal(r.Body)
|
||||
}
|
||||
buildPrioritySnapshot(t, f)
|
||||
first := f.request("GET", path, f.admin, nil).Body.String()
|
||||
ctx, cancel := context.WithCancel(f.ctx)
|
||||
cancel()
|
||||
if _, err := refreshAcceptanceSnapshot(ctx, f.store, f.item.OrganizationID); err == nil {
|
||||
t.Fatal("published interrupted scan")
|
||||
}
|
||||
if after := f.request("GET", path, f.admin, nil).Body.String(); after != first {
|
||||
t.Fatal(after, first)
|
||||
}
|
||||
}
|
||||
|
||||
type acceptanceRefreshTimeoutStore struct {
|
||||
store.Store
|
||||
attempts map[string]int
|
||||
slowScan time.Duration
|
||||
}
|
||||
|
||||
func (s *acceptanceRefreshTimeoutStore) ListOrganizations(context.Context) ([]domain.OrganizationRecord, error) {
|
||||
return []domain.OrganizationRecord{{ID: "first"}, {ID: "second"}}, nil
|
||||
}
|
||||
|
||||
func (s *acceptanceRefreshTimeoutStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
|
||||
s.attempts[org]++
|
||||
if org == "first" {
|
||||
if s.slowScan > 0 {
|
||||
timer := time.NewTimer(s.slowScan)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
} else if s.attempts[org] == 1 {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
return s.Store.VisitAcceptanceCandidates(ctx, org, stage, visit)
|
||||
}
|
||||
|
||||
func TestAcceptancePriorityRefreshTimeoutLetsLaterOrganizationFinish(t *testing.T) {
|
||||
db := &acceptanceRefreshTimeoutStore{Store: store.NewMemoryStore(), attempts: make(map[string]int)}
|
||||
old := domain.AcceptanceSnapshot{
|
||||
OrganizationID: "first",
|
||||
CompletedAt: time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
Counts: []domain.AcceptanceCount{{ProjectID: "project", Count: 12}},
|
||||
}
|
||||
if err := db.SaveAcceptanceSnapshot(t.Context(), old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
slot := time.Now()
|
||||
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancel()
|
||||
if err := refreshAcceptanceOrganizations(ctx, db, slot); err != nil {
|
||||
t.Fatalf("slow organization exhausted the entire window: %v", err)
|
||||
}
|
||||
first, err := db.ReadAcceptanceSnapshot(t.Context(), "first")
|
||||
if err != nil || first == nil || first.CompletedAt != old.CompletedAt || len(first.Counts) != 1 || first.Counts[0] != old.Counts[0] {
|
||||
t.Fatalf("timed-out refresh replaced the previous snapshot: %+v, %v", first, err)
|
||||
}
|
||||
second, err := db.ReadAcceptanceSnapshot(t.Context(), "second")
|
||||
if err != nil || second == nil {
|
||||
t.Fatalf("later organization did not publish its snapshot: %+v, %v", second, err)
|
||||
}
|
||||
|
||||
// Use the successful snapshot's day so this test also works around midnight.
|
||||
retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retry, cancelRetry := context.WithTimeout(t.Context(), time.Second)
|
||||
defer cancelRetry()
|
||||
if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil {
|
||||
t.Fatalf("retry failed: %v", err)
|
||||
}
|
||||
first, err = db.ReadAcceptanceSnapshot(t.Context(), "first")
|
||||
if err != nil || first == nil || first.CompletedAt == old.CompletedAt {
|
||||
t.Fatalf("retry did not publish a fresh snapshot: %+v, %v", first, err)
|
||||
}
|
||||
if db.attempts["first"] != 2 || db.attempts["second"] != 1 {
|
||||
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptancePriorityRefreshRetryGivesSlowOrganizationEnoughBudget(t *testing.T) {
|
||||
const window = 2 * time.Second
|
||||
db := &acceptanceRefreshTimeoutStore{
|
||||
Store: store.NewMemoryStore(),
|
||||
attempts: make(map[string]int),
|
||||
slowScan: 1400 * time.Millisecond,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), window)
|
||||
defer cancel()
|
||||
if err := refreshAcceptanceOrganizations(ctx, db, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := db.ReadAcceptanceSnapshot(t.Context(), "first")
|
||||
if err != nil || first != nil {
|
||||
t.Fatalf("slow organization should time out in its initial share: %+v, %v", first, err)
|
||||
}
|
||||
second, err := db.ReadAcceptanceSnapshot(t.Context(), "second")
|
||||
if err != nil || second == nil {
|
||||
t.Fatalf("fast organization should finish in the first window: %+v, %v", second, err)
|
||||
}
|
||||
|
||||
// Use the successful snapshot's day so the retry is independent of wall-clock midnight.
|
||||
retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retry, cancelRetry := context.WithTimeout(t.Context(), window)
|
||||
defer cancelRetry()
|
||||
if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err = db.ReadAcceptanceSnapshot(t.Context(), "first")
|
||||
if err != nil || first == nil {
|
||||
t.Fatalf("slow organization should finish when it no longer shares the retry budget: %+v, %v", first, err)
|
||||
}
|
||||
if db.attempts["first"] != 2 || db.attempts["second"] != 1 {
|
||||
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
|
||||
}
|
||||
}
|
||||
|
||||
@ -172,6 +172,7 @@ func TestAcceptancePriorityUnknownScopeExpiryAndAdminOnly(t *testing.T) {
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("unknown dropped from ordinary group: %d", len(got))
|
||||
}
|
||||
buildPrioritySnapshot(t, f)
|
||||
admin := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=1", f.admin, nil)
|
||||
if admin.Code != 200 || !strings.Contains(admin.Body.String(), `"unresolvedCount":1`) {
|
||||
t.Fatalf("unknown not reported %s", admin.Body)
|
||||
@ -196,6 +197,7 @@ func TestAcceptancePriorityConcurrentClaimCAS(t *testing.T) {
|
||||
target.ID = "priority-race"
|
||||
addPrioritySource(t, f, target, "priority-production")
|
||||
priorityPolicy(t, f, true, 0, time.Now().Add(time.Hour).UTC().Format(time.RFC3339), nil)
|
||||
buildPrioritySnapshot(t, f)
|
||||
adminCount := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=1", f.admin, nil)
|
||||
if !strings.Contains(adminCount.Body.String(), `"remainingCount":1`) {
|
||||
t.Fatalf("admin count %s", adminCount.Body)
|
||||
|
||||
@ -21,3 +21,16 @@ type AcceptancePriorityChange struct {
|
||||
ActorID string `bson:"actorId"`
|
||||
At string `bson:"at"`
|
||||
}
|
||||
|
||||
// Counts are partitioned by the facts, independent of mutable policy versions.
|
||||
type AcceptanceCount struct {
|
||||
ProjectID string `bson:"projectId"`
|
||||
SupplierID string `bson:"supplierId"`
|
||||
Count int `bson:"count"`
|
||||
}
|
||||
|
||||
type AcceptanceSnapshot struct {
|
||||
OrganizationID string `bson:"_id"`
|
||||
CompletedAt string `bson:"completedAt"`
|
||||
Counts []AcceptanceCount `bson:"counts"`
|
||||
}
|
||||
|
||||
112
internal/store/acceptance_priority_snapshot.go
Normal file
112
internal/store/acceptance_priority_snapshot.go
Normal file
@ -0,0 +1,112 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// A single cursor avoids re-sorting and skipping the backlog for every page.
|
||||
func (s *MongoStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
|
||||
filter := humanItemMongoFilter(org, domain.HumanItemQuery{PluginID: "annotation-qa"})
|
||||
filter["currentStageId"] = stage
|
||||
filter["assignedUserId"] = bson.M{"$in": bson.A{"", nil}}
|
||||
filter["status"] = bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemSubmitted, domain.HumanItemRework}}
|
||||
projection := bson.M{"_id": 0, "batchId": 1, "projectId": 1, "pluginId": 1, "processingScope": 1, "currentStageId": 1}
|
||||
cur, err := s.humanItems.Find(ctx, filter, options.Find().SetProjection(projection).SetBatchSize(100))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cur.Close(ctx)
|
||||
page := make([]domain.HumanItemRecord, 0, 100)
|
||||
for cur.Next(ctx) {
|
||||
var row domain.HumanItemRecord
|
||||
if err := cur.Decode(&row); err != nil {
|
||||
return err
|
||||
}
|
||||
page = append(page, row)
|
||||
if len(page) == cap(page) {
|
||||
if err := visit(page); err != nil {
|
||||
return err
|
||||
}
|
||||
page = page[:0]
|
||||
}
|
||||
}
|
||||
if err := cur.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(page) > 0 {
|
||||
return visit(page)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
|
||||
for offset := 0; ; offset += 100 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
page, err := s.ListReviewCandidates(ctx, org, "", "", false, offset, 100, stage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(page) > 0 {
|
||||
if err := visit(page); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(page) < 100 {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MongoStore) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
|
||||
var v domain.AcceptanceSnapshot
|
||||
err := s.db.Collection("acceptanceStatistics").FindOne(ctx, bson.M{"_id": org}).Decode(&v)
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, nil
|
||||
}
|
||||
return &v, err
|
||||
}
|
||||
|
||||
func (s *MongoStore) SaveAcceptanceSnapshot(ctx context.Context, v domain.AcceptanceSnapshot) error {
|
||||
_, err := s.db.Collection("acceptanceStatistics").ReplaceOne(ctx, bson.M{"_id": v.OrganizationID}, v, options.Replace().SetUpsert(true))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) ReadAcceptanceSnapshot(_ context.Context, org string) (*domain.AcceptanceSnapshot, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v, ok := s.acceptanceSnapshots[org]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
v.Counts = slices.Clone(v.Counts)
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) SaveAcceptanceSnapshot(_ context.Context, v domain.AcceptanceSnapshot) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.acceptanceSnapshots == nil {
|
||||
s.acceptanceSnapshots = map[string]domain.AcceptanceSnapshot{}
|
||||
}
|
||||
v.Counts = slices.Clone(v.Counts)
|
||||
s.acceptanceSnapshots[v.OrganizationID] = v
|
||||
return nil
|
||||
}
|
||||
|
||||
// One cluster-wide attempt per slot, including across process restarts.
|
||||
func (s *MongoStore) ClaimAcceptanceRefresh(ctx context.Context, slot string) (bool, error) {
|
||||
result, err := s.db.Collection("acceptanceStatisticsSchedule").UpdateOne(ctx, bson.M{"_id": "nightly", "slot": bson.M{"$ne": slot}}, bson.M{"$set": bson.M{"slot": slot}}, options.Update().SetUpsert(true))
|
||||
if mongo.IsDuplicateKeyError(err) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil && (result.ModifiedCount == 1 || result.UpsertedCount == 1), err
|
||||
}
|
||||
@ -14,6 +14,7 @@ import (
|
||||
)
|
||||
|
||||
type MemoryStore struct {
|
||||
acceptanceSnapshots map[string]domain.AcceptanceSnapshot
|
||||
acceptancePriorities map[string]domain.AcceptancePriority
|
||||
outputSnapshots outputsnapshot.MemoryBackend
|
||||
acceptanceAuthorizations map[string]domain.SupplierAcceptanceAuthorization
|
||||
|
||||
@ -248,6 +248,9 @@ type Store interface {
|
||||
ListSuppliers(context.Context, string) ([]domain.SupplierRecord, error)
|
||||
GetSupplier(context.Context, string) (*domain.SupplierRecord, error)
|
||||
ResolveAcceptanceSourceSuppliers(context.Context, string, []string) (map[string]string, error)
|
||||
ReadAcceptanceSnapshot(context.Context, string) (*domain.AcceptanceSnapshot, error)
|
||||
SaveAcceptanceSnapshot(context.Context, domain.AcceptanceSnapshot) error
|
||||
VisitAcceptanceCandidates(context.Context, string, string, func([]domain.HumanItemRecord) error) error
|
||||
ListAcceptancePriorities(context.Context, string) ([]domain.AcceptancePriority, error)
|
||||
SaveAcceptancePriority(context.Context, domain.AcceptancePriority, int) (*domain.AcceptancePriority, error)
|
||||
GetSupplierAcceptanceAuthorization(context.Context, string, string) (*domain.SupplierAcceptanceAuthorization, error)
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { formatDateTime } from "../lib/datetime";
|
||||
import { projectsApi } from "../services/api/projects";
|
||||
import { apiRequest } from "../services/api/client";
|
||||
import type { Project } from "../types";
|
||||
@ -8,7 +9,8 @@ import { userErrorMessage } from "../lib/user-error-message.js";
|
||||
type Policy = { supplierId: string; enabled: boolean; projectIds: string[] | null; expiresAt: string; version: number };
|
||||
type PriorityState = { policy: Policy; active: boolean; remainingCount: number | null; unresolvedCount: number | null };
|
||||
type Statistics = {
|
||||
status: "completed" | "incomplete";
|
||||
status: "completed" | "incomplete" | "pending";
|
||||
completedAt?: string;
|
||||
policyVersion: number;
|
||||
remainingCount: number | null;
|
||||
unresolvedCount: number | null;
|
||||
@ -76,6 +78,10 @@ export function SupplierAcceptancePriorityPanel({ supplierId }: { supplierId: st
|
||||
)
|
||||
.then((next) => {
|
||||
if (!current) return;
|
||||
if (next.policyVersion === policyVersion && next.status === "pending") {
|
||||
setStatistics(next);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
next.policyVersion !== policyVersion ||
|
||||
next.status !== "completed" ||
|
||||
@ -146,21 +152,28 @@ export function SupplierAcceptancePriorityPanel({ supplierId }: { supplierId: st
|
||||
{value && (
|
||||
<>
|
||||
<div role="status" aria-label="验收优先统计" className="flex flex-wrap gap-4 text-sm">
|
||||
<span>
|
||||
待领取{" "}
|
||||
{statistics?.status === "completed" && statistics.policyVersion === value.policy.version
|
||||
? statistics.remainingCount
|
||||
: statistics
|
||||
? "未完成"
|
||||
: "统计中…"}
|
||||
</span>
|
||||
{statistics?.status === "pending" ? (
|
||||
<span>等待每日 01:00(北京时间)统计</span>
|
||||
) : (
|
||||
<span>
|
||||
待领取{" "}
|
||||
{statistics?.status === "completed" && statistics.policyVersion === value.policy.version
|
||||
? statistics.remainingCount
|
||||
: statistics
|
||||
? "未完成"
|
||||
: "统计中…"}
|
||||
</span>
|
||||
)}
|
||||
{statistics?.status === "completed" && statistics.policyVersion === value.policy.version ? (
|
||||
(statistics.unresolvedCount ?? 0) > 0 && (
|
||||
<span className="text-amber-600">归属待核对 {statistics.unresolvedCount}</span>
|
||||
)
|
||||
) : statistics ? (
|
||||
) : statistics?.status === "incomplete" ? (
|
||||
<span>归属待核对 未完成</span>
|
||||
) : null}
|
||||
{statistics?.status === "completed" && statistics.completedAt && (
|
||||
<span>统计于 {formatDateTime(statistics.completedAt, "Asia/Shanghai")}(北京时间)</span>
|
||||
)}
|
||||
{statistics?.status === "incomplete" && (
|
||||
<button type="button" onClick={() => setStatisticsAttempt((n) => n + 1)} className="underline">
|
||||
重试统计
|
||||
|
||||
@ -33,6 +33,10 @@ try {
|
||||
if (!u.pathname.startsWith("/api/")) return route.continue();
|
||||
if (u.pathname.endsWith("/acceptance-priority/statistics")) {
|
||||
const policyVersion = Number(u.searchParams.get("version"));
|
||||
if (statsMode === "pending")
|
||||
return route.fulfill({
|
||||
json: { status: "pending", policyVersion, remainingCount: null, unresolvedCount: null },
|
||||
});
|
||||
if (statsMode === "incomplete")
|
||||
return route.fulfill({
|
||||
json: { status: "incomplete", policyVersion, remainingCount: null, unresolvedCount: null },
|
||||
@ -80,6 +84,14 @@ try {
|
||||
);
|
||||
assert(state.remainingCount > 0, JSON.stringify(state));
|
||||
await panel.getByText(`待领取 ${state.remainingCount}`, { exact: true }).waitFor();
|
||||
await panel.getByText(/统计于.*北京时间/).waitFor();
|
||||
statsMode = "pending";
|
||||
await page.reload();
|
||||
await page
|
||||
.getByRole("button", { name: /^优先供应商/ })
|
||||
.first()
|
||||
.click();
|
||||
await panel.getByText("等待每日 01:00(北京时间)统计").waitFor();
|
||||
await panel.getByRole("checkbox", { name: "泛宇草稿项目", exact: true }).check();
|
||||
await panel.getByLabel("到期时间").fill("2090-01-01T12:00");
|
||||
statsMode = "slow";
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user