Merge pull request '验收优先统计改为每天凌晨 1 点刷新' (#80) from yao.wang/droboticflow:dev into dev
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions

This commit is contained in:
zhanghongming 2026-11-18 22:33:36 +08:00
commit 90336acc34
14 changed files with 575 additions and 97 deletions

View File

@ -235,6 +235,8 @@ func main() {
os.Exit(1)
}
go api.RunAcceptanceStatisticsSchedule(ctx, dataStore)
aiService := ai.NewService(cfg.AIAPIBaseURL, cfg.AIAPIKey, cfg.AIModel)
handler := spaHandler(api.NewHandlerWithConfig(dataStore, aiService, cfg), cfg.DistDir)
server := &http.Server{

View File

@ -43,3 +43,11 @@
- 成功的 PUT 响应确认配置 CAS 已落库,不再附带可能失败的统计。传输中断仍需刷新核对结果,不能自动重放保存或把网络超时当作确定未保存。
随机领取、连续审查、服务端权限复核、匿名字段、任务版本 CAS 及原有列表/计数接口均不变。真实供应商、项目与有效期仍须管理员另行配置;管理入口修复与实际启用验收前,不得对外宣称优先验收已生效。
## 每日统计
管理员面板读取组织最近一次成功保存的汇总,按当前已保存配置的项目范围展示“待领取”和“归属待核对”,同时显示北京时间的统计完成时间。首次统计前返回 `status: pending`、两个计数为 null。配置变化可以直接用同一份分项目汇总计算对应范围。统计读取失败仍可重试。
API 实例在北京时间 01:00–01:30 首轮刷新,01:35–02:05 补跑。每轮由一个实例串行处理,每 100 条候选任务批量解析来源。每轮先筛出当天未完成的组织,再按剩余窗口时间和仍待处理的组织数分配预算,每组织最多 15 分钟。提前完成时,未用的时间留给后续组织。
一个组织完整处理后原子替换其汇总。读取失败或超时后继续处理下一个组织。窗口内重启可以参与尚未被领取的窗口,两轮均未完成时保留最近成功值,次日再试。完成日志记录候选条数和用时。统计反映扫描期间看到的数据,业务领取和配置保存按原流程实时处理。

View File

@ -200,107 +200,44 @@ func (s *server) handleAcceptancePriorityStatistics(w http.ResponseWriter, r *ht
renderJSON(w, 404, map[string]string{"error": "供应商不存在"})
return
}
resolver, err := s.newAcceptancePriorityResolver(ctx, org)
snapshot, err := s.store.ReadAcceptanceSnapshot(ctx, org)
if err != nil {
incomplete()
return
}
value := domain.AcceptancePriority{OrganizationID: org, SupplierID: id}
for _, v := range resolver.rules {
if v.SupplierID == id {
value = v
}
}
if value.Version != expected {
rules, err := s.store.ListAcceptancePriorities(ctx, org)
if err != nil {
incomplete()
return
}
project := ""
if len(value.ProjectIDs) == 1 {
project = value.ProjectIDs[0]
}
remaining, unresolved := 0, 0
for offset := 0; ; offset += 100 {
if ctx.Err() != nil {
incomplete()
return
}
rows, e := s.store.ListReviewCandidates(r.Context(), org, project, "", false, offset, 100, fanyuV5PrePackageAcceptanceStageID)
if e != nil {
incomplete()
return
}
pageSize := len(rows)
if len(value.ProjectIDs) > 0 {
scoped := []domain.HumanItemRecord{}
for _, item := range rows {
if slices.Contains(value.ProjectIDs, item.ProjectID) {
scoped = append(scoped, item)
}
}
rows = scoped
}
batchIDs := []string{}
for _, item := range rows {
batchIDs = append(batchIDs, item.BatchID)
}
batchRows, e := s.store.ListHumanBatchAccessRecords(r.Context(), org, batchIDs)
if e != nil {
incomplete()
return
}
batches := map[string]*domain.HumanBatchRecord{}
for i := range batchRows {
batches[batchRows[i].ID] = &batchRows[i]
}
if e := resolver.prime(rows); e != nil {
incomplete()
return
}
for _, item := range rows {
if len(value.ProjectIDs) > 0 && !slices.Contains(value.ProjectIDs, item.ProjectID) {
continue
}
b := batches[item.BatchID]
if !domain.SharedAcceptanceItem(item, b) {
continue
}
sid, e := resolver.supplier(item)
if e != nil {
incomplete()
return
}
if sid == "" {
unresolved++
} else if sid == id {
remaining++
}
}
if pageSize < 100 {
var policy domain.AcceptancePriority
for _, rule := range rules {
if rule.SupplierID == id {
policy = rule
break
}
}
if ctx.Err() != nil {
if policy.Version != expected {
incomplete()
return
}
latest, e := s.store.ListAcceptancePriorities(ctx, org)
if e != nil {
incomplete()
if snapshot == nil {
renderJSON(w, 200, map[string]any{"status": "pending", "policyVersion": expected, "remainingCount": nil, "unresolvedCount": nil})
return
}
version := 0
for _, v := range latest {
if v.SupplierID == id {
version = v.Version
remaining, unresolved := 0, 0
for _, group := range snapshot.Counts {
if len(policy.ProjectIDs) > 0 && !slices.Contains(policy.ProjectIDs, group.ProjectID) {
continue
}
switch group.SupplierID {
case "":
unresolved += group.Count
case id:
remaining += group.Count
}
}
if version != expected {
incomplete()
return
}
renderJSON(w, 200, map[string]any{"status": "completed", "policyVersion": expected, "remainingCount": remaining, "unresolvedCount": unresolved})
renderJSON(w, 200, map[string]any{"status": "completed", "policyVersion": expected, "remainingCount": remaining, "unresolvedCount": unresolved, "completedAt": snapshot.CompletedAt})
}
// Preserve the existing first-eligible-page random semantics inside each group.

View File

@ -21,6 +21,7 @@ func TestAcceptancePriorityBrowser(t *testing.T) {
item := f.item
item.ID = "priority-browser"
addPrioritySource(t, f, item, "priority-production")
buildPrioritySnapshot(t, f)
h := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Test-only authentication at the loopback fixture boundary.
user := f.admin

View File

@ -2,6 +2,7 @@ package api
import (
"context"
"droboticflow/internal/domain"
"droboticflow/internal/store"
"fmt"
"go.mongodb.org/mongo-driver/bson"
@ -9,6 +10,7 @@ import (
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"os"
"strings"
"sync/atomic"
"testing"
"time"
@ -45,9 +47,31 @@ func TestAcceptancePriorityConfigMongo(t *testing.T) {
if e = s.UpsertUser(ctx, f.admin); e != nil {
t.Fatal(e)
}
org := f.item.OrganizationID
template, err := f.store.GetHumanBatch(ctx, f.item.BatchID)
if err != nil || template == nil {
t.Fatal("missing fixture batch", err)
}
for i := 0; i < 56; i++ {
b := *template
b.ID = fmt.Sprintf("target-%d", i)
b.SourceBatchID = fmt.Sprintf("source-%d", i)
if err := s.UpsertHumanBatch(ctx, b); err != nil {
t.Fatal(err)
}
b.ID = b.SourceBatchID
b.SourceBatchID = ""
if err := s.UpsertHumanBatch(ctx, b); err != nil {
t.Fatal(err)
}
if err := s.UpsertHumanItem(ctx, domain.HumanItemRecord{ID: b.ID, BatchID: b.ID, OrganizationID: org, ProjectID: b.ProjectID, SupplierID: supplier.ID, Status: domain.HumanItemAccepted}); err != nil {
t.Fatal(err)
}
}
scans.Store(0)
docs := make([]any, 5600)
for i := range docs {
docs[i] = bson.M{"id": fmt.Sprintf("candidate-%06d", i), "organizationId": f.item.OrganizationID, "currentStageId": "prepackage_acceptance", "status": "ready", "pluginId": "annotation-qa", "subject": bson.M{"payload": bson.M{"synthetic": "fixture"}}}
docs[i] = bson.M{"id": fmt.Sprintf("candidate-%06d", i), "organizationId": f.item.OrganizationID, "currentStageId": "prepackage_acceptance", "status": "ready", "projectId": template.ProjectID, "batchId": fmt.Sprintf("target-%d", i/100), "pluginId": "annotation-qa", "subject": bson.M{"payload": bson.M{"synthetic": "fixture"}}}
}
if _, e = db.Collection("humanItems").InsertMany(ctx, docs); e != nil {
t.Fatal(e)
@ -72,4 +96,24 @@ func TestAcceptancePriorityConfigMongo(t *testing.T) {
if e != nil || len(rows) != 1 || rows[0].Version != 2 || rows[0].Enabled || len(rows[0].History) != 2 {
t.Fatal("confirmed writes did not survive new store")
}
at := time.Now()
n, err := refreshAcceptanceSnapshot(ctx, s, org)
if err != nil || n != len(docs) {
t.Fatalf("refresh rows=%d err=%v", n, err)
}
t.Logf("nightly complete: %d candidates, 56 source batches, %s", n, time.Since(at))
scans.Store(0)
restarted := store.NewMongoStore(db)
f.handler = NewHandler(restarted, nil)
r := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=2", f.admin, nil)
if !strings.Contains(r.Body.String(), `"remainingCount":5600`) || scans.Load() != 0 {
t.Fatal("snapshot read", r.Body, scans.Load())
}
for i, replica := range []*store.MongoStore{s, restarted} {
claimed, err := replica.ClaimAcceptanceRefresh(ctx, "2026-11-18T01:00:00+08:00")
if claimed != (i == 0) || err != nil {
t.Fatalf("replica %d claimed=%v err=%v", i, claimed, err)
}
}
}

View File

@ -0,0 +1,163 @@
package api
import (
"context"
"fmt"
"log"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
var acceptanceZone = time.FixedZone("Asia/Shanghai", 8*60*60)
const (
acceptanceWindow = 30 * time.Minute
acceptanceRetryOffset = acceptanceWindow + 5*time.Minute
acceptanceOrganizationBudget = 15 * time.Minute
acceptanceSnapshotLookupBudget = 2 * time.Second
)
// Give a failed or interrupted run one more window at 01:35. Deploying
// during the day never starts a catch-up scan during business hours.
func acceptanceRefreshSlot(now time.Time) time.Time {
now = now.In(acceptanceZone)
first := time.Date(now.Year(), now.Month(), now.Day(), 1, 0, 0, 0, acceptanceZone)
for _, slot := range []time.Time{first, first.Add(acceptanceRetryOffset)} {
if now.Before(slot.Add(acceptanceWindow)) {
return slot
}
}
return first.AddDate(0, 0, 1)
}
func RunAcceptanceStatisticsSchedule(ctx context.Context, db *store.MongoStore) {
for ctx.Err() == nil {
slot := acceptanceRefreshSlot(time.Now())
if !waitUntil(ctx, slot) {
return
}
window, cancel := context.WithDeadline(ctx, slot.Add(acceptanceWindow))
claimed, err := db.ClaimAcceptanceRefresh(window, slot.Format(time.RFC3339))
if err == nil && claimed {
err = refreshAcceptanceOrganizations(window, db, slot)
}
cancel()
if err != nil {
log.Printf("acceptance statistics window failed: %v", err)
}
if !waitUntil(ctx, slot.Add(acceptanceWindow)) {
return
}
}
}
func waitUntil(ctx context.Context, at time.Time) bool {
t := time.NewTimer(time.Until(at))
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return ctx.Err() == nil
}
}
func refreshAcceptanceOrganizations(ctx context.Context, db store.Store, slot time.Time) error {
orgs, err := db.ListOrganizations(ctx)
if err != nil {
return err
}
day := slot.In(acceptanceZone).Format(time.DateOnly)
pending := make([]string, 0, len(orgs))
for _, org := range orgs {
if err := ctx.Err(); err != nil {
return err
}
lookup, cancel := context.WithTimeout(ctx, acceptanceSnapshotLookupBudget)
last, err := db.ReadAcceptanceSnapshot(lookup, org.ID)
cancel()
if err != nil {
log.Printf("acceptance statistics org=%q could not read previous snapshot: %v", org.ID, err)
continue
}
if last != nil {
completed, err := time.Parse(time.RFC3339Nano, last.CompletedAt)
if err == nil && completed.In(acceptanceZone).Format(time.DateOnly) == day {
continue
}
}
pending = append(pending, org.ID)
}
for i, org := range pending {
if err := ctx.Err(); err != nil {
return err
}
// Share only among organizations that still need a scan this window.
remaining := acceptanceWindow
if deadline, ok := ctx.Deadline(); ok {
remaining = time.Until(deadline)
}
budget := min(acceptanceOrganizationBudget, remaining/time.Duration(len(pending)-i))
orgCtx, cancel := context.WithTimeout(ctx, budget)
start := time.Now()
n, err := refreshAcceptanceSnapshot(orgCtx, db, org)
cancel()
log.Printf("acceptance statistics org=%q candidates=%d elapsed=%s completed=%t", org, n, time.Since(start), err == nil)
if err != nil {
log.Printf("acceptance statistics org=%q refresh failed: %v", org, err)
}
}
return ctx.Err()
}
func refreshAcceptanceSnapshot(ctx context.Context, db store.Store, org string) (int, error) {
type key struct{ project, supplier string }
counts := map[key]int{}
total := 0
err := db.VisitAcceptanceCandidates(ctx, org, fanyuV5PrePackageAcceptanceStageID, func(page []domain.HumanItemRecord) error {
total += len(page)
ids := make([]string, 0, len(page))
seen := map[string]bool{}
for _, row := range page {
if row.BatchID != "" && !seen[row.BatchID] {
ids = append(ids, row.BatchID)
seen[row.BatchID] = true
}
}
if len(ids) == 0 {
return nil
}
batches, err := db.ListHumanBatchAccessRecords(ctx, org, ids)
if err != nil {
return err
}
byID := map[string]*domain.HumanBatchRecord{}
for i := range batches {
byID[batches[i].ID] = &batches[i]
}
owners, err := db.ResolveAcceptanceSourceSuppliers(ctx, org, ids)
if err != nil {
return err
}
for _, row := range page {
if domain.SharedAcceptanceItem(row, byID[row.BatchID]) {
counts[key{row.ProjectID, owners[row.BatchID]}]++
}
}
return nil
})
if err != nil {
return total, fmt.Errorf("scan acceptance candidates: %w", err)
}
if err := ctx.Err(); err != nil {
return total, err
}
v := domain.AcceptanceSnapshot{OrganizationID: org, CompletedAt: time.Now().UTC().Format(time.RFC3339Nano)}
for k, n := range counts {
v.Counts = append(v.Counts, domain.AcceptanceCount{ProjectID: k.project, SupplierID: k.supplier, Count: n})
}
return total, db.SaveAcceptanceSnapshot(ctx, v)
}

View File

@ -7,6 +7,7 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
@ -26,6 +27,20 @@ func (s *priorityStatsGuard) ListReviewCandidates(ctx context.Context, org, proj
}
return nil, errors.New("statistics unavailable")
}
func (s *priorityStatsGuard) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
s.calls.Add(1)
if s.wait {
<-ctx.Done()
return nil, ctx.Err()
}
return nil, errors.New("snapshot unavailable")
}
func buildPrioritySnapshot(t *testing.T, f combinedDraftFixture) {
t.Helper()
if _, err := refreshAcceptanceSnapshot(f.ctx, f.store, f.item.OrganizationID); err != nil {
t.Fatal(err)
}
}
func TestAcceptancePriorityConfigDoesNotWaitForStatistics(t *testing.T) {
f := priorityFixture(t)
g := &priorityStatsGuard{Store: f.store}
@ -83,6 +98,7 @@ func TestAcceptancePriorityStatisticsFailureAndDeadline(t *testing.T) {
}
func TestAcceptancePriorityStatisticsVersionAndBoundary(t *testing.T) {
f := priorityFixture(t)
buildPrioritySnapshot(t, f)
path := "/api/suppliers/priority-production/acceptance-priority/statistics"
for _, actor := range []domain.UserRecord{f.annotator, f.reviewer} {
if r := f.request("GET", path+"?version=0", actor, nil); r.Code != 403 {
@ -110,7 +126,7 @@ type priorityConcurrentStatsStore struct {
release chan struct{}
}
func (s *priorityConcurrentStatsStore) ListReviewCandidates(ctx context.Context, org, project, user string, mine bool, offset, limit int, stages ...string) ([]domain.HumanItemRecord, error) {
func (s *priorityConcurrentStatsStore) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
select {
case s.started <- struct{}{}:
default:
@ -120,7 +136,7 @@ func (s *priorityConcurrentStatsStore) ListReviewCandidates(ctx context.Context,
return nil, ctx.Err()
case <-s.release:
}
return s.Store.ListReviewCandidates(ctx, org, project, user, mine, offset, limit, stages...)
return s.Store.ReadAcceptanceSnapshot(ctx, org)
}
func TestAcceptancePrioritySaveWhileStatisticsIsPending(t *testing.T) {
f := priorityFixture(t)
@ -144,3 +160,154 @@ func TestAcceptancePrioritySaveWhileStatisticsIsPending(t *testing.T) {
t.Fatal("old version statistics published after save")
}
}
func TestAcceptancePriorityRefreshWindow(t *testing.T) {
for _, tc := range []struct{ now, want string }{
{"00:59", "18 01:00"},
{"01:00", "18 01:00"},
{"01:29", "18 01:00"},
{"01:30", "18 01:35"},
{"01:34", "18 01:35"},
{"01:35", "18 01:35"},
{"02:04", "18 01:35"},
{"02:05", "19 01:00"},
} {
now, _ := time.ParseInLocation("2006-01-02 15:04", "2026-11-18 "+tc.now, acceptanceZone)
slot := acceptanceRefreshSlot(now.UTC())
if slot.Format("02 15:04") != tc.want {
t.Fatal(tc, slot)
}
}
}
func TestAcceptancePrioritySnapshotFirstRunAndFailure(t *testing.T) {
f := priorityFixture(t)
path := "/api/suppliers/priority-production/acceptance-priority/statistics?version=0"
if r := f.request("GET", path, f.admin, nil); !strings.Contains(r.Body.String(), `"status":"pending"`) {
t.Fatal(r.Body)
}
buildPrioritySnapshot(t, f)
first := f.request("GET", path, f.admin, nil).Body.String()
ctx, cancel := context.WithCancel(f.ctx)
cancel()
if _, err := refreshAcceptanceSnapshot(ctx, f.store, f.item.OrganizationID); err == nil {
t.Fatal("published interrupted scan")
}
if after := f.request("GET", path, f.admin, nil).Body.String(); after != first {
t.Fatal(after, first)
}
}
type acceptanceRefreshTimeoutStore struct {
store.Store
attempts map[string]int
slowScan time.Duration
}
func (s *acceptanceRefreshTimeoutStore) ListOrganizations(context.Context) ([]domain.OrganizationRecord, error) {
return []domain.OrganizationRecord{{ID: "first"}, {ID: "second"}}, nil
}
func (s *acceptanceRefreshTimeoutStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
s.attempts[org]++
if org == "first" {
if s.slowScan > 0 {
timer := time.NewTimer(s.slowScan)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
}
} else if s.attempts[org] == 1 {
<-ctx.Done()
return ctx.Err()
}
}
return s.Store.VisitAcceptanceCandidates(ctx, org, stage, visit)
}
func TestAcceptancePriorityRefreshTimeoutLetsLaterOrganizationFinish(t *testing.T) {
db := &acceptanceRefreshTimeoutStore{Store: store.NewMemoryStore(), attempts: make(map[string]int)}
old := domain.AcceptanceSnapshot{
OrganizationID: "first",
CompletedAt: time.Now().Add(-24 * time.Hour).UTC().Format(time.RFC3339Nano),
Counts: []domain.AcceptanceCount{{ProjectID: "project", Count: 12}},
}
if err := db.SaveAcceptanceSnapshot(t.Context(), old); err != nil {
t.Fatal(err)
}
slot := time.Now()
ctx, cancel := context.WithTimeout(t.Context(), time.Second)
defer cancel()
if err := refreshAcceptanceOrganizations(ctx, db, slot); err != nil {
t.Fatalf("slow organization exhausted the entire window: %v", err)
}
first, err := db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first == nil || first.CompletedAt != old.CompletedAt || len(first.Counts) != 1 || first.Counts[0] != old.Counts[0] {
t.Fatalf("timed-out refresh replaced the previous snapshot: %+v, %v", first, err)
}
second, err := db.ReadAcceptanceSnapshot(t.Context(), "second")
if err != nil || second == nil {
t.Fatalf("later organization did not publish its snapshot: %+v, %v", second, err)
}
// Use the successful snapshot's day so this test also works around midnight.
retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt)
if err != nil {
t.Fatal(err)
}
retry, cancelRetry := context.WithTimeout(t.Context(), time.Second)
defer cancelRetry()
if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil {
t.Fatalf("retry failed: %v", err)
}
first, err = db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first == nil || first.CompletedAt == old.CompletedAt {
t.Fatalf("retry did not publish a fresh snapshot: %+v, %v", first, err)
}
if db.attempts["first"] != 2 || db.attempts["second"] != 1 {
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
}
}
func TestAcceptancePriorityRefreshRetryGivesSlowOrganizationEnoughBudget(t *testing.T) {
const window = 2 * time.Second
db := &acceptanceRefreshTimeoutStore{
Store: store.NewMemoryStore(),
attempts: make(map[string]int),
slowScan: 1400 * time.Millisecond,
}
ctx, cancel := context.WithTimeout(t.Context(), window)
defer cancel()
if err := refreshAcceptanceOrganizations(ctx, db, time.Now()); err != nil {
t.Fatal(err)
}
first, err := db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first != nil {
t.Fatalf("slow organization should time out in its initial share: %+v, %v", first, err)
}
second, err := db.ReadAcceptanceSnapshot(t.Context(), "second")
if err != nil || second == nil {
t.Fatalf("fast organization should finish in the first window: %+v, %v", second, err)
}
// Use the successful snapshot's day so the retry is independent of wall-clock midnight.
retryDay, err := time.Parse(time.RFC3339Nano, second.CompletedAt)
if err != nil {
t.Fatal(err)
}
retry, cancelRetry := context.WithTimeout(t.Context(), window)
defer cancelRetry()
if err := refreshAcceptanceOrganizations(retry, db, retryDay); err != nil {
t.Fatal(err)
}
first, err = db.ReadAcceptanceSnapshot(t.Context(), "first")
if err != nil || first == nil {
t.Fatalf("slow organization should finish when it no longer shares the retry budget: %+v, %v", first, err)
}
if db.attempts["first"] != 2 || db.attempts["second"] != 1 {
t.Fatalf("retry must only scan unfinished organizations: %v", db.attempts)
}
}

View File

@ -172,6 +172,7 @@ func TestAcceptancePriorityUnknownScopeExpiryAndAdminOnly(t *testing.T) {
if len(got) != 2 {
t.Fatalf("unknown dropped from ordinary group: %d", len(got))
}
buildPrioritySnapshot(t, f)
admin := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=1", f.admin, nil)
if admin.Code != 200 || !strings.Contains(admin.Body.String(), `"unresolvedCount":1`) {
t.Fatalf("unknown not reported %s", admin.Body)
@ -196,6 +197,7 @@ func TestAcceptancePriorityConcurrentClaimCAS(t *testing.T) {
target.ID = "priority-race"
addPrioritySource(t, f, target, "priority-production")
priorityPolicy(t, f, true, 0, time.Now().Add(time.Hour).UTC().Format(time.RFC3339), nil)
buildPrioritySnapshot(t, f)
adminCount := f.request("GET", "/api/suppliers/priority-production/acceptance-priority/statistics?version=1", f.admin, nil)
if !strings.Contains(adminCount.Body.String(), `"remainingCount":1`) {
t.Fatalf("admin count %s", adminCount.Body)

View File

@ -21,3 +21,16 @@ type AcceptancePriorityChange struct {
ActorID string `bson:"actorId"`
At string `bson:"at"`
}
// Counts are partitioned by the facts, independent of mutable policy versions.
type AcceptanceCount struct {
ProjectID string `bson:"projectId"`
SupplierID string `bson:"supplierId"`
Count int `bson:"count"`
}
type AcceptanceSnapshot struct {
OrganizationID string `bson:"_id"`
CompletedAt string `bson:"completedAt"`
Counts []AcceptanceCount `bson:"counts"`
}

View File

@ -0,0 +1,112 @@
package store
import (
"context"
"errors"
"slices"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
// A single cursor avoids re-sorting and skipping the backlog for every page.
func (s *MongoStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
filter := humanItemMongoFilter(org, domain.HumanItemQuery{PluginID: "annotation-qa"})
filter["currentStageId"] = stage
filter["assignedUserId"] = bson.M{"$in": bson.A{"", nil}}
filter["status"] = bson.M{"$in": bson.A{domain.HumanItemReady, domain.HumanItemSubmitted, domain.HumanItemRework}}
projection := bson.M{"_id": 0, "batchId": 1, "projectId": 1, "pluginId": 1, "processingScope": 1, "currentStageId": 1}
cur, err := s.humanItems.Find(ctx, filter, options.Find().SetProjection(projection).SetBatchSize(100))
if err != nil {
return err
}
defer cur.Close(ctx)
page := make([]domain.HumanItemRecord, 0, 100)
for cur.Next(ctx) {
var row domain.HumanItemRecord
if err := cur.Decode(&row); err != nil {
return err
}
page = append(page, row)
if len(page) == cap(page) {
if err := visit(page); err != nil {
return err
}
page = page[:0]
}
}
if err := cur.Err(); err != nil {
return err
}
if len(page) > 0 {
return visit(page)
}
return nil
}
func (s *MemoryStore) VisitAcceptanceCandidates(ctx context.Context, org, stage string, visit func([]domain.HumanItemRecord) error) error {
for offset := 0; ; offset += 100 {
if err := ctx.Err(); err != nil {
return err
}
page, err := s.ListReviewCandidates(ctx, org, "", "", false, offset, 100, stage)
if err != nil {
return err
}
if len(page) > 0 {
if err := visit(page); err != nil {
return err
}
}
if len(page) < 100 {
return nil
}
}
}
func (s *MongoStore) ReadAcceptanceSnapshot(ctx context.Context, org string) (*domain.AcceptanceSnapshot, error) {
var v domain.AcceptanceSnapshot
err := s.db.Collection("acceptanceStatistics").FindOne(ctx, bson.M{"_id": org}).Decode(&v)
if errors.Is(err, mongo.ErrNoDocuments) {
return nil, nil
}
return &v, err
}
func (s *MongoStore) SaveAcceptanceSnapshot(ctx context.Context, v domain.AcceptanceSnapshot) error {
_, err := s.db.Collection("acceptanceStatistics").ReplaceOne(ctx, bson.M{"_id": v.OrganizationID}, v, options.Replace().SetUpsert(true))
return err
}
func (s *MemoryStore) ReadAcceptanceSnapshot(_ context.Context, org string) (*domain.AcceptanceSnapshot, error) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.acceptanceSnapshots[org]
if !ok {
return nil, nil
}
v.Counts = slices.Clone(v.Counts)
return &v, nil
}
func (s *MemoryStore) SaveAcceptanceSnapshot(_ context.Context, v domain.AcceptanceSnapshot) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.acceptanceSnapshots == nil {
s.acceptanceSnapshots = map[string]domain.AcceptanceSnapshot{}
}
v.Counts = slices.Clone(v.Counts)
s.acceptanceSnapshots[v.OrganizationID] = v
return nil
}
// One cluster-wide attempt per slot, including across process restarts.
func (s *MongoStore) ClaimAcceptanceRefresh(ctx context.Context, slot string) (bool, error) {
result, err := s.db.Collection("acceptanceStatisticsSchedule").UpdateOne(ctx, bson.M{"_id": "nightly", "slot": bson.M{"$ne": slot}}, bson.M{"$set": bson.M{"slot": slot}}, options.Update().SetUpsert(true))
if mongo.IsDuplicateKeyError(err) {
return false, nil
}
return err == nil && (result.ModifiedCount == 1 || result.UpsertedCount == 1), err
}

View File

@ -14,6 +14,7 @@ import (
)
type MemoryStore struct {
acceptanceSnapshots map[string]domain.AcceptanceSnapshot
acceptancePriorities map[string]domain.AcceptancePriority
outputSnapshots outputsnapshot.MemoryBackend
acceptanceAuthorizations map[string]domain.SupplierAcceptanceAuthorization

View File

@ -248,6 +248,9 @@ type Store interface {
ListSuppliers(context.Context, string) ([]domain.SupplierRecord, error)
GetSupplier(context.Context, string) (*domain.SupplierRecord, error)
ResolveAcceptanceSourceSuppliers(context.Context, string, []string) (map[string]string, error)
ReadAcceptanceSnapshot(context.Context, string) (*domain.AcceptanceSnapshot, error)
SaveAcceptanceSnapshot(context.Context, domain.AcceptanceSnapshot) error
VisitAcceptanceCandidates(context.Context, string, string, func([]domain.HumanItemRecord) error) error
ListAcceptancePriorities(context.Context, string) ([]domain.AcceptancePriority, error)
SaveAcceptancePriority(context.Context, domain.AcceptancePriority, int) (*domain.AcceptancePriority, error)
GetSupplierAcceptanceAuthorization(context.Context, string, string) (*domain.SupplierAcceptanceAuthorization, error)

View File

@ -1,4 +1,5 @@
import { useEffect, useState } from "react";
import { formatDateTime } from "../lib/datetime";
import { projectsApi } from "../services/api/projects";
import { apiRequest } from "../services/api/client";
import type { Project } from "../types";
@ -8,7 +9,8 @@ import { userErrorMessage } from "../lib/user-error-message.js";
type Policy = { supplierId: string; enabled: boolean; projectIds: string[] | null; expiresAt: string; version: number };
type PriorityState = { policy: Policy; active: boolean; remainingCount: number | null; unresolvedCount: number | null };
type Statistics = {
status: "completed" | "incomplete";
status: "completed" | "incomplete" | "pending";
completedAt?: string;
policyVersion: number;
remainingCount: number | null;
unresolvedCount: number | null;
@ -76,6 +78,10 @@ export function SupplierAcceptancePriorityPanel({ supplierId }: { supplierId: st
)
.then((next) => {
if (!current) return;
if (next.policyVersion === policyVersion && next.status === "pending") {
setStatistics(next);
return;
}
if (
next.policyVersion !== policyVersion ||
next.status !== "completed" ||
@ -146,21 +152,28 @@ export function SupplierAcceptancePriorityPanel({ supplierId }: { supplierId: st
{value && (
<>
<div role="status" aria-label="验收优先统计" className="flex flex-wrap gap-4 text-sm">
<span>
待领取{" "}
{statistics?.status === "completed" && statistics.policyVersion === value.policy.version
? statistics.remainingCount
: statistics
? "未完成"
: "统计中…"}
</span>
{statistics?.status === "pending" ? (
<span>等待每日 01:00(北京时间)统计</span>
) : (
<span>
待领取{" "}
{statistics?.status === "completed" && statistics.policyVersion === value.policy.version
? statistics.remainingCount
: statistics
? "未完成"
: "统计中…"}
</span>
)}
{statistics?.status === "completed" && statistics.policyVersion === value.policy.version ? (
(statistics.unresolvedCount ?? 0) > 0 && (
<span className="text-amber-600">归属待核对 {statistics.unresolvedCount}</span>
)
) : statistics ? (
) : statistics?.status === "incomplete" ? (
<span>归属待核对 未完成</span>
) : null}
{statistics?.status === "completed" && statistics.completedAt && (
<span>统计于 {formatDateTime(statistics.completedAt, "Asia/Shanghai")}(北京时间)</span>
)}
{statistics?.status === "incomplete" && (
<button type="button" onClick={() => setStatisticsAttempt((n) => n + 1)} className="underline">
重试统计

View File

@ -33,6 +33,10 @@ try {
if (!u.pathname.startsWith("/api/")) return route.continue();
if (u.pathname.endsWith("/acceptance-priority/statistics")) {
const policyVersion = Number(u.searchParams.get("version"));
if (statsMode === "pending")
return route.fulfill({
json: { status: "pending", policyVersion, remainingCount: null, unresolvedCount: null },
});
if (statsMode === "incomplete")
return route.fulfill({
json: { status: "incomplete", policyVersion, remainingCount: null, unresolvedCount: null },
@ -80,6 +84,14 @@ try {
);
assert(state.remainingCount > 0, JSON.stringify(state));
await panel.getByText(`待领取 ${state.remainingCount}`, { exact: true }).waitFor();
await panel.getByText(/统计于.*北京时间/).waitFor();
statsMode = "pending";
await page.reload();
await page
.getByRole("button", { name: /^优先供应商/ })
.first()
.click();
await panel.getByText("等待每日 01:00(北京时间)统计").waitFor();
await panel.getByRole("checkbox", { name: "泛宇草稿项目", exact: true }).check();
await panel.getByLabel("到期时间").fill("2090-01-01T12:00");
statsMode = "slow";