282 lines
7.5 KiB
Go
282 lines
7.5 KiB
Go
package store
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"maps"
|
|
"sync"
|
|
"time"
|
|
"unsafe"
|
|
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/queryserving"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
const queryCountMemoryBudget = 64 << 20
|
|
|
|
// Cache keys include factEpoch so synced mutations invalidate immediately; a
|
|
// 15s TTL avoids 2s re-fetch/decompression thrashing while epochs are unchanged.
|
|
const queryCountTTL = 15 * time.Second
|
|
|
|
var errQueryCountCacheCapacity = errors.New("count candidate cache capacity exceeded")
|
|
|
|
type queryCountEntry struct {
|
|
rows []domain.HumanItemRecord
|
|
expires time.Time
|
|
bytes int
|
|
}
|
|
type queryCountFlight struct {
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
done chan struct{}
|
|
waiters int
|
|
rows []domain.HumanItemRecord
|
|
err error
|
|
}
|
|
type queryCountCache struct {
|
|
mu sync.Mutex
|
|
entries map[string]queryCountEntry
|
|
flights map[string]*queryCountFlight
|
|
bytes int
|
|
}
|
|
|
|
func cloneQueryCountRow(v domain.HumanItemRecord) domain.HumanItemRecord {
|
|
v.Subject.Payload = nil
|
|
v.PoolSearch = nil
|
|
v.CanClaim = nil
|
|
v.TraceSummary = nil
|
|
v.ReviewHistory = nil
|
|
v.LegacyAssignment = maps.Clone(v.LegacyAssignment)
|
|
if v.AcceptanceClaim != nil {
|
|
r := *v.AcceptanceClaim
|
|
v.AcceptanceClaim = &r
|
|
}
|
|
if v.AccountHandoff != nil {
|
|
r := *v.AccountHandoff
|
|
v.AccountHandoff = &r
|
|
}
|
|
if v.ProcessingJob != nil {
|
|
r := *v.ProcessingJob
|
|
r.Input = nil
|
|
v.ProcessingJob = &r
|
|
}
|
|
return v
|
|
}
|
|
func (s *MongoStore) countFactEpoch(ctx context.Context) (int64, error) {
|
|
var st struct {
|
|
Epoch int64 `bson:"factEpoch"`
|
|
}
|
|
e := s.db.Collection(queryReplicaControl).FindOne(ctx, bson.M{"_id": "state"}, options.FindOne().SetProjection(bson.M{"factEpoch": 1})).Decode(&st)
|
|
return st.Epoch, e
|
|
}
|
|
func (c *queryCountCache) remember(key string, rows []domain.HumanItemRecord, bytes int) {
|
|
if bytes > queryCountMemoryBudget {
|
|
return
|
|
}
|
|
if c.entries == nil {
|
|
c.entries = map[string]queryCountEntry{}
|
|
}
|
|
if prev, ok := c.entries[key]; ok {
|
|
delete(c.entries, key)
|
|
c.bytes -= prev.bytes
|
|
}
|
|
now := time.Now()
|
|
for k, e := range c.entries {
|
|
if !now.Before(e.expires) {
|
|
delete(c.entries, k)
|
|
c.bytes -= e.bytes
|
|
}
|
|
}
|
|
for len(c.entries) >= 32 || (len(c.entries) > 0 && c.bytes+bytes > queryCountMemoryBudget) {
|
|
var oldestKey string
|
|
var oldestExpires time.Time
|
|
first := true
|
|
for k, e := range c.entries {
|
|
if first || e.expires.Before(oldestExpires) {
|
|
oldestKey = k
|
|
oldestExpires = e.expires
|
|
first = false
|
|
}
|
|
}
|
|
if oldestKey == "" {
|
|
break
|
|
}
|
|
c.bytes -= c.entries[oldestKey].bytes
|
|
delete(c.entries, oldestKey)
|
|
}
|
|
if c.bytes < 0 {
|
|
c.bytes = 0
|
|
}
|
|
c.entries[key] = queryCountEntry{rows: rows, bytes: bytes, expires: now.Add(queryCountTTL)}
|
|
c.bytes += bytes
|
|
}
|
|
|
|
// The cache stores query candidates, NEVER authorization or a final count.
|
|
// The API still evaluates current permissions on private per-request copies.
|
|
// The target fact epoch is committed atomically with fact changes, so a cold
|
|
// miss or invalidation reads the query DB, never the authority DB.
|
|
func (s *MongoStore) visitCachedReplicaCounts(ctx context.Context, org string, q domain.HumanItemQuery, visit func([]domain.HumanItemRecord) error) error {
|
|
epoch, e := s.countFactEpoch(ctx)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
q.Limit = 0
|
|
q.Cursor = nil
|
|
key := queryserving.Key("replica-count-candidates/v1", s.db.Name(), org, epoch, q)
|
|
c := &s.queryReplicaCounts
|
|
c.mu.Lock()
|
|
if c.entries == nil {
|
|
c.entries = map[string]queryCountEntry{}
|
|
c.flights = map[string]*queryCountFlight{}
|
|
}
|
|
var rows []domain.HumanItemRecord
|
|
if entry, ok := c.entries[key]; ok && time.Now().Before(entry.expires) {
|
|
rows = entry.rows
|
|
c.mu.Unlock()
|
|
} else {
|
|
f := c.flights[key]
|
|
if f == nil && len(c.flights) >= 16 {
|
|
c.mu.Unlock()
|
|
return s.visitUncachedHumanPoolCounts(ctx, org, q, visit)
|
|
}
|
|
if f == nil || f.ctx.Err() != nil {
|
|
work, cancel := context.WithTimeout(context.Background(), 8*time.Second)
|
|
f = &queryCountFlight{ctx: work, cancel: cancel, done: make(chan struct{})}
|
|
c.flights[key] = f
|
|
go func() {
|
|
result, bytes, err := s.loadReplicaCountRows(work, org, q, key)
|
|
cacheable := true
|
|
if err == nil {
|
|
next, errEpoch := s.countFactEpoch(work)
|
|
if errEpoch != nil {
|
|
err = errEpoch
|
|
} else if next != epoch {
|
|
cacheable = false
|
|
}
|
|
}
|
|
c.mu.Lock()
|
|
f.rows, f.err = result, err
|
|
if c.flights[key] == f {
|
|
delete(c.flights, key)
|
|
if err == nil && cacheable && work.Err() == nil {
|
|
c.remember(key, result, bytes)
|
|
}
|
|
}
|
|
close(f.done)
|
|
c.mu.Unlock()
|
|
cancel()
|
|
}()
|
|
}
|
|
f.waiters++
|
|
c.mu.Unlock()
|
|
select {
|
|
case <-ctx.Done():
|
|
e = ctx.Err()
|
|
case <-f.done:
|
|
rows, e = f.rows, f.err
|
|
}
|
|
c.mu.Lock()
|
|
f.waiters--
|
|
if f.waiters == 0 {
|
|
f.cancel()
|
|
if c.flights[key] == f {
|
|
delete(c.flights, key)
|
|
}
|
|
}
|
|
c.mu.Unlock()
|
|
if errors.Is(e, errQueryCountCacheCapacity) {
|
|
return s.visitUncachedHumanPoolCounts(ctx, org, q, visit)
|
|
}
|
|
if e != nil {
|
|
return e
|
|
}
|
|
}
|
|
buffer := make([]domain.HumanItemRecord, min(len(rows), humanPoolCountBatchSize))
|
|
for start := 0; start < len(rows); start += len(buffer) {
|
|
if e := ctx.Err(); e != nil {
|
|
return e
|
|
}
|
|
end := min(start+len(buffer), len(rows))
|
|
page := buffer[:end-start]
|
|
for i := range page {
|
|
page[i] = cloneQueryCountRow(rows[start+i])
|
|
}
|
|
if e := visit(page); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
return ctx.Err()
|
|
}
|
|
func (s *MongoStore) loadReplicaCountRows(ctx context.Context, org string, q domain.HumanItemQuery, key string) ([]domain.HumanItemRecord, int, error) {
|
|
// BSON retains private eligibility metadata that JSON intentionally omits.
|
|
if cache := s.SharedQueryCache(); cache != nil {
|
|
if raw, ok := cache.Get(ctx, key); ok {
|
|
if rows, e := decodeQueryCountRows(raw); e == nil {
|
|
return rows, queryCountRowsBytes(rows), nil
|
|
}
|
|
}
|
|
}
|
|
rows := []domain.HumanItemRecord{}
|
|
e := s.visitUncachedHumanPoolCounts(ctx, org, q, func(page []domain.HumanItemRecord) error {
|
|
if len(rows)+len(page) > 50000 {
|
|
return errQueryCountCacheCapacity
|
|
}
|
|
for _, row := range page {
|
|
rows = append(rows, cloneQueryCountRow(row))
|
|
}
|
|
return nil
|
|
})
|
|
if e != nil {
|
|
return nil, 0, e
|
|
}
|
|
if cache := s.SharedQueryCache(); cache != nil {
|
|
if raw, e := encodeQueryCountRows(rows); e == nil && len(raw) <= queryserving.MaxValueBytes {
|
|
cache.Put(ctx, key, raw, queryCountTTL)
|
|
}
|
|
}
|
|
return rows, queryCountRowsBytes(rows), nil
|
|
}
|
|
func queryCountRowsBytes(rows []domain.HumanItemRecord) int {
|
|
n := len(rows) * int(unsafe.Sizeof(domain.HumanItemRecord{}))
|
|
for _, r := range rows {
|
|
n += len(r.ID) + len(r.ProjectID) + len(r.LegacyTaskID) + len(r.BatchID) + len(r.AssignedUserID) + len(r.LatestAuthorUserID) + 512
|
|
}
|
|
return n
|
|
}
|
|
func encodeQueryCountRows(rows []domain.HumanItemRecord) ([]byte, error) {
|
|
raw, e := bson.Marshal(bson.M{"rows": rows})
|
|
if e != nil || len(raw) > 128<<20 {
|
|
return nil, errors.New("count cache exceeds budget")
|
|
}
|
|
var b bytes.Buffer
|
|
w := gzip.NewWriter(&b)
|
|
if _, e = w.Write(raw); e != nil {
|
|
return nil, e
|
|
}
|
|
if e = w.Close(); e != nil {
|
|
return nil, e
|
|
}
|
|
return b.Bytes(), nil
|
|
}
|
|
func decodeQueryCountRows(raw []byte) ([]domain.HumanItemRecord, error) {
|
|
r, e := gzip.NewReader(bytes.NewReader(raw))
|
|
if e != nil {
|
|
return nil, e
|
|
}
|
|
defer r.Close()
|
|
b, e := io.ReadAll(io.LimitReader(r, 128<<20+1))
|
|
if e != nil || len(b) > 128<<20 {
|
|
return nil, errors.New("invalid count cache")
|
|
}
|
|
var d struct {
|
|
Rows []domain.HumanItemRecord `bson:"rows"`
|
|
}
|
|
e = bson.Unmarshal(b, &d)
|
|
return d.Rows, e
|
|
}
|