forked from longtao.wu/droboticflow
212 lines
6.8 KiB
Go
212 lines
6.8 KiB
Go
package outputsnapshot
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/queryserving"
|
|
"droboticflow/internal/workforcestats"
|
|
"encoding/json"
|
|
"reflect"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type queryTestCache struct {
|
|
mu sync.Mutex
|
|
data map[string][]byte
|
|
registry map[string]map[string][]byte
|
|
}
|
|
|
|
func (c *queryTestCache) Get(_ context.Context, k string) ([]byte, bool) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
v, ok := c.data[k]
|
|
return append([]byte(nil), v...), ok
|
|
}
|
|
func (c *queryTestCache) Put(_ context.Context, k string, v []byte, _ time.Duration) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.data == nil {
|
|
c.data = map[string][]byte{}
|
|
}
|
|
c.data[k] = append([]byte(nil), v...)
|
|
}
|
|
|
|
func (c *queryTestCache) RememberQuery(_ context.Context, scope, id string, value []byte) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if c.registry == nil {
|
|
c.registry = map[string]map[string][]byte{}
|
|
}
|
|
if c.registry[scope] == nil {
|
|
c.registry[scope] = map[string][]byte{}
|
|
}
|
|
c.registry[scope][id] = append([]byte(nil), value...)
|
|
}
|
|
func (c *queryTestCache) RecentQueries(_ context.Context, scope string) [][]byte {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
out := [][]byte{}
|
|
for _, v := range c.registry[scope] {
|
|
out = append(out, append([]byte(nil), v...))
|
|
}
|
|
return out
|
|
}
|
|
|
|
type queryCachedBackend struct {
|
|
MemoryBackend
|
|
cache queryTestCache
|
|
blobs atomic.Int64
|
|
}
|
|
|
|
func (b *queryCachedBackend) SharedQueryCache() queryserving.Cache { return &b.cache }
|
|
func (b *queryCachedBackend) OutputSnapshotBlob(ctx context.Context, g Generation) ([]byte, error) {
|
|
b.blobs.Add(1)
|
|
return b.MemoryBackend.OutputSnapshotBlob(ctx, g)
|
|
}
|
|
|
|
func TestQueryServingSharedOutputCacheKeepsScopeLocationAndGeneration(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
backend := &queryCachedBackend{}
|
|
source := func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
|
|
return testData(now), nil
|
|
}
|
|
one := NewReader(backend, source)
|
|
defer one.Close()
|
|
if _, err := one.Prewarm(t.Context(), "org", []string{"p"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, info := ready(t, one, "")
|
|
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"}
|
|
r, _, err := one.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
|
if err != nil || r.Totals.Submitted.Count != 12 {
|
|
t.Fatal("initial report", err)
|
|
}
|
|
calls := backend.blobs.Load()
|
|
two := NewReader(backend, source)
|
|
defer two.Close()
|
|
q.Location = time.FixedZone("UTC", 0)
|
|
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
|
if err != nil || r.Totals.Submitted.Count != 12 || backend.blobs.Load() != calls {
|
|
t.Fatal("same immutable scope failed cross-reader reuse", err)
|
|
}
|
|
r.Tasks = nil
|
|
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
|
if err != nil || len(r.Tasks) == 0 {
|
|
t.Fatal("shared response mutated by another caller")
|
|
}
|
|
q.ScopeUserID = "not-authorized-user"
|
|
r, _, err = two.Query(t.Context(), "org", []string{"p"}, info.ID, q)
|
|
if err != nil || r.Totals.Submitted.Count != 0 {
|
|
t.Fatal("personal cache leaked another user's data", err)
|
|
}
|
|
if _, _, err = two.Query(t.Context(), "other-org", []string{"p"}, info.ID, q); err != ErrExpired {
|
|
t.Fatal("foreign organization borrowed generation", err)
|
|
}
|
|
if _, _, err = two.Query(t.Context(), "org", []string{"not-visible"}, info.ID, q); err != ErrExpired {
|
|
t.Fatal("changed project scope borrowed generation", err)
|
|
}
|
|
}
|
|
|
|
func TestQueryServingActiveReportsWarmBeforeGenerationPublication(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
raw := testData(now)
|
|
backend := &queryCachedBackend{}
|
|
source := func(ctx context.Context, _ string, _ []string, yield func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error) {
|
|
return nil, yield(raw)
|
|
}
|
|
reader := NewStreamReader(backend, source)
|
|
defer reader.Close()
|
|
var clock atomic.Int64
|
|
clock.Store(now.UnixNano())
|
|
reader.now = func() time.Time { return time.Unix(0, clock.Load()).UTC() }
|
|
if _, e := reader.Prewarm(t.Context(), "org", []string{"p"}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
var first Info
|
|
firstDeadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
_, info, e := reader.resolve(t.Context(), "org", []string{"p"}, "")
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
if info.ID != "" {
|
|
first = info
|
|
break
|
|
}
|
|
if time.Now().After(firstDeadline) {
|
|
t.Fatal("first generation not ready")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
queries := []workforcestats.OutputQuery{
|
|
{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"},
|
|
{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u", ProductionQuality: true, ReportRole: "annotation"},
|
|
{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchSupplier: true, ScopeSupplierID: "v"},
|
|
}
|
|
for _, q := range queries {
|
|
if _, _, e := reader.Query(t.Context(), "org", []string{"p"}, first.ID, q); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
clock.Store(now.Add(RefreshInterval + time.Second).UnixNano())
|
|
if _, e := reader.Prewarm(t.Context(), "org", []string{"p"}); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
var next Generation
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
state, e := backend.OutputSnapshotState(t.Context())
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
current := state.Heads[scopeKey("org", []string{"p"})].Current
|
|
if current != nil && current.ID != first.ID {
|
|
next = *current
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("next generation not published")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if next.Metrics.PreparedQueries != len(queries) {
|
|
t.Fatal("published without active report preparation", next.Metrics)
|
|
}
|
|
fresh := NewStreamReader(backend, source)
|
|
defer fresh.Close()
|
|
fresh.now = reader.now
|
|
reads := backend.blobs.Load()
|
|
for _, q := range queries {
|
|
got, info, e := fresh.Query(t.Context(), "org", []string{"p"}, next.ID, q)
|
|
if e != nil || info.ID != next.ID {
|
|
t.Fatal("prepared generation", e, info)
|
|
}
|
|
dataset := workforcestats.BuildOutputDataset(raw, next.AsOf)
|
|
before, _ := json.Marshal(dataset)
|
|
want := workforcestats.QueryOutputDataset(dataset, q)
|
|
after, _ := json.Marshal(dataset)
|
|
if !reflect.DeepEqual(before, after) || !reflect.DeepEqual(got, &want) {
|
|
t.Fatal("prepared report changed calculation or source data")
|
|
}
|
|
}
|
|
if backend.blobs.Load() != reads {
|
|
t.Fatal("first query decoded source after prepared publication")
|
|
}
|
|
}
|
|
|
|
func TestQueryServingActiveQueryDoesNotSubstituteCustomTimeZones(t *testing.T) {
|
|
custom := time.FixedZone("UTC", 3600)
|
|
if reportLocationRules(custom, 2028) != "" {
|
|
t.Fatal("custom zone treated as canonical UTC")
|
|
}
|
|
location, err := workforcestats.ReportingLocation("America/New_York")
|
|
if err != nil || reportLocationRules(location, 2028) == "" {
|
|
t.Fatal("canonical DST rules not reusable", err)
|
|
}
|
|
if reportLocationRules(time.UTC, 2028) == "" {
|
|
t.Fatal("canonical UTC not reusable")
|
|
}
|
|
}
|