301 lines
10 KiB
Go

package outputsnapshot
import (
"context"
"crypto/sha256"
"droboticflow/internal/domain"
"droboticflow/internal/workforcestats"
"encoding/hex"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func testData(now time.Time) workforcestats.OutputSnapshot {
d := workforcestats.OutputSnapshot{Projects: []domain.ProjectRecord{{ID: "p", Name: "project"}}, Users: []domain.UserRecord{{ID: "u", Name: "user"}}, Suppliers: []domain.SupplierRecord{{ID: "v", Name: "vendor"}}}
for i := 0; i < 12; i++ {
id := fmt.Sprintf("t%d", i)
d.Tasks = append(d.Tasks, domain.AnnotationTaskRecord{ID: id, ProjectID: "p", SourceIsFullVideo: true, SourceDurationSeconds: 900})
d.Events = append(d.Events, domain.AnnotationWorklogEventRecord{ID: id + "-event", TaskID: id, UserID: "u", SupplierID: "v", Revision: 1, SubmissionKind: "first", EventType: "combined_submitted", CreatedAt: now.Add(-time.Hour).Format(time.RFC3339Nano)})
}
return d
}
func eventually(t *testing.T, f func() bool) {
t.Helper()
until := time.Now().Add(3 * time.Second)
for !f() {
if time.Now().After(until) {
t.Fatal("condition did not converge")
}
time.Sleep(time.Millisecond)
}
}
func ready(t *testing.T, r *Reader, pin string) (*workforcestats.OutputDataset, Info) {
t.Helper()
var data *workforcestats.OutputDataset
var info Info
eventually(t, func() bool {
var e error
data, info, e = r.Read(t.Context(), "org", []string{"p"}, pin)
if e != nil {
t.Fatal(e)
}
return data != nil
})
return data, info
}
func TestSharedReadersCoalesceColdBuildAndReuseForPages(t *testing.T) {
var calls atomic.Int32
gate := make(chan struct{})
b := &MemoryBackend{}
now := time.Now().UTC().Truncate(time.Millisecond)
source := func(ctx context.Context, _ string, _ []string) (workforcestats.OutputSnapshot, error) {
calls.Add(1)
select {
case <-gate:
return testData(now), nil
case <-ctx.Done():
return workforcestats.OutputSnapshot{}, ctx.Err()
}
}
readers := []*Reader{}
for i := 0; i < 4; i++ {
r := NewReader(b, source)
readers = append(readers, r)
t.Cleanup(r.Close)
}
var wg sync.WaitGroup
for _, r := range readers {
wg.Add(1)
go func(r *Reader) {
defer wg.Done()
d, _, e := r.Read(t.Context(), "org", []string{"p"}, "")
if e != nil || d != nil {
t.Errorf("cold read: %v %v", d, e)
}
}(r)
}
wg.Wait()
eventually(t, func() bool { return calls.Load() == 1 })
close(gate)
_, first := ready(t, readers[0], "")
for _, r := range readers {
for i := 0; i < 8; i++ {
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), MatchUser: true, ScopeUserID: "u"}
report, info, e := r.Query(t.Context(), "org", []string{"p"}, first.ID, q)
if e != nil || info.ID != first.ID || report.Totals.Submitted.Count != 12 {
t.Fatalf("shared read %+v %v", info, e)
}
}
}
if calls.Load() != 1 {
t.Fatalf("%d source builds", calls.Load())
}
}
func TestRefreshFailureKeepsCompleteGenerationAndPinsExpire(t *testing.T) {
b := &MemoryBackend{}
base := time.Now().UTC().Truncate(time.Millisecond)
var clock atomic.Int64
clock.Store(base.UnixMilli())
var calls atomic.Int32
var failing atomic.Bool
r := NewReader(b, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
calls.Add(1)
if failing.Load() {
return workforcestats.OutputSnapshot{}, errors.New("simulated read failure")
}
return testData(base), nil
})
r.now = func() time.Time { return time.UnixMilli(clock.Load()) }
defer r.Close()
_, one := ready(t, r, "")
failing.Store(true)
clock.Add((RefreshInterval + time.Second).Milliseconds())
data, info, e := r.Read(t.Context(), "org", []string{"p"}, one.ID)
if e != nil || data == nil || info.ID != one.ID {
t.Fatal("stale refresh blocked or lost complete data")
}
eventually(t, func() bool {
s, _ := b.OutputSnapshotState(t.Context())
return s.Heads[scopeKey("org", []string{"p"})].Failed
})
for i := 0; i < 10; i++ {
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
}
if calls.Load() != 2 {
t.Fatalf("failure retry storm %d", calls.Load())
}
failing.Store(false)
clock.Add((RetryInterval + time.Second).Milliseconds())
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
var two Info
eventually(t, func() bool { _, two = ready(t, r, ""); return two.ID != one.ID })
_, pin, e := r.Read(t.Context(), "org", []string{"p"}, one.ID)
if e != nil || pin.ID != one.ID {
t.Fatal("previous generation pin lost")
}
clock.Add((RefreshInterval + time.Second).Milliseconds())
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
eventually(t, func() bool { _, g := ready(t, r, ""); return g.ID != two.ID })
if _, _, e = r.Read(t.Context(), "org", []string{"p"}, one.ID); !errors.Is(e, ErrExpired) {
t.Fatal("recycled pin silently changed version")
}
if _, _, e = r.Read(t.Context(), "org", []string{"different"}, two.ID); !errors.Is(e, ErrExpired) {
t.Fatal("scope change reused old snapshot")
}
}
func TestLeaseRecoveryRejectsOldOwnerAndStorageIsBounded(t *testing.T) {
b := &MemoryBackend{}
now := time.Now().UTC().Truncate(time.Millisecond)
first, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
if e != nil || !ok {
t.Fatal(e)
}
// Failed process never releases. A new owner can take over after the lease.
now = now.Add(LeaseDuration + time.Second)
second, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
if e != nil || !ok {
t.Fatal(e)
}
g := Generation{ID: "second", Epoch: second.Epoch, Slot: second.Slot, AsOf: now, Bytes: 1}
if ok, e = b.PublishOutputSnapshot(t.Context(), first, Generation{Slot: first.Slot, Epoch: first.Epoch}, []byte{1}, now); e != nil || ok {
t.Fatal("old owner published")
}
_ = b.FailOutputSnapshot(t.Context(), first, now, Failure{})
if ok, e = b.PublishOutputSnapshot(t.Context(), second, g, []byte{1}, now); e != nil || !ok {
t.Fatalf("old failure disturbed successor: %v", e)
}
for i := 0; i < 50; i++ {
now = now.Add(RefreshInterval + time.Second)
l, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
if e != nil || !ok {
t.Fatal(e)
}
g := Generation{ID: fmt.Sprint(i), Epoch: l.Epoch, Slot: l.Slot, AsOf: now, Bytes: 1}
if ok, e = b.PublishOutputSnapshot(t.Context(), l, g, []byte{1}, now); e != nil || !ok {
t.Fatal(e)
}
}
b.mu.Lock()
defer b.mu.Unlock()
populated := 0
for _, slot := range b.slots {
if slot.data != nil {
populated++
}
}
if populated > 3 {
t.Fatalf("unbounded generations: %d", populated)
}
}
func TestCorruptCurrentFallsBackButPinnedDataNeverChanges(t *testing.T) {
b := &MemoryBackend{}
now := time.Now().UTC().Truncate(time.Millisecond)
key := scopeKey("org", []string{"p"})
var old Generation
for i := 0; i < 2; i++ {
l, _, e := b.AcquireOutputSnapshot(t.Context(), key, now)
if e != nil {
t.Fatal(e)
}
payload, _, _ := encode(workforcestats.BuildOutputDataset(testData(now), now))
h := sha256.Sum256(payload)
g := Generation{ID: fmt.Sprint(i), Epoch: l.Epoch, Slot: l.Slot, AsOf: now, Bytes: len(payload), SHA256: hex.EncodeToString(h[:])}
_, _ = b.PublishOutputSnapshot(t.Context(), l, g, payload, now)
if i == 0 {
old = g
}
now = now.Add(RefreshInterval + time.Second)
}
state, _ := b.OutputSnapshotState(t.Context())
current := state.Heads[key].Current
b.mu.Lock()
b.slots[current.Slot].data[0] ^= 1
b.mu.Unlock()
r := NewReader(b, nil)
r.now = func() time.Time { return current.AsOf.Add(time.Second) }
defer r.Close()
d, info, e := r.Read(t.Context(), "org", []string{"p"}, "")
if e != nil || d == nil || info.ID != old.ID {
t.Fatal("did not retain previous valid snapshot")
}
if _, _, e = r.Read(t.Context(), "org", []string{"p"}, current.ID); !errors.Is(e, ErrUnavailable) {
t.Fatal("corrupt pin silently substituted")
}
}
func TestBuildCancellationAndPayloadLimits(t *testing.T) {
b := &MemoryBackend{}
started := make(chan struct{})
r := NewReader(b, func(ctx context.Context, _ string, _ []string) (workforcestats.OutputSnapshot, error) {
close(started)
<-ctx.Done()
return workforcestats.OutputSnapshot{}, ctx.Err()
})
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
<-started
r.Close()
state, _ := b.OutputSnapshotState(t.Context())
head := state.Heads[scopeKey("org", []string{"p"})]
if state.Owner != "" || head.Current != nil || !head.Failed {
t.Fatal("cancel left a visible incomplete generation or live owner")
}
// Limit the decoded data too, not only a highly compressible stored blob.
huge := workforcestats.OutputDataset{Projects: map[string]string{"p": strings.Repeat("x", MaxDecodedBytes+1)}}
if _, _, e := encode(huge); !errors.Is(e, ErrCapacity) {
t.Fatal("decoded payload limit not enforced")
}
}
func TestSnapshotScopeCountCannotGrowWithoutBound(t *testing.T) {
b := &MemoryBackend{}
now := time.Now().UTC().Truncate(time.Millisecond)
for i := 0; i < MaxScopes; i++ {
l, ok, e := b.AcquireOutputSnapshot(t.Context(), fmt.Sprint(i), now)
if e != nil || !ok {
t.Fatal(e)
}
g := Generation{ID: fmt.Sprint(i), Slot: l.Slot, Epoch: l.Epoch, AsOf: now, Bytes: 1}
if ok, e = b.PublishOutputSnapshot(t.Context(), l, g, []byte{1}, now); e != nil || !ok {
t.Fatal(e)
}
}
if _, _, e := b.AcquireOutputSnapshot(t.Context(), "extra", now); !errors.Is(e, ErrCapacity) {
t.Fatal("scope limit not enforced")
}
}
func TestReportMemoScopeGenerationAndMutationIsolation(t *testing.T) {
now := time.Now().UTC().Truncate(time.Millisecond)
r := NewReader(&MemoryBackend{}, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
return testData(now), nil
})
defer r.Close()
_, generation := ready(t, r, "")
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"}
first, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
if err != nil || first == nil {
t.Fatalf("query: %v", err)
}
first.Totals.Submitted.Count = -100
q.Now = now.Add(time.Minute)
second, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
if err != nil || second.Totals.Submitted.Count != 12 {
t.Fatalf("memo mutated: %+v %v", second, err)
}
q.ScopeUserID = "other"
other, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
if err != nil || other.Totals.Submitted.Count != 0 {
t.Fatalf("scope leaked: %+v %v", other, err)
}
r.mu.Lock()
entries, bytes := len(r.queryMemo), r.queryMemoBytes
r.mu.Unlock()
if entries != 2 || bytes > queryMemoBudget {
t.Fatalf("memo entries=%d bytes=%d", entries, bytes)
}
}