forked from longtao.wu/droboticflow
301 lines
10 KiB
Go
301 lines
10 KiB
Go
package outputsnapshot
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"droboticflow/internal/domain"
|
|
"droboticflow/internal/workforcestats"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func testData(now time.Time) workforcestats.OutputSnapshot {
|
|
d := workforcestats.OutputSnapshot{Projects: []domain.ProjectRecord{{ID: "p", Name: "project"}}, Users: []domain.UserRecord{{ID: "u", Name: "user"}}, Suppliers: []domain.SupplierRecord{{ID: "v", Name: "vendor"}}}
|
|
for i := 0; i < 12; i++ {
|
|
id := fmt.Sprintf("t%d", i)
|
|
d.Tasks = append(d.Tasks, domain.AnnotationTaskRecord{ID: id, ProjectID: "p", SourceIsFullVideo: true, SourceDurationSeconds: 900})
|
|
d.Events = append(d.Events, domain.AnnotationWorklogEventRecord{ID: id + "-event", TaskID: id, UserID: "u", SupplierID: "v", Revision: 1, SubmissionKind: "first", EventType: "combined_submitted", CreatedAt: now.Add(-time.Hour).Format(time.RFC3339Nano)})
|
|
}
|
|
return d
|
|
}
|
|
func eventually(t *testing.T, f func() bool) {
|
|
t.Helper()
|
|
until := time.Now().Add(3 * time.Second)
|
|
for !f() {
|
|
if time.Now().After(until) {
|
|
t.Fatal("condition did not converge")
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
}
|
|
func ready(t *testing.T, r *Reader, pin string) (*workforcestats.OutputDataset, Info) {
|
|
t.Helper()
|
|
var data *workforcestats.OutputDataset
|
|
var info Info
|
|
eventually(t, func() bool {
|
|
var e error
|
|
data, info, e = r.Read(t.Context(), "org", []string{"p"}, pin)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
return data != nil
|
|
})
|
|
return data, info
|
|
}
|
|
func TestSharedReadersCoalesceColdBuildAndReuseForPages(t *testing.T) {
|
|
var calls atomic.Int32
|
|
gate := make(chan struct{})
|
|
b := &MemoryBackend{}
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
source := func(ctx context.Context, _ string, _ []string) (workforcestats.OutputSnapshot, error) {
|
|
calls.Add(1)
|
|
select {
|
|
case <-gate:
|
|
return testData(now), nil
|
|
case <-ctx.Done():
|
|
return workforcestats.OutputSnapshot{}, ctx.Err()
|
|
}
|
|
}
|
|
readers := []*Reader{}
|
|
for i := 0; i < 4; i++ {
|
|
r := NewReader(b, source)
|
|
readers = append(readers, r)
|
|
t.Cleanup(r.Close)
|
|
}
|
|
var wg sync.WaitGroup
|
|
for _, r := range readers {
|
|
wg.Add(1)
|
|
go func(r *Reader) {
|
|
defer wg.Done()
|
|
d, _, e := r.Read(t.Context(), "org", []string{"p"}, "")
|
|
if e != nil || d != nil {
|
|
t.Errorf("cold read: %v %v", d, e)
|
|
}
|
|
}(r)
|
|
}
|
|
wg.Wait()
|
|
eventually(t, func() bool { return calls.Load() == 1 })
|
|
close(gate)
|
|
_, first := ready(t, readers[0], "")
|
|
for _, r := range readers {
|
|
for i := 0; i < 8; i++ {
|
|
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), MatchUser: true, ScopeUserID: "u"}
|
|
report, info, e := r.Query(t.Context(), "org", []string{"p"}, first.ID, q)
|
|
if e != nil || info.ID != first.ID || report.Totals.Submitted.Count != 12 {
|
|
t.Fatalf("shared read %+v %v", info, e)
|
|
}
|
|
}
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Fatalf("%d source builds", calls.Load())
|
|
}
|
|
}
|
|
func TestRefreshFailureKeepsCompleteGenerationAndPinsExpire(t *testing.T) {
|
|
b := &MemoryBackend{}
|
|
base := time.Now().UTC().Truncate(time.Millisecond)
|
|
var clock atomic.Int64
|
|
clock.Store(base.UnixMilli())
|
|
var calls atomic.Int32
|
|
var failing atomic.Bool
|
|
r := NewReader(b, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
|
|
calls.Add(1)
|
|
if failing.Load() {
|
|
return workforcestats.OutputSnapshot{}, errors.New("simulated read failure")
|
|
}
|
|
return testData(base), nil
|
|
})
|
|
r.now = func() time.Time { return time.UnixMilli(clock.Load()) }
|
|
defer r.Close()
|
|
_, one := ready(t, r, "")
|
|
failing.Store(true)
|
|
clock.Add((RefreshInterval + time.Second).Milliseconds())
|
|
data, info, e := r.Read(t.Context(), "org", []string{"p"}, one.ID)
|
|
if e != nil || data == nil || info.ID != one.ID {
|
|
t.Fatal("stale refresh blocked or lost complete data")
|
|
}
|
|
eventually(t, func() bool {
|
|
s, _ := b.OutputSnapshotState(t.Context())
|
|
return s.Heads[scopeKey("org", []string{"p"})].Failed
|
|
})
|
|
for i := 0; i < 10; i++ {
|
|
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
|
|
}
|
|
if calls.Load() != 2 {
|
|
t.Fatalf("failure retry storm %d", calls.Load())
|
|
}
|
|
failing.Store(false)
|
|
clock.Add((RetryInterval + time.Second).Milliseconds())
|
|
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
|
|
var two Info
|
|
eventually(t, func() bool { _, two = ready(t, r, ""); return two.ID != one.ID })
|
|
_, pin, e := r.Read(t.Context(), "org", []string{"p"}, one.ID)
|
|
if e != nil || pin.ID != one.ID {
|
|
t.Fatal("previous generation pin lost")
|
|
}
|
|
clock.Add((RefreshInterval + time.Second).Milliseconds())
|
|
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
|
|
eventually(t, func() bool { _, g := ready(t, r, ""); return g.ID != two.ID })
|
|
if _, _, e = r.Read(t.Context(), "org", []string{"p"}, one.ID); !errors.Is(e, ErrExpired) {
|
|
t.Fatal("recycled pin silently changed version")
|
|
}
|
|
if _, _, e = r.Read(t.Context(), "org", []string{"different"}, two.ID); !errors.Is(e, ErrExpired) {
|
|
t.Fatal("scope change reused old snapshot")
|
|
}
|
|
}
|
|
func TestLeaseRecoveryRejectsOldOwnerAndStorageIsBounded(t *testing.T) {
|
|
b := &MemoryBackend{}
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
first, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
|
|
if e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
// Failed process never releases. A new owner can take over after the lease.
|
|
now = now.Add(LeaseDuration + time.Second)
|
|
second, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
|
|
if e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
g := Generation{ID: "second", Epoch: second.Epoch, Slot: second.Slot, AsOf: now, Bytes: 1}
|
|
if ok, e = b.PublishOutputSnapshot(t.Context(), first, Generation{Slot: first.Slot, Epoch: first.Epoch}, []byte{1}, now); e != nil || ok {
|
|
t.Fatal("old owner published")
|
|
}
|
|
_ = b.FailOutputSnapshot(t.Context(), first, now, Failure{})
|
|
if ok, e = b.PublishOutputSnapshot(t.Context(), second, g, []byte{1}, now); e != nil || !ok {
|
|
t.Fatalf("old failure disturbed successor: %v", e)
|
|
}
|
|
for i := 0; i < 50; i++ {
|
|
now = now.Add(RefreshInterval + time.Second)
|
|
l, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
|
|
if e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
g := Generation{ID: fmt.Sprint(i), Epoch: l.Epoch, Slot: l.Slot, AsOf: now, Bytes: 1}
|
|
if ok, e = b.PublishOutputSnapshot(t.Context(), l, g, []byte{1}, now); e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
populated := 0
|
|
for _, slot := range b.slots {
|
|
if slot.data != nil {
|
|
populated++
|
|
}
|
|
}
|
|
if populated > 3 {
|
|
t.Fatalf("unbounded generations: %d", populated)
|
|
}
|
|
}
|
|
func TestCorruptCurrentFallsBackButPinnedDataNeverChanges(t *testing.T) {
|
|
b := &MemoryBackend{}
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
key := scopeKey("org", []string{"p"})
|
|
var old Generation
|
|
for i := 0; i < 2; i++ {
|
|
l, _, e := b.AcquireOutputSnapshot(t.Context(), key, now)
|
|
if e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
payload, _, _ := encode(workforcestats.BuildOutputDataset(testData(now), now))
|
|
h := sha256.Sum256(payload)
|
|
g := Generation{ID: fmt.Sprint(i), Epoch: l.Epoch, Slot: l.Slot, AsOf: now, Bytes: len(payload), SHA256: hex.EncodeToString(h[:])}
|
|
_, _ = b.PublishOutputSnapshot(t.Context(), l, g, payload, now)
|
|
if i == 0 {
|
|
old = g
|
|
}
|
|
now = now.Add(RefreshInterval + time.Second)
|
|
}
|
|
state, _ := b.OutputSnapshotState(t.Context())
|
|
current := state.Heads[key].Current
|
|
b.mu.Lock()
|
|
b.slots[current.Slot].data[0] ^= 1
|
|
b.mu.Unlock()
|
|
r := NewReader(b, nil)
|
|
r.now = func() time.Time { return current.AsOf.Add(time.Second) }
|
|
defer r.Close()
|
|
d, info, e := r.Read(t.Context(), "org", []string{"p"}, "")
|
|
if e != nil || d == nil || info.ID != old.ID {
|
|
t.Fatal("did not retain previous valid snapshot")
|
|
}
|
|
if _, _, e = r.Read(t.Context(), "org", []string{"p"}, current.ID); !errors.Is(e, ErrUnavailable) {
|
|
t.Fatal("corrupt pin silently substituted")
|
|
}
|
|
}
|
|
|
|
func TestBuildCancellationAndPayloadLimits(t *testing.T) {
|
|
b := &MemoryBackend{}
|
|
started := make(chan struct{})
|
|
r := NewReader(b, func(ctx context.Context, _ string, _ []string) (workforcestats.OutputSnapshot, error) {
|
|
close(started)
|
|
<-ctx.Done()
|
|
return workforcestats.OutputSnapshot{}, ctx.Err()
|
|
})
|
|
_, _, _ = r.Read(t.Context(), "org", []string{"p"}, "")
|
|
<-started
|
|
r.Close()
|
|
state, _ := b.OutputSnapshotState(t.Context())
|
|
head := state.Heads[scopeKey("org", []string{"p"})]
|
|
if state.Owner != "" || head.Current != nil || !head.Failed {
|
|
t.Fatal("cancel left a visible incomplete generation or live owner")
|
|
}
|
|
// Limit the decoded data too, not only a highly compressible stored blob.
|
|
huge := workforcestats.OutputDataset{Projects: map[string]string{"p": strings.Repeat("x", MaxDecodedBytes+1)}}
|
|
if _, _, e := encode(huge); !errors.Is(e, ErrCapacity) {
|
|
t.Fatal("decoded payload limit not enforced")
|
|
}
|
|
}
|
|
func TestSnapshotScopeCountCannotGrowWithoutBound(t *testing.T) {
|
|
b := &MemoryBackend{}
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
for i := 0; i < MaxScopes; i++ {
|
|
l, ok, e := b.AcquireOutputSnapshot(t.Context(), fmt.Sprint(i), now)
|
|
if e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
g := Generation{ID: fmt.Sprint(i), Slot: l.Slot, Epoch: l.Epoch, AsOf: now, Bytes: 1}
|
|
if ok, e = b.PublishOutputSnapshot(t.Context(), l, g, []byte{1}, now); e != nil || !ok {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
if _, _, e := b.AcquireOutputSnapshot(t.Context(), "extra", now); !errors.Is(e, ErrCapacity) {
|
|
t.Fatal("scope limit not enforced")
|
|
}
|
|
}
|
|
|
|
func TestReportMemoScopeGenerationAndMutationIsolation(t *testing.T) {
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
r := NewReader(&MemoryBackend{}, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
|
|
return testData(now), nil
|
|
})
|
|
defer r.Close()
|
|
_, generation := ready(t, r, "")
|
|
q := workforcestats.OutputQuery{From: now.Add(-24 * time.Hour), To: now.Add(time.Hour), Location: time.UTC, MatchUser: true, ScopeUserID: "u"}
|
|
first, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
|
|
if err != nil || first == nil {
|
|
t.Fatalf("query: %v", err)
|
|
}
|
|
first.Totals.Submitted.Count = -100
|
|
q.Now = now.Add(time.Minute)
|
|
second, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
|
|
if err != nil || second.Totals.Submitted.Count != 12 {
|
|
t.Fatalf("memo mutated: %+v %v", second, err)
|
|
}
|
|
q.ScopeUserID = "other"
|
|
other, _, err := r.Query(t.Context(), "org", []string{"p"}, generation.ID, q)
|
|
if err != nil || other.Totals.Submitted.Count != 0 {
|
|
t.Fatalf("scope leaked: %+v %v", other, err)
|
|
}
|
|
r.mu.Lock()
|
|
entries, bytes := len(r.queryMemo), r.queryMemoBytes
|
|
r.mu.Unlock()
|
|
if entries != 2 || bytes > queryMemoBudget {
|
|
t.Fatalf("memo entries=%d bytes=%d", entries, bytes)
|
|
}
|
|
}
|