droboticflow/internal/outputsnapshot/reader_diagnostics_test.go

77 lines
2.5 KiB
Go

package outputsnapshot
import (
"bytes"
"context"
"droboticflow/internal/workforcestats"
"errors"
"log"
"strings"
"testing"
"time"
)
func TestFailedBuildReportsStageAndCause(t *testing.T) {
var logs bytes.Buffer
previous := log.Writer()
log.SetOutput(&logs)
defer log.SetOutput(previous)
backend := &MemoryBackend{}
r := NewReader(backend, func(context.Context, string, []string) (workforcestats.OutputSnapshot, error) {
return workforcestats.OutputSnapshot{}, errors.New("annotation output read runs: query time limit exceeded")
})
defer r.Close()
r.build(scopeKey("org", []string{"p"}), "org", []string{"p"})
text := logs.String()
if !strings.Contains(text, "output_snapshot_build_failed") || !strings.Contains(text, "stage=source") || !strings.Contains(text, "read runs: query time limit exceeded") {
t.Fatalf("build failure lost its cause: %q", text)
}
state, e := backend.OutputSnapshotState(t.Context())
if e != nil {
t.Fatal(e)
}
h := state.Heads[scopeKey("org", []string{"p"})]
if !h.Failed || state.Owner != "" || !h.NextAttempt.After(time.Now()) {
t.Fatal("diagnostics changed release/backoff behavior")
}
}
func TestFailureDiagnosticsSurviveLeaseTakeoverAndClearAfterSuccess(t *testing.T) {
b := &MemoryBackend{}
now := time.Now().UTC()
l, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
if e != nil || !ok {
t.Fatal(e)
}
f := Failure{Stage: "source", Code: "timeout", At: now, DurationMS: 15000}
if e = b.FailOutputSnapshot(t.Context(), l, now, f); e != nil {
t.Fatal(e)
}
st, _ := b.OutputSnapshotState(t.Context())
if st.Heads["scope"].ConsecutiveFailures != 1 {
t.Fatal("failure not durable")
}
now = now.Add(RetryInterval + time.Second)
next, ok, e := b.AcquireOutputSnapshot(t.Context(), "scope", now)
if e != nil || !ok {
t.Fatal(e)
}
st, _ = b.OutputSnapshotState(t.Context())
if st.Heads["scope"].LastFailure == nil || st.Heads["scope"].ConsecutiveFailures != 1 {
t.Fatal("retry erased failure evidence")
}
_ = b.FailOutputSnapshot(t.Context(), l, now, f)
st, _ = b.OutputSnapshotState(t.Context())
if st.Heads["scope"].ConsecutiveFailures != 1 {
t.Fatal("old owner changed failure state")
}
g := Generation{ID: "recovered", Slot: next.Slot, Epoch: next.Epoch, AsOf: now, Bytes: 1}
if ok, e = b.PublishOutputSnapshot(t.Context(), next, g, []byte{1}, now); e != nil || !ok {
t.Fatal(e)
}
st, _ = b.OutputSnapshotState(t.Context())
if st.Heads["scope"].LastFailure != nil || st.Heads["scope"].ConsecutiveFailures != 0 {
t.Fatal("success did not clear active failure")
}
}