178 lines
6.5 KiB
Go

// Package outputsnapshot shares bounded, immutable reporting generations across
// API instances. The Backend owns atomic lease fencing and generation publication.
package outputsnapshot
import (
"context"
"droboticflow/internal/workforcestats"
"errors"
"fmt"
"time"
)
func GenerationID(l Lease) string { return fmt.Sprintf("of1-%s-%d", l.Owner, l.Epoch) }
const (
Schema = "annotation-output-facts/v5"
RefreshInterval = 120 * time.Second
RetryInterval = 120 * time.Second
MaxAge = 15 * time.Minute
LeaseDuration = 60 * time.Second
BuildTimeout = 90 * time.Second
MaxScopes = 8
SlotCount = 16
MaxEncodedBytes = 4 << 20
MaxDecodedBytes = 32 << 20
)
var (
ErrExpired = errors.New("snapshot expired")
ErrUnavailable = errors.New("snapshot unavailable")
ErrCapacity = errors.New("snapshot capacity exceeded")
)
// DurationMS covers source reads, calculation, encoding and optional active
// query preparation, before publication.
type BuildMetrics struct {
PreparedQueries int `bson:"preparedQueries,omitempty" json:"preparedQueries,omitempty"`
PrewarmMS int64 `bson:"prewarmMs,omitempty" json:"prewarmMs,omitempty"`
PeakEncodedBytes int `bson:"peakEncodedBytes,omitempty" json:"peakEncodedBytes,omitempty"`
PeakDecodedBytes int `bson:"peakDecodedBytes,omitempty" json:"peakDecodedBytes,omitempty"`
DurationMS int64 `bson:"durationMs" json:"durationMs"`
DecodedBytes int `bson:"decodedBytes" json:"decodedBytes"`
Parts int `bson:"parts" json:"parts"`
Source *workforcestats.OutputSourceStats `bson:"source,omitempty" json:"source,omitempty"`
}
// Failure stores bounded machine-readable categories; raw errors stay in logs.
type Failure struct {
Stage string `bson:"stage" json:"stage"`
Code string `bson:"code" json:"code"`
At time.Time `bson:"at" json:"at"`
DurationMS int64 `bson:"durationMs" json:"durationMs"`
}
type Generation struct {
Shards int `bson:"shards,omitempty" json:"-"`
ManifestBytes int `bson:"manifestBytes,omitempty" json:"-"`
Metrics *BuildMetrics `bson:"metrics,omitempty" json:"-"`
ID string `bson:"id" json:"id"`
Slot int `bson:"slot" json:"-"`
Epoch int64 `bson:"epoch" json:"-"`
AsOf time.Time `bson:"asOf" json:"asOf"`
SHA256 string `bson:"sha256" json:"-"`
Bytes int `bson:"bytes" json:"-"`
}
type Head struct {
ConsecutiveFailures int64 `bson:"consecutiveFailures,omitempty"`
LastFailure *Failure `bson:"lastFailure,omitempty"`
Current *Generation `bson:"current,omitempty"`
Previous *Generation `bson:"previous,omitempty"`
NextAttempt time.Time `bson:"nextAttempt"`
Failed bool `bson:"failed"`
}
type State struct {
ActiveEpoch int64 `bson:"activeEpoch,omitempty"`
ActiveKey string `bson:"activeKey,omitempty"`
Heads map[string]Head `bson:"heads"`
Owner string `bson:"owner"`
Epoch int64 `bson:"epoch"`
Until time.Time `bson:"until"`
}
type Lease struct {
Key, Owner string
Epoch int64
Slot int
Previous *Generation
}
type Backend interface {
OutputSnapshotState(context.Context) (State, error)
AcquireOutputSnapshot(context.Context, string, time.Time) (Lease, bool, error)
RenewOutputSnapshot(context.Context, Lease, time.Time) (bool, error)
PublishOutputSnapshot(context.Context, Lease, Generation, []byte, time.Time) (bool, error)
FailOutputSnapshot(context.Context, Lease, time.Time, Failure) error
OutputSnapshotBlob(context.Context, Generation) ([]byte, error)
}
type Info struct {
ID string `json:"id"`
AsOf time.Time `json:"asOf"`
Refreshing bool `json:"refreshing"`
Stale bool `json:"stale"`
}
type Source func(context.Context, string, []string) (workforcestats.OutputSnapshot, error)
// PrepareLease mutates only the newly acquired coordinator state, never a
// visible generation. A recycled previous generation yields ErrExpired to pins.
func PrepareLease(s *State, key string, now time.Time) (Lease, error) {
if s.Heads == nil {
s.Heads = map[string]Head{}
}
for k, h := range s.Heads {
if k != key && (h.Current != nil && now.Sub(h.Current.AsOf) > MaxAge || h.Current == nil && !now.Before(h.NextAttempt)) {
delete(s.Heads, k)
}
}
if _, ok := s.Heads[key]; !ok && len(s.Heads) >= MaxScopes {
return Lease{}, ErrCapacity
}
used := map[int]bool{}
for _, h := range s.Heads {
if h.Current != nil {
used[h.Current.Slot] = true
}
if h.Previous != nil {
used[h.Previous.Slot] = true
}
}
slot := -1
for i := 0; i < SlotCount; i++ {
if !used[i] {
slot = i
break
}
}
if slot < 0 {
for k, h := range s.Heads {
if h.Previous != nil {
slot = h.Previous.Slot
h.Previous = nil
s.Heads[k] = h
break
}
}
}
if slot < 0 {
return Lease{}, ErrCapacity
}
h := s.Heads[key]
old := h.Current
s.Heads[key] = h
return Lease{Key: key, Owner: s.Owner, Epoch: s.Epoch, Slot: slot, Previous: old}, nil
}
const StateCollection = "annotationOutputSnapshotState"
const SlotsCollection = "annotationOutputSnapshotSlots"
func IsPrivateCollection(name string) bool { return name == StateCollection || name == SlotsCollection }
// ShardedBackend stages immutable, epoch-addressed blobs. Only publication of
// the manifest makes a complete generation visible to readers.
type ShardedBackend interface {
StageOutputSnapshotShard(context.Context, Lease, int, []byte, time.Time) (bool, error)
OutputSnapshotShard(context.Context, Generation, int) ([]byte, error)
}
type StreamSource func(context.Context, string, []string, func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error)
func ValidPublication(l Lease, g Generation, data []byte) bool {
if l.Slot < 0 || l.Slot >= SlotCount || g.Slot != l.Slot || g.Epoch != l.Epoch || len(data) > MaxEncodedBytes {
return false
}
if g.Shards > 0 {
return g.ManifestBytes == len(data) && g.Shards <= MaxShards && g.Bytes >= 0
}
return g.Shards == 0 && g.Bytes == len(data)
}
// Bound manifest size and total per-generation derived storage independently of
// historical source size. Oversized single parts remain explicit failures.
const MaxShards = 4096