forked from longtao.wu/droboticflow
178 lines
6.5 KiB
Go
178 lines
6.5 KiB
Go
// Package outputsnapshot shares bounded, immutable reporting generations across
|
|
// API instances. The Backend owns atomic lease fencing and generation publication.
|
|
package outputsnapshot
|
|
|
|
import (
|
|
"context"
|
|
"droboticflow/internal/workforcestats"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
func GenerationID(l Lease) string { return fmt.Sprintf("of1-%s-%d", l.Owner, l.Epoch) }
|
|
|
|
const (
|
|
Schema = "annotation-output-facts/v5"
|
|
RefreshInterval = 120 * time.Second
|
|
RetryInterval = 120 * time.Second
|
|
MaxAge = 15 * time.Minute
|
|
LeaseDuration = 60 * time.Second
|
|
BuildTimeout = 90 * time.Second
|
|
MaxScopes = 8
|
|
SlotCount = 16
|
|
MaxEncodedBytes = 4 << 20
|
|
MaxDecodedBytes = 32 << 20
|
|
)
|
|
|
|
var (
|
|
ErrExpired = errors.New("snapshot expired")
|
|
ErrUnavailable = errors.New("snapshot unavailable")
|
|
ErrCapacity = errors.New("snapshot capacity exceeded")
|
|
)
|
|
|
|
// DurationMS covers source reads, calculation, encoding and optional active
|
|
// query preparation, before publication.
|
|
type BuildMetrics struct {
|
|
PreparedQueries int `bson:"preparedQueries,omitempty" json:"preparedQueries,omitempty"`
|
|
PrewarmMS int64 `bson:"prewarmMs,omitempty" json:"prewarmMs,omitempty"`
|
|
PeakEncodedBytes int `bson:"peakEncodedBytes,omitempty" json:"peakEncodedBytes,omitempty"`
|
|
PeakDecodedBytes int `bson:"peakDecodedBytes,omitempty" json:"peakDecodedBytes,omitempty"`
|
|
DurationMS int64 `bson:"durationMs" json:"durationMs"`
|
|
DecodedBytes int `bson:"decodedBytes" json:"decodedBytes"`
|
|
Parts int `bson:"parts" json:"parts"`
|
|
Source *workforcestats.OutputSourceStats `bson:"source,omitempty" json:"source,omitempty"`
|
|
}
|
|
|
|
// Failure stores bounded machine-readable categories; raw errors stay in logs.
|
|
type Failure struct {
|
|
Stage string `bson:"stage" json:"stage"`
|
|
Code string `bson:"code" json:"code"`
|
|
At time.Time `bson:"at" json:"at"`
|
|
DurationMS int64 `bson:"durationMs" json:"durationMs"`
|
|
}
|
|
type Generation struct {
|
|
Shards int `bson:"shards,omitempty" json:"-"`
|
|
ManifestBytes int `bson:"manifestBytes,omitempty" json:"-"`
|
|
Metrics *BuildMetrics `bson:"metrics,omitempty" json:"-"`
|
|
ID string `bson:"id" json:"id"`
|
|
Slot int `bson:"slot" json:"-"`
|
|
Epoch int64 `bson:"epoch" json:"-"`
|
|
AsOf time.Time `bson:"asOf" json:"asOf"`
|
|
SHA256 string `bson:"sha256" json:"-"`
|
|
Bytes int `bson:"bytes" json:"-"`
|
|
}
|
|
type Head struct {
|
|
ConsecutiveFailures int64 `bson:"consecutiveFailures,omitempty"`
|
|
LastFailure *Failure `bson:"lastFailure,omitempty"`
|
|
Current *Generation `bson:"current,omitempty"`
|
|
Previous *Generation `bson:"previous,omitempty"`
|
|
NextAttempt time.Time `bson:"nextAttempt"`
|
|
Failed bool `bson:"failed"`
|
|
}
|
|
type State struct {
|
|
ActiveEpoch int64 `bson:"activeEpoch,omitempty"`
|
|
ActiveKey string `bson:"activeKey,omitempty"`
|
|
Heads map[string]Head `bson:"heads"`
|
|
Owner string `bson:"owner"`
|
|
Epoch int64 `bson:"epoch"`
|
|
Until time.Time `bson:"until"`
|
|
}
|
|
type Lease struct {
|
|
Key, Owner string
|
|
Epoch int64
|
|
Slot int
|
|
Previous *Generation
|
|
}
|
|
type Backend interface {
|
|
OutputSnapshotState(context.Context) (State, error)
|
|
AcquireOutputSnapshot(context.Context, string, time.Time) (Lease, bool, error)
|
|
RenewOutputSnapshot(context.Context, Lease, time.Time) (bool, error)
|
|
PublishOutputSnapshot(context.Context, Lease, Generation, []byte, time.Time) (bool, error)
|
|
FailOutputSnapshot(context.Context, Lease, time.Time, Failure) error
|
|
OutputSnapshotBlob(context.Context, Generation) ([]byte, error)
|
|
}
|
|
type Info struct {
|
|
ID string `json:"id"`
|
|
AsOf time.Time `json:"asOf"`
|
|
Refreshing bool `json:"refreshing"`
|
|
Stale bool `json:"stale"`
|
|
}
|
|
type Source func(context.Context, string, []string) (workforcestats.OutputSnapshot, error)
|
|
|
|
// PrepareLease mutates only the newly acquired coordinator state, never a
|
|
// visible generation. A recycled previous generation yields ErrExpired to pins.
|
|
func PrepareLease(s *State, key string, now time.Time) (Lease, error) {
|
|
if s.Heads == nil {
|
|
s.Heads = map[string]Head{}
|
|
}
|
|
for k, h := range s.Heads {
|
|
if k != key && (h.Current != nil && now.Sub(h.Current.AsOf) > MaxAge || h.Current == nil && !now.Before(h.NextAttempt)) {
|
|
delete(s.Heads, k)
|
|
}
|
|
}
|
|
if _, ok := s.Heads[key]; !ok && len(s.Heads) >= MaxScopes {
|
|
return Lease{}, ErrCapacity
|
|
}
|
|
used := map[int]bool{}
|
|
for _, h := range s.Heads {
|
|
if h.Current != nil {
|
|
used[h.Current.Slot] = true
|
|
}
|
|
if h.Previous != nil {
|
|
used[h.Previous.Slot] = true
|
|
}
|
|
}
|
|
slot := -1
|
|
for i := 0; i < SlotCount; i++ {
|
|
if !used[i] {
|
|
slot = i
|
|
break
|
|
}
|
|
}
|
|
if slot < 0 {
|
|
for k, h := range s.Heads {
|
|
if h.Previous != nil {
|
|
slot = h.Previous.Slot
|
|
h.Previous = nil
|
|
s.Heads[k] = h
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if slot < 0 {
|
|
return Lease{}, ErrCapacity
|
|
}
|
|
h := s.Heads[key]
|
|
old := h.Current
|
|
s.Heads[key] = h
|
|
return Lease{Key: key, Owner: s.Owner, Epoch: s.Epoch, Slot: slot, Previous: old}, nil
|
|
}
|
|
|
|
const StateCollection = "annotationOutputSnapshotState"
|
|
const SlotsCollection = "annotationOutputSnapshotSlots"
|
|
|
|
func IsPrivateCollection(name string) bool { return name == StateCollection || name == SlotsCollection }
|
|
|
|
// ShardedBackend stages immutable, epoch-addressed blobs. Only publication of
|
|
// the manifest makes a complete generation visible to readers.
|
|
type ShardedBackend interface {
|
|
StageOutputSnapshotShard(context.Context, Lease, int, []byte, time.Time) (bool, error)
|
|
OutputSnapshotShard(context.Context, Generation, int) ([]byte, error)
|
|
}
|
|
type StreamSource func(context.Context, string, []string, func(workforcestats.OutputSnapshot) error) (*workforcestats.OutputSourceStats, error)
|
|
|
|
func ValidPublication(l Lease, g Generation, data []byte) bool {
|
|
if l.Slot < 0 || l.Slot >= SlotCount || g.Slot != l.Slot || g.Epoch != l.Epoch || len(data) > MaxEncodedBytes {
|
|
return false
|
|
}
|
|
if g.Shards > 0 {
|
|
return g.ManifestBytes == len(data) && g.Shards <= MaxShards && g.Bytes >= 0
|
|
}
|
|
return g.Shards == 0 && g.Bytes == len(data)
|
|
}
|
|
|
|
// Bound manifest size and total per-generation derived storage independently of
|
|
// historical source size. Oversized single parts remain explicit failures.
|
|
const MaxShards = 4096
|