500 lines
15 KiB
Go

package outputsnapshot
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"droboticflow/internal/queryserving"
"droboticflow/internal/workforcestats"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"golang.org/x/sync/singleflight"
"io"
"log"
"sort"
"sync"
"time"
)
type Reader struct {
queryMemo map[string][]byte
registeredQueries map[string]time.Time
queryMemoBytes int
queryFlights singleflight.Group
querySlots chan struct{}
backend Backend
source Source
stream StreamSource
now func() time.Time
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
mu sync.Mutex
closed bool
attempts map[string]time.Time
cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache.
cacheBytes int
reads chan struct{}
}
func NewReader(b Backend, source Source) *Reader {
ctx, cancel := context.WithCancel(context.Background())
return &Reader{registeredQueries: map[string]time.Time{}, queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
}
func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() }
// Prewarm uses the existing fenced publication protocol, without a customer
// request or mutation. Repeated workers cannot duplicate a successful refresh.
func (r *Reader) Prewarm(ctx context.Context, org string, projects []string) (Info, error) {
_, info, err := r.resolve(ctx, org, projects, "")
return info, err
}
func scopeKey(org string, projects []string) string {
ids := append([]string(nil), projects...)
sort.Strings(ids)
data, _ := json.Marshal([]any{Schema, org, ids})
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
// Read returns nil while the initial generation is being built. Existing complete
// generations remain available during refresh. Pins never silently change data.
func (r *Reader) resolve(ctx context.Context, org string, projects []string, pin string) (*Generation, Info, error) {
key, now := scopeKey(org, projects), r.now()
state, err := r.backend.OutputSnapshotState(ctx)
if err != nil {
return nil, Info{}, err
}
head := state.Heads[key]
if _, exists := state.Heads[key]; !exists {
live := 0
for _, h := range state.Heads {
if h.Current != nil && now.Sub(h.Current.AsOf) <= MaxAge || h.Current == nil && now.Before(h.NextAttempt) {
live++
}
}
if live >= MaxScopes {
return nil, Info{}, ErrCapacity
}
}
var gen *Generation
if pin != "" {
for _, g := range []*Generation{head.Current, head.Previous} {
if g != nil && g.ID == pin {
gen = g
}
}
if gen == nil {
return nil, Info{}, ErrExpired
}
} else {
gen = head.Current
}
if state.Owner == "" || !now.Before(state.Until) {
r.request(key, org, projects, head, now)
}
info := Info{Refreshing: state.Owner != "" && now.Before(state.Until)}
if gen == nil {
if head.Failed && now.Before(head.NextAttempt) {
return nil, info, ErrUnavailable
}
return nil, info, nil
}
if now.Sub(gen.AsOf) > MaxAge {
if pin != "" {
return nil, info, ErrExpired
}
if head.Failed && now.Before(head.NextAttempt) {
return nil, info, ErrUnavailable
}
return nil, info, nil
}
info.ID = gen.ID
info.AsOf = gen.AsOf
info.Stale = now.Sub(gen.AsOf) >= RefreshInterval
return gen, info, nil
}
func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) {
gen, info, err := r.resolve(ctx, org, projects, pin)
if err != nil || gen == nil {
return nil, info, err
}
return r.readResolved(ctx, gen, info, pin)
}
func (r *Reader) readResolved(ctx context.Context, gen *Generation, info Info, pin string) (*workforcestats.OutputDataset, Info, error) {
now := r.now()
if gen.Shards > 0 {
return nil, info, ErrCapacity
} // Query streams shards; Read retains its bounded single-dataset contract.
select {
case r.reads <- struct{}{}:
case <-ctx.Done():
return nil, info, ctx.Err()
}
defer func() { <-r.reads }()
data, err := r.load(ctx, *gen)
if err != nil && pin == "" {
state, stateErr := r.backend.OutputSnapshotState(ctx)
if stateErr == nil {
for _, head := range state.Heads {
if head.Current != nil && head.Current.ID == gen.ID && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge {
gen = head.Previous
data, err = r.load(ctx, *gen)
break
}
}
}
}
if err != nil {
return nil, info, ErrUnavailable
}
info.ID = gen.ID
info.AsOf = gen.AsOf
info.Stale = now.Sub(gen.AsOf) >= RefreshInterval
return data, info, nil
}
func (r *Reader) request(key, org string, projects []string, head Head, now time.Time) {
if head.Current != nil && now.Sub(head.Current.AsOf) < RefreshInterval {
return
}
if now.Before(head.NextAttempt) {
return
}
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return
}
if t := r.attempts[key]; now.Before(t) {
r.mu.Unlock()
return
}
if len(r.attempts) >= MaxScopes*2 {
r.attempts = map[string]time.Time{}
}
r.attempts[key] = now.Add(2 * time.Second)
r.wg.Add(1)
r.mu.Unlock()
ids := append([]string(nil), projects...)
go func() { defer r.wg.Done(); r.build(key, org, ids) }()
}
func (r *Reader) build(key, org string, projects []string) {
ctx, cancel := context.WithTimeout(r.ctx, BuildTimeout)
defer cancel()
acquiredAt := r.now()
lease, ok, err := r.backend.AcquireOutputSnapshot(ctx, key, acquiredAt)
if err != nil || !ok {
return
}
published := false
started := time.Now()
stage := "source"
var buildErr error
defer func() {
if !published {
if buildErr == nil {
buildErr = ctx.Err()
}
if buildErr == nil {
buildErr = fmt.Errorf("snapshot lease lost or publication rejected")
}
log.Printf("output_snapshot_build_failed stage=%s epoch=%d elapsed_ms=%d error=%v", stage, lease.Epoch, time.Since(started).Milliseconds(), buildErr)
cleanup, stop := context.WithTimeout(context.Background(), 3*time.Second)
defer stop()
if err := r.backend.FailOutputSnapshot(cleanup, lease, r.now(), Failure{Stage: stage, Code: failureCode(buildErr), At: r.now(), DurationMS: time.Since(started).Milliseconds()}); err != nil {
log.Printf("output_snapshot_cleanup_failed epoch=%d error=%v", lease.Epoch, err)
}
}
}()
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
r.keepBuildLease(ctx, lease, acquiredAt, cancel, stop)
}()
defer func() { close(stop); <-done }()
asOf := r.now().UTC().Truncate(time.Millisecond)
if r.stream != nil {
stage = "partitioned_build"
gen, payload, err := r.buildShards(ctx, lease, org, projects, asOf, started)
if err != nil {
buildErr = err
return
}
stage = "query_prewarm"
warmStart := time.Now()
gen.Metrics.PreparedQueries = r.prewarmActiveQueries(ctx, key, gen, payload)
gen.Metrics.PrewarmMS = time.Since(warmStart).Milliseconds()
gen.Metrics.DurationMS = time.Since(started).Milliseconds()
stage = "publish"
published, buildErr = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now())
return
}
raw, err := r.source(ctx, org, projects)
if err != nil || ctx.Err() != nil {
buildErr = err
return
}
stage = "calculate"
data := workforcestats.BuildOutputDataset(raw, asOf)
if ctx.Err() != nil {
return
}
stage = "encode"
payload, decodedBytes, err := encode(data)
if err != nil {
buildErr = err
return
}
hash := sha256.Sum256(payload)
gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)}
stage = "query_prewarm"
warmStart := time.Now()
gen.Metrics.PreparedQueries = r.prewarmActiveQueries(ctx, key, gen, payload)
gen.Metrics.PrewarmMS = time.Since(warmStart).Milliseconds()
gen.Metrics.DurationMS = time.Since(started).Milliseconds()
stage = "publish"
published, buildErr = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now())
if published {
health := snapshotHealth(State{Heads: map[string]Head{key: {Current: &gen}}}, key, r.now())
if len(health.Warnings) > 0 {
log.Printf("output_snapshot_capacity_warning epoch=%d warnings=%v duration_ms=%d encoded_bytes=%d decoded_bytes=%d", lease.Epoch, health.Warnings, gen.Metrics.DurationMS, gen.Bytes, gen.Metrics.DecodedBytes)
}
}
}
func encode(d workforcestats.OutputDataset) ([]byte, int, error) {
raw, err := json.Marshal(d)
if err != nil {
return nil, len(raw), err
}
if len(raw) > MaxDecodedBytes {
return nil, len(raw), ErrCapacity
}
var b bytes.Buffer
w, _ := gzip.NewWriterLevel(&b, gzip.BestSpeed)
_, err = w.Write(raw)
if err != nil {
return nil, len(raw), err
}
if err = w.Close(); err != nil {
return nil, len(raw), err
}
if b.Len() > MaxEncodedBytes {
return nil, len(raw), ErrCapacity
}
return b.Bytes(), len(raw), nil
}
func (r *Reader) load(ctx context.Context, g Generation) (*workforcestats.OutputDataset, error) {
r.mu.Lock()
payload := r.cache[g.ID]
r.mu.Unlock()
if payload == nil {
var err error
payload, err = r.backend.OutputSnapshotBlob(ctx, g)
if err != nil {
return nil, err
}
if len(payload) > MaxEncodedBytes || len(payload) != g.Bytes {
return nil, ErrCapacity
}
hash := sha256.Sum256(payload)
if hex.EncodeToString(hash[:]) != g.SHA256 {
return nil, ErrUnavailable
}
r.mu.Lock()
if _, exists := r.cache[g.ID]; !exists {
if r.cacheBytes+len(payload) > 8<<20 || len(r.cache) >= MaxScopes*2 {
r.cache = map[string][]byte{}
r.cacheBytes = 0
}
r.cache[g.ID] = payload
r.cacheBytes += len(payload)
}
r.mu.Unlock()
}
z, err := gzip.NewReader(bytes.NewReader(payload))
if err != nil {
return nil, err
}
defer z.Close()
raw, err := io.ReadAll(io.LimitReader(z, MaxDecodedBytes+1))
if err != nil {
return nil, err
}
if len(raw) > MaxDecodedBytes {
return nil, ErrCapacity
}
var d workforcestats.OutputDataset
if err = json.Unmarshal(raw, &d); err != nil {
return nil, err
}
if !d.AsOf.Equal(g.AsOf) {
return nil, ErrUnavailable
}
return &d, nil
}
// Query bounds the entire decode + report calculation, not only Mongo reads.
func (r *Reader) Query(ctx context.Context, org string, projects []string, pin string, q workforcestats.OutputQuery) (*workforcestats.OutputReport, Info, error) {
gen, info, err := r.resolve(ctx, org, projects, pin)
if err != nil || gen == nil {
return nil, info, err
}
// Scope permissions are supplied afresh by the handler. The complete query,
// generation and location identity participate in the key, including personal
// and supplier isolation. No permissions or mutable datasets are cached.
q.Now = gen.AsOf
shared := r.sharedCache()
scope := scopeKey(org, projects)
r.rememberActiveQuery(ctx, scope, q)
sharedKey := outputSharedQueryKey(scope, gen.ID, q)
keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)})
key := string(keyData)
select {
case r.querySlots <- struct{}{}:
case <-ctx.Done():
return nil, info, ctx.Err()
}
cached := r.memoReport(key)
<-r.querySlots
if cached != nil {
return cached, info, nil
}
type answer struct {
report *workforcestats.OutputReport
info Info
}
result := r.queryFlights.DoChan(key, func() (any, error) {
r.mu.Lock()
if r.closed {
r.mu.Unlock()
return nil, ErrUnavailable
}
r.wg.Add(1)
r.mu.Unlock()
defer r.wg.Done()
buildCtx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
defer cancel()
select {
case r.querySlots <- struct{}{}:
case <-buildCtx.Done():
return nil, buildCtx.Err()
}
defer func() { <-r.querySlots }()
if report := r.memoReport(key); report != nil {
return answer{report, info}, nil
}
var sharedReport workforcestats.OutputReport
if queryserving.GetJSON(buildCtx, shared, sharedKey, &sharedReport) {
r.rememberReport(key, &sharedReport)
return answer{&sharedReport, info}, nil
}
report, actual, err := r.queryResolved(buildCtx, gen, info, pin, q)
if err != nil || report == nil {
return nil, err
}
// Corrupt-current fallback must not poison the current generation's key.
if actual.ID == gen.ID {
r.rememberReport(key, report)
queryserving.PutJSON(buildCtx, shared, sharedKey, report, MaxAge)
}
return answer{report, actual}, nil
})
select {
case <-ctx.Done():
return nil, info, ctx.Err()
case value := <-result:
if value.Err != nil {
return nil, info, value.Err
}
valueAnswer := value.Val.(answer)
select {
case r.querySlots <- struct{}{}:
case <-ctx.Done():
return nil, info, ctx.Err()
}
defer func() { <-r.querySlots }()
// Every caller receives an independent value; UI/report handlers may paginate
// or modify the report without mutating another user's response.
raw, err := json.Marshal(valueAnswer.report)
if err != nil {
return nil, info, err
}
var report workforcestats.OutputReport
if err = json.Unmarshal(raw, &report); err != nil {
return nil, info, err
}
return &report, valueAnswer.info, nil
}
}
const queryMemoBudget = 4 << 20
func (r *Reader) memoReport(key string) *workforcestats.OutputReport {
r.mu.Lock()
payload := r.queryMemo[key]
r.mu.Unlock()
if payload == nil {
return nil
}
reader, err := gzip.NewReader(bytes.NewReader(payload))
if err != nil {
return nil
}
defer reader.Close()
var report workforcestats.OutputReport
if json.NewDecoder(io.LimitReader(reader, MaxDecodedBytes+1)).Decode(&report) != nil {
return nil
}
return &report
}
func (r *Reader) rememberReport(key string, report *workforcestats.OutputReport) {
raw, err := json.Marshal(report)
if err != nil || len(raw) > MaxDecodedBytes {
return
}
var encoded bytes.Buffer
writer, _ := gzip.NewWriterLevel(&encoded, gzip.BestSpeed)
if _, err = writer.Write(raw); err != nil {
return
}
if writer.Close() != nil || encoded.Len() > queryMemoBudget {
return
}
r.mu.Lock()
defer r.mu.Unlock()
if r.queryMemo[key] != nil {
return
}
cost := encoded.Len() + len(key) + 128
if cost > queryMemoBudget {
return
}
if len(r.queryMemo) >= 128 || r.queryMemoBytes+cost > queryMemoBudget {
r.queryMemo = map[string][]byte{}
r.queryMemoBytes = 0
}
r.queryMemo[key] = encoded.Bytes()
r.queryMemoBytes += cost
}
func failureCode(err error) string {
switch {
case errors.Is(err, ErrCapacity), errors.Is(err, workforcestats.ErrOutputSourceBudget):
return "capacity"
case errors.Is(err, context.DeadlineExceeded):
return "timeout"
case errors.Is(err, context.Canceled):
return "canceled"
default:
return "build_failed"
}
}