forked from longtao.wu/droboticflow
500 lines
15 KiB
Go
500 lines
15 KiB
Go
package outputsnapshot
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"droboticflow/internal/queryserving"
|
|
"droboticflow/internal/workforcestats"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"golang.org/x/sync/singleflight"
|
|
"io"
|
|
"log"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type Reader struct {
|
|
queryMemo map[string][]byte
|
|
registeredQueries map[string]time.Time
|
|
queryMemoBytes int
|
|
queryFlights singleflight.Group
|
|
querySlots chan struct{}
|
|
backend Backend
|
|
source Source
|
|
stream StreamSource
|
|
now func() time.Time
|
|
ctx context.Context
|
|
cancel context.CancelFunc
|
|
wg sync.WaitGroup
|
|
mu sync.Mutex
|
|
closed bool
|
|
attempts map[string]time.Time
|
|
cache map[string][]byte // Compressed bytes only; no unbounded decoded object cache.
|
|
cacheBytes int
|
|
reads chan struct{}
|
|
}
|
|
|
|
func NewReader(b Backend, source Source) *Reader {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
return &Reader{registeredQueries: map[string]time.Time{}, queryMemo: map[string][]byte{}, querySlots: make(chan struct{}, 2), backend: b, source: source, now: time.Now, ctx: ctx, cancel: cancel, attempts: map[string]time.Time{}, cache: map[string][]byte{}, reads: make(chan struct{}, 2)}
|
|
}
|
|
func (r *Reader) Close() { r.mu.Lock(); r.closed = true; r.cancel(); r.mu.Unlock(); r.wg.Wait() }
|
|
|
|
// Prewarm uses the existing fenced publication protocol, without a customer
|
|
// request or mutation. Repeated workers cannot duplicate a successful refresh.
|
|
func (r *Reader) Prewarm(ctx context.Context, org string, projects []string) (Info, error) {
|
|
_, info, err := r.resolve(ctx, org, projects, "")
|
|
return info, err
|
|
}
|
|
func scopeKey(org string, projects []string) string {
|
|
ids := append([]string(nil), projects...)
|
|
sort.Strings(ids)
|
|
data, _ := json.Marshal([]any{Schema, org, ids})
|
|
h := sha256.Sum256(data)
|
|
return hex.EncodeToString(h[:])
|
|
}
|
|
|
|
// Read returns nil while the initial generation is being built. Existing complete
|
|
// generations remain available during refresh. Pins never silently change data.
|
|
func (r *Reader) resolve(ctx context.Context, org string, projects []string, pin string) (*Generation, Info, error) {
|
|
key, now := scopeKey(org, projects), r.now()
|
|
state, err := r.backend.OutputSnapshotState(ctx)
|
|
if err != nil {
|
|
return nil, Info{}, err
|
|
}
|
|
head := state.Heads[key]
|
|
if _, exists := state.Heads[key]; !exists {
|
|
live := 0
|
|
for _, h := range state.Heads {
|
|
if h.Current != nil && now.Sub(h.Current.AsOf) <= MaxAge || h.Current == nil && now.Before(h.NextAttempt) {
|
|
live++
|
|
}
|
|
}
|
|
if live >= MaxScopes {
|
|
return nil, Info{}, ErrCapacity
|
|
}
|
|
}
|
|
var gen *Generation
|
|
if pin != "" {
|
|
for _, g := range []*Generation{head.Current, head.Previous} {
|
|
if g != nil && g.ID == pin {
|
|
gen = g
|
|
}
|
|
}
|
|
if gen == nil {
|
|
return nil, Info{}, ErrExpired
|
|
}
|
|
} else {
|
|
gen = head.Current
|
|
}
|
|
if state.Owner == "" || !now.Before(state.Until) {
|
|
r.request(key, org, projects, head, now)
|
|
}
|
|
info := Info{Refreshing: state.Owner != "" && now.Before(state.Until)}
|
|
if gen == nil {
|
|
if head.Failed && now.Before(head.NextAttempt) {
|
|
return nil, info, ErrUnavailable
|
|
}
|
|
return nil, info, nil
|
|
}
|
|
if now.Sub(gen.AsOf) > MaxAge {
|
|
if pin != "" {
|
|
return nil, info, ErrExpired
|
|
}
|
|
if head.Failed && now.Before(head.NextAttempt) {
|
|
return nil, info, ErrUnavailable
|
|
}
|
|
return nil, info, nil
|
|
}
|
|
info.ID = gen.ID
|
|
info.AsOf = gen.AsOf
|
|
info.Stale = now.Sub(gen.AsOf) >= RefreshInterval
|
|
return gen, info, nil
|
|
}
|
|
|
|
func (r *Reader) Read(ctx context.Context, org string, projects []string, pin string) (*workforcestats.OutputDataset, Info, error) {
|
|
gen, info, err := r.resolve(ctx, org, projects, pin)
|
|
if err != nil || gen == nil {
|
|
return nil, info, err
|
|
}
|
|
return r.readResolved(ctx, gen, info, pin)
|
|
}
|
|
func (r *Reader) readResolved(ctx context.Context, gen *Generation, info Info, pin string) (*workforcestats.OutputDataset, Info, error) {
|
|
now := r.now()
|
|
if gen.Shards > 0 {
|
|
return nil, info, ErrCapacity
|
|
} // Query streams shards; Read retains its bounded single-dataset contract.
|
|
|
|
select {
|
|
case r.reads <- struct{}{}:
|
|
case <-ctx.Done():
|
|
return nil, info, ctx.Err()
|
|
}
|
|
defer func() { <-r.reads }()
|
|
data, err := r.load(ctx, *gen)
|
|
if err != nil && pin == "" {
|
|
state, stateErr := r.backend.OutputSnapshotState(ctx)
|
|
if stateErr == nil {
|
|
for _, head := range state.Heads {
|
|
if head.Current != nil && head.Current.ID == gen.ID && head.Previous != nil && now.Sub(head.Previous.AsOf) <= MaxAge {
|
|
gen = head.Previous
|
|
data, err = r.load(ctx, *gen)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err != nil {
|
|
return nil, info, ErrUnavailable
|
|
}
|
|
info.ID = gen.ID
|
|
info.AsOf = gen.AsOf
|
|
info.Stale = now.Sub(gen.AsOf) >= RefreshInterval
|
|
return data, info, nil
|
|
}
|
|
func (r *Reader) request(key, org string, projects []string, head Head, now time.Time) {
|
|
if head.Current != nil && now.Sub(head.Current.AsOf) < RefreshInterval {
|
|
return
|
|
}
|
|
if now.Before(head.NextAttempt) {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
if r.closed {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
if t := r.attempts[key]; now.Before(t) {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
if len(r.attempts) >= MaxScopes*2 {
|
|
r.attempts = map[string]time.Time{}
|
|
}
|
|
r.attempts[key] = now.Add(2 * time.Second)
|
|
r.wg.Add(1)
|
|
r.mu.Unlock()
|
|
ids := append([]string(nil), projects...)
|
|
go func() { defer r.wg.Done(); r.build(key, org, ids) }()
|
|
}
|
|
func (r *Reader) build(key, org string, projects []string) {
|
|
ctx, cancel := context.WithTimeout(r.ctx, BuildTimeout)
|
|
defer cancel()
|
|
acquiredAt := r.now()
|
|
lease, ok, err := r.backend.AcquireOutputSnapshot(ctx, key, acquiredAt)
|
|
if err != nil || !ok {
|
|
return
|
|
}
|
|
published := false
|
|
started := time.Now()
|
|
stage := "source"
|
|
var buildErr error
|
|
defer func() {
|
|
if !published {
|
|
if buildErr == nil {
|
|
buildErr = ctx.Err()
|
|
}
|
|
if buildErr == nil {
|
|
buildErr = fmt.Errorf("snapshot lease lost or publication rejected")
|
|
}
|
|
log.Printf("output_snapshot_build_failed stage=%s epoch=%d elapsed_ms=%d error=%v", stage, lease.Epoch, time.Since(started).Milliseconds(), buildErr)
|
|
cleanup, stop := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer stop()
|
|
if err := r.backend.FailOutputSnapshot(cleanup, lease, r.now(), Failure{Stage: stage, Code: failureCode(buildErr), At: r.now(), DurationMS: time.Since(started).Milliseconds()}); err != nil {
|
|
log.Printf("output_snapshot_cleanup_failed epoch=%d error=%v", lease.Epoch, err)
|
|
}
|
|
}
|
|
}()
|
|
stop := make(chan struct{})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
r.keepBuildLease(ctx, lease, acquiredAt, cancel, stop)
|
|
}()
|
|
defer func() { close(stop); <-done }()
|
|
asOf := r.now().UTC().Truncate(time.Millisecond)
|
|
if r.stream != nil {
|
|
stage = "partitioned_build"
|
|
gen, payload, err := r.buildShards(ctx, lease, org, projects, asOf, started)
|
|
if err != nil {
|
|
buildErr = err
|
|
return
|
|
}
|
|
stage = "query_prewarm"
|
|
warmStart := time.Now()
|
|
gen.Metrics.PreparedQueries = r.prewarmActiveQueries(ctx, key, gen, payload)
|
|
gen.Metrics.PrewarmMS = time.Since(warmStart).Milliseconds()
|
|
gen.Metrics.DurationMS = time.Since(started).Milliseconds()
|
|
stage = "publish"
|
|
published, buildErr = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now())
|
|
return
|
|
}
|
|
|
|
raw, err := r.source(ctx, org, projects)
|
|
if err != nil || ctx.Err() != nil {
|
|
buildErr = err
|
|
return
|
|
}
|
|
stage = "calculate"
|
|
data := workforcestats.BuildOutputDataset(raw, asOf)
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
stage = "encode"
|
|
payload, decodedBytes, err := encode(data)
|
|
if err != nil {
|
|
buildErr = err
|
|
return
|
|
}
|
|
hash := sha256.Sum256(payload)
|
|
gen := Generation{Metrics: &BuildMetrics{DurationMS: time.Since(started).Milliseconds(), DecodedBytes: decodedBytes, Parts: len(data.Parts), Source: raw.SourceStats}, ID: GenerationID(lease), Slot: lease.Slot, Epoch: lease.Epoch, AsOf: asOf, SHA256: hex.EncodeToString(hash[:]), Bytes: len(payload)}
|
|
stage = "query_prewarm"
|
|
warmStart := time.Now()
|
|
gen.Metrics.PreparedQueries = r.prewarmActiveQueries(ctx, key, gen, payload)
|
|
gen.Metrics.PrewarmMS = time.Since(warmStart).Milliseconds()
|
|
gen.Metrics.DurationMS = time.Since(started).Milliseconds()
|
|
stage = "publish"
|
|
published, buildErr = r.backend.PublishOutputSnapshot(ctx, lease, gen, payload, r.now())
|
|
if published {
|
|
health := snapshotHealth(State{Heads: map[string]Head{key: {Current: &gen}}}, key, r.now())
|
|
if len(health.Warnings) > 0 {
|
|
log.Printf("output_snapshot_capacity_warning epoch=%d warnings=%v duration_ms=%d encoded_bytes=%d decoded_bytes=%d", lease.Epoch, health.Warnings, gen.Metrics.DurationMS, gen.Bytes, gen.Metrics.DecodedBytes)
|
|
}
|
|
}
|
|
|
|
}
|
|
func encode(d workforcestats.OutputDataset) ([]byte, int, error) {
|
|
raw, err := json.Marshal(d)
|
|
if err != nil {
|
|
return nil, len(raw), err
|
|
}
|
|
if len(raw) > MaxDecodedBytes {
|
|
return nil, len(raw), ErrCapacity
|
|
}
|
|
var b bytes.Buffer
|
|
w, _ := gzip.NewWriterLevel(&b, gzip.BestSpeed)
|
|
_, err = w.Write(raw)
|
|
if err != nil {
|
|
return nil, len(raw), err
|
|
}
|
|
if err = w.Close(); err != nil {
|
|
return nil, len(raw), err
|
|
}
|
|
if b.Len() > MaxEncodedBytes {
|
|
return nil, len(raw), ErrCapacity
|
|
}
|
|
return b.Bytes(), len(raw), nil
|
|
}
|
|
func (r *Reader) load(ctx context.Context, g Generation) (*workforcestats.OutputDataset, error) {
|
|
r.mu.Lock()
|
|
payload := r.cache[g.ID]
|
|
r.mu.Unlock()
|
|
if payload == nil {
|
|
var err error
|
|
payload, err = r.backend.OutputSnapshotBlob(ctx, g)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(payload) > MaxEncodedBytes || len(payload) != g.Bytes {
|
|
return nil, ErrCapacity
|
|
}
|
|
hash := sha256.Sum256(payload)
|
|
if hex.EncodeToString(hash[:]) != g.SHA256 {
|
|
return nil, ErrUnavailable
|
|
}
|
|
r.mu.Lock()
|
|
if _, exists := r.cache[g.ID]; !exists {
|
|
if r.cacheBytes+len(payload) > 8<<20 || len(r.cache) >= MaxScopes*2 {
|
|
r.cache = map[string][]byte{}
|
|
r.cacheBytes = 0
|
|
}
|
|
r.cache[g.ID] = payload
|
|
r.cacheBytes += len(payload)
|
|
}
|
|
r.mu.Unlock()
|
|
}
|
|
z, err := gzip.NewReader(bytes.NewReader(payload))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer z.Close()
|
|
raw, err := io.ReadAll(io.LimitReader(z, MaxDecodedBytes+1))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(raw) > MaxDecodedBytes {
|
|
return nil, ErrCapacity
|
|
}
|
|
var d workforcestats.OutputDataset
|
|
if err = json.Unmarshal(raw, &d); err != nil {
|
|
return nil, err
|
|
}
|
|
if !d.AsOf.Equal(g.AsOf) {
|
|
return nil, ErrUnavailable
|
|
}
|
|
return &d, nil
|
|
}
|
|
|
|
// Query bounds the entire decode + report calculation, not only Mongo reads.
|
|
func (r *Reader) Query(ctx context.Context, org string, projects []string, pin string, q workforcestats.OutputQuery) (*workforcestats.OutputReport, Info, error) {
|
|
gen, info, err := r.resolve(ctx, org, projects, pin)
|
|
if err != nil || gen == nil {
|
|
return nil, info, err
|
|
}
|
|
// Scope permissions are supplied afresh by the handler. The complete query,
|
|
// generation and location identity participate in the key, including personal
|
|
// and supplier isolation. No permissions or mutable datasets are cached.
|
|
q.Now = gen.AsOf
|
|
shared := r.sharedCache()
|
|
scope := scopeKey(org, projects)
|
|
r.rememberActiveQuery(ctx, scope, q)
|
|
sharedKey := outputSharedQueryKey(scope, gen.ID, q)
|
|
keyData, _ := json.Marshal([]any{gen.ID, q, fmt.Sprintf("%p", q.Location)})
|
|
key := string(keyData)
|
|
select {
|
|
case r.querySlots <- struct{}{}:
|
|
case <-ctx.Done():
|
|
return nil, info, ctx.Err()
|
|
}
|
|
cached := r.memoReport(key)
|
|
<-r.querySlots
|
|
if cached != nil {
|
|
return cached, info, nil
|
|
}
|
|
type answer struct {
|
|
report *workforcestats.OutputReport
|
|
info Info
|
|
}
|
|
result := r.queryFlights.DoChan(key, func() (any, error) {
|
|
r.mu.Lock()
|
|
if r.closed {
|
|
r.mu.Unlock()
|
|
return nil, ErrUnavailable
|
|
}
|
|
r.wg.Add(1)
|
|
r.mu.Unlock()
|
|
defer r.wg.Done()
|
|
buildCtx, cancel := context.WithTimeout(r.ctx, 10*time.Second)
|
|
defer cancel()
|
|
select {
|
|
case r.querySlots <- struct{}{}:
|
|
case <-buildCtx.Done():
|
|
return nil, buildCtx.Err()
|
|
}
|
|
defer func() { <-r.querySlots }()
|
|
if report := r.memoReport(key); report != nil {
|
|
return answer{report, info}, nil
|
|
}
|
|
var sharedReport workforcestats.OutputReport
|
|
if queryserving.GetJSON(buildCtx, shared, sharedKey, &sharedReport) {
|
|
r.rememberReport(key, &sharedReport)
|
|
return answer{&sharedReport, info}, nil
|
|
}
|
|
report, actual, err := r.queryResolved(buildCtx, gen, info, pin, q)
|
|
if err != nil || report == nil {
|
|
return nil, err
|
|
}
|
|
// Corrupt-current fallback must not poison the current generation's key.
|
|
if actual.ID == gen.ID {
|
|
r.rememberReport(key, report)
|
|
queryserving.PutJSON(buildCtx, shared, sharedKey, report, MaxAge)
|
|
}
|
|
return answer{report, actual}, nil
|
|
})
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, info, ctx.Err()
|
|
case value := <-result:
|
|
if value.Err != nil {
|
|
return nil, info, value.Err
|
|
}
|
|
valueAnswer := value.Val.(answer)
|
|
select {
|
|
case r.querySlots <- struct{}{}:
|
|
case <-ctx.Done():
|
|
return nil, info, ctx.Err()
|
|
}
|
|
defer func() { <-r.querySlots }()
|
|
// Every caller receives an independent value; UI/report handlers may paginate
|
|
// or modify the report without mutating another user's response.
|
|
raw, err := json.Marshal(valueAnswer.report)
|
|
if err != nil {
|
|
return nil, info, err
|
|
}
|
|
var report workforcestats.OutputReport
|
|
if err = json.Unmarshal(raw, &report); err != nil {
|
|
return nil, info, err
|
|
}
|
|
return &report, valueAnswer.info, nil
|
|
}
|
|
}
|
|
|
|
const queryMemoBudget = 4 << 20
|
|
|
|
func (r *Reader) memoReport(key string) *workforcestats.OutputReport {
|
|
r.mu.Lock()
|
|
payload := r.queryMemo[key]
|
|
r.mu.Unlock()
|
|
if payload == nil {
|
|
return nil
|
|
}
|
|
reader, err := gzip.NewReader(bytes.NewReader(payload))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer reader.Close()
|
|
var report workforcestats.OutputReport
|
|
if json.NewDecoder(io.LimitReader(reader, MaxDecodedBytes+1)).Decode(&report) != nil {
|
|
return nil
|
|
}
|
|
return &report
|
|
}
|
|
func (r *Reader) rememberReport(key string, report *workforcestats.OutputReport) {
|
|
raw, err := json.Marshal(report)
|
|
if err != nil || len(raw) > MaxDecodedBytes {
|
|
return
|
|
}
|
|
var encoded bytes.Buffer
|
|
writer, _ := gzip.NewWriterLevel(&encoded, gzip.BestSpeed)
|
|
if _, err = writer.Write(raw); err != nil {
|
|
return
|
|
}
|
|
if writer.Close() != nil || encoded.Len() > queryMemoBudget {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.queryMemo[key] != nil {
|
|
return
|
|
}
|
|
cost := encoded.Len() + len(key) + 128
|
|
if cost > queryMemoBudget {
|
|
return
|
|
}
|
|
if len(r.queryMemo) >= 128 || r.queryMemoBytes+cost > queryMemoBudget {
|
|
r.queryMemo = map[string][]byte{}
|
|
r.queryMemoBytes = 0
|
|
}
|
|
r.queryMemo[key] = encoded.Bytes()
|
|
r.queryMemoBytes += cost
|
|
}
|
|
|
|
func failureCode(err error) string {
|
|
switch {
|
|
case errors.Is(err, ErrCapacity), errors.Is(err, workforcestats.ErrOutputSourceBudget):
|
|
return "capacity"
|
|
case errors.Is(err, context.DeadlineExceeded):
|
|
return "timeout"
|
|
case errors.Is(err, context.Canceled):
|
|
return "canceled"
|
|
default:
|
|
return "build_failed"
|
|
}
|
|
}
|