d-robotics 4a364b1352 🐛 Fix review binding races and unblock worker dispatch
(cherry picked from commit ea9dd2fc79cac1a3c2c02fe34ad9491119a4a848)
2026-09-29 08:54:35 -07:00

218 lines
9.3 KiB
Go

package worker
import (
"sync"
"sync/atomic"
"time"
)
const durationBucketCount = 7
var durationBucketUpperBoundsMS = [durationBucketCount]uint64{100, 500, 1000, 5000, 15000, 60000, 300000}
type DurationBucketSnapshot struct {
UpperBoundMS uint64 `json:"upperBoundMS"`
Count uint64 `json:"count"`
}
type MetricsSnapshot struct {
Maintenance map[string]MaintenanceSnapshot `json:"maintenance,omitempty"`
ProcessAvailableTotal uint64 `json:"processAvailableTotal"`
ProcessAvailableErrors uint64 `json:"processAvailableErrors"`
ProcessAvailableDurationMS uint64 `json:"processAvailableDurationMs"`
ProcessAvailableDurationBuckets []DurationBucketSnapshot `json:"processAvailableDurationBuckets"`
DispatchedRunsTotal uint64 `json:"dispatchedRunsTotal"`
NodeExecutionsStartedTotal uint64 `json:"nodeExecutionsStartedTotal"`
NodeExecutionsCompletedTotal uint64 `json:"nodeExecutionsCompletedTotal"`
NodeExecutionsFailedTotal uint64 `json:"nodeExecutionsFailedTotal"`
NodeExecutionDurationMS uint64 `json:"nodeExecutionDurationMs"`
NodeExecutionDurationBuckets []DurationBucketSnapshot `json:"nodeExecutionDurationBuckets"`
LastDispatchQueueDepth uint64 `json:"lastDispatchQueueDepth"`
LastDispatchRunningCount uint64 `json:"lastDispatchRunningCount"`
LastDispatchSelectedCount uint64 `json:"lastDispatchSelectedCount"`
LastCycleAt string `json:"lastCycleAt,omitempty"`
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
CycleDurationMS uint64 `json:"cycleDurationMs"`
StuckRunCount uint64 `json:"stuckRunCount"`
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"`
OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"`
}
type WorkerMetrics struct {
maintenanceMu sync.Mutex
maintenance map[string]MaintenanceSnapshot
processAvailableTotal atomic.Uint64
processAvailableErrors atomic.Uint64
processAvailableDurationMS atomic.Uint64
processAvailableDurationBuckets [durationBucketCount]atomic.Uint64
dispatchedRunsTotal atomic.Uint64
nodeExecutionsStartedTotal atomic.Uint64
nodeExecutionsCompletedTotal atomic.Uint64
nodeExecutionsFailedTotal atomic.Uint64
nodeExecutionDurationMS atomic.Uint64
nodeExecutionDurationBuckets [durationBucketCount]atomic.Uint64
lastDispatchQueueDepth atomic.Uint64
lastDispatchRunningCount atomic.Uint64
lastDispatchSelectedCount atomic.Uint64
lastCycleUnixNano atomic.Int64
lastSuccessfulCycleUnixNano atomic.Int64
lastSuccessfulCycleDurationMS atomic.Uint64
cycleDurationMS atomic.Uint64
stuckRunCount atomic.Uint64
orphanRuntimeCount atomic.Uint64
orphanRuntimeCleanedCount atomic.Uint64
orphanRuntimeCleanupErrorCount atomic.Uint64
}
type MaintenanceSnapshot struct {
Running bool `json:"running"`
StartedAt string `json:"startedAt,omitempty"`
FinishedAt string `json:"finishedAt,omitempty"`
DurationMS uint64 `json:"durationMs"`
Errors uint64 `json:"errors"`
}
func (metrics *WorkerMetrics) StartMaintenance(name string, started time.Time) {
metrics.maintenanceMu.Lock()
defer metrics.maintenanceMu.Unlock()
if metrics.maintenance == nil {
metrics.maintenance = map[string]MaintenanceSnapshot{}
}
s := metrics.maintenance[name]
s.Running = true
s.StartedAt = started.UTC().Format(time.RFC3339Nano)
metrics.maintenance[name] = s
}
func (metrics *WorkerMetrics) FinishMaintenance(name string, duration time.Duration, err error) {
metrics.maintenanceMu.Lock()
defer metrics.maintenanceMu.Unlock()
s := metrics.maintenance[name]
s.Running = false
s.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano)
s.DurationMS = uint64(duration.Milliseconds())
if err != nil {
s.Errors++
}
metrics.maintenance[name] = s
}
func (metrics *WorkerMetrics) maintenanceSnapshot() map[string]MaintenanceSnapshot {
metrics.maintenanceMu.Lock()
defer metrics.maintenanceMu.Unlock()
result := make(map[string]MaintenanceSnapshot, len(metrics.maintenance))
for name, value := range metrics.maintenance {
result[name] = value
}
return result
}
func (metrics *WorkerMetrics) RecordProcessAvailable(durationMS uint64, err error) {
now := time.Now().UTC()
metrics.processAvailableTotal.Add(1)
metrics.processAvailableDurationMS.Add(durationMS)
recordDurationBucket(metrics.processAvailableDurationBuckets[:], durationMS)
metrics.lastCycleUnixNano.Store(now.UnixNano())
metrics.cycleDurationMS.Store(durationMS)
if err != nil {
metrics.processAvailableErrors.Add(1)
return
}
metrics.lastSuccessfulCycleDurationMS.Store(durationMS)
metrics.lastSuccessfulCycleUnixNano.Store(now.UnixNano())
}
func (metrics *WorkerMetrics) RecordDispatch(queueDepth int, runningCount int, selectedCount int) {
metrics.lastDispatchQueueDepth.Store(uint64(max(queueDepth, 0)))
metrics.lastDispatchRunningCount.Store(uint64(max(runningCount, 0)))
metrics.lastDispatchSelectedCount.Store(uint64(max(selectedCount, 0)))
if selectedCount > 0 {
metrics.dispatchedRunsTotal.Add(uint64(selectedCount))
}
}
func (metrics *WorkerMetrics) RecordNodeExecutionStarted() {
metrics.nodeExecutionsStartedTotal.Add(1)
}
func (metrics *WorkerMetrics) RecordNodeExecutionCompleted(durationMS uint64) {
metrics.nodeExecutionsCompletedTotal.Add(1)
metrics.nodeExecutionDurationMS.Add(durationMS)
recordDurationBucket(metrics.nodeExecutionDurationBuckets[:], durationMS)
}
func (metrics *WorkerMetrics) RecordNodeExecutionFailed(durationMS uint64) {
metrics.nodeExecutionsFailedTotal.Add(1)
metrics.nodeExecutionDurationMS.Add(durationMS)
recordDurationBucket(metrics.nodeExecutionDurationBuckets[:], durationMS)
}
func (metrics *WorkerMetrics) RecordStuckRuns(count int) {
metrics.stuckRunCount.Store(uint64(max(count, 0)))
}
func (metrics *WorkerMetrics) RecordOrphanRuntimes(count int) {
metrics.orphanRuntimeCount.Store(uint64(max(count, 0)))
}
func (metrics *WorkerMetrics) RecordOrphanCleanup(cleaned int, errors int) {
metrics.orphanRuntimeCleanedCount.Add(uint64(max(cleaned, 0)))
metrics.orphanRuntimeCleanupErrorCount.Add(uint64(max(errors, 0)))
}
func metricTimestamp(value int64) string {
if value <= 0 {
return ""
}
return time.Unix(0, value).UTC().Format(time.RFC3339Nano)
}
func (metrics *WorkerMetrics) Snapshot() MetricsSnapshot {
return MetricsSnapshot{
Maintenance: metrics.maintenanceSnapshot(),
ProcessAvailableTotal: metrics.processAvailableTotal.Load(),
ProcessAvailableErrors: metrics.processAvailableErrors.Load(),
ProcessAvailableDurationMS: metrics.processAvailableDurationMS.Load(),
ProcessAvailableDurationBuckets: snapshotDurationBuckets(metrics.processAvailableDurationBuckets[:]),
DispatchedRunsTotal: metrics.dispatchedRunsTotal.Load(),
NodeExecutionsStartedTotal: metrics.nodeExecutionsStartedTotal.Load(),
NodeExecutionsCompletedTotal: metrics.nodeExecutionsCompletedTotal.Load(),
NodeExecutionsFailedTotal: metrics.nodeExecutionsFailedTotal.Load(),
NodeExecutionDurationMS: metrics.nodeExecutionDurationMS.Load(),
NodeExecutionDurationBuckets: snapshotDurationBuckets(metrics.nodeExecutionDurationBuckets[:]),
LastDispatchQueueDepth: metrics.lastDispatchQueueDepth.Load(),
LastDispatchRunningCount: metrics.lastDispatchRunningCount.Load(),
LastDispatchSelectedCount: metrics.lastDispatchSelectedCount.Load(),
LastCycleAt: metricTimestamp(metrics.lastCycleUnixNano.Load()),
LastSuccessfulCycleAt: metricTimestamp(metrics.lastSuccessfulCycleUnixNano.Load()),
CycleDurationMS: metrics.cycleDurationMS.Load(),
StuckRunCount: metrics.stuckRunCount.Load(),
OrphanRuntimeCount: metrics.orphanRuntimeCount.Load(),
OrphanRuntimeCleanedCount: metrics.orphanRuntimeCleanedCount.Load(),
OrphanRuntimeCleanupErrorCount: metrics.orphanRuntimeCleanupErrorCount.Load(),
}
}
func recordDurationBucket(buckets []atomic.Uint64, durationMS uint64) {
for index, upperBoundMS := range durationBucketUpperBoundsMS {
if durationMS <= upperBoundMS {
for cumulativeIndex := index; cumulativeIndex < len(buckets); cumulativeIndex++ {
buckets[cumulativeIndex].Add(1)
}
return
}
}
}
func snapshotDurationBuckets(buckets []atomic.Uint64) []DurationBucketSnapshot {
snapshot := make([]DurationBucketSnapshot, 0, len(durationBucketUpperBoundsMS))
for index, upperBoundMS := range durationBucketUpperBoundsMS {
snapshot = append(snapshot, DurationBucketSnapshot{
UpperBoundMS: upperBoundMS,
Count: buckets[index].Load(),
})
}
return snapshot
}