forked from longtao.wu/droboticflow
218 lines
9.3 KiB
Go
218 lines
9.3 KiB
Go
package worker
|
|
|
|
import (
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
const durationBucketCount = 7
|
|
|
|
var durationBucketUpperBoundsMS = [durationBucketCount]uint64{100, 500, 1000, 5000, 15000, 60000, 300000}
|
|
|
|
type DurationBucketSnapshot struct {
|
|
UpperBoundMS uint64 `json:"upperBoundMS"`
|
|
Count uint64 `json:"count"`
|
|
}
|
|
|
|
type MetricsSnapshot struct {
|
|
Maintenance map[string]MaintenanceSnapshot `json:"maintenance,omitempty"`
|
|
ProcessAvailableTotal uint64 `json:"processAvailableTotal"`
|
|
ProcessAvailableErrors uint64 `json:"processAvailableErrors"`
|
|
ProcessAvailableDurationMS uint64 `json:"processAvailableDurationMs"`
|
|
ProcessAvailableDurationBuckets []DurationBucketSnapshot `json:"processAvailableDurationBuckets"`
|
|
DispatchedRunsTotal uint64 `json:"dispatchedRunsTotal"`
|
|
NodeExecutionsStartedTotal uint64 `json:"nodeExecutionsStartedTotal"`
|
|
NodeExecutionsCompletedTotal uint64 `json:"nodeExecutionsCompletedTotal"`
|
|
NodeExecutionsFailedTotal uint64 `json:"nodeExecutionsFailedTotal"`
|
|
NodeExecutionDurationMS uint64 `json:"nodeExecutionDurationMs"`
|
|
NodeExecutionDurationBuckets []DurationBucketSnapshot `json:"nodeExecutionDurationBuckets"`
|
|
LastDispatchQueueDepth uint64 `json:"lastDispatchQueueDepth"`
|
|
LastDispatchRunningCount uint64 `json:"lastDispatchRunningCount"`
|
|
LastDispatchSelectedCount uint64 `json:"lastDispatchSelectedCount"`
|
|
LastCycleAt string `json:"lastCycleAt,omitempty"`
|
|
LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"`
|
|
CycleDurationMS uint64 `json:"cycleDurationMs"`
|
|
StuckRunCount uint64 `json:"stuckRunCount"`
|
|
OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"`
|
|
OrphanRuntimeCleanedCount uint64 `json:"orphanRuntimeCleanedCount"`
|
|
OrphanRuntimeCleanupErrorCount uint64 `json:"orphanRuntimeCleanupErrorCount"`
|
|
}
|
|
|
|
type WorkerMetrics struct {
|
|
maintenanceMu sync.Mutex
|
|
maintenance map[string]MaintenanceSnapshot
|
|
processAvailableTotal atomic.Uint64
|
|
processAvailableErrors atomic.Uint64
|
|
processAvailableDurationMS atomic.Uint64
|
|
processAvailableDurationBuckets [durationBucketCount]atomic.Uint64
|
|
dispatchedRunsTotal atomic.Uint64
|
|
nodeExecutionsStartedTotal atomic.Uint64
|
|
nodeExecutionsCompletedTotal atomic.Uint64
|
|
nodeExecutionsFailedTotal atomic.Uint64
|
|
nodeExecutionDurationMS atomic.Uint64
|
|
nodeExecutionDurationBuckets [durationBucketCount]atomic.Uint64
|
|
lastDispatchQueueDepth atomic.Uint64
|
|
lastDispatchRunningCount atomic.Uint64
|
|
lastDispatchSelectedCount atomic.Uint64
|
|
lastCycleUnixNano atomic.Int64
|
|
lastSuccessfulCycleUnixNano atomic.Int64
|
|
lastSuccessfulCycleDurationMS atomic.Uint64
|
|
cycleDurationMS atomic.Uint64
|
|
stuckRunCount atomic.Uint64
|
|
orphanRuntimeCount atomic.Uint64
|
|
orphanRuntimeCleanedCount atomic.Uint64
|
|
orphanRuntimeCleanupErrorCount atomic.Uint64
|
|
}
|
|
|
|
type MaintenanceSnapshot struct {
|
|
Running bool `json:"running"`
|
|
StartedAt string `json:"startedAt,omitempty"`
|
|
FinishedAt string `json:"finishedAt,omitempty"`
|
|
DurationMS uint64 `json:"durationMs"`
|
|
Errors uint64 `json:"errors"`
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) StartMaintenance(name string, started time.Time) {
|
|
metrics.maintenanceMu.Lock()
|
|
defer metrics.maintenanceMu.Unlock()
|
|
if metrics.maintenance == nil {
|
|
metrics.maintenance = map[string]MaintenanceSnapshot{}
|
|
}
|
|
s := metrics.maintenance[name]
|
|
s.Running = true
|
|
s.StartedAt = started.UTC().Format(time.RFC3339Nano)
|
|
metrics.maintenance[name] = s
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) FinishMaintenance(name string, duration time.Duration, err error) {
|
|
metrics.maintenanceMu.Lock()
|
|
defer metrics.maintenanceMu.Unlock()
|
|
s := metrics.maintenance[name]
|
|
s.Running = false
|
|
s.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano)
|
|
s.DurationMS = uint64(duration.Milliseconds())
|
|
if err != nil {
|
|
s.Errors++
|
|
}
|
|
metrics.maintenance[name] = s
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) maintenanceSnapshot() map[string]MaintenanceSnapshot {
|
|
metrics.maintenanceMu.Lock()
|
|
defer metrics.maintenanceMu.Unlock()
|
|
result := make(map[string]MaintenanceSnapshot, len(metrics.maintenance))
|
|
for name, value := range metrics.maintenance {
|
|
result[name] = value
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordProcessAvailable(durationMS uint64, err error) {
|
|
now := time.Now().UTC()
|
|
metrics.processAvailableTotal.Add(1)
|
|
metrics.processAvailableDurationMS.Add(durationMS)
|
|
recordDurationBucket(metrics.processAvailableDurationBuckets[:], durationMS)
|
|
metrics.lastCycleUnixNano.Store(now.UnixNano())
|
|
metrics.cycleDurationMS.Store(durationMS)
|
|
if err != nil {
|
|
metrics.processAvailableErrors.Add(1)
|
|
return
|
|
}
|
|
metrics.lastSuccessfulCycleDurationMS.Store(durationMS)
|
|
metrics.lastSuccessfulCycleUnixNano.Store(now.UnixNano())
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordDispatch(queueDepth int, runningCount int, selectedCount int) {
|
|
metrics.lastDispatchQueueDepth.Store(uint64(max(queueDepth, 0)))
|
|
metrics.lastDispatchRunningCount.Store(uint64(max(runningCount, 0)))
|
|
metrics.lastDispatchSelectedCount.Store(uint64(max(selectedCount, 0)))
|
|
if selectedCount > 0 {
|
|
metrics.dispatchedRunsTotal.Add(uint64(selectedCount))
|
|
}
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordNodeExecutionStarted() {
|
|
metrics.nodeExecutionsStartedTotal.Add(1)
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordNodeExecutionCompleted(durationMS uint64) {
|
|
metrics.nodeExecutionsCompletedTotal.Add(1)
|
|
metrics.nodeExecutionDurationMS.Add(durationMS)
|
|
recordDurationBucket(metrics.nodeExecutionDurationBuckets[:], durationMS)
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordNodeExecutionFailed(durationMS uint64) {
|
|
metrics.nodeExecutionsFailedTotal.Add(1)
|
|
metrics.nodeExecutionDurationMS.Add(durationMS)
|
|
recordDurationBucket(metrics.nodeExecutionDurationBuckets[:], durationMS)
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordStuckRuns(count int) {
|
|
metrics.stuckRunCount.Store(uint64(max(count, 0)))
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordOrphanRuntimes(count int) {
|
|
metrics.orphanRuntimeCount.Store(uint64(max(count, 0)))
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) RecordOrphanCleanup(cleaned int, errors int) {
|
|
metrics.orphanRuntimeCleanedCount.Add(uint64(max(cleaned, 0)))
|
|
metrics.orphanRuntimeCleanupErrorCount.Add(uint64(max(errors, 0)))
|
|
}
|
|
|
|
func metricTimestamp(value int64) string {
|
|
if value <= 0 {
|
|
return ""
|
|
}
|
|
return time.Unix(0, value).UTC().Format(time.RFC3339Nano)
|
|
}
|
|
|
|
func (metrics *WorkerMetrics) Snapshot() MetricsSnapshot {
|
|
return MetricsSnapshot{
|
|
Maintenance: metrics.maintenanceSnapshot(),
|
|
ProcessAvailableTotal: metrics.processAvailableTotal.Load(),
|
|
ProcessAvailableErrors: metrics.processAvailableErrors.Load(),
|
|
ProcessAvailableDurationMS: metrics.processAvailableDurationMS.Load(),
|
|
ProcessAvailableDurationBuckets: snapshotDurationBuckets(metrics.processAvailableDurationBuckets[:]),
|
|
DispatchedRunsTotal: metrics.dispatchedRunsTotal.Load(),
|
|
NodeExecutionsStartedTotal: metrics.nodeExecutionsStartedTotal.Load(),
|
|
NodeExecutionsCompletedTotal: metrics.nodeExecutionsCompletedTotal.Load(),
|
|
NodeExecutionsFailedTotal: metrics.nodeExecutionsFailedTotal.Load(),
|
|
NodeExecutionDurationMS: metrics.nodeExecutionDurationMS.Load(),
|
|
NodeExecutionDurationBuckets: snapshotDurationBuckets(metrics.nodeExecutionDurationBuckets[:]),
|
|
LastDispatchQueueDepth: metrics.lastDispatchQueueDepth.Load(),
|
|
LastDispatchRunningCount: metrics.lastDispatchRunningCount.Load(),
|
|
LastDispatchSelectedCount: metrics.lastDispatchSelectedCount.Load(),
|
|
LastCycleAt: metricTimestamp(metrics.lastCycleUnixNano.Load()),
|
|
LastSuccessfulCycleAt: metricTimestamp(metrics.lastSuccessfulCycleUnixNano.Load()),
|
|
CycleDurationMS: metrics.cycleDurationMS.Load(),
|
|
StuckRunCount: metrics.stuckRunCount.Load(),
|
|
OrphanRuntimeCount: metrics.orphanRuntimeCount.Load(),
|
|
OrphanRuntimeCleanedCount: metrics.orphanRuntimeCleanedCount.Load(),
|
|
OrphanRuntimeCleanupErrorCount: metrics.orphanRuntimeCleanupErrorCount.Load(),
|
|
}
|
|
}
|
|
|
|
func recordDurationBucket(buckets []atomic.Uint64, durationMS uint64) {
|
|
for index, upperBoundMS := range durationBucketUpperBoundsMS {
|
|
if durationMS <= upperBoundMS {
|
|
for cumulativeIndex := index; cumulativeIndex < len(buckets); cumulativeIndex++ {
|
|
buckets[cumulativeIndex].Add(1)
|
|
}
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func snapshotDurationBuckets(buckets []atomic.Uint64) []DurationBucketSnapshot {
|
|
snapshot := make([]DurationBucketSnapshot, 0, len(durationBucketUpperBoundsMS))
|
|
for index, upperBoundMS := range durationBucketUpperBoundsMS {
|
|
snapshot = append(snapshot, DurationBucketSnapshot{
|
|
UpperBoundMS: upperBoundMS,
|
|
Count: buckets[index].Load(),
|
|
})
|
|
}
|
|
return snapshot
|
|
}
|