forked from longtao.wu/droboticflow
106 lines
3.7 KiB
YAML
106 lines
3.7 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: drosync-slo
|
|
namespace: __NAMESPACE__
|
|
labels:
|
|
app.kubernetes.io/name: drosync
|
|
spec:
|
|
groups:
|
|
- name: drosync.availability
|
|
rules:
|
|
- alert: DrosyncCoordinatorDown
|
|
expr: max(up{service="drosync-coordinator"}) < 1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: drosync coordinator is unavailable
|
|
- alert: DrosyncCoordinatorStalled
|
|
expr: time() - max(drosync_coordinator_last_cycle_timestamp_seconds) > 120
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: drosync coordinator has not completed a cycle for two minutes
|
|
- alert: DrosyncCoordinatorErrorBudgetBurn
|
|
expr: |
|
|
(
|
|
sum(rate(drosync_coordinator_errors_total[1h]))
|
|
/
|
|
clamp_min(sum(rate(drosync_coordinator_cycles_total[1h])), 0.000001)
|
|
) > 0.001
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: drosync coordinator exceeds the 99.9 percent monthly error budget
|
|
- alert: DrosyncAuditSealMissing
|
|
expr: |
|
|
drosync_coordinator_enabled == 1
|
|
and
|
|
increase(drosync_coordinator_audit_seals_total[30h]) < 1
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: drosync did not produce a daily audit seal
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: drosync-grafana-dashboard
|
|
namespace: __NAMESPACE__
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
data:
|
|
drosync-slo.json: |-
|
|
{
|
|
"title": "drosync production v1",
|
|
"uid": "drosync-v1",
|
|
"schemaVersion": 39,
|
|
"refresh": "30s",
|
|
"time": {"from": "now-24h", "to": "now"},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "stat",
|
|
"title": "30 day coordinator availability",
|
|
"targets": [{"expr": "100 * avg_over_time(up{service=\"drosync-coordinator\"}[30d])"}],
|
|
"fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 99.9}]}}},
|
|
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 0}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "Seconds since successful cycle",
|
|
"targets": [{"expr": "time() - max(drosync_coordinator_last_success_timestamp_seconds)"}],
|
|
"fieldConfig": {"defaults": {"unit": "s"}},
|
|
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 0}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "Committed items",
|
|
"targets": [{"expr": "sum(rate(drosync_coordinator_items_committed_total[5m]))", "legendFormat": "items per second"}],
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "timeseries",
|
|
"title": "Coordinator error ratio",
|
|
"targets": [{"expr": "sum(rate(drosync_coordinator_errors_total[5m])) / clamp_min(sum(rate(drosync_coordinator_cycles_total[5m])), 0.000001)", "legendFormat": "error ratio"}],
|
|
"fieldConfig": {"defaults": {"unit": "percentunit"}},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Coordinator cycle duration",
|
|
"targets": [{"expr": "max(drosync_coordinator_cycle_duration_seconds)", "legendFormat": "seconds"}],
|
|
"fieldConfig": {"defaults": {"unit": "s"}},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}
|
|
}
|
|
]
|
|
}
|