droboticflow-new/k8s/drosync-observability.template.yaml
2026-07-28 20:43:46 +08:00

106 lines
3.7 KiB
YAML

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: drosync-slo
namespace: __NAMESPACE__
labels:
app.kubernetes.io/name: drosync
spec:
groups:
- name: drosync.availability
rules:
- alert: DrosyncCoordinatorDown
expr: max(up{service="drosync-coordinator"}) < 1
for: 5m
labels:
severity: critical
annotations:
summary: drosync coordinator is unavailable
- alert: DrosyncCoordinatorStalled
expr: time() - max(drosync_coordinator_last_cycle_timestamp_seconds) > 120
for: 5m
labels:
severity: critical
annotations:
summary: drosync coordinator has not completed a cycle for two minutes
- alert: DrosyncCoordinatorErrorBudgetBurn
expr: |
(
sum(rate(drosync_coordinator_errors_total[1h]))
/
clamp_min(sum(rate(drosync_coordinator_cycles_total[1h])), 0.000001)
) > 0.001
for: 15m
labels:
severity: warning
annotations:
summary: drosync coordinator exceeds the 99.9 percent monthly error budget
- alert: DrosyncAuditSealMissing
expr: |
drosync_coordinator_enabled == 1
and
increase(drosync_coordinator_audit_seals_total[30h]) < 1
for: 1h
labels:
severity: critical
annotations:
summary: drosync did not produce a daily audit seal
---
apiVersion: v1
kind: ConfigMap
metadata:
name: drosync-grafana-dashboard
namespace: __NAMESPACE__
labels:
grafana_dashboard: "1"
data:
drosync-slo.json: |-
{
"title": "drosync production v1",
"uid": "drosync-v1",
"schemaVersion": 39,
"refresh": "30s",
"time": {"from": "now-24h", "to": "now"},
"panels": [
{
"id": 1,
"type": "stat",
"title": "30 day coordinator availability",
"targets": [{"expr": "100 * avg_over_time(up{service=\"drosync-coordinator\"}[30d])"}],
"fieldConfig": {"defaults": {"unit": "percent", "thresholds": {"steps": [{"color": "red", "value": null}, {"color": "green", "value": 99.9}]}}},
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 0}
},
{
"id": 2,
"type": "stat",
"title": "Seconds since successful cycle",
"targets": [{"expr": "time() - max(drosync_coordinator_last_success_timestamp_seconds)"}],
"fieldConfig": {"defaults": {"unit": "s"}},
"gridPos": {"h": 8, "w": 6, "x": 6, "y": 0}
},
{
"id": 3,
"type": "timeseries",
"title": "Committed items",
"targets": [{"expr": "sum(rate(drosync_coordinator_items_committed_total[5m]))", "legendFormat": "items per second"}],
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
},
{
"id": 4,
"type": "timeseries",
"title": "Coordinator error ratio",
"targets": [{"expr": "sum(rate(drosync_coordinator_errors_total[5m])) / clamp_min(sum(rate(drosync_coordinator_cycles_total[5m])), 0.000001)", "legendFormat": "error ratio"}],
"fieldConfig": {"defaults": {"unit": "percentunit"}},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}
},
{
"id": 5,
"type": "timeseries",
"title": "Coordinator cycle duration",
"targets": [{"expr": "max(drosync_coordinator_cycle_duration_seconds)", "legendFormat": "seconds"}],
"fieldConfig": {"defaults": {"unit": "s"}},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}
}
]
}