rules/alerts/meta.yml

groups:
  - name: meta.alerts
    rules:
      - alert: PrometheusRuleEvaluationFailures
        expr: increase(prometheus_rule_evaluation_failures_total[5m]) > 0
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: Prometheus is failing to evaluate rules
          description: prometheus_rule_evaluation_failures_total has been increasing over every 5 minute window for 10 minutes.
          runbook_url: https://runbooks.larkspur.example/PrometheusRuleEvaluationFailures
      - alert: AlertmanagerNotificationsFailing
        expr: |
          (
            sum by (service) (rate(alertmanager_notifications_failed_total[5m]))
            /
            sum by (service) (rate(alertmanager_notifications_total[5m]))
          ) > 0.10
          and
          sum by (service) (rate(alertmanager_notifications_total[5m])) > 0
        for: 15m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: Alertmanager is failing more than 10 percent of notifications
          description: The 5 minute failure ratio of alertmanager notifications has stayed above 0.10 for 15 minutes.
          runbook_url: https://runbooks.larkspur.example/AlertmanagerNotificationsFailing
      # Dead-man's switch. It is firing whenever Prometheus can evaluate rules.
      # A for duration would hide a stalled pipeline. alerts/comparison flags
      # the intentional always-firing expression.
      # pint disable alerts/for
      # pint disable alerts/comparison
      - alert: Watchdog
        expr: vector(1)
        labels:
          severity: info
          team: platform
        annotations:
          summary: Watchdog is firing
          description: This alert is always firing. The deadman receiver must see it. If it stops arriving, the alert pipeline is down.
          runbook_url: https://runbooks.larkspur.example/Watchdog