monitoring: prometheus: add simple alerting rule

files/alerting_rules.yml

@@ -0,0 +1,12 @@
+groups:
+  - name: Basic
+    rules:
+        # from https://awesome-prometheus-alerts.grep.to/rules.html#rule-prometheus-self-monitoring-1-2
+        - alert: PrometheusTargetMissing
+          expr: up == 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Prometheus target missing (instance {{ $labels.instance }})
+            description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

playbook_provision_monitoring.yml

@@ -14,6 +14,17 @@
           - prometheus
           - prometheus-alertmanager
 
+    - name: configure prometheus alerting rules
+      copy:
+        src: files/alerting_rules.yml
+        dest: /etc/prometheus/alerting_rules.yml
+        owner: root
+        group: root
+        mode: 0644
+        validate: "/usr/bin/promtool check rules %s"
+      notify:
+        - reload prometheus
+
     - name: configure prometheus
       template:
         src: templates/prometheus.yml

templates/prometheus.yml

@@ -9,7 +9,7 @@ alerting:
     - targets: ['localhost:9093']
 
 rule_files:
-  # - "first_rules.yml"
+  - "/etc/prometheus/alerting_rules.yml"
 
 scrape_configs:
   - job_name: 'prometheus'