From 5a21b2cd88628147bdc273a51b54d89d3bc8d7dc Mon Sep 17 00:00:00 2001
From: Gregor Michels <hirnpfirsich@brainpeach.de>
Date: Wed, 13 Jul 2022 01:27:07 +0200
Subject: [PATCH] monitoring: prometheus: add simple alerting rule

---
 files/alerting_rules.yml          | 12 ++++++++++++
 playbook_provision_monitoring.yml | 11 +++++++++++
 templates/prometheus.yml          |  2 +-
 3 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 files/alerting_rules.yml

diff --git a/files/alerting_rules.yml b/files/alerting_rules.yml
new file mode 100644
index 0000000..94c4577
--- /dev/null
+++ b/files/alerting_rules.yml
@@ -0,0 +1,12 @@
+groups:
+  - name: Basic
+    rules:
+        # from https://awesome-prometheus-alerts.grep.to/rules.html#rule-prometheus-self-monitoring-1-2
+        - alert: PrometheusTargetMissing
+          expr: up == 0
+          for: 0m
+          labels:
+            severity: critical
+          annotations:
+            summary: Prometheus target missing (instance {{ $labels.instance }})
+            description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
diff --git a/playbook_provision_monitoring.yml b/playbook_provision_monitoring.yml
index 45f7504..9b12a8a 100644
--- a/playbook_provision_monitoring.yml
+++ b/playbook_provision_monitoring.yml
@@ -14,6 +14,17 @@
           - prometheus
           - prometheus-alertmanager
 
+    - name: configure prometheus alerting rules
+      copy:
+        src: files/alerting_rules.yml
+        dest: /etc/prometheus/alerting_rules.yml
+        owner: root
+        group: root
+        mode: 0644
+        validate: "/usr/bin/promtool check rules %s"
+      notify:
+        - reload prometheus
+
     - name: configure prometheus
       template:
         src: templates/prometheus.yml
diff --git a/templates/prometheus.yml b/templates/prometheus.yml
index 777310d..666ad74 100644
--- a/templates/prometheus.yml
+++ b/templates/prometheus.yml
@@ -9,7 +9,7 @@ alerting:
     - targets: ['localhost:9093']
 
 rule_files:
-  # - "first_rules.yml"
+  - "/etc/prometheus/alerting_rules.yml"
 
 scrape_configs:
   - job_name: 'prometheus'