monitoring: configure alertmanager to send mails

playbook_provision_monitoring.yml

@@ -14,6 +14,32 @@
           - prometheus
           - prometheus-alertmanager
 
+    # stolen from usr/share/prometheus/alertmanager/generate-ui.sh
+    # script calls apt without "-y" therefore we need to install them beforehand
+    - name: install dependencies for alertmanager ui generation
+      package:
+        name:
+          - libjs-bootstrap4
+          - fonts-font-awesome
+          - curl
+          - uglifyjs
+          - golang-github-prometheus-alertmanager-dev
+
+    - name: configure alertmanager
+      template:
+        src: templates/alertmanager.yml.j2
+        dest: /etc/prometheus/alertmanager.yml
+        validate: "/usr/bin/amtool check-config %s"
+      notify:
+        - reload prometheus-alertmanager
+
+    - name: generate alertmanager ui
+      shell:
+        cmd: /usr/share/prometheus/alertmanager/generate-ui.sh
+        creates: "/usr/share/prometheus/alertmanager/ui/index.html"
+      notify:
+        - restart prometheus-alertmanager
+
     - name: configure prometheus alerting rules
       copy:
         src: files/alerting_rules.yml
@@ -69,6 +95,16 @@
         name: prometheus
         state: reloaded
 
+    - name: reload prometheus-alertmanager
+      service:
+        name: prometheus-alertmanager
+        state: reloaded
+
+    - name: restart prometheus-alertmanager
+      service:
+        name: prometheus-alertmanager
+        state: restarted
+
     - name: restart grafana
       service:
         name: grafana-server

templates/alertmanager.yml.j2

@@ -0,0 +1,44 @@
+# Sample configuration.
+# See https://prometheus.io/docs/alerting/configuration/ for documentation.
+
+global:
+  # The smarthost and SMTP sender used for mail notifications.
+  smtp_smarthost: 'harald.brainpeach.de:587'
+  smtp_from: 'ffl-eae-adp-mon01@brainpeach.de'
+  smtp_auth_username: 'ffl-eae-adp-mon01@brainpeach.de'
+  smtp_auth_password: '{{ lookup("passwordstore", "mailboxes/ffl-eae-adp-mon01@brainpeach.de") }}'
+
+# The directory from which notification templates are read.
+templates: 
+- '/etc/prometheus/alertmanager_templates/*.tmpl'
+
+# The root route on which each incoming alert enters.
+route:
+  # The labels by which incoming alerts are grouped together. For example,
+  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
+  # be batched into a single group.
+  group_by: ['alertname']
+
+  # When a new group of alerts is created by an incoming alert, wait at
+  # least 'group_wait' to send the initial notification.
+  # This way ensures that you get multiple alerts for the same group that start
+  # firing shortly after another are batched together on the first 
+  # notification.
+  group_wait: 30s
+
+  # When the first notification was sent, wait 'group_interval' to send a batch
+  # of new alerts that started firing for that group.
+  group_interval: 5m
+
+  # If an alert has successfully been sent, wait 'repeat_interval' to
+  # resend them.
+  repeat_interval: 3h 
+
+  # A default receiver
+  receiver: admins
+
+receivers:
+- name: 'admins'
+  email_configs:
+  - to: 'hirnpfirsich@brainpeach.de'
+    send_resolved: true