monitoring: add end to end tests to monitor internet reachability

via imcp (blackbox exporter)

There are two exporters.
One lives inside `monitoring01` and uses the "normal" route into the
internet without a vpn (job: `e2e_default_v4`).

The other one lives inside `mon-e2e-clients01` and routes into the
internet via the vpn (job: `e2e_clients_v4`).

files/blackbox.yml

@@ -0,0 +1,36 @@
+modules:
+  http_2xx:
+    prober: http
+  http_post_2xx:
+    prober: http
+    http:
+      method: POST
+  tcp_connect:
+    prober: tcp
+  pop3s_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^+OK"
+      tls: true
+      tls_config:
+        insecure_skip_verify: false
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+  irc_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: "NICK prober"
+      - send: "USER prober prober prober :prober"
+      - expect: "PING :([^ ]+)"
+        send: "PONG ${1}"
+      - expect: "^:[^ ]+ 001"
+  icmp_v4:
+    prober: icmp
+    icmp:
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false

playbook_provision_monitoring.yml

@@ -19,6 +19,39 @@
         state: started
         enabled: yes
 
+- name: provision blackbox exporters
+  hosts:
+    - mon-e2e-clients01
+    - monitoring01
+  tasks:
+    - name: install blackbox exporter
+      package:
+        name: prometheus-blackbox-exporter
+
+    - name: add net raw capability to blackbox exporter
+      capabilities:
+        path: /usr/bin/prometheus-blackbox-exporter
+        capability: cap_net_raw+ep
+      notify:
+        - restart blackbox-exporter
+
+    - name: configure blackbox-exporter
+      copy:
+        src: files/blackbox.yml
+        dest: /etc/prometheus/blackbox.yml
+        owner: root
+        group: root
+        mode: 0644
+        validate: "prometheus-blackbox-exporter --config.file='%s' --config.check"
+      notify:
+        - restart blackbox-exporter
+
+  handlers:
+    - name: restart blackbox-exporter
+      service:
+        name: prometheus-blackbox-exporter
+        state: restarted
+
 - name: provision monitoring
   hosts:
     - monitoring01

templates/prometheus.yml

@@ -28,3 +28,46 @@ scrape_configs:
 {% endfor %}
 
 {% endfor %}
+
+  - job_name: 'blackbox'
+    static_configs:
+      - targets:
+        - {{ hostvars['mon-e2e-clients01']['ip'] }}:9115
+        - {{ hostvars['monitoring01']['ip'] }}:9115
+
+  - job_name: 'e2e_clients_v4'
+    metrics_path: /probe
+    params:
+      module: [icmp_v4]
+    static_configs:
+      - targets:
+        - freifunk-leipzig.de
+        - harald.brainpeach.de
+        - 195.201.165.118       # freifunk-leipzig.de without dns query
+        - 88.198.195.242        # harald.brainpeach.de without dns query
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: {{ hostvars['mon-e2e-clients01']['ip'] }}:9115
+
+  - job_name: 'e2e_default_v4'
+    metrics_path: /probe
+    params:
+      module: [icmp_v4]
+    static_configs:
+      - targets:
+        - 192.168.0.1           # gigacube
+        - freifunk-leipzig.de
+        - harald.brainpeach.de
+        - 195.201.165.118       # freifunk-leipzig.de without dns query
+        - 88.198.195.242        # harald.brainpeach.de without dns query
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: {{ hostvars['monitoring01']['ip'] }}:9115