monitoring: add alerting rules for disks running out of space

files/alerting_rules.yml

@@ -28,3 +28,29 @@ groups:
           annotations:
             summary: The public wifi lost its ability to route into the internet
             description: "check the vpn connection"
+
+  - name: ServerSpecific
+    rules:
+        # https://awesome-prometheus-alerts.grep.to/rules#rule-host-and-hardware-1-7
+        #
+        # Please add ignored mountpoints in node_exporter parameters like
+        # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
+        # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
+        - alert: HostOutOfDiskSpace
+          expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Host out of disk space (instance {{ $labels.instance }})
+            description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+        # https://awesome-prometheus-alerts.grep.to/rules#rule-host-and-hardware-1-9
+        - alert: HostOutOfInodes
+          expr: node_filesystem_files_free / node_filesystem_files * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: Host out of inodes (instance {{ $labels.instance }})
+            description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"