2023-04-12

5 files changed, 359 insertions, 420 deletions

diff --git a/modules/nixos/monitoring/default.nix b/modules/nixos/monitoring/default.nix
index a492a47..37e34d9 100644
--- a/modules/nixos/monitoring/default.nix
+++ b/modules/nixos/monitoring/default.nix
@@ -124,146 +124,77 @@ in {
       prometheus = {
         # It would be nice if these could be generated dynamically. That would
         # require a complete rework of how configurations are defined, though.
-        scrapeConfigs = let
-          mkTargets = hosts: port: map (host: "${host.hostname}:${toString port}") hosts;
-        in
-          with my.configurations;
-          with config.services.prometheus.exporters; [
-            {
-              job_name = "promtail";
-              static_configs = [
-                {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                      varda
-                      yavanna
-                    ]
-                    config.nixfiles.modules.promtail.port;
-                }
-              ];
-            }
-            {
-              job_name = "ntfy";
-              static_configs = [
-                {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                    ]
-                    config.nixfiles.modules.ntfy.prometheus.port;
-                }
-              ];
-            }
-            {
-              job_name = "soju";
-              static_configs = [
-                {
-                  targets = [
-                    "127.0.0.1:${toString config.nixfiles.modules.soju.prometheus.port}"
-                  ];
-                }
-              ];
-            }
-            {
-              job_name = "endlessh-go";
-              static_configs = [
-                {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                      varda
-                      yavanna
-                    ]
-                    config.services.endlessh-go.prometheus.port;
-                }
-              ];
-            }
-            {
-              job_name = "nginx";
-              static_configs = [
-                {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                      yavanna
-                    ]
-                    nginx.port;
-                }
-              ];
-            }
-            {
-              job_name = "node";
-              static_configs = [
-                {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                      varda
-                      yavanna
-                    ]
-                    node.port;
-                }
-              ];
-            }
-            {
-              job_name = "postgres";
+        scrapeConfigs = with my.configurations;
+          mapAttrsToList
+          (
+            name: value: {
+              job_name = name;
               static_configs = [
                 {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                    ]
-                    postgres.port;
+                  targets = with value;
+                    map (host:
+                      concatStringsSep ":" [
+                        (
+                          if isAttrs host
+                          then host.hostname
+                          else host
+                        )
+                        (toString port)
+                      ])
+                    hosts;
                 }
               ];
-            }
-            {
-              job_name = "redis";
-              static_configs = [
+              relabel_configs = [
                 {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                    ]
-                    redis.port;
+                  source_labels = ["__address__"];
+                  regex = "([^:]+):\\d+";
+                  target_label = "instance";
                 }
               ];
             }
-            {
-              job_name = "unbound";
-              static_configs = [
-                {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                    ]
-                    unbound.port;
-                }
-              ];
-            }
-            {
-              job_name = "wireguard";
-              static_configs = [
-                {
-                  targets =
-                    mkTargets
-                    [
-                      manwe
-                    ]
-                    wireguard.port;
-                }
-              ];
-            }
-          ];
+          )
+          {
+            promtail = {
+              hosts = [manwe varda yavanna];
+              inherit (config.nixfiles.modules.promtail) port;
+            };
+            ntfy = {
+              hosts = [manwe];
+              inherit (config.nixfiles.modules.ntfy.prometheus) port;
+            };
+            soju = {
+              hosts = ["127.0.0.1"];
+              inherit (config.nixfiles.modules.soju.prometheus) port;
+            };
+            endlessh-go = {
+              hosts = [manwe varda yavanna];
+              inherit (config.services.endlessh-go.prometheus) port;
+            };
+            nginx = {
+              hosts = [manwe yavanna];
+              inherit (config.services.prometheus.exporters.nginx) port;
+            };
+            node = {
+              hosts = [manwe varda yavanna];
+              inherit (config.services.prometheus.exporters.node) port;
+            };
+            postgres = {
+              hosts = [manwe];
+              inherit (config.services.prometheus.exporters.postgres) port;
+            };
+            redis = {
+              hosts = [manwe];
+              inherit (config.services.prometheus.exporters.redis) port;
+            };
+            unbound = {
+              hosts = [manwe];
+              inherit (config.services.prometheus.exporters.unbound) port;
+            };
+            wireguard = {
+              hosts = [manwe];
+              inherit (config.services.prometheus.exporters.wireguard) port;
+            };
+          };
 
         ruleFiles = [
           ./rules/nginx.yaml
diff --git a/modules/nixos/monitoring/rules/nginx.yaml b/modules/nixos/monitoring/rules/nginx.yaml
index 59229a8..f00d372 100644
--- a/modules/nixos/monitoring/rules/nginx.yaml
+++ b/modules/nixos/monitoring/rules/nginx.yaml
@@ -6,47 +6,55 @@ groups:
       - alert: NginxHighHttp4xxErrorRate
         expr: >-
           sum(rate(nginx_http_requests_total{status=~"^4.."}[1m]))
-          / sum(rate(nginx_http_requests_total[1m])) * 100
+          /
+          sum(rate(nginx_http_requests_total[1m])) * 100
           > 5
         for: 1m
         labels:
           severity: critical
         annotations:
-          summary: NGINX high HTTP 4xx error rate (instance {{ $labels.instance }})
+          summary: NGINX high HTTP 4xx error rate at {{ $labels.instance }}.
           description: |-
-            Too many HTTP requests with status 4xx (> 5%).
+            Too many HTTP requests with a 4xx status code.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: NginxHighHttp5xxErrorRate
         expr: >-
           sum(rate(nginx_http_requests_total{status=~"^5.."}[1m]))
-          / sum(rate(nginx_http_requests_total[1m])) * 100
+          /
+          sum(rate(nginx_http_requests_total[1m])) * 100
           > 5
         for: 1m
         labels:
           severity: critical
         annotations:
-          summary: NGINX high HTTP 5xx error rate (instance {{ $labels.instance }})
+          summary: NGINX high HTTP 5xx error rate at {{ $labels.instance }}.
           description: |-
-            Too many HTTP requests with status 5xx (> 5%).
+            Too many HTTP requests with a 5xx status code.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: NginxLatencyHigh
         expr: >-
-          histogram_quantile(0.99,
-            sum(rate(nginx_http_request_duration_seconds_bucket[2m])) by (host, node))
+          histogram_quantile(
+            0.99,
+            sum(
+              rate(
+                nginx_http_request_duration_seconds_bucket[2m]
+              )
+            ) by (host, node)
+          )
           > 3
         for: 2m
         labels:
           severity: warning
         annotations:
-          summary: NGINX high latency (instance {{ $labels.instance }})
+          summary: NGINX high latency at {{ $labels.instance }}.
           description: |-
-            NGINX 99% latency is higher than 3 seconds.
+            NGINX 99% of latency spikes is higher than 3 seconds.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
diff --git a/modules/nixos/monitoring/rules/node.yaml b/modules/nixos/monitoring/rules/node.yaml
index 81d7810..98217b3 100644
--- a/modules/nixos/monitoring/rules/node.yaml
+++ b/modules/nixos/monitoring/rules/node.yaml
@@ -13,12 +13,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host out of memory (instance {{ $labels.instance }})
+          summary: Host out of memory at {{ $labels.instance }}.
           description: |-
-            Node memory is filling up (< 10% left).
+            Node memory is filling up.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostMemoryUnderMemoryPressure
         expr: >-
@@ -27,13 +27,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host memory under memory pressure (instance {{ $labels.instance }})
+          summary: Host memory under memory pressure at {{ $labels.instance }}.
           description: |-
-            The node is under heavy memory pressure. High rate of major page
-            faults.
+            The node is under heavy memory pressure. High rate of major page faults.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostUnusualNetworkThroughputIn
         expr: >-
@@ -47,10 +46,10 @@ groups:
         annotations:
           summary: Host unusual network throughput in (instance {{ $labels.instance }})
           description: |-
-            Host network interfaces are probably receiving too much data (> 100 MB/s).
+            Host network interfaces are probably receiving too much data.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostUnusualNetworkThroughputOut
         expr: >-
@@ -62,12 +61,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host unusual network throughput out (instance {{ $labels.instance }})
+          summary: Host unusual network throughput out at {{ $labels.instance }}.
           description: |-
-            Host network interfaces are probably sending too much data (> 100 MB/s).
+            Host network interfaces are probably sending too much data.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostUnusualDiskReadRate
         expr: >-
@@ -79,12 +78,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host unusual disk read rate (instance {{ $labels.instance }})
+          summary: Host unusual disk read rate at {{ $labels.instance }}.
           description: |-
-            Disk is probably reading too much data (> 50 MB/s).
+            Disk is probably reading too much data.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostUnusualDiskWriteRate
         expr: >-
@@ -96,12 +95,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host unusual disk write rate (instance {{ $labels.instance }})
+          summary: Host unusual disk write rate at {{ $labels.instance }}.
           description: |-
-            Disk is probably writing too much data (> 50 MB/s).
+            Disk is probably writing too much data.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostOutOfDiskSpace
         expr: >-
@@ -109,18 +108,18 @@ groups:
           / node_filesystem_size_bytes
           < 10
           and
-          ON (instance, device, mountpoint) node_filesystem_readonly
+          on (instance, device, mountpoint) node_filesystem_readonly
           == 0
         for: 2m
         labels:
           severity: warning
         annotations:
-          summary: Host out of disk space (instance {{ $labels.instance }})
+          summary: Host out of disk space at {{ $labels.instance }}.
           description: |-
-            Disk is almost full (< 10% left).
+            Disk is almost full.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostDiskWillFillIn24Hours
         expr: >-
@@ -136,13 +135,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
+          summary: Host disk will fill in 24 hours at {{ $labels.instance }}.
           description: |-
-            Filesystem is predicted to run out of space within the next 24 hours
-            at current write rate.
+            Filesystem is predicted to run out of space within the next 24 hours at current write rate.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostOutOfInodes
         expr: >-
@@ -156,12 +154,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host out of inodes (instance {{ $labels.instance }})
+          summary: Host out of inodes at {{ $labels.instance }}.
           description: |-
-            Disk is almost running out of available inodes (< 10% left).
+            Disk is almost running out of available inodes.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostInodesWillFillIn24Hours
         expr: >-
@@ -178,13 +176,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host inodes will fill in 24 hours (instance {{ $labels.instance }})
+          summary: Host inodes will fill in 24 hours at {{ $labels.instance }}.
           description: |-
-            Filesystem is predicted to run out of inodes within the next 24
-            hours at current write rate.
+            Filesystem is predicted to run out of inodes within the next 24 hours at current write rate.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostUnusualDiskReadLatency
         expr: >-
@@ -198,12 +195,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host unusual disk read latency (instance {{ $labels.instance }})
+          summary: Host unusual disk read latency at {{ $labels.instance }}.
           description: |-
-            Disk latency is growing (read operations > 100ms).
+            Disk latency is growing.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostUnusualDiskWriteLatency
         expr: >-
@@ -217,12 +214,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host unusual disk write latency (instance {{ $labels.instance }})
+          summary: Host unusual disk write latency at {{ $labels.instance }}.
           description: |-
-            Disk latency is growing (write operations > 100ms).
+            Disk latency is growing.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostHighCpuLoad
         expr: >-
@@ -234,23 +231,23 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host high CPU load (instance {{ $labels.instance }})
+          summary: Host high CPU load at {{ $labels.instance }}.
           description: |-
-            CPU load is > 80%.
+            CPU load is high.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostCpuStealNoisyNeighbor
         expr: >-
           avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m]))
           * 100
-          > 10
+          > 15
         for: 0m
         labels:
           severity: warning
         annotations:
-          summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
+          summary: Host CPU steal noisy neighbor at {{ $labels.instance }}.
           description: |-
             CPU steal is > 10%. A noisy neighbor is killing VM performances or a
             spot instance may be out of credit.
@@ -262,18 +259,17 @@ groups:
         expr: |-
           avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m]))
           * 100
-          > 10
+          > 15
         for: 0m
         labels:
           severity: warning
         annotations:
-          summary: Host CPU high iowait (instance {{ $labels.instance }})
+          summary: Host CPU high I/O wait at {{ $labels.instance }}.
           description: |-
-            CPU iowait > 10%. A high iowait means that you are disk or network
-            bound.
+            CPU I/O wait is high. A high I/O wait means that you are disk or network bound.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostUnusualDiskIo
         expr: >-
@@ -282,12 +278,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host unusual disk IO (instance {{ $labels.instance }})
+          summary: Host unusual disk I/O at {{ $labels.instance }}.
           description: |-
-            Time spent in IO is too high on {{ $labels.instance }}. Check storage for issues.
+            Time spent over I/O is too high. Check storage for issues.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostSwapIsFillingUp
         expr: >-
@@ -298,12 +294,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host swap is filling up (instance {{ $labels.instance }})
+          summary: Host swap is filling up at {{ $labels.instance }}.
           description: |-
-            Swap is filling up (> 80%).
+            Swap is filling up.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostSystemdServiceCrashed
         expr: >-
@@ -312,12 +308,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host systemd service crashed (instance {{ $labels.instance }})
+          summary: Host systemd service crashed at {{ $labels.instance }}.
           description: |-
             Systemd service crashed.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostPhysicalComponentTooHot
         expr: >-
@@ -326,12 +322,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host physical component too hot (instance {{ $labels.instance }})
+          summary: Host physical component too hot at {{ $labels.instance }}.
           description: |-
             Physical hardware component too hot.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostNodeOvertemperatureAlarm
         expr: >-
@@ -340,12 +336,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Host node overtemperature alarm (instance {{ $labels.instance }})
+          summary: Host node overtemperature alarm at {{ $labels.instance }}.
           description: |-
             Physical node temperature alarm triggered.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostRaidArrayGotInactive
         expr: >-
@@ -354,14 +350,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Host RAID array got inactive (instance {{ $labels.instance }})
+          summary: Host RAID array got inactive at {{ $labels.instance }}.
           description: |-
-            RAID array {{ $labels.device }} is in degraded state due to one or
-            more disks failures. Number of spare drives is insufficient to fix
-            issue automatically.
+            RAID array is in a degraded state due to one or more disks failures. Number of spare drives is insufficient to fix the issue automatically.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostRaidDiskFailure
         expr: >-
@@ -370,14 +364,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host RAID disk failure (instance {{ $labels.instance }})
+          summary: Host RAID disk failure at {{ $labels.instance }}.
           description: |-
-            At least one device in RAID array on {{ $labels.instance }} failed.
-            Array {{ $labels.md_device }} needs attention and possibly a disk
-            swap.
+            At least one device in RAID array is failed. Possibly, a disk swap is required.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostOomKillDetected
         expr: >-
@@ -386,12 +378,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host OOM kill detected (instance {{ $labels.instance }})
+          summary: Host OOM kill detected at {{ $labels.instance }}.
           description: |-
             OOM kill detected.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostEdacCorrectableErrorsDetected
         expr: >-
@@ -400,13 +392,12 @@ groups:
         labels:
           severity: info
         annotations:
-          summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})
+          summary: Host EDAC correctable errors detected at {{ $labels.instance }}.
           description: |-
-            Host {{ $labels.instance }} has had {{ printf "%.0f" $value }}
-            correctable memory errors reported by EDAC in the last 5 minutes.
+            Host has had {{ printf "%.0f" $value }} correctable memory errors reported by EDAC in the last 5 minutes.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostEdacUncorrectableErrorsDetected
         expr: >-
@@ -415,66 +406,67 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
+          summary: Host EDAC uncorrectable errors detected at {{ $labels.instance }}.
           description: |-
-            Host {{ $labels.instance }} has had {{ printf "%.0f" $value }}
-            uncorrectable memory errors reported by EDAC in the last 5
-            minutes.
+            Host has had {{ printf "%.0f" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostNetworkReceiveErrors
-        expr: "rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01"
+        expr: >-
+          rate(node_network_receive_errs_total{device!~"^wg.*"}[2m])
+          /
+          rate(node_network_receive_packets_total{device!~"^wg.*"}[2m])
+          > 0.01
         for: 2m
         labels:
           severity: warning
         annotations:
-          summary: Host Network Receive Errors (instance {{ $labels.instance }})
+          summary: Host Network Receive Errors at {{ $labels.instance }}.
           description: |-
-            Host {{ $labels.instance }} interface {{ $labels.device }} has
-            encountered {{ printf "%.0f" $value }} receive errors in the last
-            two minutes.
+            Host interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostNetworkTransmitErrors
-        expr: "rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01"
+        expr: >-
+          rate(node_network_transmit_errs_total{device!~"^wg.*"}[2m])
+          /
+          rate(node_network_transmit_packets_total{device!~"^wg.*"}[2m])
+          > 0.1
         for: 2m
         labels:
           severity: warning
         annotations:
-          summary: Host Network Transmit Errors (instance {{ $labels.instance }})
+          summary: Host network transmit errors at {{ $labels.instance }}.
           description: |-
-            Host {{ $labels.instance }} interface {{ $labels.device }} has
-            encountered {{ printf "%.0f" $value }} transmit errors in the last
-            two minutes.
+            Host interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostNetworkInterfaceSaturated
         expr: >-
           (
-            rate(node_network_receive_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])
+            rate(node_network_receive_bytes_total{device!~"^wg.*"}[1m])
             +
-            rate(node_network_transmit_bytes_total{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}[1m])
+            rate(node_network_transmit_bytes_total{device!~"^wg.*"}[1m])
           )
-          / node_network_speed_bytes{device!~"^tap.*|^vnet.*|^veth.*|^tun.*"}
+          / node_network_speed_bytes{device!~"^wg.*"}
           > 0.8
           < 10000
         for: 1m
         labels:
           severity: warning
         annotations:
-          summary: Host Network Interface Saturated (instance {{ $labels.instance }})
+          summary: Host network interface saturated at {{ $labels.instance }}.
           description: |-
-            The network interface "{{ $labels.device }}" on "{{ $labels.instance }}"
-            is getting overloaded.
+            The network interface {{ $labels.device }} is getting overloaded.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostNetworkBondDegraded
         expr: >-
@@ -483,43 +475,53 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host Network Bond Degraded (instance {{ $labels.instance }})
+          summary: Host network bond degraded at {{ $labels.instance }}.
           description: |-
-            Bond "{{ $labels.device }}" degraded on "{{ $labels.instance }}".
+            Bond {{ $labels.device }} degraded.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostConntrackLimit
         expr: >-
-          node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
+          node_nf_conntrack_entries
+          /
+          node_nf_conntrack_entries_limit
+          > 0.8
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: Host conntrack limit (instance {{ $labels.instance }})
+          summary: Host conntrack limit at {{ $labels.instance }}.
           description: |-
             The number of conntrack is approaching limit.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostClockSkew
         expr: >-
-          (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0)
+          (
+            node_timex_offset_seconds > 0.05
+            and
+            deriv(node_timex_offset_seconds[5m]) >= 0
+          )
           or
-          (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
+          (
+            node_timex_offset_seconds < -0.05
+            and
+            deriv(node_timex_offset_seconds[5m]) <= 0
+          )
         for: 2m
         labels:
           severity: warning
         annotations:
-          summary: Host clock skew (instance {{ $labels.instance }})
+          summary: Host clock skew at {{ $labels.instance }}.
           description: |-
-            Clock skew detected. Clock is out of sync. Ensure NTP is configured
-            correctly on this host.
+            Clock skew is detected and the clock is out of sync. Ensure that NTP is configured correctly on this host.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostClockNotSynchronising
         expr: >-
@@ -530,12 +532,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Host clock not synchronising (instance {{ $labels.instance }})
+          summary: Host clock not synchronising at {{ $labels.instance }}.
           description: |-
-            Clock not synchronising. Ensure NTP is configured on this host.
+            Clock is not synchronising. Ensure that NTP is configured correctly on this host.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: HostRequiresReboot
         expr: >-
@@ -544,9 +546,9 @@ groups:
         labels:
           severity: info
         annotations:
-          summary: Host requires reboot (instance {{ $labels.instance }})
+          summary: Host requires reboot at {{ $labels.instance }}.
           description: |-
-            Instance {{ $labels.instance }} requires a reboot.
+            Instance requires a reboot.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
diff --git a/modules/nixos/monitoring/rules/postgres.yaml b/modules/nixos/monitoring/rules/postgres.yaml
index 5d360fa..6aee560 100644
--- a/modules/nixos/monitoring/rules/postgres.yaml
+++ b/modules/nixos/monitoring/rules/postgres.yaml
@@ -10,12 +10,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: PostgreSQL is down (instance {{ $labels.instance }})
+          summary: PostgreSQL is down at {{ $labels.instance }}.
           description: |-
-            Postgresql instance is down.
+            PostgreSQL instance is down.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlRestarted
         expr: >-
@@ -24,12 +24,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: PostgreSQL restarted (instance {{ $labels.instance }})
+          summary: PostgreSQL restarted at {{ $labels.instance }}.
           description: |-
             PostgreSQL restarted.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlExporterError
         expr: >-
@@ -38,12 +38,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: PostgreSQL exporter error (instance {{ $labels.instance }})
+          summary: PostgreSQL exporter errors at {{ $labels.instance }}.
           description: |-
-            PostgreSQL exporter is showing errors. A query may be buggy in query.yaml.
+            PostgreSQL exporter is showing errors.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlTableNotAutoVacuumed
         expr: >-
@@ -55,12 +55,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL table not auto vacuumed (instance {{ $labels.instance }})
+          summary: PostgreSQL table not auto vacuumed at {{ $labels.instance }}.
           description: |-
             Table {{ $labels.relname }} has not been auto vacuumed for 10 days.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlTableNotAutoAnalyzed
         expr: >-
@@ -72,57 +72,60 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL table not auto analyzed (instance {{ $labels.instance }})
+          summary: PostgreSQL table not auto analyzed at {{ $labels.instance }}.
           description: |-
             Table {{ $labels.relname }} has not been auto analyzed for 10 days.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}"
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}"
 
       - alert: PostgresqlTooManyConnections
         expr: >-
-          sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"})
-          > pg_settings_max_connections * 0.8
+          sum by (datname) (
+            pg_stat_activity_count{datname!~"template.*|postgres"}
+          ) > pg_settings_max_connections * 0.8
         for: 2m
         labels:
           severity: warning
         annotations:
-          summary: Postgresql too many connections (instance {{ $labels.instance }})
+          summary: PostgreSQL with too many connections at {{ $labels.instance }}.
           description: |-
-            PostgreSQL instance has too many connections (> 80%).
+            PostgreSQL instance {{ $labels.instance }} has too many connections.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlNotEnoughConnections
         expr: >-
-          sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"})
-          < 1
+          sum by (datname) (
+            pg_stat_activity_count{datname!~"template.*|postgres"}
+          ) < 1
         for: 2m
         labels:
           severity: warning
         annotations:
-          summary: Postgresql not enough connections (instance {{ $labels.instance }})
+          summary: PostgreSQL with not enough connections at {{ $labels.instance }}.
           description: |-
-            PostgreSQL instance should have more connections (> 1).
+            PostgreSQL instance {{ $labels.instance }} should have more connections.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlDeadLocks
         expr: >-
-          increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m])
-          > 5
+          increase(
+            pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]
+          ) > 5
         for: 0m
         labels:
           severity: warning
         annotations:
-          summary: Postgresql dead locks (instance {{ $labels.instance }})
+          summary: PostgreSQL dead-locks at instance {{ $labels.instance }}.
           description: |-
-            PostgreSQL has dead-locks.
+            PostgreSQL shows dead-locks.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlHighRollbackRate
         expr: >-
@@ -136,17 +139,17 @@ groups:
                 (rate(pg_stat_database_xact_commit{datname!~"template.*|postgres",datid!="0"}[3m]))
               )
             )
-          > 0.02
+          > 0.10
         for: 0m
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL is at a high rollback rate (instance {{ $labels.instance }})
+          summary: PostgreSQL at a high rollback rate at {{ $labels.instance }}.
           description: |-
-            Ratio of transactions being aborted compared to committed is > 2%.
+            Ratio of transactions being aborted compared to committed is too big.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlCommitRateLow
         expr: >-
@@ -156,12 +159,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: PostgreSQL commit rate low (instance {{ $labels.instance }})
+          summary: PostgreSQL commit rate low at instance {{ $labels.instance }}.
           description: |-
             PostgreSQL seems to be processing very few transactions.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlLowXidConsumption
         expr: >-
@@ -171,12 +174,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL low XID consumption (instance {{ $labels.instance }})
+          summary: PostgreSQL low XID consumption at instance {{ $labels.instance }}.
           description: |-
             PostgreSQL seems to be consuming transaction IDs very slowly.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlHighRateStatementTimeout
         expr: >-
@@ -190,8 +193,8 @@ groups:
           description: |-
             PostgreSQL transactions showing high rate of statement timeouts.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlHighRateDeadlock
         expr: >-
@@ -201,12 +204,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: PostgreSQL high rate deadlock (instance {{ $labels.instance }})
+          summary: PostgreSQL high rate dead-lock at {{ $labels.instance }}.
           description: |-
-            PostgreSQL detected deadlocks.
+            PostgreSQL has detected dead-locks.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlUnusedReplicationSlot
         expr: >-
@@ -215,12 +218,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL unused replication slot (instance {{ $labels.instance }})
+          summary: PostgreSQL unused replication slot at {{ $labels.instance }}.
           description: |-
-            Unused Replication Slots.
+            Unused replication slots.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlTooManyDeadTuples
         expr: >-
@@ -234,12 +237,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL too many dead tuples (instance {{ $labels.instance }})
+          summary: PostgreSQL too many dead tuples at {{ $labels.instance }}.
           description: |-
             PostgreSQL number of dead tuples is too large.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlSslCompressionActive
         expr: >-
@@ -248,13 +251,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Postgresql SSL compression active (instance {{ $labels.instance }})
+          summary: PostgreSQL SSL compression active at {{ $labels.instance }}.
           description: |-
-            Database connections with SSL compression is enabled. This may add a
-            significant jitter in the replication delay.
+            Database connections with an SSL compression is enabled. This may add a significant jitter in the replication delay.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlTooManyLocksAcquired
         expr: >-
@@ -268,12 +270,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: PostgreSQL too many locks acquired (instance {{ $labels.instance }})
+          summary: PostgreSQL too many locks acquired at {{ $labels.instance }}.
           description: |-
             Too many locks acquired on the database.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlBloatIndexHigh
         expr: >-
@@ -284,13 +286,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL bloat index high (> 80%) (instance {{ $labels.instance }})
+          summary: PostgreSQL index bloat high at {{ $labels.instance }}.
           description: |-
-            The index {{ $labels.idxname }} is bloated. You should execute
-            `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`
+            The index {{ $labels.idxname }} is bloated. You should execute `REINDEX INDEX CONCURRENTLY {{ $labels.idxname }};`.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: PostgresqlBloatTableHigh
         expr: >-
@@ -301,10 +302,9 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: PostgreSQL bloat table high (> 80%) (instance {{ $labels.instance }})
+          summary: PostgreSQL table bloat high at instance {{ $labels.instance }}.
           description: |-
-            The table {{ $labels.relname }} is bloated. You should execute
-            `VACUUM {{ $labels.relname }};`
+            The table {{ $labels.relname }} is bloated. You should execute `VACUUM {{ $labels.relname }};`.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
diff --git a/modules/nixos/monitoring/rules/redis.yaml b/modules/nixos/monitoring/rules/redis.yaml
index f6d1fe1..c07c819 100644
--- a/modules/nixos/monitoring/rules/redis.yaml
+++ b/modules/nixos/monitoring/rules/redis.yaml
@@ -10,12 +10,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis down (instance {{ $labels.instance }})
+          summary: Redis down at {{ $labels.instance }}.
           description: |-
             Redis instance is down.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisMissingMaster
         expr: >-
@@ -25,12 +25,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis missing master (instance {{ $labels.instance }})
+          summary: Redis missing master at {{ $labels.instance }}).
           description: |-
-            Redis cluster has no node marked as master.
+            Redis cluster has no node marked as a master.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisTooManyMasters
         expr: >-
@@ -39,12 +39,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis too many masters (instance {{ $labels.instance }})
+          summary: Redis too many masters at {{ $labels.instance }}.
           description: |-
-            Redis cluster has too many nodes marked as master.
+            Redis cluster has too many nodes marked as a master.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisDisconnectedSlaves
         expr: >-
@@ -56,12 +56,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis disconnected slaves (instance {{ $labels.instance }})
+          summary: Redis disconnected slaves at {{ $labels.instance }}.
           description: |-
             Redis is not replicating for all slaves.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisReplicationBroken
         expr: >-
@@ -70,12 +70,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis replication broken (instance {{ $labels.instance }})
+          summary: Redis replication broken at {{ $labels.instance }}.
           description: |-
             Redis instance lost a slave.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisClusterFlapping
         expr: >-
@@ -84,14 +84,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis cluster flapping (instance {{ $labels.instance }})
+          summary: Redis cluster flapping at {{ $labels.instance }}.
           description: |-
-            Changes have been detected in the Redis replica connection. This can
-            occur when replica nodes lose connection to the master and reconnect
-            (a.k.a flapping).
+            Changes have been detected in the Redis replica connection. This can occur when replica nodes lose connection to the master and reconnect (a.k.a flapping).
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisMissingBackup
         expr: >-
@@ -101,12 +99,12 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis missing backup (instance {{ $labels.instance }})
+          summary: Redis missing backup at {{ $labels.instance }}.
           description: |-
             Redis has not been backed up for 24 hours.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisOutOfSystemMemory
         expr: >-
@@ -118,12 +116,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Redis out of system memory (instance {{ $labels.instance }})
+          summary: Redis out of system memory at {{ $labels.instance }}.
           description: |-
-            Redis is running out of system memory (> 90%).
+            Redis is running out of system memory.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisOutOfConfiguredMaxmemory
         expr: >-
@@ -139,12 +137,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Redis out of configured maxmemory (instance {{ $labels.instance }})
+          summary: Redis out of configured maxmemory at {{ $labels.instance }}.
           description: |-
-            Redis is running out of configured maxmemory (> 90%).
+            Redis is running out of configured maxmemory.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisTooManyConnections
         expr: >-
@@ -153,12 +151,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Redis too many connections (instance {{ $labels.instance }})
+          summary: Redis too many connections at {{ $labels.instance }}.
           description: |-
             Redis instance has too many connections.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisNotEnoughConnections
         expr: >-
@@ -167,12 +165,12 @@ groups:
         labels:
           severity: warning
         annotations:
-          summary: Redis not enough connections (instance {{ $labels.instance }})
+          summary: Redis not enough connections at {{ $labels.instance }}.
           description: |-
-            Redis instance should have more connections (> 1).
+            Redis instance should have more connections.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}
 
       - alert: RedisRejectedConnections
         expr: >-
@@ -181,9 +179,9 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: Redis rejected connections (instance {{ $labels.instance }})
+          summary: Redis rejected connections at {{ $labels.instance }}.
           description: |-
             Some connections to Redis have been rejected.
 
-              VALUE = {{ $value }}
-              LABELS = {{ $labels }}
+            VALUE = {{ $value }}
+            LABELS = {{ $labels }}