2024-04-21

1 files changed, 243 insertions, 0 deletions

diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix
new file mode 100644
index 0000000..6e5b782
--- /dev/null
+++ b/modules/monitoring/default.nix
@@ -0,0 +1,243 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+with lib;
+let
+  cfg = config.nixfiles.modules.monitoring;
+in
+{
+  options.nixfiles.modules.monitoring.enable = mkEnableOption ''
+    a glue to provision a monitoring stack
+  '';
+
+  config = mkIf cfg.enable {
+    nixfiles.modules = {
+      alertmanager.enable = true;
+      grafana.enable = true;
+      loki.enable = false; # FIXME High RAM usage.
+      prometheus.enable = true;
+    };
+
+    services = {
+      grafana = {
+        declarativePlugins = with pkgs.grafanaPlugins; [
+          redis-app
+          redis-datasource
+          redis-explorer-app
+        ];
+
+        provision = {
+          enable = true;
+
+          # https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources
+          datasources.settings.datasources = with config.nixfiles.modules; [
+            (mkIf alertmanager.enable {
+              name = "Alertmanager";
+              type = "alertmanager";
+              access = "proxy";
+              url = "https://${alertmanager.domain}";
+              jsonData.implementation = "prometheus";
+            })
+            (mkIf loki.enable {
+              name = "Loki";
+              type = "loki";
+              access = "proxy";
+              url = "https://${loki.domain}";
+              isDefault = true;
+            })
+            (mkIf prometheus.enable {
+              name = "Prometheus";
+              type = "prometheus";
+              access = "proxy";
+              url = "https://${prometheus.domain}";
+            })
+            (mkIf redis.enable {
+              name = "Redis";
+              type = "redis-datasource";
+              access = "proxy";
+              url = with config.services.redis.servers.default; "redis://${bind}:${toString port}";
+              jsonData.client = "standalone";
+            })
+          ];
+          datasources.settings.deleteDatasources = with config.nixfiles.modules; [
+            (mkIf (!alertmanager.enable) {
+              name = "Alertmanager";
+              orgId = 1;
+            })
+            (mkIf (!loki.enable) {
+              name = "Loki";
+              orgId = 1;
+            })
+            (mkIf (!prometheus.enable) {
+              name = "Prometheus";
+              orgId = 1;
+            })
+            (mkIf (!redis.enable) {
+              name = "Redis";
+              orgId = 1;
+            })
+          ];
+
+          # https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards
+          dashboards.settings.providers = [
+            {
+              name = "node";
+              options.path = ./dashboards/node.json;
+            }
+            {
+              name = "ntfy";
+              options.path = ./dashboards/ntfy.json;
+            }
+            {
+              name = "endlessh";
+              options.path = ./dashboards/endlessh.json;
+            }
+            {
+              name = "unbound";
+              options.path = ./dashboards/unbound.json;
+            }
+            {
+              name = "nginx";
+              options.path = ./dashboards/nginx.json;
+            }
+            {
+              name = "postgersql";
+              options.path = ./dashboards/postgresql.json;
+            }
+            {
+              name = "redis";
+              options.path = ./dashboards/redis.json;
+            }
+          ];
+
+          # https://grafana.com/docs/grafana/latest/administration/provisioning/#alerting
+          alerting.contactPoints.settings.contactPoints = [
+            {
+              name = "Alertmanager";
+              receivers = [
+                {
+                  uid = 1;
+                  type = "prometheus-alertmanager";
+                  settings.url = "https://${config.nixfiles.modules.alertmanager.domain}";
+                }
+              ];
+            }
+          ];
+        };
+      };
+
+      loki.configuration.ruler = {
+        external_url = "https://${config.nixfiles.modules.grafana.domain}";
+        alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}";
+        enable_alertmanager_v2 = true;
+      };
+
+      prometheus = {
+        scrapeConfigs =
+          with my.configurations;
+          mapAttrsToList
+            (name: value: {
+              job_name = name;
+              static_configs = [
+                {
+                  targets =
+                    with value;
+                    map (
+                      host:
+                      concatStringsSep ":" [
+                        (if isAttrs host then host.hostname else host)
+                        (toString port)
+                      ]
+                    ) hosts;
+                }
+              ];
+              relabel_configs = [
+                {
+                  source_labels = [ "__address__" ];
+                  regex = "([^:]+):\\d+";
+                  target_label = "instance";
+                }
+              ] ++ optionals (hasAttr "relabel" value) value.relabel;
+            })
+            {
+              promtail = {
+                hosts = [
+                  manwe
+                  varda
+                  yavanna
+                ];
+                inherit (config.nixfiles.modules.promtail) port;
+              };
+              ntfy = {
+                hosts = [ manwe ];
+                inherit (config.nixfiles.modules.ntfy.prometheus) port;
+              };
+              soju = {
+                hosts = [ "127.0.0.1" ];
+                inherit (config.nixfiles.modules.soju.prometheus) port;
+              };
+              endlessh-go = {
+                hosts = [
+                  manwe
+                  varda
+                  yavanna
+                ];
+                inherit (config.services.endlessh-go.prometheus) port;
+              };
+              nginx = {
+                hosts = [
+                  manwe
+                  yavanna
+                ];
+                inherit (config.services.prometheus.exporters.nginx) port;
+              };
+              node = {
+                hosts = [
+                  manwe
+                  varda
+                  yavanna
+                ];
+                inherit (config.services.prometheus.exporters.node) port;
+              };
+              postgres = {
+                hosts = [ manwe ];
+                inherit (config.services.prometheus.exporters.postgres) port;
+              };
+              redis = {
+                hosts = [ manwe ];
+                inherit (config.services.prometheus.exporters.redis) port;
+              };
+              unbound = {
+                hosts = [ manwe ];
+                inherit (config.services.prometheus.exporters.unbound) port;
+              };
+              wireguard = {
+                hosts = [ manwe ];
+                inherit (config.services.prometheus.exporters.wireguard) port;
+              };
+              exportarr-lidarr = {
+                hosts = [ yavanna ];
+                inherit (config.services.prometheus.exporters.exportarr-lidarr) port;
+              };
+            };
+
+        ruleFiles = [
+          ./rules/nginx.yaml
+          ./rules/node.yaml
+          ./rules/postgres.yaml
+          ./rules/redis.yaml
+        ];
+
+        alertmanagers = [
+          {
+            scheme = "https";
+            static_configs = [ { targets = [ config.nixfiles.modules.alertmanager.domain ]; } ];
+          }
+        ];
+      };
+    };
+  };
+}