{ config, lib, pkgs, ... }: with lib; let cfg = config.nixfiles.modules.monitoring; in { options.nixfiles.modules.monitoring.enable = mkEnableOption '' a glue to provision a monitoring stack ''; config = mkIf cfg.enable { nixfiles.modules = { alertmanager.enable = true; grafana.enable = true; loki.enable = false; # FIXME High RAM usage. prometheus.enable = true; }; services = { grafana = { declarativePlugins = with pkgs.grafanaPlugins; [ redis-app redis-datasource redis-explorer-app ]; provision = { enable = true; # https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources datasources.settings.datasources = with config.nixfiles.modules; [ (mkIf alertmanager.enable { name = "Alertmanager"; type = "alertmanager"; access = "proxy"; url = "https://${alertmanager.domain}"; jsonData.implementation = "prometheus"; }) (mkIf loki.enable { name = "Loki"; type = "loki"; access = "proxy"; url = "https://${loki.domain}"; isDefault = true; }) (mkIf prometheus.enable { name = "Prometheus"; type = "prometheus"; access = "proxy"; url = "https://${prometheus.domain}"; }) (mkIf redis.enable { name = "Redis"; type = "redis-datasource"; access = "proxy"; url = with config.services.redis.servers.default; "redis://${bind}:${toString port}"; jsonData.client = "standalone"; }) ]; datasources.settings.deleteDatasources = with config.nixfiles.modules; [ (mkIf (!alertmanager.enable) { name = "Alertmanager"; orgId = 1; }) (mkIf (!loki.enable) { name = "Loki"; orgId = 1; }) (mkIf (!prometheus.enable) { name = "Prometheus"; orgId = 1; }) (mkIf (!redis.enable) { name = "Redis"; orgId = 1; }) ]; # https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards dashboards.settings.providers = [ { name = "node"; options.path = ./dashboards/node.json; } { name = "ntfy"; options.path = ./dashboards/ntfy.json; } { name = "endlessh"; options.path = ./dashboards/endlessh.json; } { name = "unbound"; options.path = ./dashboards/unbound.json; } { name = "nginx"; options.path = ./dashboards/nginx.json; } { name = "postgersql"; options.path = ./dashboards/postgresql.json; } { name = "redis"; options.path = ./dashboards/redis.json; } ]; # https://grafana.com/docs/grafana/latest/administration/provisioning/#alerting alerting.contactPoints.settings.contactPoints = [ { name = "Alertmanager"; receivers = [ { uid = 1; type = "prometheus-alertmanager"; settings.url = "https://${config.nixfiles.modules.alertmanager.domain}"; } ]; } ]; }; }; loki.configuration.ruler = { external_url = "https://${config.nixfiles.modules.grafana.domain}"; alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}"; enable_alertmanager_v2 = true; }; prometheus = { scrapeConfigs = with my.configurations; mapAttrsToList (name: value: { job_name = name; static_configs = [ { targets = with value; map ( host: concatStringsSep ":" [ (if isAttrs host then host.hostname else host) (toString port) ] ) hosts; } ]; relabel_configs = [ { source_labels = [ "__address__" ]; regex = "([^:]+):\\d+"; target_label = "instance"; } ] ++ optionals (hasAttr "relabel" value) value.relabel; }) { promtail = { hosts = [ manwe tulkas varda yavanna ]; inherit (config.nixfiles.modules.promtail) port; }; ntfy = { hosts = [ manwe ]; inherit (config.nixfiles.modules.ntfy.prometheus) port; }; soju = { hosts = [ "127.0.0.1" ]; inherit (config.nixfiles.modules.soju.prometheus) port; }; endlessh-go = { hosts = [ manwe tulkas varda yavanna ]; inherit (config.services.endlessh-go.prometheus) port; }; nginx = { hosts = [ manwe yavanna ]; inherit (config.services.prometheus.exporters.nginx) port; }; node = { hosts = [ manwe tulkas varda yavanna ]; inherit (config.services.prometheus.exporters.node) port; }; postgres = { hosts = [ manwe ]; inherit (config.services.prometheus.exporters.postgres) port; }; redis = { hosts = [ manwe ]; inherit (config.services.prometheus.exporters.redis) port; }; unbound = { hosts = [ manwe ]; inherit (config.services.prometheus.exporters.unbound) port; }; wireguard = { hosts = [ manwe ]; inherit (config.services.prometheus.exporters.wireguard) port; }; exportarr-lidarr = { hosts = [ yavanna ]; inherit (config.services.prometheus.exporters.exportarr-lidarr) port; }; }; ruleFiles = [ ./rules/nginx.yaml ./rules/node.yaml ./rules/postgres.yaml ./rules/redis.yaml ]; alertmanagers = [ { scheme = "https"; static_configs = [ { targets = [ config.nixfiles.modules.alertmanager.domain ]; } ]; } ]; }; }; }; }