{ config, lib, pkgs, ... }: with lib; let cfg = config.nixfiles.modules.monitoring; in { options.nixfiles.modules.monitoring.enable = mkEnableOption '' a glue to provision a monitoring stack ''; config = mkIf cfg.enable { nixfiles.modules = { alertmanager.enable = true; grafana.enable = true; loki.enable = true; prometheus.enable = true; }; services = { grafana = { declarativePlugins = with pkgs.grafanaPlugins; [ redis-app redis-datasource redis-explorer-app ]; provision = { enable = true; # https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources datasources.settings.datasources = with config.nixfiles.modules; [ { name = "Alertmanager"; type = "alertmanager"; access = "proxy"; jsonData.implementation = "prometheus"; url = "https://${alertmanager.domain}"; } { name = "Loki"; type = "loki"; access = "proxy"; url = "https://${loki.domain}"; isDefault = true; } { name = "Prometheus"; type = "prometheus"; access = "proxy"; url = "https://${prometheus.domain}"; } (mkIf config.nixfiles.modules.redis.enable { name = "Redis"; type = "redis-datasource"; access = "proxy"; url = with config.services.redis.servers.default; "redis://${bind}:${toString port}"; jsonData.client = "standalone"; }) ]; datasources.settings.deleteDatasources = [ { name = "PostgreSQL"; orgId = 1; } ]; # https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards dashboards.settings.providers = [ { name = "node"; options.path = ./dashboards/node.json; } { name = "ntfy"; options.path = ./dashboards/ntfy.json; } { name = "endlessh"; options.path = ./dashboards/endlessh.json; } { name = "unbound"; options.path = ./dashboards/unbound.json; } { name = "nginx"; options.path = ./dashboards/nginx.json; } { name = "postgersql"; options.path = ./dashboards/postgresql.json; } { name = "redis"; options.path = ./dashboards/redis.json; } { name = "redis-streaming"; options.path = ./dashboards/redis-streaming.json; } ]; # https://grafana.com/docs/grafana/latest/administration/provisioning/#alerting alerting.contactPoints.settings.contactPoints = [ { name = "Alertmanager"; receivers = [ { uid = 1; type = "prometheus-alertmanager"; settings.url = "https://${config.nixfiles.modules.alertmanager.domain}"; } ]; } ]; }; }; loki.configuration.ruler.alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}"; prometheus = { scrapeConfigs = with my.configurations; mapAttrsToList ( name: value: { job_name = name; static_configs = [ { targets = with value; map (host: concatStringsSep ":" [ ( if isAttrs host then host.hostname else host ) (toString port) ]) hosts; } ]; relabel_configs = [ { source_labels = ["__address__"]; regex = "([^:]+):\\d+"; target_label = "instance"; } ] ++ optionals (hasAttr "relabel" value) value.relabel; } ) { promtail = { hosts = [manwe varda yavanna]; inherit (config.nixfiles.modules.promtail) port; }; ntfy = { hosts = [manwe]; inherit (config.nixfiles.modules.ntfy.prometheus) port; }; soju = { hosts = ["127.0.0.1"]; inherit (config.nixfiles.modules.soju.prometheus) port; }; endlessh-go = { hosts = [manwe varda yavanna]; inherit (config.services.endlessh-go.prometheus) port; }; nginx = { hosts = [manwe yavanna]; inherit (config.services.prometheus.exporters.nginx) port; }; node = { hosts = [manwe varda yavanna]; inherit (config.services.prometheus.exporters.node) port; }; postgres = { hosts = [manwe]; inherit (config.services.prometheus.exporters.postgres) port; }; redis = { hosts = [manwe]; inherit (config.services.prometheus.exporters.redis) port; }; unbound = { hosts = [manwe]; inherit (config.services.prometheus.exporters.unbound) port; }; wireguard = { hosts = [manwe]; inherit (config.services.prometheus.exporters.wireguard) port; }; }; ruleFiles = [ ./rules/nginx.yaml ./rules/node.yaml ./rules/postgres.yaml ./rules/redis.yaml ]; alertmanagers = [ { scheme = "https"; static_configs = [ {targets = [config.nixfiles.modules.alertmanager.domain];} ]; } ]; }; }; }; }