{ config, lib, pkgs, ... }: with lib; let cfg = config.nixfiles.modules.monitoring; in { options.nixfiles.modules.monitoring.enable = mkEnableOption '' a custom monitoring stack bas on the Grafana Labs toolkit ''; config = mkIf cfg.enable { nixfiles.modules = { grafana.enable = true; loki.enable = true; prometheus.enable = true; alertmanager.enable = true; }; services = { grafana.provision = { enable = true; # https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources datasources.settings.datasources = with config.nixfiles.modules; [ { name = "Prometheus"; type = "prometheus"; access = "proxy"; url = "https://${prometheus.domain}"; isDefault = true; } { name = "Loki"; type = "loki"; access = "proxy"; url = "https://${loki.domain}"; } { name = "Alertmanager"; type = "alertmanager"; access = "proxy"; jsonData.implementation = "prometheus"; url = "https://${alertmanager.domain}"; } ]; # https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards dashboards.settings.providers = [ # System dashboard is imported manually from here[1]. Too bad # provisioned dashboards cannot properly integrate dynamic datasources # yet. # # [1]: https://grafana.com/grafana/dashboards/1860-node-exporter-full { name = "endlessh"; options.path = ./dashboards/endlessh.json; } { name = "unbound"; options.path = ./dashboards/unbound.json; } { name = "nginx"; options.path = ./dashboards/nginx.json; } { name = "postgersql"; options.path = ./dashboards/postgresql.json; } ]; }; loki.configuration.ruler.alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}"; prometheus = { # It would be nice if these could be generated dynamically. That would # require a complete rework of how configurations are defined, though. scrapeConfigs = let mkTargets = hosts: port: map (host: "${host.hostname}:${toString port}") hosts; in with my.configurations; with config.services.prometheus.exporters; [ { job_name = "endlessh-go"; static_configs = [ { targets = mkTargets [ manwe varda yavanna ] config.services.endlessh-go.prometheus.port; } ]; } { job_name = "nginx"; static_configs = [ { targets = mkTargets [ manwe varda yavanna ] nginx.port; } ]; } { job_name = "node"; static_configs = [ { targets = mkTargets [ manwe varda yavanna ] node.port; } ]; } { job_name = "postgres"; static_configs = [ { targets = mkTargets [ manwe ] postgres.port; } ]; } { job_name = "unbound"; static_configs = [ { targets = mkTargets [ manwe ] unbound.port; } ]; } { job_name = "wireguard"; static_configs = [ { targets = mkTargets [ manwe ] wireguard.port; } ]; } ]; alertmanagers = [ { scheme = "https"; static_configs = [ {targets = [config.nixfiles.modules.alertmanager.domain];} ]; } ]; }; }; }; }