diff options
Diffstat (limited to 'modules/monitoring/default.nix')
| -rw-r--r-- | modules/monitoring/default.nix | 196 |
1 files changed, 37 insertions, 159 deletions
diff --git a/modules/monitoring/default.nix b/modules/monitoring/default.nix index 04fcccc..7051133 100644 --- a/modules/monitoring/default.nix +++ b/modules/monitoring/default.nix @@ -14,10 +14,10 @@ in config = lib.mkIf cfg.enable { nixfiles.modules = { - alertmanager.enable = true; + loki.enable = true; grafana.enable = true; - loki.enable = false; # FIXME High RAM usage. - prometheus.enable = true; + tempo.enable = false; + mimir.enable = true; }; services = { @@ -33,25 +33,31 @@ in # https://grafana.com/docs/grafana/latest/administration/provisioning/#data-sources datasources.settings.datasources = with config.nixfiles.modules; [ - (lib.mkIf alertmanager.enable { + (lib.mkIf mimir.enable { name = "Alertmanager"; type = "alertmanager"; access = "proxy"; - url = "https://${alertmanager.domain}"; + url = "https://${mimir.domain}/alertmanager"; jsonData.implementation = "prometheus"; }) + (lib.mkIf mimir.enable { + name = "Mimir"; + type = "prometheus"; + access = "proxy"; + url = "https://${mimir.domain}/prometheus"; + isDefault = true; + }) (lib.mkIf loki.enable { name = "Loki"; type = "loki"; access = "proxy"; url = "https://${loki.domain}"; - isDefault = true; }) - (lib.mkIf prometheus.enable { - name = "Prometheus"; - type = "prometheus"; + (lib.mkIf tempo.enable { + name = "Tempo"; + type = "tempo"; access = "proxy"; - url = "https://${prometheus.domain}"; + url = "https://${tempo.domain}"; }) (lib.mkIf redis.enable { name = "Redis"; @@ -62,16 +68,20 @@ in }) ]; datasources.settings.deleteDatasources = with config.nixfiles.modules; [ - (lib.mkIf (!alertmanager.enable) { + (lib.mkIf (!mimir.enable) { name = "Alertmanager"; orgId = 1; }) + (lib.mkIf (!mimir.enable) { + name = "Mimir"; + orgId = 1; + }) (lib.mkIf (!loki.enable) { name = "Loki"; orgId = 1; }) - (lib.mkIf (!prometheus.enable) { - name = "Prometheus"; + (lib.mkIf (!tempo.enable) { + name = "Tempo"; orgId = 1; }) (lib.mkIf (!redis.enable) { @@ -80,38 +90,6 @@ in }) ]; - # https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards - dashboards.settings.providers = [ - { - name = "node"; - options.path = ./dashboards/node.json; - } - { - name = "ntfy"; - options.path = ./dashboards/ntfy.json; - } - { - name = "endlessh"; - options.path = ./dashboards/endlessh.json; - } - { - name = "unbound"; - options.path = ./dashboards/unbound.json; - } - { - name = "nginx"; - options.path = ./dashboards/nginx.json; - } - { - name = "postgersql"; - options.path = ./dashboards/postgresql.json; - } - { - name = "redis"; - options.path = ./dashboards/redis.json; - } - ]; - # https://grafana.com/docs/grafana/latest/administration/provisioning/#alerting alerting.contactPoints.settings.contactPoints = [ { @@ -120,7 +98,7 @@ in { uid = 1; type = "prometheus-alertmanager"; - settings.url = "https://${config.nixfiles.modules.alertmanager.domain}"; + settings.url = "https://${config.nixfiles.modules.mimir.domain}/alertmanager"; } ]; } @@ -128,122 +106,22 @@ in }; }; - loki.configuration.ruler = { - external_url = "https://${config.nixfiles.modules.grafana.domain}"; - alertmanager_url = "https://${config.nixfiles.modules.alertmanager.domain}"; - enable_alertmanager_v2 = true; + mimir.configuration = { + ruler = { + alertmanager_url = "https://${config.nixfiles.modules.mimir.domain}/alertmanager"; + }; + alertmanager = { + external_url = "https://${config.nixfiles.modules.mimir.domain}/alertmanager"; + }; }; - prometheus = { - scrapeConfigs = - lib.mapAttrsToList - (name: value: { - job_name = name; - static_configs = [ - { - targets = map ( - host: - lib.concatStringsSep ":" [ - (if lib.isAttrs host then host.hostname else host) - (toString value.port) - ] - ) value.hosts; - } - ]; - relabel_configs = [ - { - source_labels = [ "__address__" ]; - regex = "([^:]+):\\d+"; - target_label = "instance"; - } - ] ++ lib.optionals (lib.hasAttr "relabel" value) value.relabel; - }) - ( - with lib.my.configurations; - { - promtail = { - hosts = [ - manwe - tulkas - varda - yavanna - ]; - inherit (config.nixfiles.modules.promtail) port; - }; - ntfy = { - hosts = [ manwe ]; - inherit (config.nixfiles.modules.ntfy.prometheus) port; - }; - soju = { - hosts = [ "127.0.0.1" ]; - inherit (config.nixfiles.modules.soju.prometheus) port; - }; - endlessh-go = { - hosts = [ - manwe - tulkas - varda - yavanna - ]; - inherit (config.services.endlessh-go.prometheus) port; - }; - exportarr-prowlarr = { - hosts = [ yavanna ]; - inherit (config.services.prometheus.exporters.exportarr-prowlarr) port; - }; - exportarr-lidarr = { - hosts = [ yavanna ]; - inherit (config.services.prometheus.exporters.exportarr-lidarr) port; - }; - nginx = { - hosts = [ - manwe - yavanna - ]; - inherit (config.services.prometheus.exporters.nginx) port; - }; - node = { - hosts = [ - manwe - tulkas - varda - yavanna - ]; - inherit (config.services.prometheus.exporters.node) port; - }; - postgres = { - hosts = [ manwe ]; - inherit (config.services.prometheus.exporters.postgres) port; - }; - redis = { - hosts = [ manwe ]; - inherit (config.services.prometheus.exporters.redis) port; - }; - unbound = { - hosts = [ manwe ]; - inherit (config.services.prometheus.exporters.unbound) port; - }; - wireguard = { - hosts = [ manwe ]; - inherit (config.services.prometheus.exporters.wireguard) port; - }; - } - ); - - ruleFiles = [ - ./rules/nginx.yaml - ./rules/node.yaml - ./rules/postgres.yaml - ./rules/redis.yaml - ]; - - alertmanagers = [ - { - scheme = "https"; - static_configs = [ { targets = [ config.nixfiles.modules.alertmanager.domain ]; } ]; - } - ]; + loki.configuration = { + ruler = { + alertmanager_url = "https://${config.nixfiles.modules.mimir.domain}/alertmanager"; + }; }; + + tempo.settings = { }; }; }; } |